vendor/clang/clang-r126079

371 files changed, 72718 insertions, 36139 deletions

diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index 4591a0f3c55c..945dfb87f297 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -20,7 +20,9 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExternalASTSource.h"
+#include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/RecordLayout.h"
+#include "clang/AST/Mangle.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -50,7 +52,7 @@ ASTContext::CanonicalTemplateTemplateParm::Profile(llvm::FoldingSetNodeID &ID,
                                                TemplateTemplateParmDecl *Parm) {
   ID.AddInteger(Parm->getDepth());
   ID.AddInteger(Parm->getPosition());
-  // FIXME: Parameter pack
+  ID.AddBoolean(Parm->isParameterPack());
 
   TemplateParameterList *Params = Parm->getTemplateParameters();
   ID.AddInteger(Params->size());
@@ -65,8 +67,15 @@ ASTContext::CanonicalTemplateTemplateParm::Profile(llvm::FoldingSetNodeID &ID,
     
     if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
       ID.AddInteger(1);
-      // FIXME: Parameter pack
+      ID.AddBoolean(NTTP->isParameterPack());
       ID.AddPointer(NTTP->getType().getAsOpaquePtr());
+      if (NTTP->isExpandedParameterPack()) {
+        ID.AddBoolean(true);
+        ID.AddInteger(NTTP->getNumExpansionTypes());
+        for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I)
+          ID.AddPointer(NTTP->getExpansionType(I).getAsOpaquePtr());
+      } else 
+        ID.AddBoolean(false);
       continue;
     }
     
@@ -78,7 +87,7 @@ ASTContext::CanonicalTemplateTemplateParm::Profile(llvm::FoldingSetNodeID &ID,
 
 TemplateTemplateParmDecl *
 ASTContext::getCanonicalTemplateTemplateParmDecl(
-                                                 TemplateTemplateParmDecl *TTP) {
+                                          TemplateTemplateParmDecl *TTP) const {
   // Check if we already have a canonical template template parameter.
   llvm::FoldingSetNodeID ID;
   CanonicalTemplateTemplateParm::Profile(ID, TTP);
@@ -102,14 +111,40 @@ ASTContext::getCanonicalTemplateTemplateParmDecl(
                                                TTP->getIndex(), 0, false,
                                                TTP->isParameterPack()));
     else if (NonTypeTemplateParmDecl *NTTP
-             = dyn_cast<NonTypeTemplateParmDecl>(*P))
-      CanonParams.push_back(
-            NonTypeTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
-                                            SourceLocation(), NTTP->getDepth(),
-                                            NTTP->getPosition(), 0, 
-                                            getCanonicalType(NTTP->getType()),
-                                            0));
-    else
+             = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
+      QualType T = getCanonicalType(NTTP->getType());
+      TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(T);
+      NonTypeTemplateParmDecl *Param;
+      if (NTTP->isExpandedParameterPack()) {
+        llvm::SmallVector<QualType, 2> ExpandedTypes;
+        llvm::SmallVector<TypeSourceInfo *, 2> ExpandedTInfos;
+        for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I) {
+          ExpandedTypes.push_back(getCanonicalType(NTTP->getExpansionType(I)));
+          ExpandedTInfos.push_back(
+                                getTrivialTypeSourceInfo(ExpandedTypes.back()));
+        }
+        
+        Param = NonTypeTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
+                                                SourceLocation(), 
+                                                NTTP->getDepth(),
+                                                NTTP->getPosition(), 0, 
+                                                T,
+                                                TInfo,
+                                                ExpandedTypes.data(),
+                                                ExpandedTypes.size(),
+                                                ExpandedTInfos.data());
+      } else {
+        Param = NonTypeTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
+                                                SourceLocation(), 
+                                                NTTP->getDepth(),
+                                                NTTP->getPosition(), 0, 
+                                                T,
+                                                NTTP->isParameterPack(),
+                                                TInfo);
+      }
+      CanonParams.push_back(Param);
+
+    } else
       CanonParams.push_back(getCanonicalTemplateTemplateParmDecl(
                                            cast<TemplateTemplateParmDecl>(*P)));
   }
@@ -117,7 +152,9 @@ ASTContext::getCanonicalTemplateTemplateParmDecl(
   TemplateTemplateParmDecl *CanonTTP
     = TemplateTemplateParmDecl::Create(*this, getTranslationUnitDecl(), 
                                        SourceLocation(), TTP->getDepth(),
-                                       TTP->getPosition(), 0,
+                                       TTP->getPosition(), 
+                                       TTP->isParameterPack(),
+                                       0,
                          TemplateParameterList::Create(*this, SourceLocation(),
                                                        SourceLocation(),
                                                        CanonParams.data(),
@@ -160,12 +197,13 @@ ASTContext::ASTContext(const LangOptions& LOpts, SourceManager &SM,
   CFConstantStringTypeDecl(0), NSConstantStringTypeDecl(0),
   ObjCFastEnumerationStateTypeDecl(0), FILEDecl(0), jmp_bufDecl(0),
   sigjmp_bufDecl(0), BlockDescriptorType(0), BlockDescriptorExtendedType(0),
+  cudaConfigureCallDecl(0),
   NullTypeSourceInfo(QualType()),
   SourceMgr(SM), LangOpts(LOpts), ABI(createCXXABI(t)), Target(t),
   Idents(idents), Selectors(sels),
   BuiltinInfo(builtins),
   DeclarationNames(*this),
-  ExternalSource(0), PrintingPolicy(LOpts),
+  ExternalSource(0), Listener(0), PrintingPolicy(LOpts),
   LastSDM(0, 0),
   UniqueBlockByRefTypeID(0), UniqueBlockParmTypeID(0) {
   ObjCIdRedefinitionType = QualType();
@@ -314,9 +352,12 @@ void ASTContext::InitBuiltinTypes() {
   InitBuiltinType(Int128Ty,            BuiltinType::Int128);
   InitBuiltinType(UnsignedInt128Ty,    BuiltinType::UInt128);
 
-  if (LangOpts.CPlusPlus) // C++ 3.9.1p5
-    InitBuiltinType(WCharTy,           BuiltinType::WChar);
-  else // C99
+  if (LangOpts.CPlusPlus) { // C++ 3.9.1p5
+    if (!LangOpts.ShortWChar)
+      InitBuiltinType(WCharTy,           BuiltinType::WChar_S);
+    else  // -fshort-wchar makes wchar_t be unsigned.
+      InitBuiltinType(WCharTy,           BuiltinType::WChar_U);
+  } else // C99
     WCharTy = getFromTargetType(Target.getWCharType());
 
   if (LangOpts.CPlusPlus) // C++0x 3.9.1p5, extension for C++
@@ -329,9 +370,6 @@ void ASTContext::InitBuiltinTypes() {
   else // C99
     Char32Ty = getFromTargetType(Target.getChar32Type());
 
-  // Placeholder type for functions.
-  InitBuiltinType(OverloadTy,          BuiltinType::Overload);
-
   // Placeholder type for type-dependent expressions whose type is
   // completely unknown. No code should ever check a type against
   // DependentTy and users should never see it; however, it is here to
@@ -339,9 +377,8 @@ void ASTContext::InitBuiltinTypes() {
   // expressions.
   InitBuiltinType(DependentTy,         BuiltinType::Dependent);
 
-  // Placeholder type for C++0x auto declarations whose real type has
-  // not yet been deduced.
-  InitBuiltinType(UndeducedAutoTy, BuiltinType::UndeducedAuto);
+  // Placeholder type for functions.
+  InitBuiltinType(OverloadTy,          BuiltinType::Overload);
 
   // C99 6.2.5p11.
   FloatComplexTy      = getComplexType(FloatTy);
@@ -369,6 +406,10 @@ void ASTContext::InitBuiltinTypes() {
   InitBuiltinType(NullPtrTy,           BuiltinType::NullPtr);
 }
 
+Diagnostic &ASTContext::getDiagnostics() const {
+  return SourceMgr.getDiagnostics();
+}
+
 AttrVec& ASTContext::getDeclAttrs(const Decl *D) {
   AttrVec *&Result = DeclAttrs[D];
   if (!Result) {
@@ -525,12 +566,33 @@ const llvm::fltSemantics &ASTContext::getFloatTypeSemantics(QualType T) const {
 /// this method will assert on them.
 /// If @p RefAsPointee, references are treated like their underlying type
 /// (for alignof), else they're treated like pointers (for CodeGen).
-CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) {
+CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) const {
   unsigned Align = Target.getCharWidth();
 
-  Align = std::max(Align, D->getMaxAlignment());
+  bool UseAlignAttrOnly = false;
+  if (unsigned AlignFromAttr = D->getMaxAlignment()) {
+    Align = AlignFromAttr;
+
+    // __attribute__((aligned)) can increase or decrease alignment
+    // *except* on a struct or struct member, where it only increases
+    // alignment unless 'packed' is also specified.
+    //
+    // It is an error for [[align]] to decrease alignment, so we can
+    // ignore that possibility;  Sema should diagnose it.
+    if (isa<FieldDecl>(D)) {
+      UseAlignAttrOnly = D->hasAttr<PackedAttr>() ||
+        cast<FieldDecl>(D)->getParent()->hasAttr<PackedAttr>();
+    } else {
+      UseAlignAttrOnly = true;
+    }
+  }
 
-  if (const ValueDecl *VD = dyn_cast<ValueDecl>(D)) {
+  // If we're using the align attribute only, just ignore everything
+  // else about the declaration and its type.
+  if (UseAlignAttrOnly) {
+    // do nothing
+
+  } else if (const ValueDecl *VD = dyn_cast<ValueDecl>(D)) {
     QualType T = VD->getType();
     if (const ReferenceType* RT = T->getAs<ReferenceType>()) {
       if (RefAsPointee)
@@ -539,41 +601,61 @@ CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) {
         T = getPointerType(RT->getPointeeType());
     }
     if (!T->isIncompleteType() && !T->isFunctionType()) {
+      // Adjust alignments of declarations with array type by the
+      // large-array alignment on the target.
       unsigned MinWidth = Target.getLargeArrayMinWidth();
-      unsigned ArrayAlign = Target.getLargeArrayAlign();
-      if (isa<VariableArrayType>(T) && MinWidth != 0)
-        Align = std::max(Align, ArrayAlign);
-      if (ConstantArrayType *CT = dyn_cast<ConstantArrayType>(T)) {
-        unsigned Size = getTypeSize(CT);
-        if (MinWidth != 0 && MinWidth <= Size)
-          Align = std::max(Align, ArrayAlign);
+      const ArrayType *arrayType;
+      if (MinWidth && (arrayType = getAsArrayType(T))) {
+        if (isa<VariableArrayType>(arrayType))
+          Align = std::max(Align, Target.getLargeArrayAlign());
+        else if (isa<ConstantArrayType>(arrayType) &&
+                 MinWidth <= getTypeSize(cast<ConstantArrayType>(arrayType)))
+          Align = std::max(Align, Target.getLargeArrayAlign());
+
+        // Walk through any array types while we're at it.
+        T = getBaseElementType(arrayType);
       }
-      // Incomplete or function types default to 1.
-      while (isa<VariableArrayType>(T) || isa<IncompleteArrayType>(T))
-        T = cast<ArrayType>(T)->getElementType();
-
       Align = std::max(Align, getPreferredTypeAlign(T.getTypePtr()));
     }
-    if (const FieldDecl *FD = dyn_cast<FieldDecl>(VD)) {
-      // In the case of a field in a packed struct, we want the minimum
-      // of the alignment of the field and the alignment of the struct.
-      Align = std::min(Align,
-        getPreferredTypeAlign(FD->getParent()->getTypeForDecl()));
+
+    // Fields can be subject to extra alignment constraints, like if
+    // the field is packed, the struct is packed, or the struct has a
+    // a max-field-alignment constraint (#pragma pack).  So calculate
+    // the actual alignment of the field within the struct, and then
+    // (as we're expected to) constrain that by the alignment of the type.
+    if (const FieldDecl *field = dyn_cast<FieldDecl>(VD)) {
+      // So calculate the alignment of the field.
+      const ASTRecordLayout &layout = getASTRecordLayout(field->getParent());
+
+      // Start with the record's overall alignment.
+      unsigned fieldAlign = toBits(layout.getAlignment());
+
+      // Use the GCD of that and the offset within the record.
+      uint64_t offset = layout.getFieldOffset(field->getFieldIndex());
+      if (offset > 0) {
+        // Alignment is always a power of 2, so the GCD will be a power of 2,
+        // which means we get to do this crazy thing instead of Euclid's.
+        uint64_t lowBitOfOffset = offset & (~offset + 1);
+        if (lowBitOfOffset < fieldAlign)
+          fieldAlign = static_cast<unsigned>(lowBitOfOffset);
+      }
+
+      Align = std::min(Align, fieldAlign);
     }
   }
 
-  return CharUnits::fromQuantity(Align / Target.getCharWidth());
+  return toCharUnitsFromBits(Align);
 }
 
 std::pair<CharUnits, CharUnits>
-ASTContext::getTypeInfoInChars(const Type *T) {
+ASTContext::getTypeInfoInChars(const Type *T) const {
   std::pair<uint64_t, unsigned> Info = getTypeInfo(T);
-  return std::make_pair(CharUnits::fromQuantity(Info.first / getCharWidth()),
-                        CharUnits::fromQuantity(Info.second / getCharWidth()));
+  return std::make_pair(toCharUnitsFromBits(Info.first),
+                        toCharUnitsFromBits(Info.second));
 }
 
 std::pair<CharUnits, CharUnits>
-ASTContext::getTypeInfoInChars(QualType T) {
+ASTContext::getTypeInfoInChars(QualType T) const {
   return getTypeInfoInChars(T.getTypePtr());
 }
 
@@ -584,7 +666,7 @@ ASTContext::getTypeInfoInChars(QualType T) {
 /// alignment requirements: getPointerInfo should take an AddrSpace, this
 /// should take a QualType, &c.
 std::pair<uint64_t, unsigned>
-ASTContext::getTypeInfo(const Type *T) {
+ASTContext::getTypeInfo(const Type *T) const {
   uint64_t Width=0;
   unsigned Align=8;
   switch (T->getTypeClass()) {
@@ -652,7 +734,8 @@ ASTContext::getTypeInfo(const Type *T) {
       Width = Target.getCharWidth();
       Align = Target.getCharAlign();
       break;
-    case BuiltinType::WChar:
+    case BuiltinType::WChar_S:
+    case BuiltinType::WChar_U:
       Width = Target.getWCharWidth();
       Align = Target.getWCharAlign();
       break;
@@ -760,8 +843,8 @@ ASTContext::getTypeInfo(const Type *T) {
   case Type::ObjCInterface: {
     const ObjCInterfaceType *ObjCI = cast<ObjCInterfaceType>(T);
     const ASTRecordLayout &Layout = getASTObjCInterfaceLayout(ObjCI->getDecl());
-    Width = Layout.getSize();
-    Align = Layout.getAlignment();
+    Width = toBits(Layout.getSize());
+    Align = toBits(Layout.getAlignment());
     break;
   }
   case Type::Record:
@@ -779,8 +862,8 @@ ASTContext::getTypeInfo(const Type *T) {
 
     const RecordType *RT = cast<RecordType>(TT);
     const ASTRecordLayout &Layout = getASTRecordLayout(RT->getDecl());
-    Width = Layout.getSize();
-    Align = Layout.getAlignment();
+    Width = toBits(Layout.getSize());
+    Align = toBits(Layout.getAlignment());
     break;
   }
 
@@ -788,11 +871,26 @@ ASTContext::getTypeInfo(const Type *T) {
     return getTypeInfo(cast<SubstTemplateTypeParmType>(T)->
                        getReplacementType().getTypePtr());
 
+  case Type::Auto: {
+    const AutoType *A = cast<AutoType>(T);
+    assert(A->isDeduced() && "Cannot request the size of a dependent type");
+    return getTypeInfo(cast<AutoType>(T)->getDeducedType().getTypePtr());
+  }
+
+  case Type::Paren:
+    return getTypeInfo(cast<ParenType>(T)->getInnerType().getTypePtr());
+
   case Type::Typedef: {
     const TypedefDecl *Typedef = cast<TypedefType>(T)->getDecl();
     std::pair<uint64_t, unsigned> Info
       = getTypeInfo(Typedef->getUnderlyingType().getTypePtr());
-    Align = std::max(Typedef->getMaxAlignment(), Info.second);
+    // If the typedef has an aligned attribute on it, it overrides any computed
+    // alignment we have.  This violates the GCC documentation (which says that
+    // attribute(aligned) can only round up) but matches its implementation.
+    if (unsigned AttrAlign = Typedef->getMaxAlignment())
+      Align = AttrAlign;
+    else
+      Align = Info.second;
     Width = Info.first;
     break;
   }
@@ -811,6 +909,10 @@ ASTContext::getTypeInfo(const Type *T) {
   case Type::Elaborated:
     return getTypeInfo(cast<ElaboratedType>(T)->getNamedType().getTypePtr());
 
+  case Type::Attributed:
+    return getTypeInfo(
+                  cast<AttributedType>(T)->getEquivalentType().getTypePtr());
+
   case Type::TemplateSpecialization:
     assert(getCanonicalType(T) != T &&
            "Cannot request the size of a dependent type");
@@ -824,29 +926,39 @@ ASTContext::getTypeInfo(const Type *T) {
   return std::make_pair(Width, Align);
 }
 
+/// toCharUnitsFromBits - Convert a size in bits to a size in characters.
+CharUnits ASTContext::toCharUnitsFromBits(int64_t BitSize) const {
+  return CharUnits::fromQuantity(BitSize / getCharWidth());
+}
+
+/// toBits - Convert a size in characters to a size in characters.
+int64_t ASTContext::toBits(CharUnits CharSize) const {
+  return CharSize.getQuantity() * getCharWidth();
+}
+
 /// getTypeSizeInChars - Return the size of the specified type, in characters.
 /// This method does not work on incomplete types.
-CharUnits ASTContext::getTypeSizeInChars(QualType T) {
-  return CharUnits::fromQuantity(getTypeSize(T) / getCharWidth());
+CharUnits ASTContext::getTypeSizeInChars(QualType T) const {
+  return toCharUnitsFromBits(getTypeSize(T));
 }
-CharUnits ASTContext::getTypeSizeInChars(const Type *T) {
-  return CharUnits::fromQuantity(getTypeSize(T) / getCharWidth());
+CharUnits ASTContext::getTypeSizeInChars(const Type *T) const {
+  return toCharUnitsFromBits(getTypeSize(T));
 }
 
 /// getTypeAlignInChars - Return the ABI-specified alignment of a type, in 
 /// characters. This method does not work on incomplete types.
-CharUnits ASTContext::getTypeAlignInChars(QualType T) {
-  return CharUnits::fromQuantity(getTypeAlign(T) / getCharWidth());
+CharUnits ASTContext::getTypeAlignInChars(QualType T) const {
+  return toCharUnitsFromBits(getTypeAlign(T));
 }
-CharUnits ASTContext::getTypeAlignInChars(const Type *T) {
-  return CharUnits::fromQuantity(getTypeAlign(T) / getCharWidth());
+CharUnits ASTContext::getTypeAlignInChars(const Type *T) const {
+  return toCharUnitsFromBits(getTypeAlign(T));
 }
 
 /// getPreferredTypeAlign - Return the "preferred" alignment of the specified
 /// type for the current target in bits.  This can be different than the ABI
 /// alignment in cases where it is beneficial for performance to overalign
 /// a data type.
-unsigned ASTContext::getPreferredTypeAlign(const Type *T) {
+unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   unsigned ABIAlign = getTypeAlign(T);
 
   // Double and long long should be naturally aligned if possible.
@@ -863,7 +975,7 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) {
 /// Collect all ivars, including those synthesized, in the current class.
 ///
 void ASTContext::ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
-                                 llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
+                            llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) const {
   // FIXME. This need be removed but there are two many places which
   // assume const-ness of ObjCInterfaceDecl
   ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl *>(OI);
@@ -880,7 +992,7 @@ void ASTContext::ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
 ///
 void ASTContext::DeepCollectObjCIvars(const ObjCInterfaceDecl *OI,
                                       bool leafClass,
-                                llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
+                            llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) const {
   if (const ObjCInterfaceDecl *SuperClass = OI->getSuperClass())
     DeepCollectObjCIvars(SuperClass, false, Ivars);
   if (!leafClass) {
@@ -940,7 +1052,7 @@ void ASTContext::CollectInheritedProtocols(const Decl *CDecl,
   }
 }
 
-unsigned ASTContext::CountNonClassIvars(const ObjCInterfaceDecl *OI) {
+unsigned ASTContext::CountNonClassIvars(const ObjCInterfaceDecl *OI) const {
   unsigned count = 0;  
   // Count ivars declared in class extension.
   for (const ObjCCategoryDecl *CDecl = OI->getFirstClassExtension(); CDecl;
@@ -985,6 +1097,25 @@ void ASTContext::setObjCImplementation(ObjCCategoryDecl *CatD,
   ObjCImpls[CatD] = ImplD;
 }
 
+/// \brief Get the copy initialization expression of VarDecl,or NULL if 
+/// none exists.
+Expr *ASTContext::getBlockVarCopyInits(const VarDecl*VD) {
+  assert(VD && "Passed null params");
+  assert(VD->hasAttr<BlocksAttr>() && 
+         "getBlockVarCopyInits - not __block var");
+  llvm::DenseMap<const VarDecl*, Expr*>::iterator
+    I = BlockVarCopyInits.find(VD);
+  return (I != BlockVarCopyInits.end()) ? cast<Expr>(I->second) : 0;
+}
+
+/// \brief Set the copy inialization expression of a block var decl.
+void ASTContext::setBlockVarCopyInits(VarDecl*VD, Expr* Init) {
+  assert(VD && Init && "Passed null params");
+  assert(VD->hasAttr<BlocksAttr>() && 
+         "setBlockVarCopyInits - not __block var");
+  BlockVarCopyInits[VD] = Init;
+}
+
 /// \brief Allocate an uninitialized TypeSourceInfo.
 ///
 /// The caller should initialize the memory held by TypeSourceInfo using
@@ -994,7 +1125,7 @@ void ASTContext::setObjCImplementation(ObjCCategoryDecl *CatD,
 /// should refer to how the declarator was written in source code, not to
 /// what type semantic analysis resolved the declarator to.
 TypeSourceInfo *ASTContext::CreateTypeSourceInfo(QualType T,
-                                                 unsigned DataSize) {
+                                                 unsigned DataSize) const {
   if (!DataSize)
     DataSize = TypeLoc::getFullDataSizeForType(T);
   else
@@ -1008,19 +1139,20 @@ TypeSourceInfo *ASTContext::CreateTypeSourceInfo(QualType T,
 }
 
 TypeSourceInfo *ASTContext::getTrivialTypeSourceInfo(QualType T,
-                                                     SourceLocation L) {
+                                                     SourceLocation L) const {
   TypeSourceInfo *DI = CreateTypeSourceInfo(T);
-  DI->getTypeLoc().initialize(L);
+  DI->getTypeLoc().initialize(const_cast<ASTContext &>(*this), L);
   return DI;
 }
 
 const ASTRecordLayout &
-ASTContext::getASTObjCInterfaceLayout(const ObjCInterfaceDecl *D) {
+ASTContext::getASTObjCInterfaceLayout(const ObjCInterfaceDecl *D) const {
   return getObjCLayout(D, 0);
 }
 
 const ASTRecordLayout &
-ASTContext::getASTObjCImplementationLayout(const ObjCImplementationDecl *D) {
+ASTContext::getASTObjCImplementationLayout(
+                                        const ObjCImplementationDecl *D) const {
   return getObjCLayout(D->getClassInterface(), D);
 }
 
@@ -1028,38 +1160,38 @@ ASTContext::getASTObjCImplementationLayout(const ObjCImplementationDecl *D) {
 //                   Type creation/memoization methods
 //===----------------------------------------------------------------------===//
 
-QualType ASTContext::getExtQualType(const Type *TypeNode, Qualifiers Quals) {
-  unsigned Fast = Quals.getFastQualifiers();
-  Quals.removeFastQualifiers();
+QualType
+ASTContext::getExtQualType(const Type *baseType, Qualifiers quals) const {
+  unsigned fastQuals = quals.getFastQualifiers();
+  quals.removeFastQualifiers();
 
   // Check if we've already instantiated this type.
   llvm::FoldingSetNodeID ID;
-  ExtQuals::Profile(ID, TypeNode, Quals);
-  void *InsertPos = 0;
-  if (ExtQuals *EQ = ExtQualNodes.FindNodeOrInsertPos(ID, InsertPos)) {
-    assert(EQ->getQualifiers() == Quals);
-    QualType T = QualType(EQ, Fast);
-    return T;
+  ExtQuals::Profile(ID, baseType, quals);
+  void *insertPos = 0;
+  if (ExtQuals *eq = ExtQualNodes.FindNodeOrInsertPos(ID, insertPos)) {
+    assert(eq->getQualifiers() == quals);
+    return QualType(eq, fastQuals);
   }
 
-  ExtQuals *New = new (*this, TypeAlignment) ExtQuals(*this, TypeNode, Quals);
-  ExtQualNodes.InsertNode(New, InsertPos);
-  QualType T = QualType(New, Fast);
-  return T;
-}
-
-QualType ASTContext::getVolatileType(QualType T) {
-  QualType CanT = getCanonicalType(T);
-  if (CanT.isVolatileQualified()) return T;
+  // If the base type is not canonical, make the appropriate canonical type.
+  QualType canon;
+  if (!baseType->isCanonicalUnqualified()) {
+    SplitQualType canonSplit = baseType->getCanonicalTypeInternal().split();
+    canonSplit.second.addConsistentQualifiers(quals);
+    canon = getExtQualType(canonSplit.first, canonSplit.second);
 
-  QualifierCollector Quals;
-  const Type *TypeNode = Quals.strip(T);
-  Quals.addVolatile();
+    // Re-find the insert position.
+    (void) ExtQualNodes.FindNodeOrInsertPos(ID, insertPos);
+  }
 
-  return getExtQualType(TypeNode, Quals);
+  ExtQuals *eq = new (*this, TypeAlignment) ExtQuals(baseType, canon, quals);
+  ExtQualNodes.InsertNode(eq, insertPos);
+  return QualType(eq, fastQuals);
 }
 
-QualType ASTContext::getAddrSpaceQualType(QualType T, unsigned AddressSpace) {
+QualType
+ASTContext::getAddrSpaceQualType(QualType T, unsigned AddressSpace) const {
   QualType CanT = getCanonicalType(T);
   if (CanT.getAddressSpace() == AddressSpace)
     return T;
@@ -1079,13 +1211,13 @@ QualType ASTContext::getAddrSpaceQualType(QualType T, unsigned AddressSpace) {
 }
 
 QualType ASTContext::getObjCGCQualType(QualType T,
-                                       Qualifiers::GC GCAttr) {
+                                       Qualifiers::GC GCAttr) const {
   QualType CanT = getCanonicalType(T);
   if (CanT.getObjCGCAttr() == GCAttr)
     return T;
 
-  if (T->isPointerType()) {
-    QualType Pointee = T->getAs<PointerType>()->getPointeeType();
+  if (const PointerType *ptr = T->getAs<PointerType>()) {
+    QualType Pointee = ptr->getPointeeType();
     if (Pointee->isAnyPointerType()) {
       QualType ResultType = getObjCGCQualType(Pointee, GCAttr);
       return getPointerType(ResultType);
@@ -1106,79 +1238,28 @@ QualType ASTContext::getObjCGCQualType(QualType T,
   return getExtQualType(TypeNode, Quals);
 }
 
-static QualType getExtFunctionType(ASTContext& Context, QualType T,
-                                        const FunctionType::ExtInfo &Info) {
-  QualType ResultType;
-  if (const PointerType *Pointer = T->getAs<PointerType>()) {
-    QualType Pointee = Pointer->getPointeeType();
-    ResultType = getExtFunctionType(Context, Pointee, Info);
-    if (ResultType == Pointee)
-      return T;
-
-    ResultType = Context.getPointerType(ResultType);
-  } else if (const BlockPointerType *BlockPointer
-                                              = T->getAs<BlockPointerType>()) {
-    QualType Pointee = BlockPointer->getPointeeType();
-    ResultType = getExtFunctionType(Context, Pointee, Info);
-    if (ResultType == Pointee)
-      return T;
-
-    ResultType = Context.getBlockPointerType(ResultType);
-  } else if (const MemberPointerType *MemberPointer 
-                                            = T->getAs<MemberPointerType>()) {
-    QualType Pointee = MemberPointer->getPointeeType();
-    ResultType = getExtFunctionType(Context, Pointee, Info);
-    if (ResultType == Pointee)
-      return T;
-    
-    ResultType = Context.getMemberPointerType(ResultType, 
-                                              MemberPointer->getClass());
-   } else if (const FunctionType *F = T->getAs<FunctionType>()) {
-    if (F->getExtInfo() == Info)
-      return T;
-
-    if (const FunctionNoProtoType *FNPT = dyn_cast<FunctionNoProtoType>(F)) {
-      ResultType = Context.getFunctionNoProtoType(FNPT->getResultType(),
-                                                  Info);
-    } else {
-      const FunctionProtoType *FPT = cast<FunctionProtoType>(F);
-      ResultType
-        = Context.getFunctionType(FPT->getResultType(), FPT->arg_type_begin(),
-                                  FPT->getNumArgs(), FPT->isVariadic(),
-                                  FPT->getTypeQuals(),
-                                  FPT->hasExceptionSpec(),
-                                  FPT->hasAnyExceptionSpec(),
-                                  FPT->getNumExceptions(),
-                                  FPT->exception_begin(),
-                                  Info);
-    }
-  } else
+const FunctionType *ASTContext::adjustFunctionType(const FunctionType *T,
+                                                   FunctionType::ExtInfo Info) {
+  if (T->getExtInfo() == Info)
     return T;
 
-  return Context.getQualifiedType(ResultType, T.getLocalQualifiers());
-}
-
-QualType ASTContext::getNoReturnType(QualType T, bool AddNoReturn) {
-  FunctionType::ExtInfo Info = getFunctionExtInfo(T);
-  return getExtFunctionType(*this, T,
-                                 Info.withNoReturn(AddNoReturn));
-}
-
-QualType ASTContext::getCallConvType(QualType T, CallingConv CallConv) {
-  FunctionType::ExtInfo Info = getFunctionExtInfo(T);
-  return getExtFunctionType(*this, T,
-                            Info.withCallingConv(CallConv));
-}
+  QualType Result;
+  if (const FunctionNoProtoType *FNPT = dyn_cast<FunctionNoProtoType>(T)) {
+    Result = getFunctionNoProtoType(FNPT->getResultType(), Info);
+  } else {
+    const FunctionProtoType *FPT = cast<FunctionProtoType>(T);
+    FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+    EPI.ExtInfo = Info;
+    Result = getFunctionType(FPT->getResultType(), FPT->arg_type_begin(),
+                             FPT->getNumArgs(), EPI);
+  }
 
-QualType ASTContext::getRegParmType(QualType T, unsigned RegParm) {
-  FunctionType::ExtInfo Info = getFunctionExtInfo(T);
-  return getExtFunctionType(*this, T,
-                                 Info.withRegParm(RegParm));
+  return cast<FunctionType>(Result.getTypePtr());
 }
 
 /// getComplexType - Return the uniqued reference to the type for a complex
 /// number with the specified element type.
-QualType ASTContext::getComplexType(QualType T) {
+QualType ASTContext::getComplexType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
@@ -1196,7 +1277,7 @@ QualType ASTContext::getComplexType(QualType T) {
 
     // Get the new insert position for the node we care about.
     ComplexType *NewIP = ComplexTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   ComplexType *New = new (*this, TypeAlignment) ComplexType(T, Canonical);
   Types.push_back(New);
@@ -1206,7 +1287,7 @@ QualType ASTContext::getComplexType(QualType T) {
 
 /// getPointerType - Return the uniqued reference to the type for a pointer to
 /// the specified type.
-QualType ASTContext::getPointerType(QualType T) {
+QualType ASTContext::getPointerType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
@@ -1224,7 +1305,7 @@ QualType ASTContext::getPointerType(QualType T) {
 
     // Get the new insert position for the node we care about.
     PointerType *NewIP = PointerTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   PointerType *New = new (*this, TypeAlignment) PointerType(T, Canonical);
   Types.push_back(New);
@@ -1234,7 +1315,7 @@ QualType ASTContext::getPointerType(QualType T) {
 
 /// getBlockPointerType - Return the uniqued reference to the type for
 /// a pointer to the specified block.
-QualType ASTContext::getBlockPointerType(QualType T) {
+QualType ASTContext::getBlockPointerType(QualType T) const {
   assert(T->isFunctionType() && "block of function types only");
   // Unique pointers, to guarantee there is only one block of a particular
   // structure.
@@ -1255,7 +1336,7 @@ QualType ASTContext::getBlockPointerType(QualType T) {
     // Get the new insert position for the node we care about.
     BlockPointerType *NewIP =
       BlockPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   BlockPointerType *New
     = new (*this, TypeAlignment) BlockPointerType(T, Canonical);
@@ -1266,7 +1347,8 @@ QualType ASTContext::getBlockPointerType(QualType T) {
 
 /// getLValueReferenceType - Return the uniqued reference to the type for an
 /// lvalue reference to the specified type.
-QualType ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) {
+QualType
+ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
@@ -1289,7 +1371,7 @@ QualType ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) {
     // Get the new insert position for the node we care about.
     LValueReferenceType *NewIP =
       LValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   LValueReferenceType *New
@@ -1303,7 +1385,7 @@ QualType ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) {
 
 /// getRValueReferenceType - Return the uniqued reference to the type for an
 /// rvalue reference to the specified type.
-QualType ASTContext::getRValueReferenceType(QualType T) {
+QualType ASTContext::getRValueReferenceType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
@@ -1326,7 +1408,7 @@ QualType ASTContext::getRValueReferenceType(QualType T) {
     // Get the new insert position for the node we care about.
     RValueReferenceType *NewIP =
       RValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   RValueReferenceType *New
@@ -1338,7 +1420,7 @@ QualType ASTContext::getRValueReferenceType(QualType T) {
 
 /// getMemberPointerType - Return the uniqued reference to the type for a
 /// member pointer to the specified type, in the specified class.
-QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) {
+QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
@@ -1358,7 +1440,7 @@ QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) {
     // Get the new insert position for the node we care about.
     MemberPointerType *NewIP =
       MemberPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   MemberPointerType *New
     = new (*this, TypeAlignment) MemberPointerType(T, Cls, Canonical);
@@ -1372,7 +1454,7 @@ QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) {
 QualType ASTContext::getConstantArrayType(QualType EltTy,
                                           const llvm::APInt &ArySizeIn,
                                           ArrayType::ArraySizeModifier ASM,
-                                          unsigned EltTypeQuals) {
+                                          unsigned IndexTypeQuals) const {
   assert((EltTy->isDependentType() ||
           EltTy->isIncompleteType() || EltTy->isConstantSizeType()) &&
          "Constant array of VLAs is illegal!");
@@ -1380,55 +1462,185 @@ QualType ASTContext::getConstantArrayType(QualType EltTy,
   // Convert the array size into a canonical width matching the pointer size for
   // the target.
   llvm::APInt ArySize(ArySizeIn);
-  ArySize.zextOrTrunc(Target.getPointerWidth(EltTy.getAddressSpace()));
+  ArySize =
+    ArySize.zextOrTrunc(Target.getPointerWidth(EltTy.getAddressSpace()));
 
   llvm::FoldingSetNodeID ID;
-  ConstantArrayType::Profile(ID, EltTy, ArySize, ASM, EltTypeQuals);
+  ConstantArrayType::Profile(ID, EltTy, ArySize, ASM, IndexTypeQuals);
 
   void *InsertPos = 0;
   if (ConstantArrayType *ATP =
       ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(ATP, 0);
 
-  // If the element type isn't canonical, this won't be a canonical type either,
-  // so fill in the canonical type field.
-  QualType Canonical;
-  if (!EltTy.isCanonical()) {
-    Canonical = getConstantArrayType(getCanonicalType(EltTy), ArySize,
-                                     ASM, EltTypeQuals);
+  // If the element type isn't canonical or has qualifiers, this won't
+  // be a canonical type either, so fill in the canonical type field.
+  QualType Canon;
+  if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers()) {
+    SplitQualType canonSplit = getCanonicalType(EltTy).split();
+    Canon = getConstantArrayType(QualType(canonSplit.first, 0), ArySize,
+                                 ASM, IndexTypeQuals);
+    Canon = getQualifiedType(Canon, canonSplit.second);
+
     // Get the new insert position for the node we care about.
     ConstantArrayType *NewIP =
       ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   ConstantArrayType *New = new(*this,TypeAlignment)
-    ConstantArrayType(EltTy, Canonical, ArySize, ASM, EltTypeQuals);
+    ConstantArrayType(EltTy, Canon, ArySize, ASM, IndexTypeQuals);
   ConstantArrayTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
+/// getVariableArrayDecayedType - Turns the given type, which may be
+/// variably-modified, into the corresponding type with all the known
+/// sizes replaced with [*].
+QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
+  // Vastly most common case.
+  if (!type->isVariablyModifiedType()) return type;
+
+  QualType result;
+
+  SplitQualType split = type.getSplitDesugaredType();
+  const Type *ty = split.first;
+  switch (ty->getTypeClass()) {
+#define TYPE(Class, Base)
+#define ABSTRACT_TYPE(Class, Base)
+#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
+#include "clang/AST/TypeNodes.def"
+    llvm_unreachable("didn't desugar past all non-canonical types?");
+
+  // These types should never be variably-modified.
+  case Type::Builtin:
+  case Type::Complex:
+  case Type::Vector:
+  case Type::ExtVector:
+  case Type::DependentSizedExtVector:
+  case Type::ObjCObject:
+  case Type::ObjCInterface:
+  case Type::ObjCObjectPointer:
+  case Type::Record:
+  case Type::Enum:
+  case Type::UnresolvedUsing:
+  case Type::TypeOfExpr:
+  case Type::TypeOf:
+  case Type::Decltype:
+  case Type::DependentName:
+  case Type::InjectedClassName:
+  case Type::TemplateSpecialization:
+  case Type::DependentTemplateSpecialization:
+  case Type::TemplateTypeParm:
+  case Type::SubstTemplateTypeParmPack:
+  case Type::Auto:
+  case Type::PackExpansion:
+    llvm_unreachable("type should never be variably-modified");
+
+  // These types can be variably-modified but should never need to
+  // further decay.
+  case Type::FunctionNoProto:
+  case Type::FunctionProto:
+  case Type::BlockPointer:
+  case Type::MemberPointer:
+    return type;
+
+  // These types can be variably-modified.  All these modifications
+  // preserve structure except as noted by comments.
+  // TODO: if we ever care about optimizing VLAs, there are no-op
+  // optimizations available here.
+  case Type::Pointer:
+    result = getPointerType(getVariableArrayDecayedType(
+                              cast<PointerType>(ty)->getPointeeType()));
+    break;
+
+  case Type::LValueReference: {
+    const LValueReferenceType *lv = cast<LValueReferenceType>(ty);
+    result = getLValueReferenceType(
+                 getVariableArrayDecayedType(lv->getPointeeType()),
+                                    lv->isSpelledAsLValue());
+    break;
+  }
+
+  case Type::RValueReference: {
+    const RValueReferenceType *lv = cast<RValueReferenceType>(ty);
+    result = getRValueReferenceType(
+                 getVariableArrayDecayedType(lv->getPointeeType()));
+    break;
+  }
+
+  case Type::ConstantArray: {
+    const ConstantArrayType *cat = cast<ConstantArrayType>(ty);
+    result = getConstantArrayType(
+                 getVariableArrayDecayedType(cat->getElementType()),
+                                  cat->getSize(),
+                                  cat->getSizeModifier(),
+                                  cat->getIndexTypeCVRQualifiers());
+    break;
+  }
+
+  case Type::DependentSizedArray: {
+    const DependentSizedArrayType *dat = cast<DependentSizedArrayType>(ty);
+    result = getDependentSizedArrayType(
+                 getVariableArrayDecayedType(dat->getElementType()),
+                                        dat->getSizeExpr(),
+                                        dat->getSizeModifier(),
+                                        dat->getIndexTypeCVRQualifiers(),
+                                        dat->getBracketsRange());
+    break;
+  }
+
+  // Turn incomplete types into [*] types.
+  case Type::IncompleteArray: {
+    const IncompleteArrayType *iat = cast<IncompleteArrayType>(ty);
+    result = getVariableArrayType(
+                 getVariableArrayDecayedType(iat->getElementType()),
+                                  /*size*/ 0,
+                                  ArrayType::Normal,
+                                  iat->getIndexTypeCVRQualifiers(),
+                                  SourceRange());
+    break;
+  }
+
+  // Turn VLA types into [*] types.
+  case Type::VariableArray: {
+    const VariableArrayType *vat = cast<VariableArrayType>(ty);
+    result = getVariableArrayType(
+                 getVariableArrayDecayedType(vat->getElementType()),
+                                  /*size*/ 0,
+                                  ArrayType::Star,
+                                  vat->getIndexTypeCVRQualifiers(),
+                                  vat->getBracketsRange());
+    break;
+  }
+  }
+
+  // Apply the top-level qualifiers from the original.
+  return getQualifiedType(result, split.second);
+}
+
 /// getVariableArrayType - Returns a non-unique reference to the type for a
 /// variable array of the specified element type.
 QualType ASTContext::getVariableArrayType(QualType EltTy,
                                           Expr *NumElts,
                                           ArrayType::ArraySizeModifier ASM,
-                                          unsigned EltTypeQuals,
-                                          SourceRange Brackets) {
+                                          unsigned IndexTypeQuals,
+                                          SourceRange Brackets) const {
   // Since we don't unique expressions, it isn't possible to unique VLA's
   // that have an expression provided for their size.
-  QualType CanonType;
+  QualType Canon;
   
-  if (!EltTy.isCanonical()) {
-    if (NumElts)
-      NumElts->Retain();
-    CanonType = getVariableArrayType(getCanonicalType(EltTy), NumElts, ASM,
-                                     EltTypeQuals, Brackets);
+  // Be sure to pull qualifiers off the element type.
+  if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers()) {
+    SplitQualType canonSplit = getCanonicalType(EltTy).split();
+    Canon = getVariableArrayType(QualType(canonSplit.first, 0), NumElts, ASM,
+                                 IndexTypeQuals, Brackets);
+    Canon = getQualifiedType(Canon, canonSplit.second);
   }
   
   VariableArrayType *New = new(*this, TypeAlignment)
-    VariableArrayType(EltTy, CanonType, NumElts, ASM, EltTypeQuals, Brackets);
+    VariableArrayType(EltTy, Canon, NumElts, ASM, IndexTypeQuals, Brackets);
 
   VariableArrayTypes.push_back(New);
   Types.push_back(New);
@@ -1438,109 +1650,118 @@ QualType ASTContext::getVariableArrayType(QualType EltTy,
 /// getDependentSizedArrayType - Returns a non-unique reference to
 /// the type for a dependently-sized array of the specified element
 /// type.
-QualType ASTContext::getDependentSizedArrayType(QualType EltTy,
-                                                Expr *NumElts,
+QualType ASTContext::getDependentSizedArrayType(QualType elementType,
+                                                Expr *numElements,
                                                 ArrayType::ArraySizeModifier ASM,
-                                                unsigned EltTypeQuals,
-                                                SourceRange Brackets) {
-  assert((!NumElts || NumElts->isTypeDependent() || 
-          NumElts->isValueDependent()) &&
+                                                unsigned elementTypeQuals,
+                                                SourceRange brackets) const {
+  assert((!numElements || numElements->isTypeDependent() || 
+          numElements->isValueDependent()) &&
          "Size must be type- or value-dependent!");
 
-  void *InsertPos = 0;
-  DependentSizedArrayType *Canon = 0;
-  llvm::FoldingSetNodeID ID;
-
-  if (NumElts) {
-    // Dependently-sized array types that do not have a specified
-    // number of elements will have their sizes deduced from an
-    // initializer.
-    DependentSizedArrayType::Profile(ID, *this, getCanonicalType(EltTy), ASM,
-                                     EltTypeQuals, NumElts);
-
-    Canon = DependentSizedArrayTypes.FindNodeOrInsertPos(ID, InsertPos);
+  // Dependently-sized array types that do not have a specified number
+  // of elements will have their sizes deduced from a dependent
+  // initializer.  We do no canonicalization here at all, which is okay
+  // because they can't be used in most locations.
+  if (!numElements) {
+    DependentSizedArrayType *newType
+      = new (*this, TypeAlignment)
+          DependentSizedArrayType(*this, elementType, QualType(),
+                                  numElements, ASM, elementTypeQuals,
+                                  brackets);
+    Types.push_back(newType);
+    return QualType(newType, 0);
   }
 
-  DependentSizedArrayType *New;
-  if (Canon) {
-    // We already have a canonical version of this array type; use it as
-    // the canonical type for a newly-built type.
-    New = new (*this, TypeAlignment)
-      DependentSizedArrayType(*this, EltTy, QualType(Canon, 0),
-                              NumElts, ASM, EltTypeQuals, Brackets);
-  } else {
-    QualType CanonEltTy = getCanonicalType(EltTy);
-    if (CanonEltTy == EltTy) {
-      New = new (*this, TypeAlignment)
-        DependentSizedArrayType(*this, EltTy, QualType(),
-                                NumElts, ASM, EltTypeQuals, Brackets);
-
-      if (NumElts) {
-        DependentSizedArrayType *CanonCheck
-          = DependentSizedArrayTypes.FindNodeOrInsertPos(ID, InsertPos);
-        assert(!CanonCheck && "Dependent-sized canonical array type broken");
-        (void)CanonCheck;
-        DependentSizedArrayTypes.InsertNode(New, InsertPos);
-      }
-    } else {
-      QualType Canon = getDependentSizedArrayType(CanonEltTy, NumElts,
-                                                  ASM, EltTypeQuals,
-                                                  SourceRange());
-      New = new (*this, TypeAlignment)
-        DependentSizedArrayType(*this, EltTy, Canon,
-                                NumElts, ASM, EltTypeQuals, Brackets);
-    }
-  }
+  // Otherwise, we actually build a new type every time, but we
+  // also build a canonical type.
 
-  Types.push_back(New);
-  return QualType(New, 0);
-}
+  SplitQualType canonElementType = getCanonicalType(elementType).split();
 
-QualType ASTContext::getIncompleteArrayType(QualType EltTy,
+  void *insertPos = 0;
+  llvm::FoldingSetNodeID ID;
+  DependentSizedArrayType::Profile(ID, *this,
+                                   QualType(canonElementType.first, 0),
+                                   ASM, elementTypeQuals, numElements);
+
+  // Look for an existing type with these properties.
+  DependentSizedArrayType *canonTy =
+    DependentSizedArrayTypes.FindNodeOrInsertPos(ID, insertPos);
+
+  // If we don't have one, build one.
+  if (!canonTy) {
+    canonTy = new (*this, TypeAlignment)
+      DependentSizedArrayType(*this, QualType(canonElementType.first, 0),
+                              QualType(), numElements, ASM, elementTypeQuals,
+                              brackets);
+    DependentSizedArrayTypes.InsertNode(canonTy, insertPos);
+    Types.push_back(canonTy);
+  }
+
+  // Apply qualifiers from the element type to the array.
+  QualType canon = getQualifiedType(QualType(canonTy,0),
+                                    canonElementType.second);
+
+  // If we didn't need extra canonicalization for the element type,
+  // then just use that as our result.
+  if (QualType(canonElementType.first, 0) == elementType)
+    return canon;
+
+  // Otherwise, we need to build a type which follows the spelling
+  // of the element type.
+  DependentSizedArrayType *sugaredType
+    = new (*this, TypeAlignment)
+        DependentSizedArrayType(*this, elementType, canon, numElements,
+                                ASM, elementTypeQuals, brackets);
+  Types.push_back(sugaredType);
+  return QualType(sugaredType, 0);
+}
+
+QualType ASTContext::getIncompleteArrayType(QualType elementType,
                                             ArrayType::ArraySizeModifier ASM,
-                                            unsigned EltTypeQuals) {
+                                            unsigned elementTypeQuals) const {
   llvm::FoldingSetNodeID ID;
-  IncompleteArrayType::Profile(ID, EltTy, ASM, EltTypeQuals);
+  IncompleteArrayType::Profile(ID, elementType, ASM, elementTypeQuals);
 
-  void *InsertPos = 0;
-  if (IncompleteArrayType *ATP =
-       IncompleteArrayTypes.FindNodeOrInsertPos(ID, InsertPos))
-    return QualType(ATP, 0);
+  void *insertPos = 0;
+  if (IncompleteArrayType *iat =
+       IncompleteArrayTypes.FindNodeOrInsertPos(ID, insertPos))
+    return QualType(iat, 0);
 
   // If the element type isn't canonical, this won't be a canonical type
-  // either, so fill in the canonical type field.
-  QualType Canonical;
+  // either, so fill in the canonical type field.  We also have to pull
+  // qualifiers off the element type.
+  QualType canon;
 
-  if (!EltTy.isCanonical()) {
-    Canonical = getIncompleteArrayType(getCanonicalType(EltTy),
-                                       ASM, EltTypeQuals);
+  if (!elementType.isCanonical() || elementType.hasLocalQualifiers()) {
+    SplitQualType canonSplit = getCanonicalType(elementType).split();
+    canon = getIncompleteArrayType(QualType(canonSplit.first, 0),
+                                   ASM, elementTypeQuals);
+    canon = getQualifiedType(canon, canonSplit.second);
 
     // Get the new insert position for the node we care about.
-    IncompleteArrayType *NewIP =
-      IncompleteArrayTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    IncompleteArrayType *existing =
+      IncompleteArrayTypes.FindNodeOrInsertPos(ID, insertPos);
+    assert(!existing && "Shouldn't be in the map!"); (void) existing;
   }
 
-  IncompleteArrayType *New = new (*this, TypeAlignment)
-    IncompleteArrayType(EltTy, Canonical, ASM, EltTypeQuals);
+  IncompleteArrayType *newType = new (*this, TypeAlignment)
+    IncompleteArrayType(elementType, canon, ASM, elementTypeQuals);
 
-  IncompleteArrayTypes.InsertNode(New, InsertPos);
-  Types.push_back(New);
-  return QualType(New, 0);
+  IncompleteArrayTypes.InsertNode(newType, insertPos);
+  Types.push_back(newType);
+  return QualType(newType, 0);
 }
 
 /// getVectorType - Return the unique reference to a vector type of
 /// the specified element type and size. VectorType must be a built-in type.
 QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
-    VectorType::AltiVecSpecific AltiVecSpec) {
-  BuiltinType *baseType;
-
-  baseType = dyn_cast<BuiltinType>(getCanonicalType(vecType).getTypePtr());
-  assert(baseType != 0 && "getVectorType(): Expecting a built-in type");
+                                   VectorType::VectorKind VecKind) const {
+  assert(vecType->isBuiltinType());
 
   // Check if we've already instantiated a vector of this type.
   llvm::FoldingSetNodeID ID;
-  VectorType::Profile(ID, vecType, NumElts, Type::Vector, AltiVecSpec);
+  VectorType::Profile(ID, vecType, NumElts, Type::Vector, VecKind);
 
   void *InsertPos = 0;
   if (VectorType *VTP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos))
@@ -1550,15 +1771,14 @@ QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
   // so fill in the canonical type field.
   QualType Canonical;
   if (!vecType.isCanonical()) {
-    Canonical = getVectorType(getCanonicalType(vecType), NumElts,
-      VectorType::NotAltiVec);
+    Canonical = getVectorType(getCanonicalType(vecType), NumElts, VecKind);
 
     // Get the new insert position for the node we care about.
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   VectorType *New = new (*this, TypeAlignment)
-    VectorType(vecType, NumElts, Canonical, AltiVecSpec);
+    VectorType(vecType, NumElts, Canonical, VecKind);
   VectorTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
@@ -1566,16 +1786,14 @@ QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
 
 /// getExtVectorType - Return the unique reference to an extended vector type of
 /// the specified element type and size. VectorType must be a built-in type.
-QualType ASTContext::getExtVectorType(QualType vecType, unsigned NumElts) {
-  BuiltinType *baseType;
-
-  baseType = dyn_cast<BuiltinType>(getCanonicalType(vecType).getTypePtr());
-  assert(baseType != 0 && "getExtVectorType(): Expecting a built-in type");
+QualType
+ASTContext::getExtVectorType(QualType vecType, unsigned NumElts) const {
+  assert(vecType->isBuiltinType());
 
   // Check if we've already instantiated a vector of this type.
   llvm::FoldingSetNodeID ID;
   VectorType::Profile(ID, vecType, NumElts, Type::ExtVector,
-                      VectorType::NotAltiVec);
+                      VectorType::GenericVector);
   void *InsertPos = 0;
   if (VectorType *VTP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(VTP, 0);
@@ -1588,7 +1806,7 @@ QualType ASTContext::getExtVectorType(QualType vecType, unsigned NumElts) {
 
     // Get the new insert position for the node we care about.
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
   ExtVectorType *New = new (*this, TypeAlignment)
     ExtVectorType(vecType, NumElts, Canonical);
@@ -1597,9 +1815,10 @@ QualType ASTContext::getExtVectorType(QualType vecType, unsigned NumElts) {
   return QualType(New, 0);
 }
 
-QualType ASTContext::getDependentSizedExtVectorType(QualType vecType,
-                                                    Expr *SizeExpr,
-                                                    SourceLocation AttrLoc) {
+QualType
+ASTContext::getDependentSizedExtVectorType(QualType vecType,
+                                           Expr *SizeExpr,
+                                           SourceLocation AttrLoc) const {
   llvm::FoldingSetNodeID ID;
   DependentSizedExtVectorType::Profile(ID, *this, getCanonicalType(vecType),
                                        SizeExpr);
@@ -1640,8 +1859,9 @@ QualType ASTContext::getDependentSizedExtVectorType(QualType vecType,
 
 /// getFunctionNoProtoType - Return a K&R style C function type like 'int()'.
 ///
-QualType ASTContext::getFunctionNoProtoType(QualType ResultTy,
-                                            const FunctionType::ExtInfo &Info) {
+QualType
+ASTContext::getFunctionNoProtoType(QualType ResultTy,
+                                   const FunctionType::ExtInfo &Info) const {
   const CallingConv CallConv = Info.getCC();
   // Unique functions, to guarantee there is only one function of a particular
   // structure.
@@ -1663,7 +1883,7 @@ QualType ASTContext::getFunctionNoProtoType(QualType ResultTy,
     // Get the new insert position for the node we care about.
     FunctionNoProtoType *NewIP =
       FunctionNoProtoTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   FunctionNoProtoType *New = new (*this, TypeAlignment)
@@ -1675,19 +1895,14 @@ QualType ASTContext::getFunctionNoProtoType(QualType ResultTy,
 
 /// getFunctionType - Return a normal function type with a typed argument
 /// list.  isVariadic indicates whether the argument list includes '...'.
-QualType ASTContext::getFunctionType(QualType ResultTy,const QualType *ArgArray,
-                                     unsigned NumArgs, bool isVariadic,
-                                     unsigned TypeQuals, bool hasExceptionSpec,
-                                     bool hasAnyExceptionSpec, unsigned NumExs,
-                                     const QualType *ExArray,
-                                     const FunctionType::ExtInfo &Info) {
-  const CallingConv CallConv= Info.getCC();
+QualType
+ASTContext::getFunctionType(QualType ResultTy,
+                            const QualType *ArgArray, unsigned NumArgs,
+                            const FunctionProtoType::ExtProtoInfo &EPI) const {
   // Unique functions, to guarantee there is only one function of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
-  FunctionProtoType::Profile(ID, ResultTy, ArgArray, NumArgs, isVariadic,
-                             TypeQuals, hasExceptionSpec, hasAnyExceptionSpec,
-                             NumExs, ExArray, Info);
+  FunctionProtoType::Profile(ID, ResultTy, ArgArray, NumArgs, EPI);
 
   void *InsertPos = 0;
   if (FunctionProtoType *FTP =
@@ -1695,11 +1910,13 @@ QualType ASTContext::getFunctionType(QualType ResultTy,const QualType *ArgArray,
     return QualType(FTP, 0);
 
   // Determine whether the type being created is already canonical or not.
-  bool isCanonical = !hasExceptionSpec && ResultTy.isCanonical();
+  bool isCanonical = !EPI.HasExceptionSpec && ResultTy.isCanonical();
   for (unsigned i = 0; i != NumArgs && isCanonical; ++i)
     if (!ArgArray[i].isCanonicalAsParam())
       isCanonical = false;
 
+  const CallingConv CallConv = EPI.ExtInfo.getCC();
+
   // If this type isn't canonical, get the canonical version of it.
   // The exception spec is not part of the canonical type.
   QualType Canonical;
@@ -1709,28 +1926,33 @@ QualType ASTContext::getFunctionType(QualType ResultTy,const QualType *ArgArray,
     for (unsigned i = 0; i != NumArgs; ++i)
       CanonicalArgs.push_back(getCanonicalParamType(ArgArray[i]));
 
+    FunctionProtoType::ExtProtoInfo CanonicalEPI = EPI;
+    if (CanonicalEPI.HasExceptionSpec) {
+      CanonicalEPI.HasExceptionSpec = false;
+      CanonicalEPI.HasAnyExceptionSpec = false;
+      CanonicalEPI.NumExceptions = 0;
+    }
+    CanonicalEPI.ExtInfo
+      = CanonicalEPI.ExtInfo.withCallingConv(getCanonicalCallConv(CallConv));
+
     Canonical = getFunctionType(getCanonicalType(ResultTy),
                                 CanonicalArgs.data(), NumArgs,
-                                isVariadic, TypeQuals, false,
-                                false, 0, 0,
-                     Info.withCallingConv(getCanonicalCallConv(CallConv)));
+                                CanonicalEPI);
 
     // Get the new insert position for the node we care about.
     FunctionProtoType *NewIP =
       FunctionProtoTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(NewIP == 0 && "Shouldn't be in the map!"); NewIP = NewIP;
+    assert(NewIP == 0 && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   // FunctionProtoType objects are allocated with extra bytes after them
   // for two variable size arrays (for parameter and exception types) at the
   // end of them.
-  FunctionProtoType *FTP =
-    (FunctionProtoType*)Allocate(sizeof(FunctionProtoType) +
-                                 NumArgs*sizeof(QualType) +
-                                 NumExs*sizeof(QualType), TypeAlignment);
-  new (FTP) FunctionProtoType(ResultTy, ArgArray, NumArgs, isVariadic,
-                              TypeQuals, hasExceptionSpec, hasAnyExceptionSpec,
-                              ExArray, NumExs, Canonical, Info);
+  size_t Size = sizeof(FunctionProtoType) +
+                NumArgs * sizeof(QualType) +
+                EPI.NumExceptions * sizeof(QualType);
+  FunctionProtoType *FTP = (FunctionProtoType*) Allocate(Size, TypeAlignment);
+  new (FTP) FunctionProtoType(ResultTy, ArgArray, NumArgs, Canonical, EPI);
   Types.push_back(FTP);
   FunctionProtoTypes.InsertNode(FTP, InsertPos);
   return QualType(FTP, 0);
@@ -1752,7 +1974,7 @@ static bool NeedsInjectedClassNameType(const RecordDecl *D) {
 /// getInjectedClassNameType - Return the unique reference to the
 /// injected class name type for the specified templated declaration.
 QualType ASTContext::getInjectedClassNameType(CXXRecordDecl *Decl,
-                                              QualType TST) {
+                                              QualType TST) const {
   assert(NeedsInjectedClassNameType(Decl));
   if (Decl->TypeForDecl) {
     assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
@@ -1761,16 +1983,17 @@ QualType ASTContext::getInjectedClassNameType(CXXRecordDecl *Decl,
     Decl->TypeForDecl = PrevDecl->TypeForDecl;
     assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
   } else {
-    Decl->TypeForDecl =
+    Type *newType =
       new (*this, TypeAlignment) InjectedClassNameType(Decl, TST);
-    Types.push_back(Decl->TypeForDecl);
+    Decl->TypeForDecl = newType;
+    Types.push_back(newType);
   }
   return QualType(Decl->TypeForDecl, 0);
 }
 
 /// getTypeDeclType - Return the unique reference to the type for the
 /// specified type declaration.
-QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) {
+QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const {
   assert(Decl && "Passed null for Decl param");
   assert(!Decl->TypeForDecl && "TypeForDecl present in slow case");
 
@@ -1791,56 +2014,81 @@ QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) {
     return getEnumType(Enum);
   } else if (const UnresolvedUsingTypenameDecl *Using =
                dyn_cast<UnresolvedUsingTypenameDecl>(Decl)) {
-    Decl->TypeForDecl = new (*this, TypeAlignment) UnresolvedUsingType(Using);
+    Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Using);
+    Decl->TypeForDecl = newType;
+    Types.push_back(newType);
   } else
     llvm_unreachable("TypeDecl without a type?");
 
-  Types.push_back(Decl->TypeForDecl);
   return QualType(Decl->TypeForDecl, 0);
 }
 
 /// getTypedefType - Return the unique reference to the type for the
 /// specified typename decl.
 QualType
-ASTContext::getTypedefType(const TypedefDecl *Decl, QualType Canonical) {
+ASTContext::getTypedefType(const TypedefDecl *Decl, QualType Canonical) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (Canonical.isNull())
     Canonical = getCanonicalType(Decl->getUnderlyingType());
-  Decl->TypeForDecl = new(*this, TypeAlignment)
+  TypedefType *newType = new(*this, TypeAlignment)
     TypedefType(Type::Typedef, Decl, Canonical);
-  Types.push_back(Decl->TypeForDecl);
-  return QualType(Decl->TypeForDecl, 0);
+  Decl->TypeForDecl = newType;
+  Types.push_back(newType);
+  return QualType(newType, 0);
 }
 
-QualType ASTContext::getRecordType(const RecordDecl *Decl) {
+QualType ASTContext::getRecordType(const RecordDecl *Decl) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (const RecordDecl *PrevDecl = Decl->getPreviousDeclaration())
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0); 
 
-  Decl->TypeForDecl = new (*this, TypeAlignment) RecordType(Decl);
-  Types.push_back(Decl->TypeForDecl);
-  return QualType(Decl->TypeForDecl, 0);
+  RecordType *newType = new (*this, TypeAlignment) RecordType(Decl);
+  Decl->TypeForDecl = newType;
+  Types.push_back(newType);
+  return QualType(newType, 0);
 }
 
-QualType ASTContext::getEnumType(const EnumDecl *Decl) {
+QualType ASTContext::getEnumType(const EnumDecl *Decl) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (const EnumDecl *PrevDecl = Decl->getPreviousDeclaration())
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0); 
 
-  Decl->TypeForDecl = new (*this, TypeAlignment) EnumType(Decl);
-  Types.push_back(Decl->TypeForDecl);
-  return QualType(Decl->TypeForDecl, 0);
+  EnumType *newType = new (*this, TypeAlignment) EnumType(Decl);
+  Decl->TypeForDecl = newType;
+  Types.push_back(newType);
+  return QualType(newType, 0);
 }
 
+QualType ASTContext::getAttributedType(AttributedType::Kind attrKind,
+                                       QualType modifiedType,
+                                       QualType equivalentType) {
+  llvm::FoldingSetNodeID id;
+  AttributedType::Profile(id, attrKind, modifiedType, equivalentType);
+
+  void *insertPos = 0;
+  AttributedType *type = AttributedTypes.FindNodeOrInsertPos(id, insertPos);
+  if (type) return QualType(type, 0);
+
+  QualType canon = getCanonicalType(equivalentType);
+  type = new (*this, TypeAlignment)
+           AttributedType(canon, attrKind, modifiedType, equivalentType);
+
+  Types.push_back(type);
+  AttributedTypes.InsertNode(type, insertPos);
+
+  return QualType(type, 0);
+}
+
+
 /// \brief Retrieve a substitution-result type.
 QualType
 ASTContext::getSubstTemplateTypeParmType(const TemplateTypeParmType *Parm,
-                                         QualType Replacement) {
+                                         QualType Replacement) const {
   assert(Replacement.isCanonical()
          && "replacement types must always be canonical");
 
@@ -1860,12 +2108,48 @@ ASTContext::getSubstTemplateTypeParmType(const TemplateTypeParmType *Parm,
   return QualType(SubstParm, 0);
 }
 
+/// \brief Retrieve a 
+QualType ASTContext::getSubstTemplateTypeParmPackType(
+                                          const TemplateTypeParmType *Parm,
+                                              const TemplateArgument &ArgPack) {
+#ifndef NDEBUG
+  for (TemplateArgument::pack_iterator P = ArgPack.pack_begin(), 
+                                    PEnd = ArgPack.pack_end();
+       P != PEnd; ++P) {
+    assert(P->getKind() == TemplateArgument::Type &&"Pack contains a non-type");
+    assert(P->getAsType().isCanonical() && "Pack contains non-canonical type");
+  }
+#endif
+  
+  llvm::FoldingSetNodeID ID;
+  SubstTemplateTypeParmPackType::Profile(ID, Parm, ArgPack);
+  void *InsertPos = 0;
+  if (SubstTemplateTypeParmPackType *SubstParm
+        = SubstTemplateTypeParmPackTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(SubstParm, 0);
+  
+  QualType Canon;
+  if (!Parm->isCanonicalUnqualified()) {
+    Canon = getCanonicalType(QualType(Parm, 0));
+    Canon = getSubstTemplateTypeParmPackType(cast<TemplateTypeParmType>(Canon),
+                                             ArgPack);
+    SubstTemplateTypeParmPackTypes.FindNodeOrInsertPos(ID, InsertPos);
+  }
+
+  SubstTemplateTypeParmPackType *SubstParm
+    = new (*this, TypeAlignment) SubstTemplateTypeParmPackType(Parm, Canon,
+                                                               ArgPack);
+  Types.push_back(SubstParm);
+  SubstTemplateTypeParmTypes.InsertNode(SubstParm, InsertPos);
+  return QualType(SubstParm, 0);  
+}
+
 /// \brief Retrieve the template type parameter type for a template
 /// parameter or parameter pack with the given depth, index, and (optionally)
 /// name.
 QualType ASTContext::getTemplateTypeParmType(unsigned Depth, unsigned Index,
                                              bool ParameterPack,
-                                             IdentifierInfo *Name) {
+                                             IdentifierInfo *Name) const {
   llvm::FoldingSetNodeID ID;
   TemplateTypeParmType::Profile(ID, Depth, Index, ParameterPack, Name);
   void *InsertPos = 0;
@@ -1898,7 +2182,7 @@ TypeSourceInfo *
 ASTContext::getTemplateSpecializationTypeInfo(TemplateName Name,
                                               SourceLocation NameLoc,
                                         const TemplateArgumentListInfo &Args,
-                                              QualType CanonType) {
+                                              QualType CanonType) const {
   QualType TST = getTemplateSpecializationType(Name, Args, CanonType);
 
   TypeSourceInfo *DI = CreateTypeSourceInfo(TST);
@@ -1915,7 +2199,7 @@ ASTContext::getTemplateSpecializationTypeInfo(TemplateName Name,
 QualType
 ASTContext::getTemplateSpecializationType(TemplateName Template,
                                           const TemplateArgumentListInfo &Args,
-                                          QualType Canon) {
+                                          QualType Canon) const {
   unsigned NumArgs = Args.size();
 
   llvm::SmallVector<TemplateArgument, 4> ArgVec;
@@ -1931,7 +2215,7 @@ QualType
 ASTContext::getTemplateSpecializationType(TemplateName Template,
                                           const TemplateArgument *Args,
                                           unsigned NumArgs,
-                                          QualType Canon) {
+                                          QualType Canon) const {
   if (!Canon.isNull())
     Canon = getCanonicalType(Canon);
   else
@@ -1955,7 +2239,7 @@ ASTContext::getTemplateSpecializationType(TemplateName Template,
 QualType
 ASTContext::getCanonicalTemplateSpecializationType(TemplateName Template,
                                                    const TemplateArgument *Args,
-                                                   unsigned NumArgs) {
+                                                   unsigned NumArgs) const {
   // Build the canonical template specialization type.
   TemplateName CanonTemplate = getCanonicalTemplateName(Template);
   llvm::SmallVector<TemplateArgument, 4> CanonArgs;
@@ -1993,7 +2277,7 @@ ASTContext::getCanonicalTemplateSpecializationType(TemplateName Template,
 QualType
 ASTContext::getElaboratedType(ElaboratedTypeKeyword Keyword,
                               NestedNameSpecifier *NNS,
-                              QualType NamedType) {
+                              QualType NamedType) const {
   llvm::FoldingSetNodeID ID;
   ElaboratedType::Profile(ID, Keyword, NNS, NamedType);
 
@@ -2016,10 +2300,34 @@ ASTContext::getElaboratedType(ElaboratedTypeKeyword Keyword,
   return QualType(T, 0);
 }
 
+QualType
+ASTContext::getParenType(QualType InnerType) const {
+  llvm::FoldingSetNodeID ID;
+  ParenType::Profile(ID, InnerType);
+
+  void *InsertPos = 0;
+  ParenType *T = ParenTypes.FindNodeOrInsertPos(ID, InsertPos);
+  if (T)
+    return QualType(T, 0);
+
+  QualType Canon = InnerType;
+  if (!Canon.isCanonical()) {
+    Canon = getCanonicalType(InnerType);
+    ParenType *CheckT = ParenTypes.FindNodeOrInsertPos(ID, InsertPos);
+    assert(!CheckT && "Paren canonical type broken");
+    (void)CheckT;
+  }
+
+  T = new (*this) ParenType(InnerType, Canon);
+  Types.push_back(T);
+  ParenTypes.InsertNode(T, InsertPos);
+  return QualType(T, 0);
+}
+
 QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
                                           NestedNameSpecifier *NNS,
                                           const IdentifierInfo *Name,
-                                          QualType Canon) {
+                                          QualType Canon) const {
   assert(NNS->isDependent() && "nested-name-specifier must be dependent");
 
   if (Canon.isNull()) {
@@ -2052,7 +2360,7 @@ ASTContext::getDependentTemplateSpecializationType(
                                  ElaboratedTypeKeyword Keyword,
                                  NestedNameSpecifier *NNS,
                                  const IdentifierInfo *Name,
-                                 const TemplateArgumentListInfo &Args) {
+                                 const TemplateArgumentListInfo &Args) const {
   // TODO: avoid this copy
   llvm::SmallVector<TemplateArgument, 16> ArgCopy;
   for (unsigned I = 0, E = Args.size(); I != E; ++I)
@@ -2068,7 +2376,7 @@ ASTContext::getDependentTemplateSpecializationType(
                                  NestedNameSpecifier *NNS,
                                  const IdentifierInfo *Name,
                                  unsigned NumArgs,
-                                 const TemplateArgument *Args) {
+                                 const TemplateArgument *Args) const {
   assert(NNS->isDependent() && "nested-name-specifier must be dependent");
 
   llvm::FoldingSetNodeID ID;
@@ -2114,6 +2422,33 @@ ASTContext::getDependentTemplateSpecializationType(
   return QualType(T, 0);
 }
 
+QualType ASTContext::getPackExpansionType(QualType Pattern,
+                                      llvm::Optional<unsigned> NumExpansions) {
+  llvm::FoldingSetNodeID ID;
+  PackExpansionType::Profile(ID, Pattern, NumExpansions);
+
+  assert(Pattern->containsUnexpandedParameterPack() &&
+         "Pack expansions must expand one or more parameter packs");
+  void *InsertPos = 0;
+  PackExpansionType *T
+    = PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
+  if (T)
+    return QualType(T, 0);
+
+  QualType Canon;
+  if (!Pattern.isCanonical()) {
+    Canon = getPackExpansionType(getCanonicalType(Pattern), NumExpansions);
+
+    // Find the insert position again.
+    PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
+  }
+
+  T = new (*this) PackExpansionType(Pattern, Canon, NumExpansions);
+  Types.push_back(T);
+  PackExpansionTypes.InsertNode(T, InsertPos);
+  return QualType(T, 0);  
+}
+
 /// CmpProtocolNames - Comparison predicate for sorting protocols
 /// alphabetically.
 static bool CmpProtocolNames(const ObjCProtocolDecl *LHS,
@@ -2145,7 +2480,7 @@ static void SortAndUniqueProtocols(ObjCProtocolDecl **Protocols,
 
 QualType ASTContext::getObjCObjectType(QualType BaseType,
                                        ObjCProtocolDecl * const *Protocols,
-                                       unsigned NumProtocols) {
+                                       unsigned NumProtocols) const {
   // If the base type is an interface and there aren't any protocols
   // to add, then the interface type will do just fine.
   if (!NumProtocols && isa<ObjCInterfaceType>(BaseType))
@@ -2193,7 +2528,7 @@ QualType ASTContext::getObjCObjectType(QualType BaseType,
 
 /// getObjCObjectPointerType - Return a ObjCObjectPointerType type for
 /// the given object type.
-QualType ASTContext::getObjCObjectPointerType(QualType ObjectT) {
+QualType ASTContext::getObjCObjectPointerType(QualType ObjectT) const {
   llvm::FoldingSetNodeID ID;
   ObjCObjectPointerType::Profile(ID, ObjectT);
 
@@ -2223,7 +2558,7 @@ QualType ASTContext::getObjCObjectPointerType(QualType ObjectT) {
 
 /// getObjCInterfaceType - Return the unique reference to the type for the
 /// specified ObjC interface decl. The list of protocols is optional.
-QualType ASTContext::getObjCInterfaceType(const ObjCInterfaceDecl *Decl) {
+QualType ASTContext::getObjCInterfaceType(const ObjCInterfaceDecl *Decl) const {
   if (Decl->TypeForDecl)
     return QualType(Decl->TypeForDecl, 0);
 
@@ -2240,7 +2575,7 @@ QualType ASTContext::getObjCInterfaceType(const ObjCInterfaceDecl *Decl) {
 /// multiple declarations that refer to "typeof(x)" all contain different
 /// DeclRefExpr's. This doesn't effect the type checker, since it operates
 /// on canonical type's (which are always unique).
-QualType ASTContext::getTypeOfExprType(Expr *tofExpr) {
+QualType ASTContext::getTypeOfExprType(Expr *tofExpr) const {
   TypeOfExprType *toe;
   if (tofExpr->isTypeDependent()) {
     llvm::FoldingSetNodeID ID;
@@ -2275,7 +2610,7 @@ QualType ASTContext::getTypeOfExprType(Expr *tofExpr) {
 /// memory savings. Since typeof(t) is fairly uncommon, space shouldn't be
 /// an issue. This doesn't effect the type checker, since it operates
 /// on canonical type's (which are always unique).
-QualType ASTContext::getTypeOfType(QualType tofType) {
+QualType ASTContext::getTypeOfType(QualType tofType) const {
   QualType Canonical = getCanonicalType(tofType);
   TypeOfType *tot = new (*this, TypeAlignment) TypeOfType(tofType, Canonical);
   Types.push_back(tot);
@@ -2284,7 +2619,7 @@ QualType ASTContext::getTypeOfType(QualType tofType) {
 
 /// getDecltypeForExpr - Given an expr, will return the decltype for that
 /// expression, according to the rules in C++0x [dcl.type.simple]p4
-static QualType getDecltypeForExpr(const Expr *e, ASTContext &Context) {
+static QualType getDecltypeForExpr(const Expr *e, const ASTContext &Context) {
   if (e->isTypeDependent())
     return Context.DependentTy;
 
@@ -2308,7 +2643,7 @@ static QualType getDecltypeForExpr(const Expr *e, ASTContext &Context) {
 
   // Otherwise, where T is the type of e, if e is an lvalue, decltype(e) is
   // defined as T&, otherwise decltype(e) is defined as T.
-  if (e->isLvalue(Context) == Expr::LV_Valid)
+  if (e->isLValue())
     T = Context.getLValueReferenceType(T);
 
   return T;
@@ -2319,7 +2654,7 @@ static QualType getDecltypeForExpr(const Expr *e, ASTContext &Context) {
 /// memory savings. Since decltype(t) is fairly uncommon, space shouldn't be
 /// an issue. This doesn't effect the type checker, since it operates
 /// on canonical type's (which are always unique).
-QualType ASTContext::getDecltypeType(Expr *e) {
+QualType ASTContext::getDecltypeType(Expr *e) const {
   DecltypeType *dt;
   if (e->isTypeDependent()) {
     llvm::FoldingSetNodeID ID;
@@ -2348,9 +2683,17 @@ QualType ASTContext::getDecltypeType(Expr *e) {
   return QualType(dt, 0);
 }
 
+/// getAutoType - Unlike many "get<Type>" functions, we don't unique
+/// AutoType AST's.
+QualType ASTContext::getAutoType(QualType DeducedType) const {
+  AutoType *at = new (*this, TypeAlignment) AutoType(DeducedType);
+  Types.push_back(at);
+  return QualType(at, 0);
+}
+
 /// getTagDeclType - Return the unique reference to the type for the
 /// specified TagDecl (struct/union/class/enum) decl.
-QualType ASTContext::getTagDeclType(const TagDecl *Decl) {
+QualType ASTContext::getTagDeclType(const TagDecl *Decl) const {
   assert (Decl);
   // FIXME: What is the design on getTagDeclType when it requires casting
   // away const?  mutable?
@@ -2388,12 +2731,12 @@ QualType ASTContext::getPointerDiffType() const {
 //                              Type Operators
 //===----------------------------------------------------------------------===//
 
-CanQualType ASTContext::getCanonicalParamType(QualType T) {
+CanQualType ASTContext::getCanonicalParamType(QualType T) const {
   // Push qualifiers into arrays, and then discard any remaining
   // qualifiers.
   T = getCanonicalType(T);
+  T = getVariableArrayDecayedType(T);
   const Type *Ty = T.getTypePtr();
-
   QualType Result;
   if (isa<ArrayType>(Ty)) {
     Result = getArrayDecayedType(QualType(Ty,0));
@@ -2406,97 +2749,59 @@ CanQualType ASTContext::getCanonicalParamType(QualType T) {
   return CanQualType::CreateUnsafe(Result);
 }
 
-/// getCanonicalType - Return the canonical (structural) type corresponding to
-/// the specified potentially non-canonical type.  The non-canonical version
-/// of a type may have many "decorated" versions of types.  Decorators can
-/// include typedefs, 'typeof' operators, etc. The returned type is guaranteed
-/// to be free of any of these, allowing two canonical types to be compared
-/// for exact equality with a simple pointer comparison.
-CanQualType ASTContext::getCanonicalType(QualType T) {
-  QualifierCollector Quals;
-  const Type *Ptr = Quals.strip(T);
-  QualType CanType = Ptr->getCanonicalTypeInternal();
-
-  // The canonical internal type will be the canonical type *except*
-  // that we push type qualifiers down through array types.
-
-  // If there are no new qualifiers to push down, stop here.
-  if (!Quals.hasQualifiers())
-    return CanQualType::CreateUnsafe(CanType);
-
-  // If the type qualifiers are on an array type, get the canonical
-  // type of the array with the qualifiers applied to the element
-  // type.
-  ArrayType *AT = dyn_cast<ArrayType>(CanType);
-  if (!AT)
-    return CanQualType::CreateUnsafe(getQualifiedType(CanType, Quals));
-
-  // Get the canonical version of the element with the extra qualifiers on it.
-  // This can recursively sink qualifiers through multiple levels of arrays.
-  QualType NewEltTy = getQualifiedType(AT->getElementType(), Quals);
-  NewEltTy = getCanonicalType(NewEltTy);
-
-  if (ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(AT))
-    return CanQualType::CreateUnsafe(
-             getConstantArrayType(NewEltTy, CAT->getSize(),
-                                  CAT->getSizeModifier(),
-                                  CAT->getIndexTypeCVRQualifiers()));
-  if (IncompleteArrayType *IAT = dyn_cast<IncompleteArrayType>(AT))
-    return CanQualType::CreateUnsafe(
-             getIncompleteArrayType(NewEltTy, IAT->getSizeModifier(),
-                                    IAT->getIndexTypeCVRQualifiers()));
-
-  if (DependentSizedArrayType *DSAT = dyn_cast<DependentSizedArrayType>(AT))
-    return CanQualType::CreateUnsafe(
-             getDependentSizedArrayType(NewEltTy,
-                                        DSAT->getSizeExpr() ?
-                                          DSAT->getSizeExpr()->Retain() : 0,
-                                        DSAT->getSizeModifier(),
-                                        DSAT->getIndexTypeCVRQualifiers(),
-                        DSAT->getBracketsRange())->getCanonicalTypeInternal());
-
-  VariableArrayType *VAT = cast<VariableArrayType>(AT);
-  return CanQualType::CreateUnsafe(getVariableArrayType(NewEltTy,
-                                                        VAT->getSizeExpr() ?
-                                              VAT->getSizeExpr()->Retain() : 0,
-                                                        VAT->getSizeModifier(),
-                                              VAT->getIndexTypeCVRQualifiers(),
-                                                     VAT->getBracketsRange()));
-}
 
-QualType ASTContext::getUnqualifiedArrayType(QualType T,
-                                             Qualifiers &Quals) {
-  Quals = T.getQualifiers();
-  const ArrayType *AT = getAsArrayType(T);
+QualType ASTContext::getUnqualifiedArrayType(QualType type,
+                                             Qualifiers &quals) {
+  SplitQualType splitType = type.getSplitUnqualifiedType();
+
+  // FIXME: getSplitUnqualifiedType() actually walks all the way to
+  // the unqualified desugared type and then drops it on the floor.
+  // We then have to strip that sugar back off with
+  // getUnqualifiedDesugaredType(), which is silly.
+  const ArrayType *AT =
+    dyn_cast<ArrayType>(splitType.first->getUnqualifiedDesugaredType());
+
+  // If we don't have an array, just use the results in splitType.
   if (!AT) {
-    return T.getUnqualifiedType();
+    quals = splitType.second;
+    return QualType(splitType.first, 0);
   }
 
-  QualType Elt = AT->getElementType();
-  QualType UnqualElt = getUnqualifiedArrayType(Elt, Quals);
-  if (Elt == UnqualElt)
-    return T;
+  // Otherwise, recurse on the array's element type.
+  QualType elementType = AT->getElementType();
+  QualType unqualElementType = getUnqualifiedArrayType(elementType, quals);
+
+  // If that didn't change the element type, AT has no qualifiers, so we
+  // can just use the results in splitType.
+  if (elementType == unqualElementType) {
+    assert(quals.empty()); // from the recursive call
+    quals = splitType.second;
+    return QualType(splitType.first, 0);
+  }
+
+  // Otherwise, add in the qualifiers from the outermost type, then
+  // build the type back up.
+  quals.addConsistentQualifiers(splitType.second);
 
   if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(AT)) {
-    return getConstantArrayType(UnqualElt, CAT->getSize(),
+    return getConstantArrayType(unqualElementType, CAT->getSize(),
                                 CAT->getSizeModifier(), 0);
   }
 
   if (const IncompleteArrayType *IAT = dyn_cast<IncompleteArrayType>(AT)) {
-    return getIncompleteArrayType(UnqualElt, IAT->getSizeModifier(), 0);
+    return getIncompleteArrayType(unqualElementType, IAT->getSizeModifier(), 0);
   }
 
   if (const VariableArrayType *VAT = dyn_cast<VariableArrayType>(AT)) {
-    return getVariableArrayType(UnqualElt,
-                                VAT->getSizeExpr() ?
-                                VAT->getSizeExpr()->Retain() : 0,
+    return getVariableArrayType(unqualElementType,
+                                VAT->getSizeExpr(),
                                 VAT->getSizeModifier(),
                                 VAT->getIndexTypeCVRQualifiers(),
                                 VAT->getBracketsRange());
   }
 
   const DependentSizedArrayType *DSAT = cast<DependentSizedArrayType>(AT);
-  return getDependentSizedArrayType(UnqualElt, DSAT->getSizeExpr()->Retain(),
+  return getDependentSizedArrayType(unqualElementType, DSAT->getSizeExpr(),
                                     DSAT->getSizeModifier(), 0,
                                     SourceRange());
 }
@@ -2543,8 +2848,9 @@ bool ASTContext::UnwrapSimilarPointerTypes(QualType &T1, QualType &T2) {
   return false;
 }
 
-DeclarationNameInfo ASTContext::getNameForTemplate(TemplateName Name,
-                                                   SourceLocation NameLoc) {
+DeclarationNameInfo
+ASTContext::getNameForTemplate(TemplateName Name,
+                               SourceLocation NameLoc) const {
   if (TemplateDecl *TD = Name.getAsTemplateDecl())
     // DNInfo work in progress: CHECKME: what about DNLoc?
     return DeclarationNameInfo(TD->getDeclName(), NameLoc);
@@ -2570,7 +2876,7 @@ DeclarationNameInfo ASTContext::getNameForTemplate(TemplateName Name,
   return DeclarationNameInfo((*Storage->begin())->getDeclName(), NameLoc);
 }
 
-TemplateName ASTContext::getCanonicalTemplateName(TemplateName Name) {
+TemplateName ASTContext::getCanonicalTemplateName(TemplateName Name) const {
   if (TemplateDecl *Template = Name.getAsTemplateDecl()) {
     if (TemplateTemplateParmDecl *TTP 
                               = dyn_cast<TemplateTemplateParmDecl>(Template))
@@ -2580,6 +2886,15 @@ TemplateName ASTContext::getCanonicalTemplateName(TemplateName Name) {
     return TemplateName(cast<TemplateDecl>(Template->getCanonicalDecl()));
   }
 
+  if (SubstTemplateTemplateParmPackStorage *SubstPack
+                                  = Name.getAsSubstTemplateTemplateParmPack()) {
+    TemplateTemplateParmDecl *CanonParam
+      = getCanonicalTemplateTemplateParmDecl(SubstPack->getParameterPack());
+    TemplateArgument CanonArgPack
+      = getCanonicalTemplateArgument(SubstPack->getArgumentPack());
+    return getSubstTemplateTemplateParmPack(CanonParam, CanonArgPack);
+  }
+      
   assert(!Name.getAsOverloadedTemplate());
 
   DependentTemplateName *DTN = Name.getAsDependentTemplateName();
@@ -2594,7 +2909,7 @@ bool ASTContext::hasSameTemplateName(TemplateName X, TemplateName Y) {
 }
 
 TemplateArgument
-ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) {
+ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) const {
   switch (Arg.getKind()) {
     case TemplateArgument::Null:
       return Arg;
@@ -2607,7 +2922,12 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) {
 
     case TemplateArgument::Template:
       return TemplateArgument(getCanonicalTemplateName(Arg.getAsTemplate()));
-      
+
+    case TemplateArgument::TemplateExpansion:
+      return TemplateArgument(getCanonicalTemplateName(
+                                         Arg.getAsTemplateOrTemplatePattern()),
+                              Arg.getNumTemplateExpansions());
+
     case TemplateArgument::Integral:
       return TemplateArgument(*Arg.getAsIntegral(),
                               getCanonicalType(Arg.getIntegralType()));
@@ -2616,17 +2936,18 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) {
       return TemplateArgument(getCanonicalType(Arg.getAsType()));
 
     case TemplateArgument::Pack: {
-      // FIXME: Allocate in ASTContext
-      TemplateArgument *CanonArgs = new TemplateArgument[Arg.pack_size()];
+      if (Arg.pack_size() == 0)
+        return Arg;
+      
+      TemplateArgument *CanonArgs
+        = new (*this) TemplateArgument[Arg.pack_size()];
       unsigned Idx = 0;
       for (TemplateArgument::pack_iterator A = Arg.pack_begin(),
                                         AEnd = Arg.pack_end();
            A != AEnd; (void)++A, ++Idx)
         CanonArgs[Idx] = getCanonicalTemplateArgument(*A);
 
-      TemplateArgument Result;
-      Result.setArgumentPack(CanonArgs, Arg.pack_size(), false);
-      return Result;
+      return TemplateArgument(CanonArgs, Arg.pack_size());
     }
   }
 
@@ -2636,7 +2957,7 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) {
 }
 
 NestedNameSpecifier *
-ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) {
+ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const {
   if (!NNS)
     return 0;
 
@@ -2655,9 +2976,35 @@ ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) {
   case NestedNameSpecifier::TypeSpec:
   case NestedNameSpecifier::TypeSpecWithTemplate: {
     QualType T = getCanonicalType(QualType(NNS->getAsType(), 0));
-    return NestedNameSpecifier::Create(*this, 0,
-                 NNS->getKind() == NestedNameSpecifier::TypeSpecWithTemplate,
-                                       T.getTypePtr());
+    
+    // If we have some kind of dependent-named type (e.g., "typename T::type"),
+    // break it apart into its prefix and identifier, then reconsititute those
+    // as the canonical nested-name-specifier. This is required to canonicalize
+    // a dependent nested-name-specifier involving typedefs of dependent-name
+    // types, e.g.,
+    //   typedef typename T::type T1;
+    //   typedef typename T1::type T2;
+    if (const DependentNameType *DNT = T->getAs<DependentNameType>()) {
+      NestedNameSpecifier *Prefix
+        = getCanonicalNestedNameSpecifier(DNT->getQualifier());
+      return NestedNameSpecifier::Create(*this, Prefix, 
+                           const_cast<IdentifierInfo *>(DNT->getIdentifier()));
+    }    
+
+    // Do the same thing as above, but with dependent-named specializations.
+    if (const DependentTemplateSpecializationType *DTST
+          = T->getAs<DependentTemplateSpecializationType>()) {
+      NestedNameSpecifier *Prefix
+        = getCanonicalNestedNameSpecifier(DTST->getQualifier());
+      TemplateName Name
+        = getDependentTemplateName(Prefix, DTST->getIdentifier());
+      T = getTemplateSpecializationType(Name, 
+                                        DTST->getArgs(), DTST->getNumArgs());
+      T = getCanonicalType(T);
+    }
+    
+    return NestedNameSpecifier::Create(*this, 0, false,
+                                       const_cast<Type*>(T.getTypePtr()));
   }
 
   case NestedNameSpecifier::Global:
@@ -2670,7 +3017,7 @@ ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) {
 }
 
 
-const ArrayType *ASTContext::getAsArrayType(QualType T) {
+const ArrayType *ASTContext::getAsArrayType(QualType T) const {
   // Handle the non-qualified case efficiently.
   if (!T.hasLocalQualifiers()) {
     // Handle the common positive case fast.
@@ -2679,8 +3026,7 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) {
   }
 
   // Handle the common negative case fast.
-  QualType CType = T->getCanonicalTypeInternal();
-  if (!isa<ArrayType>(CType))
+  if (!isa<ArrayType>(T.getCanonicalType()))
     return 0;
 
   // Apply any qualifiers from the array type to the element type.  This
@@ -2691,19 +3037,17 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) {
   // sugar such as a typedef in the way.  If we have type qualifiers on the type
   // we must propagate them down into the element type.
 
-  QualifierCollector Qs;
-  const Type *Ty = Qs.strip(T.getDesugaredType());
+  SplitQualType split = T.getSplitDesugaredType();
+  Qualifiers qs = split.second;
 
   // If we have a simple case, just return now.
-  const ArrayType *ATy = dyn_cast<ArrayType>(Ty);
-  if (ATy == 0 || Qs.empty())
+  const ArrayType *ATy = dyn_cast<ArrayType>(split.first);
+  if (ATy == 0 || qs.empty())
     return ATy;
 
   // Otherwise, we have an array and we have qualifiers on it.  Push the
   // qualifiers into the array element type and return a new array type.
-  // Get the canonical version of the element with the extra qualifiers on it.
-  // This can recursively sink qualifiers through multiple levels of arrays.
-  QualType NewEltTy = getQualifiedType(ATy->getElementType(), Qs);
+  QualType NewEltTy = getQualifiedType(ATy->getElementType(), qs);
 
   if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(ATy))
     return cast<ArrayType>(getConstantArrayType(NewEltTy, CAT->getSize(),
@@ -2718,29 +3062,26 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) {
         = dyn_cast<DependentSizedArrayType>(ATy))
     return cast<ArrayType>(
                      getDependentSizedArrayType(NewEltTy,
-                                                DSAT->getSizeExpr() ?
-                                              DSAT->getSizeExpr()->Retain() : 0,
+                                                DSAT->getSizeExpr(),
                                                 DSAT->getSizeModifier(),
                                               DSAT->getIndexTypeCVRQualifiers(),
                                                 DSAT->getBracketsRange()));
 
   const VariableArrayType *VAT = cast<VariableArrayType>(ATy);
   return cast<ArrayType>(getVariableArrayType(NewEltTy,
-                                              VAT->getSizeExpr() ?
-                                              VAT->getSizeExpr()->Retain() : 0,
+                                              VAT->getSizeExpr(),
                                               VAT->getSizeModifier(),
                                               VAT->getIndexTypeCVRQualifiers(),
                                               VAT->getBracketsRange()));
 }
 
-
 /// getArrayDecayedType - Return the properly qualified result of decaying the
 /// specified array type to a pointer.  This operation is non-trivial when
 /// handling typedefs etc.  The canonical type of "T" must be an array type,
 /// this returns a pointer to a properly qualified element of the array.
 ///
 /// See C99 6.7.5.3p7 and C99 6.3.2.1p3.
-QualType ASTContext::getArrayDecayedType(QualType Ty) {
+QualType ASTContext::getArrayDecayedType(QualType Ty) const {
   // Get the element type with 'getAsArrayType' so that we don't lose any
   // typedefs in the element type of the array.  This also handles propagation
   // of type qualifiers from the array type into the element type if present
@@ -2754,20 +3095,22 @@ QualType ASTContext::getArrayDecayedType(QualType Ty) {
   return getQualifiedType(PtrTy, PrettyArrayType->getIndexTypeQualifiers());
 }
 
-QualType ASTContext::getBaseElementType(QualType QT) {
-  QualifierCollector Qs;
-  while (const ArrayType *AT = getAsArrayType(QualType(Qs.strip(QT), 0)))
-    QT = AT->getElementType();
-  return Qs.apply(QT);
+QualType ASTContext::getBaseElementType(const ArrayType *array) const {
+  return getBaseElementType(array->getElementType());
 }
 
-QualType ASTContext::getBaseElementType(const ArrayType *AT) {
-  QualType ElemTy = AT->getElementType();
+QualType ASTContext::getBaseElementType(QualType type) const {
+  Qualifiers qs;
+  while (true) {
+    SplitQualType split = type.getSplitDesugaredType();
+    const ArrayType *array = split.first->getAsArrayTypeUnsafe();
+    if (!array) break;
 
-  if (const ArrayType *AT = getAsArrayType(ElemTy))
-    return getBaseElementType(AT);
+    type = array->getElementType();
+    qs.addConsistentQualifiers(split.second);
+  }
 
-  return ElemTy;
+  return getQualifiedType(type, qs);
 }
 
 /// getConstantArrayElementCount - Returns number of constant array elements.
@@ -2825,7 +3168,7 @@ QualType ASTContext::getFloatingTypeOfSizeWithinDomain(QualType Size,
 /// point types, ignoring the domain of the type (i.e. 'double' ==
 /// '_Complex double').  If LHS > RHS, return 1.  If LHS == RHS, return 0. If
 /// LHS < RHS, return -1.
-int ASTContext::getFloatingTypeOrder(QualType LHS, QualType RHS) {
+int ASTContext::getFloatingTypeOrder(QualType LHS, QualType RHS) const {
   FloatingRank LHSR = getFloatingRank(LHS);
   FloatingRank RHSR = getFloatingRank(RHS);
 
@@ -2839,12 +3182,13 @@ int ASTContext::getFloatingTypeOrder(QualType LHS, QualType RHS) {
 /// getIntegerRank - Return an integer conversion rank (C99 6.3.1.1p1). This
 /// routine will assert if passed a built-in type that isn't an integer or enum,
 /// or if it is not canonicalized.
-unsigned ASTContext::getIntegerRank(Type *T) {
+unsigned ASTContext::getIntegerRank(const Type *T) const {
   assert(T->isCanonicalUnqualified() && "T should be canonicalized");
-  if (EnumType* ET = dyn_cast<EnumType>(T))
+  if (const EnumType* ET = dyn_cast<EnumType>(T))
     T = ET->getDecl()->getPromotionType().getTypePtr();
 
-  if (T->isSpecificBuiltinType(BuiltinType::WChar))
+  if (T->isSpecificBuiltinType(BuiltinType::WChar_S) ||
+      T->isSpecificBuiltinType(BuiltinType::WChar_U))
     T = getFromTargetType(Target.getWCharType()).getTypePtr();
 
   if (T->isSpecificBuiltinType(BuiltinType::Char16))
@@ -2885,7 +3229,7 @@ unsigned ASTContext::getIntegerRank(Type *T) {
 ///
 /// \returns the type this bit-field will promote to, or NULL if no
 /// promotion occurs.
-QualType ASTContext::isPromotableBitField(Expr *E) {
+QualType ASTContext::isPromotableBitField(Expr *E) const {
   if (E->isTypeDependent() || E->isValueDependent())
     return QualType();
   
@@ -2917,7 +3261,7 @@ QualType ASTContext::isPromotableBitField(Expr *E) {
 /// getPromotedIntegerType - Returns the type that Promotable will
 /// promote to: C99 6.3.1.1p2, assuming that Promotable is a promotable
 /// integer type.
-QualType ASTContext::getPromotedIntegerType(QualType Promotable) {
+QualType ASTContext::getPromotedIntegerType(QualType Promotable) const {
   assert(!Promotable.isNull());
   assert(Promotable->isPromotableIntegerType());
   if (const EnumType *ET = Promotable->getAs<EnumType>())
@@ -2933,9 +3277,9 @@ QualType ASTContext::getPromotedIntegerType(QualType Promotable) {
 /// getIntegerTypeOrder - Returns the highest ranked integer type:
 /// C99 6.3.1.8p1.  If LHS > RHS, return 1.  If LHS == RHS, return 0. If
 /// LHS < RHS, return -1.
-int ASTContext::getIntegerTypeOrder(QualType LHS, QualType RHS) {
-  Type *LHSC = getCanonicalType(LHS).getTypePtr();
-  Type *RHSC = getCanonicalType(RHS).getTypePtr();
+int ASTContext::getIntegerTypeOrder(QualType LHS, QualType RHS) const {
+  const Type *LHSC = getCanonicalType(LHS).getTypePtr();
+  const Type *RHSC = getCanonicalType(RHS).getTypePtr();
   if (LHSC == RHSC) return 0;
 
   bool LHSUnsigned = LHSC->isUnsignedIntegerType();
@@ -2972,7 +3316,7 @@ int ASTContext::getIntegerTypeOrder(QualType LHS, QualType RHS) {
 }
 
 static RecordDecl *
-CreateRecordDecl(ASTContext &Ctx, RecordDecl::TagKind TK, DeclContext *DC,
+CreateRecordDecl(const ASTContext &Ctx, RecordDecl::TagKind TK, DeclContext *DC,
                  SourceLocation L, IdentifierInfo *Id) {
   if (Ctx.getLangOptions().CPlusPlus)
     return CXXRecordDecl::Create(Ctx, TK, DC, L, Id);
@@ -2981,7 +3325,7 @@ CreateRecordDecl(ASTContext &Ctx, RecordDecl::TagKind TK, DeclContext *DC,
 }
                                     
 // getCFConstantStringType - Return the type used for constant CFStrings.
-QualType ASTContext::getCFConstantStringType() {
+QualType ASTContext::getCFConstantStringType() const {
   if (!CFConstantStringTypeDecl) {
     CFConstantStringTypeDecl =
       CreateRecordDecl(*this, TTK_Struct, TUDecl, SourceLocation(),
@@ -3023,7 +3367,7 @@ void ASTContext::setCFConstantStringType(QualType T) {
 }
 
 // getNSConstantStringType - Return the type used for constant NSStrings.
-QualType ASTContext::getNSConstantStringType() {
+QualType ASTContext::getNSConstantStringType() const {
   if (!NSConstantStringTypeDecl) {
     NSConstantStringTypeDecl =
     CreateRecordDecl(*this, TTK_Struct, TUDecl, SourceLocation(),
@@ -3062,7 +3406,7 @@ void ASTContext::setNSConstantStringType(QualType T) {
   NSConstantStringTypeDecl = Rec->getDecl();
 }
 
-QualType ASTContext::getObjCFastEnumerationStateType() {
+QualType ASTContext::getObjCFastEnumerationStateType() const {
   if (!ObjCFastEnumerationStateTypeDecl) {
     ObjCFastEnumerationStateTypeDecl =
       CreateRecordDecl(*this, TTK_Struct, TUDecl, SourceLocation(),
@@ -3087,10 +3431,6 @@ QualType ASTContext::getObjCFastEnumerationStateType() {
       Field->setAccess(AS_public);
       ObjCFastEnumerationStateTypeDecl->addDecl(Field);
     }
-    if (getLangOptions().CPlusPlus)
-      if (CXXRecordDecl *CXXRD = 
-            dyn_cast<CXXRecordDecl>(ObjCFastEnumerationStateTypeDecl))
-        CXXRD->setEmpty(false);
 
     ObjCFastEnumerationStateTypeDecl->completeDefinition();
   }
@@ -3098,7 +3438,7 @@ QualType ASTContext::getObjCFastEnumerationStateType() {
   return getTagDeclType(ObjCFastEnumerationStateTypeDecl);
 }
 
-QualType ASTContext::getBlockDescriptorType() {
+QualType ASTContext::getBlockDescriptorType() const {
   if (BlockDescriptorType)
     return getTagDeclType(BlockDescriptorType);
 
@@ -3143,7 +3483,7 @@ void ASTContext::setBlockDescriptorType(QualType T) {
   BlockDescriptorType = Rec->getDecl();
 }
 
-QualType ASTContext::getBlockDescriptorExtendedType() {
+QualType ASTContext::getBlockDescriptorExtendedType() const {
   if (BlockDescriptorExtendedType)
     return getTagDeclType(BlockDescriptorExtendedType);
 
@@ -3192,17 +3532,25 @@ void ASTContext::setBlockDescriptorExtendedType(QualType T) {
   BlockDescriptorExtendedType = Rec->getDecl();
 }
 
-bool ASTContext::BlockRequiresCopying(QualType Ty) {
+bool ASTContext::BlockRequiresCopying(QualType Ty) const {
   if (Ty->isBlockPointerType())
     return true;
   if (isObjCNSObjectType(Ty))
     return true;
   if (Ty->isObjCObjectPointerType())
     return true;
+  if (getLangOptions().CPlusPlus) {
+    if (const RecordType *RT = Ty->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      return RD->hasConstCopyConstructor(*this);
+      
+    }
+  }
   return false;
 }
 
-QualType ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) {
+QualType
+ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) const {
   //  type = struct __Block_byref_1_X {
   //    void *__isa;
   //    struct __Block_byref_1_X *__forwarding;
@@ -3264,7 +3612,7 @@ QualType ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) {
 
 QualType ASTContext::getBlockParmType(
   bool BlockHasCopyDispose,
-  llvm::SmallVectorImpl<const Expr *> &Layout) {
+  llvm::SmallVectorImpl<const Expr *> &Layout) const {
 
   // FIXME: Move up
   llvm::SmallString<36> Name;
@@ -3351,7 +3699,7 @@ static bool isTypeTypedefedAsBOOL(QualType T) {
 
 /// getObjCEncodingTypeSize returns size of type for objective-c encoding
 /// purpose.
-CharUnits ASTContext::getObjCEncodingTypeSize(QualType type) {
+CharUnits ASTContext::getObjCEncodingTypeSize(QualType type) const {
   CharUnits sz = getTypeSizeInChars(type);
 
   // Make all integer and enum types at least as large as an int
@@ -3368,10 +3716,11 @@ std::string charUnitsToString(const CharUnits &CU) {
   return llvm::itostr(CU.getQuantity());
 }
 
-/// getObjCEncodingForBlockDecl - Return the encoded type for this block
+/// getObjCEncodingForBlock - Return the encoded type for this block
 /// declaration.
-void ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr, 
-                                             std::string& S) {
+std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
+  std::string S;
+
   const BlockDecl *Decl = Expr->getBlockDecl();
   QualType BlockTy =
       Expr->getType()->getAs<BlockPointerType>()->getPointeeType();
@@ -3414,12 +3763,50 @@ void ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr,
     S += charUnitsToString(ParmOffset);
     ParmOffset += getObjCEncodingTypeSize(PType);
   }
+
+  return S;
+}
+
+void ASTContext::getObjCEncodingForFunctionDecl(const FunctionDecl *Decl,
+                                                std::string& S) {
+  // Encode result type.
+  getObjCEncodingForType(Decl->getResultType(), S);
+  CharUnits ParmOffset;
+  // Compute size of all parameters.
+  for (FunctionDecl::param_const_iterator PI = Decl->param_begin(),
+       E = Decl->param_end(); PI != E; ++PI) {
+    QualType PType = (*PI)->getType();
+    CharUnits sz = getObjCEncodingTypeSize(PType);
+    assert (sz.isPositive() && 
+        "getObjCEncodingForMethodDecl - Incomplete param type");
+    ParmOffset += sz;
+  }
+  S += charUnitsToString(ParmOffset);
+  ParmOffset = CharUnits::Zero();
+
+  // Argument types.
+  for (FunctionDecl::param_const_iterator PI = Decl->param_begin(),
+       E = Decl->param_end(); PI != E; ++PI) {
+    ParmVarDecl *PVDecl = *PI;
+    QualType PType = PVDecl->getOriginalType();
+    if (const ArrayType *AT =
+          dyn_cast<ArrayType>(PType->getCanonicalTypeInternal())) {
+      // Use array's original type only if it has known number of
+      // elements.
+      if (!isa<ConstantArrayType>(AT))
+        PType = PVDecl->getType();
+    } else if (PType->isFunctionType())
+      PType = PVDecl->getType();
+    getObjCEncodingForType(PType, S);
+    S += charUnitsToString(ParmOffset);
+    ParmOffset += getObjCEncodingTypeSize(PType);
+  }
 }
 
 /// getObjCEncodingForMethodDecl - Return the encoded type for this method
 /// declaration.
 void ASTContext::getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
-                                              std::string& S) {
+                                              std::string& S) const {
   // FIXME: This is not very efficient.
   // Encode type qualifer, 'in', 'inout', etc. for the return type.
   getObjCEncodingForTypeQualifier(Decl->getObjCDeclQualifier(), S);
@@ -3495,7 +3882,7 @@ void ASTContext::getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
 /// @endcode
 void ASTContext::getObjCEncodingForPropertyDecl(const ObjCPropertyDecl *PD,
                                                 const Decl *Container,
-                                                std::string& S) {
+                                                std::string& S) const {
   // Collect information from the property implementation decl(s).
   bool Dynamic = false;
   ObjCPropertyImplDecl *SynthesizePID = 0;
@@ -3588,19 +3975,17 @@ void ASTContext::getObjCEncodingForPropertyDecl(const ObjCPropertyDecl *PD,
 void ASTContext::getLegacyIntegralTypeEncoding (QualType &PointeeTy) const {
   if (isa<TypedefType>(PointeeTy.getTypePtr())) {
     if (const BuiltinType *BT = PointeeTy->getAs<BuiltinType>()) {
-      if (BT->getKind() == BuiltinType::ULong &&
-          ((const_cast<ASTContext *>(this))->getIntWidth(PointeeTy) == 32))
+      if (BT->getKind() == BuiltinType::ULong && getIntWidth(PointeeTy) == 32)
         PointeeTy = UnsignedIntTy;
       else
-        if (BT->getKind() == BuiltinType::Long &&
-            ((const_cast<ASTContext *>(this))->getIntWidth(PointeeTy) == 32))
+        if (BT->getKind() == BuiltinType::Long && getIntWidth(PointeeTy) == 32)
           PointeeTy = IntTy;
     }
   }
 }
 
 void ASTContext::getObjCEncodingForType(QualType T, std::string& S,
-                                        const FieldDecl *Field) {
+                                        const FieldDecl *Field) const {
   // We follow the behavior of gcc, expanding structures which are
   // directly pointed to, and expanding embedded structures. Note that
   // these rules are sufficient to prevent recursive encoding of the
@@ -3619,31 +4004,29 @@ static char ObjCEncodingForPrimitiveKind(const ASTContext *C, QualType T) {
     case BuiltinType::UShort:     return 'S';
     case BuiltinType::UInt:       return 'I';
     case BuiltinType::ULong:
-        return
-          (const_cast<ASTContext *>(C))->getIntWidth(T) == 32 ? 'L' : 'Q';
+        return C->getIntWidth(T) == 32 ? 'L' : 'Q';
     case BuiltinType::UInt128:    return 'T';
     case BuiltinType::ULongLong:  return 'Q';
     case BuiltinType::Char_S:
     case BuiltinType::SChar:      return 'c';
     case BuiltinType::Short:      return 's';
-    case BuiltinType::WChar:
+    case BuiltinType::WChar_S:
+    case BuiltinType::WChar_U:
     case BuiltinType::Int:        return 'i';
     case BuiltinType::Long:
-      return
-        (const_cast<ASTContext *>(C))->getIntWidth(T) == 32 ? 'l' : 'q';
+      return C->getIntWidth(T) == 32 ? 'l' : 'q';
     case BuiltinType::LongLong:   return 'q';
     case BuiltinType::Int128:     return 't';
     case BuiltinType::Float:      return 'f';
     case BuiltinType::Double:     return 'd';
-    case BuiltinType::LongDouble: return 'd';
+    case BuiltinType::LongDouble: return 'D';
     }
 }
 
-static void EncodeBitField(const ASTContext *Context, std::string& S,
+static void EncodeBitField(const ASTContext *Ctx, std::string& S,
                            QualType T, const FieldDecl *FD) {
   const Expr *E = FD->getBitWidth();
   assert(E && "bitfield width not there - getObjCEncodingForTypeImpl");
-  ASTContext *Ctx = const_cast<ASTContext*>(Context);
   S += 'b';
   // The NeXT runtime encodes bit fields as b followed by the number of bits.
   // The GNU runtime requires more information; bitfields are encoded as b,
@@ -3674,7 +4057,10 @@ static void EncodeBitField(const ASTContext *Context, std::string& S,
         break;
     }
     S += llvm::utostr(RL.getFieldOffset(i));
-    S += ObjCEncodingForPrimitiveKind(Context, T);
+    if (T->isEnumeralType())
+      S += 'i';
+    else
+      S += ObjCEncodingForPrimitiveKind(Ctx, T);
   }
   unsigned N = E->EvaluateAsInt(*Ctx).getZExtValue();
   S += llvm::utostr(N);
@@ -3686,7 +4072,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
                                             bool ExpandStructures,
                                             const FieldDecl *FD,
                                             bool OutermostType,
-                                            bool EncodingProperty) {
+                                            bool EncodingProperty) const {
   if (T->getAs<BuiltinType>()) {
     if (FD && FD->isBitField())
       return EncodeBitField(this, S, T, FD);
@@ -3811,8 +4197,8 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
         const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
         std::string TemplateArgsStr
           = TemplateSpecializationType::PrintTemplateArgumentList(
-                                            TemplateArgs.getFlatArgumentList(),
-                                            TemplateArgs.flat_size(),
+                                            TemplateArgs.data(),
+                                            TemplateArgs.size(),
                                             (*this).PrintingPolicy);
 
         S += TemplateArgsStr;
@@ -3846,7 +4232,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
     S += RDecl->isUnion() ? ')' : '}';
     return;
   }
-
+  
   if (T->isEnumeralType()) {
     if (FD && FD->isBitField())
       EncodeBitField(this, S, T, FD);
@@ -3949,7 +4335,14 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
   // TODO: maybe there should be a mangling for these
   if (T->getAs<MemberPointerType>())
     return;
-
+  
+  if (T->isVectorType()) {
+    // This matches gcc's encoding, even though technically it is
+    // insufficient.
+    // FIXME. We should do a better job than gcc.
+    return;
+  }
+  
   assert(0 && "@encode for type not implemented!");
 }
 
@@ -4000,8 +4393,9 @@ void ASTContext::setObjCConstantStringInterface(ObjCInterfaceDecl *Decl) {
 
 /// \brief Retrieve the template name that corresponds to a non-empty
 /// lookup.
-TemplateName ASTContext::getOverloadedTemplateName(UnresolvedSetIterator Begin,
-                                                   UnresolvedSetIterator End) {
+TemplateName
+ASTContext::getOverloadedTemplateName(UnresolvedSetIterator Begin,
+                                      UnresolvedSetIterator End) const {
   unsigned size = End - Begin;
   assert(size > 1 && "set is not overloaded!");
 
@@ -4023,9 +4417,10 @@ TemplateName ASTContext::getOverloadedTemplateName(UnresolvedSetIterator Begin,
 
 /// \brief Retrieve the template name that represents a qualified
 /// template name such as \c std::vector.
-TemplateName ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
-                                                  bool TemplateKeyword,
-                                                  TemplateDecl *Template) {
+TemplateName
+ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
+                                     bool TemplateKeyword,
+                                     TemplateDecl *Template) const {
   // FIXME: Canonicalization?
   llvm::FoldingSetNodeID ID;
   QualifiedTemplateName::Profile(ID, NNS, TemplateKeyword, Template);
@@ -4043,8 +4438,9 @@ TemplateName ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
 
 /// \brief Retrieve the template name that represents a dependent
 /// template name such as \c MetaFun::template apply.
-TemplateName ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
-                                                  const IdentifierInfo *Name) {
+TemplateName
+ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
+                                     const IdentifierInfo *Name) const {
   assert((!NNS || NNS->isDependent()) &&
          "Nested name specifier must be dependent");
 
@@ -4078,7 +4474,7 @@ TemplateName ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
 /// template name such as \c MetaFun::template operator+.
 TemplateName 
 ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
-                                     OverloadedOperatorKind Operator) {
+                                     OverloadedOperatorKind Operator) const {
   assert((!NNS || NNS->isDependent()) &&
          "Nested name specifier must be dependent");
   
@@ -4109,6 +4505,27 @@ ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
   return TemplateName(QTN);
 }
 
+TemplateName 
+ASTContext::getSubstTemplateTemplateParmPack(TemplateTemplateParmDecl *Param,
+                                       const TemplateArgument &ArgPack) const {
+  ASTContext &Self = const_cast<ASTContext &>(*this);
+  llvm::FoldingSetNodeID ID;
+  SubstTemplateTemplateParmPackStorage::Profile(ID, Self, Param, ArgPack);
+  
+  void *InsertPos = 0;
+  SubstTemplateTemplateParmPackStorage *Subst
+    = SubstTemplateTemplateParmPacks.FindNodeOrInsertPos(ID, InsertPos);
+  
+  if (!Subst) {
+    Subst = new (*this) SubstTemplateTemplateParmPackStorage(Self, Param, 
+                                                           ArgPack.pack_size(),
+                                                         ArgPack.pack_begin());
+    SubstTemplateTemplateParmPacks.InsertNode(Subst, InsertPos);
+  }
+
+  return TemplateName(Subst);
+}
+
 /// getFromTargetType - Given one of the integer types provided by
 /// TargetInfo, produce the corresponding type. The unsigned @p Type
 /// is actually a value of type @c TargetInfo::IntType.
@@ -4139,7 +4556,7 @@ CanQualType ASTContext::getFromTargetType(unsigned Type) const {
 /// FIXME: Move to Type.
 ///
 bool ASTContext::isObjCNSObjectType(QualType Ty) const {
-  if (TypedefType *TDT = dyn_cast<TypedefType>(Ty)) {
+  if (const TypedefType *TDT = dyn_cast<TypedefType>(Ty)) {
     if (TypedefDecl *TD = TDT->getDecl())
       if (TD->getAttr<ObjCNSObjectAttr>())
         return true;
@@ -4150,24 +4567,30 @@ bool ASTContext::isObjCNSObjectType(QualType Ty) const {
 /// getObjCGCAttr - Returns one of GCNone, Weak or Strong objc's
 /// garbage collection attribute.
 ///
-Qualifiers::GC ASTContext::getObjCGCAttrKind(const QualType &Ty) const {
-  Qualifiers::GC GCAttrs = Qualifiers::GCNone;
-  if (getLangOptions().ObjC1 &&
-      getLangOptions().getGCMode() != LangOptions::NonGC) {
-    GCAttrs = Ty.getObjCGCAttr();
-    // Default behavious under objective-c's gc is for objective-c pointers
-    // (or pointers to them) be treated as though they were declared
-    // as __strong.
-    if (GCAttrs == Qualifiers::GCNone) {
-      if (Ty->isObjCObjectPointerType() || Ty->isBlockPointerType())
-        GCAttrs = Qualifiers::Strong;
-      else if (Ty->isPointerType())
-        return getObjCGCAttrKind(Ty->getAs<PointerType>()->getPointeeType());
-    }
-    // Non-pointers have none gc'able attribute regardless of the attribute
-    // set on them.
-    else if (!Ty->isAnyPointerType() && !Ty->isBlockPointerType())
-      return Qualifiers::GCNone;
+Qualifiers::GC ASTContext::getObjCGCAttrKind(QualType Ty) const {
+  if (getLangOptions().getGCMode() == LangOptions::NonGC)
+    return Qualifiers::GCNone;
+
+  assert(getLangOptions().ObjC1);
+  Qualifiers::GC GCAttrs = Ty.getObjCGCAttr();
+
+  // Default behaviour under objective-C's gc is for ObjC pointers
+  // (or pointers to them) be treated as though they were declared
+  // as __strong.
+  if (GCAttrs == Qualifiers::GCNone) {
+    if (Ty->isObjCObjectPointerType() || Ty->isBlockPointerType())
+      return Qualifiers::Strong;
+    else if (Ty->isPointerType())
+      return getObjCGCAttrKind(Ty->getAs<PointerType>()->getPointeeType());
+  } else {
+    // It's not valid to set GC attributes on anything that isn't a
+    // pointer.
+#ifndef NDEBUG
+    QualType CT = Ty->getCanonicalTypeInternal();
+    while (const ArrayType *AT = dyn_cast<ArrayType>(CT))
+      CT = AT->getElementType();
+    assert(CT->isAnyPointerType() || CT->isBlockPointerType());
+#endif
   }
   return GCAttrs;
 }
@@ -4193,15 +4616,16 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
   if (hasSameUnqualifiedType(FirstVec, SecondVec))
     return true;
 
-  // AltiVec vectors types are identical to equivalent GCC vector types
+  // Treat Neon vector types and most AltiVec vector types as if they are the
+  // equivalent GCC vector types.
   const VectorType *First = FirstVec->getAs<VectorType>();
   const VectorType *Second = SecondVec->getAs<VectorType>();
-  if ((((First->getAltiVecSpecific() == VectorType::AltiVec) &&
-        (Second->getAltiVecSpecific() == VectorType::NotAltiVec)) ||
-       ((First->getAltiVecSpecific() == VectorType::NotAltiVec) &&
-        (Second->getAltiVecSpecific() == VectorType::AltiVec))) &&
+  if (First->getNumElements() == Second->getNumElements() &&
       hasSameType(First->getElementType(), Second->getElementType()) &&
-      (First->getNumElements() == Second->getNumElements()))
+      First->getVectorKind() != VectorType::AltiVecPixel &&
+      First->getVectorKind() != VectorType::AltiVecBool &&
+      Second->getVectorKind() != VectorType::AltiVecPixel &&
+      Second->getVectorKind() != VectorType::AltiVecBool)
     return true;
 
   return false;
@@ -4213,8 +4637,9 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
 
 /// ProtocolCompatibleWithProtocol - return 'true' if 'lProto' is in the
 /// inheritance hierarchy of 'rProto'.
-bool ASTContext::ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
-                                                ObjCProtocolDecl *rProto) {
+bool
+ASTContext::ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
+                                           ObjCProtocolDecl *rProto) const {
   if (lProto == rProto)
     return true;
   for (ObjCProtocolDecl::protocol_iterator PI = rProto->protocol_begin(),
@@ -4336,33 +4761,17 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
 
   if (const ObjCObjectPointerType *lhsOPT =
         lhs->getAsObjCInterfacePointerType()) {
-    if (lhsOPT->qual_empty()) {
-      bool match = false;
-      if (ObjCInterfaceDecl *lhsID = lhsOPT->getInterfaceDecl()) {
-        for (ObjCObjectPointerType::qual_iterator I = rhsQID->qual_begin(),
-             E = rhsQID->qual_end(); I != E; ++I) {
-          // when comparing an id<P> on rhs with a static type on lhs,
-          // static class must implement all of id's protocols directly or
-          // indirectly through its super class.
-          if (lhsID->ClassImplementsProtocol(*I, true)) {
-            match = true;
-            break;
-          }
-        }
-        if (!match)
-          return false;
-      }
-      return true;
-    }
-    // Both the right and left sides have qualifiers.
+    // If both the right and left sides have qualifiers.
     for (ObjCObjectPointerType::qual_iterator I = lhsOPT->qual_begin(),
          E = lhsOPT->qual_end(); I != E; ++I) {
       ObjCProtocolDecl *lhsProto = *I;
       bool match = false;
 
-      // when comparing an id<P> on lhs with a static type on rhs,
+      // when comparing an id<P> on rhs with a static type on lhs,
       // see if static class implements all of id's protocols, directly or
       // through its super class and categories.
+      // First, lhs protocols in the qualifier list must be found, direct
+      // or indirect in rhs's qualifier list or it is a mismatch.
       for (ObjCObjectPointerType::qual_iterator J = rhsQID->qual_begin(),
            E = rhsQID->qual_end(); J != E; ++J) {
         ObjCProtocolDecl *rhsProto = *J;
@@ -4375,6 +4784,35 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
       if (!match)
         return false;
     }
+    
+    // Static class's protocols, or its super class or category protocols
+    // must be found, direct or indirect in rhs's qualifier list or it is a mismatch.
+    if (ObjCInterfaceDecl *lhsID = lhsOPT->getInterfaceDecl()) {
+      llvm::SmallPtrSet<ObjCProtocolDecl *, 8> LHSInheritedProtocols;
+      CollectInheritedProtocols(lhsID, LHSInheritedProtocols);
+      // This is rather dubious but matches gcc's behavior. If lhs has
+      // no type qualifier and its class has no static protocol(s)
+      // assume that it is mismatch.
+      if (LHSInheritedProtocols.empty() && lhsOPT->qual_empty())
+        return false;
+      for (llvm::SmallPtrSet<ObjCProtocolDecl*,8>::iterator I =
+           LHSInheritedProtocols.begin(),
+           E = LHSInheritedProtocols.end(); I != E; ++I) {
+        bool match = false;
+        ObjCProtocolDecl *lhsProto = (*I);
+        for (ObjCObjectPointerType::qual_iterator J = rhsQID->qual_begin(),
+             E = rhsQID->qual_end(); J != E; ++J) {
+          ObjCProtocolDecl *rhsProto = *J;
+          if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
+              (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
+            match = true;
+            break;
+          }
+        }
+        if (!match)
+          return false;
+      }
+    }
     return true;
   }
   return false;
@@ -4600,6 +5038,49 @@ bool ASTContext::typesAreBlockPointerCompatible(QualType LHS, QualType RHS) {
   return !mergeTypes(LHS, RHS, true).isNull();
 }
 
+/// mergeTransparentUnionType - if T is a transparent union type and a member
+/// of T is compatible with SubType, return the merged type, else return
+/// QualType()
+QualType ASTContext::mergeTransparentUnionType(QualType T, QualType SubType,
+                                               bool OfBlockPointer,
+                                               bool Unqualified) {
+  if (const RecordType *UT = T->getAsUnionType()) {
+    RecordDecl *UD = UT->getDecl();
+    if (UD->hasAttr<TransparentUnionAttr>()) {
+      for (RecordDecl::field_iterator it = UD->field_begin(),
+           itend = UD->field_end(); it != itend; ++it) {
+        QualType ET = it->getType().getUnqualifiedType();
+        QualType MT = mergeTypes(ET, SubType, OfBlockPointer, Unqualified);
+        if (!MT.isNull())
+          return MT;
+      }
+    }
+  }
+
+  return QualType();
+}
+
+/// mergeFunctionArgumentTypes - merge two types which appear as function
+/// argument types
+QualType ASTContext::mergeFunctionArgumentTypes(QualType lhs, QualType rhs, 
+                                                bool OfBlockPointer,
+                                                bool Unqualified) {
+  // GNU extension: two types are compatible if they appear as a function
+  // argument, one of the types is a transparent union type and the other
+  // type is compatible with a union member
+  QualType lmerge = mergeTransparentUnionType(lhs, rhs, OfBlockPointer,
+                                              Unqualified);
+  if (!lmerge.isNull())
+    return lmerge;
+
+  QualType rmerge = mergeTransparentUnionType(rhs, lhs, OfBlockPointer,
+                                              Unqualified);
+  if (!rmerge.isNull())
+    return rmerge;
+
+  return mergeTypes(lhs, rhs, OfBlockPointer, Unqualified);
+}
+
 QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs, 
                                         bool OfBlockPointer,
                                         bool Unqualified) {
@@ -4612,12 +5093,17 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
 
   // Check return type
   QualType retType;
-  if (OfBlockPointer)
-    retType = mergeTypes(rbase->getResultType(), lbase->getResultType(), true,
-                         Unqualified);
+  if (OfBlockPointer) {
+    QualType RHS = rbase->getResultType();
+    QualType LHS = lbase->getResultType();
+    bool UnqualifiedResult = Unqualified;
+    if (!UnqualifiedResult)
+      UnqualifiedResult = (!RHS.hasQualifiers() && LHS.hasQualifiers());
+    retType = mergeTypes(RHS, LHS, true, UnqualifiedResult);
+  }
   else
-   retType = mergeTypes(lbase->getResultType(), rbase->getResultType(),
-                        false, Unqualified);
+    retType = mergeTypes(lbase->getResultType(), rbase->getResultType(), false,
+                         Unqualified);
   if (retType.isNull()) return QualType();
   
   if (Unqualified)
@@ -4634,26 +5120,33 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     allLTypes = false;
   if (getCanonicalType(retType) != RRetType)
     allRTypes = false;
+
   // FIXME: double check this
   // FIXME: should we error if lbase->getRegParmAttr() != 0 &&
   //                           rbase->getRegParmAttr() != 0 &&
   //                           lbase->getRegParmAttr() != rbase->getRegParmAttr()?
   FunctionType::ExtInfo lbaseInfo = lbase->getExtInfo();
   FunctionType::ExtInfo rbaseInfo = rbase->getExtInfo();
-  unsigned RegParm = lbaseInfo.getRegParm() == 0 ? rbaseInfo.getRegParm() :
-      lbaseInfo.getRegParm();
+
+  // Compatible functions must have compatible calling conventions
+  if (!isSameCallConv(lbaseInfo.getCC(), rbaseInfo.getCC()))
+    return QualType();
+
+  // Regparm is part of the calling convention.
+  if (lbaseInfo.getRegParm() != rbaseInfo.getRegParm())
+    return QualType();
+
+  // It's noreturn if either type is.
+  // FIXME: some uses, e.g. conditional exprs, really want this to be 'both'.
   bool NoReturn = lbaseInfo.getNoReturn() || rbaseInfo.getNoReturn();
-  if (NoReturn != lbaseInfo.getNoReturn() ||
-      RegParm != lbaseInfo.getRegParm())
+  if (NoReturn != lbaseInfo.getNoReturn())
     allLTypes = false;
-  if (NoReturn != rbaseInfo.getNoReturn() ||
-      RegParm != rbaseInfo.getRegParm())
+  if (NoReturn != rbaseInfo.getNoReturn())
     allRTypes = false;
-  CallingConv lcc = lbaseInfo.getCC();
-  CallingConv rcc = rbaseInfo.getCC();
-  // Compatible functions must have compatible calling conventions
-  if (!isSameCallConv(lcc, rcc))
-    return QualType();
+
+  FunctionType::ExtInfo einfo(NoReturn,
+                              lbaseInfo.getRegParm(),
+                              lbaseInfo.getCC());
 
   if (lproto && rproto) { // two C99 style function prototypes
     assert(!lproto->hasExceptionSpec() && !rproto->hasExceptionSpec() &&
@@ -4677,8 +5170,9 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     for (unsigned i = 0; i < lproto_nargs; i++) {
       QualType largtype = lproto->getArgType(i).getUnqualifiedType();
       QualType rargtype = rproto->getArgType(i).getUnqualifiedType();
-      QualType argtype = mergeTypes(largtype, rargtype, OfBlockPointer,
-                                    Unqualified);
+      QualType argtype = mergeFunctionArgumentTypes(largtype, rargtype,
+                                                    OfBlockPointer,
+                                                    Unqualified);
       if (argtype.isNull()) return QualType();
       
       if (Unqualified)
@@ -4697,10 +5191,10 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     }
     if (allLTypes) return lhs;
     if (allRTypes) return rhs;
-    return getFunctionType(retType, types.begin(), types.size(),
-                           lproto->isVariadic(), lproto->getTypeQuals(),
-                           false, false, 0, 0,
-                           FunctionType::ExtInfo(NoReturn, RegParm, lcc));
+
+    FunctionProtoType::ExtProtoInfo EPI = lproto->getExtProtoInfo();
+    EPI.ExtInfo = einfo;
+    return getFunctionType(retType, types.begin(), types.size(), EPI);
   }
 
   if (lproto) allRTypes = false;
@@ -4731,17 +5225,16 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
 
     if (allLTypes) return lhs;
     if (allRTypes) return rhs;
+
+    FunctionProtoType::ExtProtoInfo EPI = proto->getExtProtoInfo();
+    EPI.ExtInfo = einfo;
     return getFunctionType(retType, proto->arg_type_begin(),
-                           proto->getNumArgs(), proto->isVariadic(),
-                           proto->getTypeQuals(),
-                           false, false, 0, 0,
-                           FunctionType::ExtInfo(NoReturn, RegParm, lcc));
+                           proto->getNumArgs(), EPI);
   }
 
   if (allLTypes) return lhs;
   if (allRTypes) return rhs;
-  FunctionType::ExtInfo Info(NoReturn, RegParm, lcc);
-  return getFunctionNoProtoType(retType, Info);
+  return getFunctionNoProtoType(retType, einfo);
 }
 
 QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, 
@@ -5020,16 +5513,11 @@ QualType ASTContext::mergeObjCGCQualifiers(QualType LHS, QualType RHS) {
       // In either case, use OldReturnType to build the new function type.
       const FunctionType *F = LHS->getAs<FunctionType>();
       if (const FunctionProtoType *FPT = cast<FunctionProtoType>(F)) {
-        FunctionType::ExtInfo Info = getFunctionExtInfo(LHS);
+        FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+        EPI.ExtInfo = getFunctionExtInfo(LHS);
         QualType ResultType
           = getFunctionType(OldReturnType, FPT->arg_type_begin(),
-                                  FPT->getNumArgs(), FPT->isVariadic(),
-                                  FPT->getTypeQuals(),
-                                  FPT->hasExceptionSpec(),
-                                  FPT->hasAnyExceptionSpec(),
-                                  FPT->getNumExceptions(),
-                                  FPT->exception_begin(),
-                                  Info);
+                            FPT->getNumArgs(), EPI);
         return ResultType;
       }
     }
@@ -5080,11 +5568,11 @@ QualType ASTContext::mergeObjCGCQualifiers(QualType LHS, QualType RHS) {
 //                         Integer Predicates
 //===----------------------------------------------------------------------===//
 
-unsigned ASTContext::getIntWidth(QualType T) {
+unsigned ASTContext::getIntWidth(QualType T) const {
+  if (const EnumType *ET = dyn_cast<EnumType>(T))
+    T = ET->getDecl()->getIntegerType();
   if (T->isBooleanType())
     return 1;
-  if (EnumType *ET = dyn_cast<EnumType>(T))
-    T = ET->getDecl()->getIntegerType();
   // For builtin types, just use the standard type sizing method
   return (unsigned)getTypeSize(T);
 }
@@ -5095,7 +5583,7 @@ QualType ASTContext::getCorrespondingUnsignedType(QualType T) {
   // Turn <4 x signed int> -> <4 x unsigned int>
   if (const VectorType *VTy = T->getAs<VectorType>())
     return getVectorType(getCorrespondingUnsignedType(VTy->getElementType()),
-             VTy->getNumElements(), VTy->getAltiVecSpecific());
+                         VTy->getNumElements(), VTy->getVectorKind());
 
   // For enums, we return the unsigned version of the base type.
   if (const EnumType *ETy = T->getAs<EnumType>())
@@ -5127,25 +5615,38 @@ ExternalASTSource::~ExternalASTSource() { }
 
 void ExternalASTSource::PrintStats() { }
 
+ASTMutationListener::~ASTMutationListener() { }
+
 
 //===----------------------------------------------------------------------===//
 //                          Builtin Type Computation
 //===----------------------------------------------------------------------===//
 
 /// DecodeTypeFromStr - This decodes one type descriptor from Str, advancing the
-/// pointer over the consumed characters.  This returns the resultant type.
-static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
+/// pointer over the consumed characters.  This returns the resultant type.  If
+/// AllowTypeModifiers is false then modifier like * are not parsed, just basic
+/// types.  This allows "v2i*" to be parsed as a pointer to a v2i instead of
+/// a vector of "i*".
+///
+/// RequiresICE is filled in on return to indicate whether the value is required
+/// to be an Integer Constant Expression.
+static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
                                   ASTContext::GetBuiltinTypeError &Error,
-                                  bool AllowTypeModifiers = true) {
+                                  bool &RequiresICE,
+                                  bool AllowTypeModifiers) {
   // Modifiers.
   int HowLong = 0;
   bool Signed = false, Unsigned = false;
-
-  // Read the modifiers first.
+  RequiresICE = false;
+  
+  // Read the prefixed modifiers first.
   bool Done = false;
   while (!Done) {
     switch (*Str++) {
     default: Done = true; --Str; break;
+    case 'I':
+      RequiresICE = true;
+      break;
     case 'S':
       assert(!Unsigned && "Can't use both 'S' and 'U' modifiers!");
       assert(!Signed && "Can't use 'S' modifier multiple times!");
@@ -5223,6 +5724,12 @@ static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
   case 'F':
     Type = Context.getCFConstantStringType();
     break;
+  case 'G':
+    Type = Context.getObjCIdType();
+    break;
+  case 'H':
+    Type = Context.getObjCSelType();
+    break;
   case 'a':
     Type = Context.getBuiltinVaListType();
     assert(!Type.isNull() && "builtin va list type not initialized!");
@@ -5238,27 +5745,30 @@ static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
     // it to be a __va_list_tag*.
     Type = Context.getBuiltinVaListType();
     assert(!Type.isNull() && "builtin va list type not initialized!");
-    if (Type->isArrayType()) {
+    if (Type->isArrayType())
       Type = Context.getArrayDecayedType(Type);
-    } else {
+    else
       Type = Context.getLValueReferenceType(Type);
-    }
     break;
   case 'V': {
     char *End;
     unsigned NumElements = strtoul(Str, &End, 10);
     assert(End != Str && "Missing vector size");
-
     Str = End;
 
-    QualType ElementType = DecodeTypeFromStr(Str, Context, Error, false);
-    // FIXME: Don't know what to do about AltiVec.
+    QualType ElementType = DecodeTypeFromStr(Str, Context, Error, 
+                                             RequiresICE, false);
+    assert(!RequiresICE && "Can't require vector ICE");
+    
+    // TODO: No way to make AltiVec vectors in builtins yet.
     Type = Context.getVectorType(ElementType, NumElements,
-                                 VectorType::NotAltiVec);
+                                 VectorType::GenericVector);
     break;
   }
   case 'X': {
-    QualType ElementType = DecodeTypeFromStr(Str, Context, Error, false);
+    QualType ElementType = DecodeTypeFromStr(Str, Context, Error, RequiresICE,
+                                             false);
+    assert(!RequiresICE && "Can't require complex ICE");
     Type = Context.getComplexType(ElementType);
     break;
   }      
@@ -5282,59 +5792,70 @@ static QualType DecodeTypeFromStr(const char *&Str, ASTContext &Context,
     break;
   }
 
-  if (!AllowTypeModifiers)
-    return Type;
-
-  Done = false;
+  // If there are modifiers and if we're allowed to parse them, go for it.
+  Done = !AllowTypeModifiers;
   while (!Done) {
     switch (char c = *Str++) {
-      default: Done = true; --Str; break;
-      case '*':
-      case '&':
-        {
-          // Both pointers and references can have their pointee types
-          // qualified with an address space.
-          char *End;
-          unsigned AddrSpace = strtoul(Str, &End, 10);
-          if (End != Str && AddrSpace != 0) {
-            Type = Context.getAddrSpaceQualType(Type, AddrSpace);
-            Str = End;
-          }
-        }
-        if (c == '*')
-          Type = Context.getPointerType(Type);
-        else
-          Type = Context.getLValueReferenceType(Type);
-        break;
-      // FIXME: There's no way to have a built-in with an rvalue ref arg.
-      case 'C':
-        Type = Type.withConst();
-        break;
-      case 'D':
-        Type = Context.getVolatileType(Type);
-        break;
+    default: Done = true; --Str; break;
+    case '*':
+    case '&': {
+      // Both pointers and references can have their pointee types
+      // qualified with an address space.
+      char *End;
+      unsigned AddrSpace = strtoul(Str, &End, 10);
+      if (End != Str && AddrSpace != 0) {
+        Type = Context.getAddrSpaceQualType(Type, AddrSpace);
+        Str = End;
+      }
+      if (c == '*')
+        Type = Context.getPointerType(Type);
+      else
+        Type = Context.getLValueReferenceType(Type);
+      break;
+    }
+    // FIXME: There's no way to have a built-in with an rvalue ref arg.
+    case 'C':
+      Type = Type.withConst();
+      break;
+    case 'D':
+      Type = Context.getVolatileType(Type);
+      break;
     }
   }
+  
+  assert((!RequiresICE || Type->isIntegralOrEnumerationType()) &&
+         "Integer constant 'I' type must be an integer"); 
 
   return Type;
 }
 
 /// GetBuiltinType - Return the type for the specified builtin.
-QualType ASTContext::GetBuiltinType(unsigned id,
-                                    GetBuiltinTypeError &Error) {
-  const char *TypeStr = BuiltinInfo.GetTypeString(id);
+QualType ASTContext::GetBuiltinType(unsigned Id,
+                                    GetBuiltinTypeError &Error,
+                                    unsigned *IntegerConstantArgs) const {
+  const char *TypeStr = BuiltinInfo.GetTypeString(Id);
 
   llvm::SmallVector<QualType, 8> ArgTypes;
 
+  bool RequiresICE = false;
   Error = GE_None;
-  QualType ResType = DecodeTypeFromStr(TypeStr, *this, Error);
+  QualType ResType = DecodeTypeFromStr(TypeStr, *this, Error,
+                                       RequiresICE, true);
   if (Error != GE_None)
     return QualType();
+  
+  assert(!RequiresICE && "Result of intrinsic cannot be required to be an ICE");
+  
   while (TypeStr[0] && TypeStr[0] != '.') {
-    QualType Ty = DecodeTypeFromStr(TypeStr, *this, Error);
+    QualType Ty = DecodeTypeFromStr(TypeStr, *this, Error, RequiresICE, true);
     if (Error != GE_None)
       return QualType();
 
+    // If this argument is required to be an IntegerConstantExpression and the
+    // caller cares, fill in the bitmask we return.
+    if (RequiresICE && IntegerConstantArgs)
+      *IntegerConstantArgs |= 1 << ArgTypes.size();
+    
     // Do array -> pointer decay.  The builtin should use the decayed type.
     if (Ty->isArrayType())
       Ty = getArrayDecayedType(Ty);
@@ -5345,164 +5866,26 @@ QualType ASTContext::GetBuiltinType(unsigned id,
   assert((TypeStr[0] != '.' || TypeStr[1] == 0) &&
          "'.' should only occur at end of builtin type list!");
 
-  // handle untyped/variadic arguments "T c99Style();" or "T cppStyle(...);".
-  if (ArgTypes.size() == 0 && TypeStr[0] == '.')
-    return getFunctionNoProtoType(ResType);
-
-  // FIXME: Should we create noreturn types?
-  return getFunctionType(ResType, ArgTypes.data(), ArgTypes.size(),
-                         TypeStr[0] == '.', 0, false, false, 0, 0,
-                         FunctionType::ExtInfo());
-}
-
-QualType
-ASTContext::UsualArithmeticConversionsType(QualType lhs, QualType rhs) {
-  // Perform the usual unary conversions. We do this early so that
-  // integral promotions to "int" can allow us to exit early, in the
-  // lhs == rhs check. Also, for conversion purposes, we ignore any
-  // qualifiers.  For example, "const float" and "float" are
-  // equivalent.
-  if (lhs->isPromotableIntegerType())
-    lhs = getPromotedIntegerType(lhs);
-  else
-    lhs = lhs.getUnqualifiedType();
-  if (rhs->isPromotableIntegerType())
-    rhs = getPromotedIntegerType(rhs);
-  else
-    rhs = rhs.getUnqualifiedType();
+  FunctionType::ExtInfo EI;
+  if (BuiltinInfo.isNoReturn(Id)) EI = EI.withNoReturn(true);
 
-  // If both types are identical, no conversion is needed.
-  if (lhs == rhs)
-    return lhs;
+  bool Variadic = (TypeStr[0] == '.');
 
-  // If either side is a non-arithmetic type (e.g. a pointer), we are done.
-  // The caller can deal with this (e.g. pointer + int).
-  if (!lhs->isArithmeticType() || !rhs->isArithmeticType())
-    return lhs;
+  // We really shouldn't be making a no-proto type here, especially in C++.
+  if (ArgTypes.empty() && Variadic)
+    return getFunctionNoProtoType(ResType, EI);
 
-  // At this point, we have two different arithmetic types.
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.ExtInfo = EI;
+  EPI.Variadic = Variadic;
 
-  // Handle complex types first (C99 6.3.1.8p1).
-  if (lhs->isComplexType() || rhs->isComplexType()) {
-    // if we have an integer operand, the result is the complex type.
-    if (rhs->isIntegerType() || rhs->isComplexIntegerType()) {
-      // convert the rhs to the lhs complex type.
-      return lhs;
-    }
-    if (lhs->isIntegerType() || lhs->isComplexIntegerType()) {
-      // convert the lhs to the rhs complex type.
-      return rhs;
-    }
-    // This handles complex/complex, complex/float, or float/complex.
-    // When both operands are complex, the shorter operand is converted to the
-    // type of the longer, and that is the type of the result. This corresponds
-    // to what is done when combining two real floating-point operands.
-    // The fun begins when size promotion occur across type domains.
-    // From H&S 6.3.4: When one operand is complex and the other is a real
-    // floating-point type, the less precise type is converted, within it's
-    // real or complex domain, to the precision of the other type. For example,
-    // when combining a "long double" with a "double _Complex", the
-    // "double _Complex" is promoted to "long double _Complex".
-    int result = getFloatingTypeOrder(lhs, rhs);
-
-    if (result > 0) { // The left side is bigger, convert rhs.
-      rhs = getFloatingTypeOfSizeWithinDomain(lhs, rhs);
-    } else if (result < 0) { // The right side is bigger, convert lhs.
-      lhs = getFloatingTypeOfSizeWithinDomain(rhs, lhs);
-    }
-    // At this point, lhs and rhs have the same rank/size. Now, make sure the
-    // domains match. This is a requirement for our implementation, C99
-    // does not require this promotion.
-    if (lhs != rhs) { // Domains don't match, we have complex/float mix.
-      if (lhs->isRealFloatingType()) { // handle "double, _Complex double".
-        return rhs;
-      } else { // handle "_Complex double, double".
-        return lhs;
-      }
-    }
-    return lhs; // The domain/size match exactly.
-  }
-  // Now handle "real" floating types (i.e. float, double, long double).
-  if (lhs->isRealFloatingType() || rhs->isRealFloatingType()) {
-    // if we have an integer operand, the result is the real floating type.
-    if (rhs->isIntegerType()) {
-      // convert rhs to the lhs floating point type.
-      return lhs;
-    }
-    if (rhs->isComplexIntegerType()) {
-      // convert rhs to the complex floating point type.
-      return getComplexType(lhs);
-    }
-    if (lhs->isIntegerType()) {
-      // convert lhs to the rhs floating point type.
-      return rhs;
-    }
-    if (lhs->isComplexIntegerType()) {
-      // convert lhs to the complex floating point type.
-      return getComplexType(rhs);
-    }
-    // We have two real floating types, float/complex combos were handled above.
-    // Convert the smaller operand to the bigger result.
-    int result = getFloatingTypeOrder(lhs, rhs);
-    if (result > 0) // convert the rhs
-      return lhs;
-    assert(result < 0 && "illegal float comparison");
-    return rhs;   // convert the lhs
-  }
-  if (lhs->isComplexIntegerType() || rhs->isComplexIntegerType()) {
-    // Handle GCC complex int extension.
-    const ComplexType *lhsComplexInt = lhs->getAsComplexIntegerType();
-    const ComplexType *rhsComplexInt = rhs->getAsComplexIntegerType();
-
-    if (lhsComplexInt && rhsComplexInt) {
-      if (getIntegerTypeOrder(lhsComplexInt->getElementType(),
-                              rhsComplexInt->getElementType()) >= 0)
-        return lhs; // convert the rhs
-      return rhs;
-    } else if (lhsComplexInt && rhs->isIntegerType()) {
-      // convert the rhs to the lhs complex type.
-      return lhs;
-    } else if (rhsComplexInt && lhs->isIntegerType()) {
-      // convert the lhs to the rhs complex type.
-      return rhs;
-    }
-  }
-  // Finally, we have two differing integer types.
-  // The rules for this case are in C99 6.3.1.8
-  int compare = getIntegerTypeOrder(lhs, rhs);
-  bool lhsSigned = lhs->hasSignedIntegerRepresentation(),
-       rhsSigned = rhs->hasSignedIntegerRepresentation();
-  QualType destType;
-  if (lhsSigned == rhsSigned) {
-    // Same signedness; use the higher-ranked type
-    destType = compare >= 0 ? lhs : rhs;
-  } else if (compare != (lhsSigned ? 1 : -1)) {
-    // The unsigned type has greater than or equal rank to the
-    // signed type, so use the unsigned type
-    destType = lhsSigned ? rhs : lhs;
-  } else if (getIntWidth(lhs) != getIntWidth(rhs)) {
-    // The two types are different widths; if we are here, that
-    // means the signed type is larger than the unsigned type, so
-    // use the signed type.
-    destType = lhsSigned ? lhs : rhs;
-  } else {
-    // The signed type is higher-ranked than the unsigned type,
-    // but isn't actually any bigger (like unsigned int and long
-    // on most 32-bit systems).  Use the unsigned type corresponding
-    // to the signed type.
-    destType = getCorrespondingUnsignedType(lhsSigned ? lhs : rhs);
-  }
-  return destType;
+  return getFunctionType(ResType, ArgTypes.data(), ArgTypes.size(), EPI);
 }
 
 GVALinkage ASTContext::GetGVALinkageForFunction(const FunctionDecl *FD) {
   GVALinkage External = GVA_StrongExternal;
 
   Linkage L = FD->getLinkage();
-  if (L == ExternalLinkage && getLangOptions().CPlusPlus &&
-      FD->getType()->getLinkage() == UniqueExternalLinkage)
-    L = UniqueExternalLinkage;
-  
   switch (L) {
   case NoLinkage:
   case InternalLinkage:
@@ -5663,4 +6046,26 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
   return true;
 }
 
+CallingConv ASTContext::getDefaultMethodCallConv() {
+  // Pass through to the C++ ABI object
+  return ABI->getDefaultMethodCallConv();
+}
+
+bool ASTContext::isNearlyEmpty(const CXXRecordDecl *RD) const {
+  // Pass through to the C++ ABI object
+  return ABI->isNearlyEmpty(RD);
+}
+
+MangleContext *ASTContext::createMangleContext() {
+  switch (Target.getCXXABI()) {
+  case CXXABI_ARM:
+  case CXXABI_Itanium:
+    return createItaniumMangleContext(*this, getDiagnostics());
+  case CXXABI_Microsoft:
+    return createMicrosoftMangleContext(*this, getDiagnostics());
+  }
+  assert(0 && "Unsupported ABI");
+  return 0;
+}
+
 CXXABI::~CXXABI() {}
diff --git a/lib/AST/ASTDiagnostic.cpp b/lib/AST/ASTDiagnostic.cpp
index 23f323d0286a..5bf8a38199b6 100644
--- a/lib/AST/ASTDiagnostic.cpp
+++ b/lib/AST/ASTDiagnostic.cpp
@@ -28,14 +28,26 @@ static QualType Desugar(ASTContext &Context, QualType QT, bool &ShouldAKA) {
     const Type *Ty = QC.strip(QT);
 
     // Don't aka just because we saw an elaborated type...
-    if (isa<ElaboratedType>(Ty)) {
-      QT = cast<ElaboratedType>(Ty)->desugar();
+    if (const ElaboratedType *ET = dyn_cast<ElaboratedType>(Ty)) {
+      QT = ET->desugar();
       continue;
     }
-
-    // ...or a substituted template type parameter.
-    if (isa<SubstTemplateTypeParmType>(Ty)) {
-      QT = cast<SubstTemplateTypeParmType>(Ty)->desugar();
+    // ... or a paren type ...
+    if (const ParenType *PT = dyn_cast<ParenType>(Ty)) {
+      QT = PT->desugar();
+      continue;
+    }
+    // ...or a substituted template type parameter ...
+    if (const SubstTemplateTypeParmType *ST =
+          dyn_cast<SubstTemplateTypeParmType>(Ty)) {
+      QT = ST->desugar();
+      continue;
+    }
+    // ... or an auto type.
+    if (const AutoType *AT = dyn_cast<AutoType>(Ty)) {
+      if (!AT->isSugared())
+        break;
+      QT = AT->desugar();
       continue;
     }
 
@@ -81,10 +93,10 @@ break; \
       break;
 
     // Don't desugar through the primary typedef of an anonymous type.
-    if (isa<TagType>(Underlying) && isa<TypedefType>(QT))
-      if (cast<TagType>(Underlying)->getDecl()->getTypedefForAnonDecl() ==
-          cast<TypedefType>(QT)->getDecl())
-        break;
+    if (const TagType *UTT = Underlying->getAs<TagType>())
+      if (const TypedefType *QTT = dyn_cast<TypedefType>(QT))
+        if (UTT->getDecl()->getTypedefForAnonDecl() == QTT->getDecl())
+          break;
 
     // Record that we actually looked through an opaque type here.
     ShouldAKA = true;
@@ -94,14 +106,17 @@ break; \
   // If we have a pointer-like type, desugar the pointee as well.
   // FIXME: Handle other pointer-like types.
   if (const PointerType *Ty = QT->getAs<PointerType>()) {
-      QT = Context.getPointerType(Desugar(Context, Ty->getPointeeType(),
-                                          ShouldAKA));
+    QT = Context.getPointerType(Desugar(Context, Ty->getPointeeType(),
+                                        ShouldAKA));
   } else if (const LValueReferenceType *Ty = QT->getAs<LValueReferenceType>()) {
-      QT = Context.getLValueReferenceType(Desugar(Context, Ty->getPointeeType(),
-                                                  ShouldAKA));
+    QT = Context.getLValueReferenceType(Desugar(Context, Ty->getPointeeType(),
+                                                ShouldAKA));
+  } else if (const RValueReferenceType *Ty = QT->getAs<RValueReferenceType>()) {
+    QT = Context.getRValueReferenceType(Desugar(Context, Ty->getPointeeType(),
+                                                ShouldAKA));
   }
 
-  return QC.apply(QT);
+  return QC.apply(Context, QT);
 }
 
 /// \brief Convert the given type to a string suitable for printing as part of 
@@ -151,13 +166,10 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
     bool ShouldAKA = false;
     QualType DesugaredTy = Desugar(Context, Ty, ShouldAKA);
     if (ShouldAKA) {
-      std::string D = DesugaredTy.getAsString(Context.PrintingPolicy);
-      if (D != S) {
-        S = "'" + S + "' (aka '";
-        S += D;
-        S += "')";
-        return S;
-      }
+      S = "'" + S + "' (aka '";
+      S += DesugaredTy.getAsString(Context.PrintingPolicy);
+      S += "')";
+      return S;
     }
   }
 
diff --git a/lib/AST/ASTImporter.cpp b/lib/AST/ASTImporter.cpp
index 2edd09c067e9..65c0a3bb619f 100644
--- a/lib/AST/ASTImporter.cpp
+++ b/lib/AST/ASTImporter.cpp
@@ -41,41 +41,42 @@ namespace {
     using StmtVisitor<ASTNodeImporter, Stmt *>::Visit;
 
     // Importing types
-    QualType VisitType(Type *T);
-    QualType VisitBuiltinType(BuiltinType *T);
-    QualType VisitComplexType(ComplexType *T);
-    QualType VisitPointerType(PointerType *T);
-    QualType VisitBlockPointerType(BlockPointerType *T);
-    QualType VisitLValueReferenceType(LValueReferenceType *T);
-    QualType VisitRValueReferenceType(RValueReferenceType *T);
-    QualType VisitMemberPointerType(MemberPointerType *T);
-    QualType VisitConstantArrayType(ConstantArrayType *T);
-    QualType VisitIncompleteArrayType(IncompleteArrayType *T);
-    QualType VisitVariableArrayType(VariableArrayType *T);
+    QualType VisitType(const Type *T);
+    QualType VisitBuiltinType(const BuiltinType *T);
+    QualType VisitComplexType(const ComplexType *T);
+    QualType VisitPointerType(const PointerType *T);
+    QualType VisitBlockPointerType(const BlockPointerType *T);
+    QualType VisitLValueReferenceType(const LValueReferenceType *T);
+    QualType VisitRValueReferenceType(const RValueReferenceType *T);
+    QualType VisitMemberPointerType(const MemberPointerType *T);
+    QualType VisitConstantArrayType(const ConstantArrayType *T);
+    QualType VisitIncompleteArrayType(const IncompleteArrayType *T);
+    QualType VisitVariableArrayType(const VariableArrayType *T);
     // FIXME: DependentSizedArrayType
     // FIXME: DependentSizedExtVectorType
-    QualType VisitVectorType(VectorType *T);
-    QualType VisitExtVectorType(ExtVectorType *T);
-    QualType VisitFunctionNoProtoType(FunctionNoProtoType *T);
-    QualType VisitFunctionProtoType(FunctionProtoType *T);
+    QualType VisitVectorType(const VectorType *T);
+    QualType VisitExtVectorType(const ExtVectorType *T);
+    QualType VisitFunctionNoProtoType(const FunctionNoProtoType *T);
+    QualType VisitFunctionProtoType(const FunctionProtoType *T);
     // FIXME: UnresolvedUsingType
-    QualType VisitTypedefType(TypedefType *T);
-    QualType VisitTypeOfExprType(TypeOfExprType *T);
+    QualType VisitTypedefType(const TypedefType *T);
+    QualType VisitTypeOfExprType(const TypeOfExprType *T);
     // FIXME: DependentTypeOfExprType
-    QualType VisitTypeOfType(TypeOfType *T);
-    QualType VisitDecltypeType(DecltypeType *T);
+    QualType VisitTypeOfType(const TypeOfType *T);
+    QualType VisitDecltypeType(const DecltypeType *T);
+    QualType VisitAutoType(const AutoType *T);
     // FIXME: DependentDecltypeType
-    QualType VisitRecordType(RecordType *T);
-    QualType VisitEnumType(EnumType *T);
+    QualType VisitRecordType(const RecordType *T);
+    QualType VisitEnumType(const EnumType *T);
     // FIXME: TemplateTypeParmType
     // FIXME: SubstTemplateTypeParmType
-    // FIXME: TemplateSpecializationType
-    QualType VisitElaboratedType(ElaboratedType *T);
+    QualType VisitTemplateSpecializationType(const TemplateSpecializationType *T);
+    QualType VisitElaboratedType(const ElaboratedType *T);
     // FIXME: DependentNameType
     // FIXME: DependentTemplateSpecializationType
-    QualType VisitObjCInterfaceType(ObjCInterfaceType *T);
-    QualType VisitObjCObjectType(ObjCObjectType *T);
-    QualType VisitObjCObjectPointerType(ObjCObjectPointerType *T);
+    QualType VisitObjCInterfaceType(const ObjCInterfaceType *T);
+    QualType VisitObjCObjectType(const ObjCObjectType *T);
+    QualType VisitObjCObjectPointerType(const ObjCObjectPointerType *T);
                             
     // Importing declarations
     bool ImportDeclParts(NamedDecl *D, DeclContext *&DC, 
@@ -83,9 +84,17 @@ namespace {
                          SourceLocation &Loc);
     void ImportDeclarationNameLoc(const DeclarationNameInfo &From,
                                   DeclarationNameInfo& To);
-    void ImportDeclContext(DeclContext *FromDC);
+    void ImportDeclContext(DeclContext *FromDC, bool ForceImport = false);
+    bool ImportDefinition(RecordDecl *From, RecordDecl *To);
+    TemplateParameterList *ImportTemplateParameterList(
+                                                 TemplateParameterList *Params);
+    TemplateArgument ImportTemplateArgument(const TemplateArgument &From);
+    bool ImportTemplateArguments(const TemplateArgument *FromArgs,
+                                 unsigned NumFromArgs,
+                               llvm::SmallVectorImpl<TemplateArgument> &ToArgs);
     bool IsStructuralMatch(RecordDecl *FromRecord, RecordDecl *ToRecord);
     bool IsStructuralMatch(EnumDecl *FromEnum, EnumDecl *ToRecord);
+    bool IsStructuralMatch(ClassTemplateDecl *From, ClassTemplateDecl *To);
     Decl *VisitDecl(Decl *D);
     Decl *VisitNamespaceDecl(NamespaceDecl *D);
     Decl *VisitTypedefDecl(TypedefDecl *D);
@@ -98,6 +107,7 @@ namespace {
     Decl *VisitCXXDestructorDecl(CXXDestructorDecl *D);
     Decl *VisitCXXConversionDecl(CXXConversionDecl *D);
     Decl *VisitFieldDecl(FieldDecl *D);
+    Decl *VisitIndirectFieldDecl(IndirectFieldDecl *D);
     Decl *VisitObjCIvarDecl(ObjCIvarDecl *D);
     Decl *VisitVarDecl(VarDecl *D);
     Decl *VisitImplicitParamDecl(ImplicitParamDecl *D);
@@ -106,9 +116,18 @@ namespace {
     Decl *VisitObjCCategoryDecl(ObjCCategoryDecl *D);
     Decl *VisitObjCProtocolDecl(ObjCProtocolDecl *D);
     Decl *VisitObjCInterfaceDecl(ObjCInterfaceDecl *D);
+    Decl *VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D);
+    Decl *VisitObjCImplementationDecl(ObjCImplementationDecl *D);
     Decl *VisitObjCPropertyDecl(ObjCPropertyDecl *D);
+    Decl *VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D);
     Decl *VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D);
     Decl *VisitObjCClassDecl(ObjCClassDecl *D);
+    Decl *VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D);
+    Decl *VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D);
+    Decl *VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D);
+    Decl *VisitClassTemplateDecl(ClassTemplateDecl *D);
+    Decl *VisitClassTemplateSpecializationDecl(
+                                            ClassTemplateSpecializationDecl *D);
                             
     // Importing statements
     Stmt *VisitStmt(Stmt *S);
@@ -137,9 +156,6 @@ namespace {
     /// \brief AST contexts for which we are checking structural equivalence.
     ASTContext &C1, &C2;
     
-    /// \brief Diagnostic object used to emit diagnostics.
-    Diagnostic &Diags;
-    
     /// \brief The set of "tentative" equivalences between two canonical 
     /// declarations, mapping from a declaration in the first context to the
     /// declaration in the second context that we believe to be equivalent.
@@ -158,10 +174,9 @@ namespace {
     bool StrictTypeSpelling;
     
     StructuralEquivalenceContext(ASTContext &C1, ASTContext &C2,
-                                 Diagnostic &Diags,
                llvm::DenseSet<std::pair<Decl *, Decl *> > &NonEquivalentDecls,
                                  bool StrictTypeSpelling = false)
-      : C1(C1), C2(C2), Diags(Diags), NonEquivalentDecls(NonEquivalentDecls),
+      : C1(C1), C2(C2), NonEquivalentDecls(NonEquivalentDecls),
         StrictTypeSpelling(StrictTypeSpelling) { }
 
     /// \brief Determine whether the two declarations are structurally
@@ -179,11 +194,11 @@ namespace {
     
   public:
     DiagnosticBuilder Diag1(SourceLocation Loc, unsigned DiagID) {
-      return Diags.Report(FullSourceLoc(Loc, C1.getSourceManager()), DiagID);
+      return C1.getDiagnostics().Report(Loc, DiagID);
     }
 
     DiagnosticBuilder Diag2(SourceLocation Loc, unsigned DiagID) {
-      return Diags.Report(FullSourceLoc(Loc, C2.getSourceManager()), DiagID);
+      return C2.getDiagnostics().Report(Loc, DiagID);
     }
   };
 }
@@ -200,9 +215,9 @@ static bool IsSameValue(const llvm::APInt &I1, const llvm::APInt &I2) {
     return I1 == I2;
   
   if (I1.getBitWidth() > I2.getBitWidth())
-    return I1 == llvm::APInt(I2).zext(I1.getBitWidth());
+    return I1 == I2.zext(I1.getBitWidth());
   
-  return llvm::APInt(I1).zext(I2.getBitWidth()) == I2;
+  return I1.zext(I2.getBitWidth()) == I2;
 }
 
 /// \brief Determine if two APSInts have the same value, zero- or sign-extending
@@ -213,9 +228,9 @@ static bool IsSameValue(const llvm::APSInt &I1, const llvm::APSInt &I2) {
   
   // Check for a bit-width mismatch.
   if (I1.getBitWidth() > I2.getBitWidth())
-    return IsSameValue(I1, llvm::APSInt(I2).extend(I1.getBitWidth()));
+    return IsSameValue(I1, I2.extend(I1.getBitWidth()));
   else if (I2.getBitWidth() > I1.getBitWidth())
-    return IsSameValue(llvm::APSInt(I1).extend(I2.getBitWidth()), I2);
+    return IsSameValue(I1.extend(I2.getBitWidth()), I2);
   
   // We have a signedness mismatch. Turn the signed value into an unsigned 
   // value.
@@ -263,7 +278,54 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      const TemplateArgument &Arg1,
                                      const TemplateArgument &Arg2) {
-  // FIXME: Implement!
+  if (Arg1.getKind() != Arg2.getKind())
+    return false;
+
+  switch (Arg1.getKind()) {
+  case TemplateArgument::Null:
+    return true;
+      
+  case TemplateArgument::Type:
+    return Context.IsStructurallyEquivalent(Arg1.getAsType(), Arg2.getAsType());
+      
+  case TemplateArgument::Integral:
+    if (!Context.IsStructurallyEquivalent(Arg1.getIntegralType(), 
+                                          Arg2.getIntegralType()))
+      return false;
+    
+    return IsSameValue(*Arg1.getAsIntegral(), *Arg2.getAsIntegral());
+      
+  case TemplateArgument::Declaration:
+    return Context.IsStructurallyEquivalent(Arg1.getAsDecl(), Arg2.getAsDecl());
+      
+  case TemplateArgument::Template:
+    return IsStructurallyEquivalent(Context, 
+                                    Arg1.getAsTemplate(), 
+                                    Arg2.getAsTemplate());
+
+  case TemplateArgument::TemplateExpansion:
+    return IsStructurallyEquivalent(Context, 
+                                    Arg1.getAsTemplateOrTemplatePattern(), 
+                                    Arg2.getAsTemplateOrTemplatePattern());
+
+  case TemplateArgument::Expression:
+    return IsStructurallyEquivalent(Context, 
+                                    Arg1.getAsExpr(), Arg2.getAsExpr());
+      
+  case TemplateArgument::Pack:
+    if (Arg1.pack_size() != Arg2.pack_size())
+      return false;
+      
+    for (unsigned I = 0, N = Arg1.pack_size(); I != N; ++I)
+      if (!IsStructurallyEquivalent(Context, 
+                                    Arg1.pack_begin()[I],
+                                    Arg2.pack_begin()[I]))
+        return false;
+      
+    return true;
+  }
+  
+  llvm_unreachable("Invalid template argument kind");
   return true;
 }
 
@@ -441,7 +503,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     if (Vec1->getNumElements() != Vec2->getNumElements())
       return false;
-    if (Vec1->getAltiVecSpecific() != Vec2->getAltiVecSpecific())
+    if (Vec1->getVectorKind() != Vec2->getVectorKind())
       return false;
     break;
   }
@@ -496,7 +558,25 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
       
     break;
+
+  case Type::Attributed:
+    if (!IsStructurallyEquivalent(Context,
+                                  cast<AttributedType>(T1)->getModifiedType(),
+                                  cast<AttributedType>(T2)->getModifiedType()))
+      return false;
+    if (!IsStructurallyEquivalent(Context,
+                                cast<AttributedType>(T1)->getEquivalentType(),
+                                cast<AttributedType>(T2)->getEquivalentType()))
+      return false;
+    break;
       
+  case Type::Paren:
+    if (!IsStructurallyEquivalent(Context,
+                                  cast<ParenType>(T1)->getInnerType(),
+                                  cast<ParenType>(T2)->getInnerType()))
+      return false;
+    break;
+
   case Type::Typedef:
     if (!IsStructurallyEquivalent(Context,
                                   cast<TypedefType>(T1)->getDecl(),
@@ -525,6 +605,13 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     break;
 
+  case Type::Auto:
+    if (!IsStructurallyEquivalent(Context,
+                                  cast<AutoType>(T1)->getDeducedType(),
+                                  cast<AutoType>(T2)->getDeducedType()))
+      return false;
+    break;
+
   case Type::Record:
   case Type::Enum:
     if (!IsStructurallyEquivalent(Context,
@@ -563,6 +650,21 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     break;
   }
 
+  case Type::SubstTemplateTypeParmPack: {
+    const SubstTemplateTypeParmPackType *Subst1
+      = cast<SubstTemplateTypeParmPackType>(T1);
+    const SubstTemplateTypeParmPackType *Subst2
+      = cast<SubstTemplateTypeParmPackType>(T2);
+    if (!IsStructurallyEquivalent(Context,
+                                  QualType(Subst1->getReplacedParameter(), 0),
+                                  QualType(Subst2->getReplacedParameter(), 0)))
+      return false;
+    if (!IsStructurallyEquivalent(Context, 
+                                  Subst1->getArgumentPack(),
+                                  Subst2->getArgumentPack()))
+      return false;
+    break;
+  }
   case Type::TemplateSpecialization: {
     const TemplateSpecializationType *Spec1
       = cast<TemplateSpecializationType>(T1);
@@ -644,7 +746,14 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     }
     break;
   }
-  
+
+  case Type::PackExpansion:
+    if (!IsStructurallyEquivalent(Context,
+                                  cast<PackExpansionType>(T1)->getPattern(),
+                                  cast<PackExpansionType>(T2)->getPattern()))
+      return false;
+    break;
+
   case Type::ObjCInterface: {
     const ObjCInterfaceType *Iface1 = cast<ObjCInterfaceType>(T1);
     const ObjCInterfaceType *Iface2 = cast<ObjCInterfaceType>(T2);
@@ -698,6 +807,33 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     return false;
   }
   
+  // If both declarations are class template specializations, we know
+  // the ODR applies, so check the template and template arguments.
+  ClassTemplateSpecializationDecl *Spec1
+    = dyn_cast<ClassTemplateSpecializationDecl>(D1);
+  ClassTemplateSpecializationDecl *Spec2
+    = dyn_cast<ClassTemplateSpecializationDecl>(D2);
+  if (Spec1 && Spec2) {
+    // Check that the specialized templates are the same.
+    if (!IsStructurallyEquivalent(Context, Spec1->getSpecializedTemplate(),
+                                  Spec2->getSpecializedTemplate()))
+      return false;
+    
+    // Check that the template arguments are the same.
+    if (Spec1->getTemplateArgs().size() != Spec2->getTemplateArgs().size())
+      return false;
+    
+    for (unsigned I = 0, N = Spec1->getTemplateArgs().size(); I != N; ++I)
+      if (!IsStructurallyEquivalent(Context, 
+                                    Spec1->getTemplateArgs().get(I),
+                                    Spec2->getTemplateArgs().get(I)))
+        return false;
+  }  
+  // If one is a class template specialization and the other is not, these
+  // structures are diferent.
+  else if (Spec1 || Spec2)
+    return false;
+
   // Compare the definitions of these two records. If either or both are
   // incomplete, we assume that they are equivalent.
   D1 = D1->getDefinition();
@@ -709,11 +845,11 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     if (CXXRecordDecl *D2CXX = dyn_cast<CXXRecordDecl>(D2)) {
       if (D1CXX->getNumBases() != D2CXX->getNumBases()) {
         Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
+          << Context.C2.getTypeDeclType(D2);
         Context.Diag2(D2->getLocation(), diag::note_odr_number_of_bases)
-        << D2CXX->getNumBases();
+          << D2CXX->getNumBases();
         Context.Diag1(D1->getLocation(), diag::note_odr_number_of_bases)
-        << D1CXX->getNumBases();
+          << D1CXX->getNumBases();
         return false;
       }
       
@@ -892,7 +1028,112 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   
   return true;
 }
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     TemplateParameterList *Params1,
+                                     TemplateParameterList *Params2) {
+  if (Params1->size() != Params2->size()) {
+    Context.Diag2(Params2->getTemplateLoc(), 
+                  diag::err_odr_different_num_template_parameters)
+      << Params1->size() << Params2->size();
+    Context.Diag1(Params1->getTemplateLoc(), 
+                  diag::note_odr_template_parameter_list);
+    return false;
+  }
   
+  for (unsigned I = 0, N = Params1->size(); I != N; ++I) {
+    if (Params1->getParam(I)->getKind() != Params2->getParam(I)->getKind()) {
+      Context.Diag2(Params2->getParam(I)->getLocation(), 
+                    diag::err_odr_different_template_parameter_kind);
+      Context.Diag1(Params1->getParam(I)->getLocation(),
+                    diag::note_odr_template_parameter_here);
+      return false;
+    }
+    
+    if (!Context.IsStructurallyEquivalent(Params1->getParam(I),
+                                          Params2->getParam(I))) {
+      
+      return false;
+    }
+  }
+  
+  return true;
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     TemplateTypeParmDecl *D1,
+                                     TemplateTypeParmDecl *D2) {
+  if (D1->isParameterPack() != D2->isParameterPack()) {
+    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+      << D2->isParameterPack();
+    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+      << D1->isParameterPack();
+    return false;
+  }
+  
+  return true;
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     NonTypeTemplateParmDecl *D1,
+                                     NonTypeTemplateParmDecl *D2) {
+  // FIXME: Enable once we have variadic templates.
+#if 0
+  if (D1->isParameterPack() != D2->isParameterPack()) {
+    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+      << D2->isParameterPack();
+    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+      << D1->isParameterPack();
+    return false;
+  }
+#endif
+  
+  // Check types.
+  if (!Context.IsStructurallyEquivalent(D1->getType(), D2->getType())) {
+    Context.Diag2(D2->getLocation(), 
+                  diag::err_odr_non_type_parameter_type_inconsistent)
+      << D2->getType() << D1->getType();
+    Context.Diag1(D1->getLocation(), diag::note_odr_value_here)
+      << D1->getType();
+    return false;
+  }
+  
+  return true;
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     TemplateTemplateParmDecl *D1,
+                                     TemplateTemplateParmDecl *D2) {
+  // FIXME: Enable once we have variadic templates.
+#if 0
+  if (D1->isParameterPack() != D2->isParameterPack()) {
+    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+    << D2->isParameterPack();
+    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+    << D1->isParameterPack();
+    return false;
+  }
+#endif
+  
+  // Check template parameter lists.
+  return IsStructurallyEquivalent(Context, D1->getTemplateParameters(),
+                                  D2->getTemplateParameters());
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     ClassTemplateDecl *D1, 
+                                     ClassTemplateDecl *D2) {
+  // Check template parameters.
+  if (!IsStructurallyEquivalent(Context,
+                                D1->getTemplateParameters(),
+                                D2->getTemplateParameters()))
+    return false;
+  
+  // Check the templated declaration.
+  return Context.IsStructurallyEquivalent(D1->getTemplatedDecl(), 
+                                          D2->getTemplatedDecl());
+}
+
 /// \brief Determine structural equivalence of two declarations.
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      Decl *D1, Decl *D2) {
@@ -988,8 +1229,47 @@ bool StructuralEquivalenceContext::Finish() {
         // Typedef/non-typedef mismatch.
         Equivalent = false;
       }
-    } 
-
+    } else if (ClassTemplateDecl *ClassTemplate1 
+                                           = dyn_cast<ClassTemplateDecl>(D1)) {
+      if (ClassTemplateDecl *ClassTemplate2 = dyn_cast<ClassTemplateDecl>(D2)) {
+        if (!::IsStructurallyEquivalent(ClassTemplate1->getIdentifier(),
+                                        ClassTemplate2->getIdentifier()) ||
+            !::IsStructurallyEquivalent(*this, ClassTemplate1, ClassTemplate2))
+          Equivalent = false;
+      } else {
+        // Class template/non-class-template mismatch.
+        Equivalent = false;
+      }
+    } else if (TemplateTypeParmDecl *TTP1= dyn_cast<TemplateTypeParmDecl>(D1)) {
+      if (TemplateTypeParmDecl *TTP2 = dyn_cast<TemplateTypeParmDecl>(D2)) {
+        if (!::IsStructurallyEquivalent(*this, TTP1, TTP2))
+          Equivalent = false;
+      } else {
+        // Kind mismatch.
+        Equivalent = false;
+      }
+    } else if (NonTypeTemplateParmDecl *NTTP1
+                                     = dyn_cast<NonTypeTemplateParmDecl>(D1)) {
+      if (NonTypeTemplateParmDecl *NTTP2
+                                      = dyn_cast<NonTypeTemplateParmDecl>(D2)) {
+        if (!::IsStructurallyEquivalent(*this, NTTP1, NTTP2))
+          Equivalent = false;
+      } else {
+        // Kind mismatch.
+        Equivalent = false;
+      }
+    } else if (TemplateTemplateParmDecl *TTP1
+                                  = dyn_cast<TemplateTemplateParmDecl>(D1)) {
+      if (TemplateTemplateParmDecl *TTP2
+                                    = dyn_cast<TemplateTemplateParmDecl>(D2)) {
+        if (!::IsStructurallyEquivalent(*this, TTP1, TTP2))
+          Equivalent = false;
+      } else {
+        // Kind mismatch.
+        Equivalent = false;
+      }
+    }
+    
     if (!Equivalent) {
       // Note that these two declarations are not equivalent (and we already
       // know about it).
@@ -1007,13 +1287,13 @@ bool StructuralEquivalenceContext::Finish() {
 // Import Types
 //----------------------------------------------------------------------------
 
-QualType ASTNodeImporter::VisitType(Type *T) {
+QualType ASTNodeImporter::VisitType(const Type *T) {
   Importer.FromDiag(SourceLocation(), diag::err_unsupported_ast_node)
     << T->getTypeClassName();
   return QualType();
 }
 
-QualType ASTNodeImporter::VisitBuiltinType(BuiltinType *T) {
+QualType ASTNodeImporter::VisitBuiltinType(const BuiltinType *T) {
   switch (T->getKind()) {
   case BuiltinType::Void: return Importer.getToContext().VoidTy;
   case BuiltinType::Bool: return Importer.getToContext().BoolTy;
@@ -1054,7 +1334,8 @@ QualType ASTNodeImporter::VisitBuiltinType(BuiltinType *T) {
     return Importer.getToContext().CharTy;
 
   case BuiltinType::SChar: return Importer.getToContext().SignedCharTy;
-  case BuiltinType::WChar:
+  case BuiltinType::WChar_S:
+  case BuiltinType::WChar_U:
     // FIXME: If not in C++, shall we translate to the C equivalent of
     // wchar_t?
     return Importer.getToContext().WCharTy;
@@ -1074,9 +1355,6 @@ QualType ASTNodeImporter::VisitBuiltinType(BuiltinType *T) {
     
   case BuiltinType::Overload: return Importer.getToContext().OverloadTy;
   case BuiltinType::Dependent: return Importer.getToContext().DependentTy;
-  case BuiltinType::UndeducedAuto: 
-    // FIXME: Make sure that the "to" context supports C++0x!
-    return Importer.getToContext().UndeducedAutoTy;
 
   case BuiltinType::ObjCId:
     // FIXME: Make sure that the "to" context supports Objective-C!
@@ -1092,7 +1370,7 @@ QualType ASTNodeImporter::VisitBuiltinType(BuiltinType *T) {
   return QualType();
 }
 
-QualType ASTNodeImporter::VisitComplexType(ComplexType *T) {
+QualType ASTNodeImporter::VisitComplexType(const ComplexType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
@@ -1100,7 +1378,7 @@ QualType ASTNodeImporter::VisitComplexType(ComplexType *T) {
   return Importer.getToContext().getComplexType(ToElementType);
 }
 
-QualType ASTNodeImporter::VisitPointerType(PointerType *T) {
+QualType ASTNodeImporter::VisitPointerType(const PointerType *T) {
   QualType ToPointeeType = Importer.Import(T->getPointeeType());
   if (ToPointeeType.isNull())
     return QualType();
@@ -1108,7 +1386,7 @@ QualType ASTNodeImporter::VisitPointerType(PointerType *T) {
   return Importer.getToContext().getPointerType(ToPointeeType);
 }
 
-QualType ASTNodeImporter::VisitBlockPointerType(BlockPointerType *T) {
+QualType ASTNodeImporter::VisitBlockPointerType(const BlockPointerType *T) {
   // FIXME: Check for blocks support in "to" context.
   QualType ToPointeeType = Importer.Import(T->getPointeeType());
   if (ToPointeeType.isNull())
@@ -1117,7 +1395,8 @@ QualType ASTNodeImporter::VisitBlockPointerType(BlockPointerType *T) {
   return Importer.getToContext().getBlockPointerType(ToPointeeType);
 }
 
-QualType ASTNodeImporter::VisitLValueReferenceType(LValueReferenceType *T) {
+QualType
+ASTNodeImporter::VisitLValueReferenceType(const LValueReferenceType *T) {
   // FIXME: Check for C++ support in "to" context.
   QualType ToPointeeType = Importer.Import(T->getPointeeTypeAsWritten());
   if (ToPointeeType.isNull())
@@ -1126,7 +1405,8 @@ QualType ASTNodeImporter::VisitLValueReferenceType(LValueReferenceType *T) {
   return Importer.getToContext().getLValueReferenceType(ToPointeeType);
 }
 
-QualType ASTNodeImporter::VisitRValueReferenceType(RValueReferenceType *T) {
+QualType
+ASTNodeImporter::VisitRValueReferenceType(const RValueReferenceType *T) {
   // FIXME: Check for C++0x support in "to" context.
   QualType ToPointeeType = Importer.Import(T->getPointeeTypeAsWritten());
   if (ToPointeeType.isNull())
@@ -1135,7 +1415,7 @@ QualType ASTNodeImporter::VisitRValueReferenceType(RValueReferenceType *T) {
   return Importer.getToContext().getRValueReferenceType(ToPointeeType);  
 }
 
-QualType ASTNodeImporter::VisitMemberPointerType(MemberPointerType *T) {
+QualType ASTNodeImporter::VisitMemberPointerType(const MemberPointerType *T) {
   // FIXME: Check for C++ support in "to" context.
   QualType ToPointeeType = Importer.Import(T->getPointeeType());
   if (ToPointeeType.isNull())
@@ -1146,7 +1426,7 @@ QualType ASTNodeImporter::VisitMemberPointerType(MemberPointerType *T) {
                                                       ClassType.getTypePtr());
 }
 
-QualType ASTNodeImporter::VisitConstantArrayType(ConstantArrayType *T) {
+QualType ASTNodeImporter::VisitConstantArrayType(const ConstantArrayType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
@@ -1157,7 +1437,8 @@ QualType ASTNodeImporter::VisitConstantArrayType(ConstantArrayType *T) {
                                                T->getIndexTypeCVRQualifiers());
 }
 
-QualType ASTNodeImporter::VisitIncompleteArrayType(IncompleteArrayType *T) {
+QualType
+ASTNodeImporter::VisitIncompleteArrayType(const IncompleteArrayType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
@@ -1167,7 +1448,7 @@ QualType ASTNodeImporter::VisitIncompleteArrayType(IncompleteArrayType *T) {
                                                 T->getIndexTypeCVRQualifiers());
 }
 
-QualType ASTNodeImporter::VisitVariableArrayType(VariableArrayType *T) {
+QualType ASTNodeImporter::VisitVariableArrayType(const VariableArrayType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
@@ -1183,17 +1464,17 @@ QualType ASTNodeImporter::VisitVariableArrayType(VariableArrayType *T) {
                                                       Brackets);
 }
 
-QualType ASTNodeImporter::VisitVectorType(VectorType *T) {
+QualType ASTNodeImporter::VisitVectorType(const VectorType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
   
   return Importer.getToContext().getVectorType(ToElementType, 
                                                T->getNumElements(),
-                                               T->getAltiVecSpecific());
+                                               T->getVectorKind());
 }
 
-QualType ASTNodeImporter::VisitExtVectorType(ExtVectorType *T) {
+QualType ASTNodeImporter::VisitExtVectorType(const ExtVectorType *T) {
   QualType ToElementType = Importer.Import(T->getElementType());
   if (ToElementType.isNull())
     return QualType();
@@ -1202,7 +1483,8 @@ QualType ASTNodeImporter::VisitExtVectorType(ExtVectorType *T) {
                                                   T->getNumElements());
 }
 
-QualType ASTNodeImporter::VisitFunctionNoProtoType(FunctionNoProtoType *T) {
+QualType
+ASTNodeImporter::VisitFunctionNoProtoType(const FunctionNoProtoType *T) {
   // FIXME: What happens if we're importing a function without a prototype 
   // into C++? Should we make it variadic?
   QualType ToResultType = Importer.Import(T->getResultType());
@@ -1213,7 +1495,7 @@ QualType ASTNodeImporter::VisitFunctionNoProtoType(FunctionNoProtoType *T) {
                                                         T->getExtInfo());
 }
 
-QualType ASTNodeImporter::VisitFunctionProtoType(FunctionProtoType *T) {
+QualType ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
   QualType ToResultType = Importer.Import(T->getResultType());
   if (ToResultType.isNull())
     return QualType();
@@ -1239,19 +1521,15 @@ QualType ASTNodeImporter::VisitFunctionProtoType(FunctionProtoType *T) {
       return QualType();
     ExceptionTypes.push_back(ExceptionType);
   }
+
+  FunctionProtoType::ExtProtoInfo EPI = T->getExtProtoInfo();
+  EPI.Exceptions = ExceptionTypes.data();
        
   return Importer.getToContext().getFunctionType(ToResultType, ArgTypes.data(),
-                                                 ArgTypes.size(),
-                                                 T->isVariadic(),
-                                                 T->getTypeQuals(),
-                                                 T->hasExceptionSpec(), 
-                                                 T->hasAnyExceptionSpec(),
-                                                 ExceptionTypes.size(),
-                                                 ExceptionTypes.data(),
-                                                 T->getExtInfo());
+                                                 ArgTypes.size(), EPI);
 }
 
-QualType ASTNodeImporter::VisitTypedefType(TypedefType *T) {
+QualType ASTNodeImporter::VisitTypedefType(const TypedefType *T) {
   TypedefDecl *ToDecl
                  = dyn_cast_or_null<TypedefDecl>(Importer.Import(T->getDecl()));
   if (!ToDecl)
@@ -1260,7 +1538,7 @@ QualType ASTNodeImporter::VisitTypedefType(TypedefType *T) {
   return Importer.getToContext().getTypeDeclType(ToDecl);
 }
 
-QualType ASTNodeImporter::VisitTypeOfExprType(TypeOfExprType *T) {
+QualType ASTNodeImporter::VisitTypeOfExprType(const TypeOfExprType *T) {
   Expr *ToExpr = Importer.Import(T->getUnderlyingExpr());
   if (!ToExpr)
     return QualType();
@@ -1268,7 +1546,7 @@ QualType ASTNodeImporter::VisitTypeOfExprType(TypeOfExprType *T) {
   return Importer.getToContext().getTypeOfExprType(ToExpr);
 }
 
-QualType ASTNodeImporter::VisitTypeOfType(TypeOfType *T) {
+QualType ASTNodeImporter::VisitTypeOfType(const TypeOfType *T) {
   QualType ToUnderlyingType = Importer.Import(T->getUnderlyingType());
   if (ToUnderlyingType.isNull())
     return QualType();
@@ -1276,7 +1554,8 @@ QualType ASTNodeImporter::VisitTypeOfType(TypeOfType *T) {
   return Importer.getToContext().getTypeOfType(ToUnderlyingType);
 }
 
-QualType ASTNodeImporter::VisitDecltypeType(DecltypeType *T) {
+QualType ASTNodeImporter::VisitDecltypeType(const DecltypeType *T) {
+  // FIXME: Make sure that the "to" context supports C++0x!
   Expr *ToExpr = Importer.Import(T->getUnderlyingExpr());
   if (!ToExpr)
     return QualType();
@@ -1284,7 +1563,20 @@ QualType ASTNodeImporter::VisitDecltypeType(DecltypeType *T) {
   return Importer.getToContext().getDecltypeType(ToExpr);
 }
 
-QualType ASTNodeImporter::VisitRecordType(RecordType *T) {
+QualType ASTNodeImporter::VisitAutoType(const AutoType *T) {
+  // FIXME: Make sure that the "to" context supports C++0x!
+  QualType FromDeduced = T->getDeducedType();
+  QualType ToDeduced;
+  if (!FromDeduced.isNull()) {
+    ToDeduced = Importer.Import(FromDeduced);
+    if (ToDeduced.isNull())
+      return QualType();
+  }
+  
+  return Importer.getToContext().getAutoType(ToDeduced);
+}
+
+QualType ASTNodeImporter::VisitRecordType(const RecordType *T) {
   RecordDecl *ToDecl
     = dyn_cast_or_null<RecordDecl>(Importer.Import(T->getDecl()));
   if (!ToDecl)
@@ -1293,7 +1585,7 @@ QualType ASTNodeImporter::VisitRecordType(RecordType *T) {
   return Importer.getToContext().getTagDeclType(ToDecl);
 }
 
-QualType ASTNodeImporter::VisitEnumType(EnumType *T) {
+QualType ASTNodeImporter::VisitEnumType(const EnumType *T) {
   EnumDecl *ToDecl
     = dyn_cast_or_null<EnumDecl>(Importer.Import(T->getDecl()));
   if (!ToDecl)
@@ -1302,7 +1594,31 @@ QualType ASTNodeImporter::VisitEnumType(EnumType *T) {
   return Importer.getToContext().getTagDeclType(ToDecl);
 }
 
-QualType ASTNodeImporter::VisitElaboratedType(ElaboratedType *T) {
+QualType ASTNodeImporter::VisitTemplateSpecializationType(
+                                       const TemplateSpecializationType *T) {
+  TemplateName ToTemplate = Importer.Import(T->getTemplateName());
+  if (ToTemplate.isNull())
+    return QualType();
+  
+  llvm::SmallVector<TemplateArgument, 2> ToTemplateArgs;
+  if (ImportTemplateArguments(T->getArgs(), T->getNumArgs(), ToTemplateArgs))
+    return QualType();
+  
+  QualType ToCanonType;
+  if (!QualType(T, 0).isCanonical()) {
+    QualType FromCanonType 
+      = Importer.getFromContext().getCanonicalType(QualType(T, 0));
+    ToCanonType =Importer.Import(FromCanonType);
+    if (ToCanonType.isNull())
+      return QualType();
+  }
+  return Importer.getToContext().getTemplateSpecializationType(ToTemplate, 
+                                                         ToTemplateArgs.data(), 
+                                                         ToTemplateArgs.size(),
+                                                               ToCanonType);
+}
+
+QualType ASTNodeImporter::VisitElaboratedType(const ElaboratedType *T) {
   NestedNameSpecifier *ToQualifier = 0;
   // Note: the qualifier in an ElaboratedType is optional.
   if (T->getQualifier()) {
@@ -1319,7 +1635,7 @@ QualType ASTNodeImporter::VisitElaboratedType(ElaboratedType *T) {
                                                    ToQualifier, ToNamedType);
 }
 
-QualType ASTNodeImporter::VisitObjCInterfaceType(ObjCInterfaceType *T) {
+QualType ASTNodeImporter::VisitObjCInterfaceType(const ObjCInterfaceType *T) {
   ObjCInterfaceDecl *Class
     = dyn_cast_or_null<ObjCInterfaceDecl>(Importer.Import(T->getDecl()));
   if (!Class)
@@ -1328,7 +1644,7 @@ QualType ASTNodeImporter::VisitObjCInterfaceType(ObjCInterfaceType *T) {
   return Importer.getToContext().getObjCInterfaceType(Class);
 }
 
-QualType ASTNodeImporter::VisitObjCObjectType(ObjCObjectType *T) {
+QualType ASTNodeImporter::VisitObjCObjectType(const ObjCObjectType *T) {
   QualType ToBaseType = Importer.Import(T->getBaseType());
   if (ToBaseType.isNull())
     return QualType();
@@ -1349,7 +1665,8 @@ QualType ASTNodeImporter::VisitObjCObjectType(ObjCObjectType *T) {
                                                    Protocols.size());
 }
 
-QualType ASTNodeImporter::VisitObjCObjectPointerType(ObjCObjectPointerType *T) {
+QualType
+ASTNodeImporter::VisitObjCObjectPointerType(const ObjCObjectPointerType *T) {
   QualType ToPointeeType = Importer.Import(T->getPointeeType());
   if (ToPointeeType.isNull())
     return QualType();
@@ -1420,7 +1737,15 @@ ASTNodeImporter::ImportDeclarationNameLoc(const DeclarationNameInfo &From,
   }
 }
 
-void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC) {
+void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC, bool ForceImport) {
+  if (Importer.isMinimalImport() && !ForceImport) {
+    if (DeclContext *ToDC = Importer.ImportContext(FromDC)) {
+      ToDC->setHasExternalLexicalStorage();
+      ToDC->setHasExternalVisibleStorage();
+    }
+    return;
+  }
+  
   for (DeclContext::decl_iterator From = FromDC->decls_begin(),
                                FromEnd = FromDC->decls_end();
        From != FromEnd;
@@ -1428,11 +1753,151 @@ void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC) {
     Importer.Import(*From);
 }
 
+bool ASTNodeImporter::ImportDefinition(RecordDecl *From, RecordDecl *To) {
+  if (To->getDefinition())
+    return false;
+  
+  To->startDefinition();
+  
+  // Add base classes.
+  if (CXXRecordDecl *ToCXX = dyn_cast<CXXRecordDecl>(To)) {
+    CXXRecordDecl *FromCXX = cast<CXXRecordDecl>(From);
+    
+    llvm::SmallVector<CXXBaseSpecifier *, 4> Bases;
+    for (CXXRecordDecl::base_class_iterator 
+                  Base1 = FromCXX->bases_begin(),
+            FromBaseEnd = FromCXX->bases_end();
+         Base1 != FromBaseEnd;
+         ++Base1) {
+      QualType T = Importer.Import(Base1->getType());
+      if (T.isNull())
+        return true;
+
+      SourceLocation EllipsisLoc;
+      if (Base1->isPackExpansion())
+        EllipsisLoc = Importer.Import(Base1->getEllipsisLoc());
+      
+      Bases.push_back(
+                    new (Importer.getToContext()) 
+                      CXXBaseSpecifier(Importer.Import(Base1->getSourceRange()),
+                                       Base1->isVirtual(),
+                                       Base1->isBaseOfClass(),
+                                       Base1->getAccessSpecifierAsWritten(),
+                                   Importer.Import(Base1->getTypeSourceInfo()),
+                                       EllipsisLoc));
+    }
+    if (!Bases.empty())
+      ToCXX->setBases(Bases.data(), Bases.size());
+  }
+  
+  ImportDeclContext(From);
+  To->completeDefinition();
+  return false;
+}
+
+TemplateParameterList *ASTNodeImporter::ImportTemplateParameterList(
+                                                TemplateParameterList *Params) {
+  llvm::SmallVector<NamedDecl *, 4> ToParams;
+  ToParams.reserve(Params->size());
+  for (TemplateParameterList::iterator P = Params->begin(), 
+                                    PEnd = Params->end();
+       P != PEnd; ++P) {
+    Decl *To = Importer.Import(*P);
+    if (!To)
+      return 0;
+    
+    ToParams.push_back(cast<NamedDecl>(To));
+  }
+  
+  return TemplateParameterList::Create(Importer.getToContext(),
+                                       Importer.Import(Params->getTemplateLoc()),
+                                       Importer.Import(Params->getLAngleLoc()),
+                                       ToParams.data(), ToParams.size(),
+                                       Importer.Import(Params->getRAngleLoc()));
+}
+
+TemplateArgument 
+ASTNodeImporter::ImportTemplateArgument(const TemplateArgument &From) {
+  switch (From.getKind()) {
+  case TemplateArgument::Null:
+    return TemplateArgument();
+     
+  case TemplateArgument::Type: {
+    QualType ToType = Importer.Import(From.getAsType());
+    if (ToType.isNull())
+      return TemplateArgument();
+    return TemplateArgument(ToType);
+  }
+      
+  case TemplateArgument::Integral: {
+    QualType ToType = Importer.Import(From.getIntegralType());
+    if (ToType.isNull())
+      return TemplateArgument();
+    return TemplateArgument(*From.getAsIntegral(), ToType);
+  }
+
+  case TemplateArgument::Declaration:
+    if (Decl *To = Importer.Import(From.getAsDecl()))
+      return TemplateArgument(To);
+    return TemplateArgument();
+      
+  case TemplateArgument::Template: {
+    TemplateName ToTemplate = Importer.Import(From.getAsTemplate());
+    if (ToTemplate.isNull())
+      return TemplateArgument();
+    
+    return TemplateArgument(ToTemplate);
+  }
+
+  case TemplateArgument::TemplateExpansion: {
+    TemplateName ToTemplate 
+      = Importer.Import(From.getAsTemplateOrTemplatePattern());
+    if (ToTemplate.isNull())
+      return TemplateArgument();
+    
+    return TemplateArgument(ToTemplate, From.getNumTemplateExpansions());
+  }
+
+  case TemplateArgument::Expression:
+    if (Expr *ToExpr = Importer.Import(From.getAsExpr()))
+      return TemplateArgument(ToExpr);
+    return TemplateArgument();
+      
+  case TemplateArgument::Pack: {
+    llvm::SmallVector<TemplateArgument, 2> ToPack;
+    ToPack.reserve(From.pack_size());
+    if (ImportTemplateArguments(From.pack_begin(), From.pack_size(), ToPack))
+      return TemplateArgument();
+    
+    TemplateArgument *ToArgs 
+      = new (Importer.getToContext()) TemplateArgument[ToPack.size()];
+    std::copy(ToPack.begin(), ToPack.end(), ToArgs);
+    return TemplateArgument(ToArgs, ToPack.size());
+  }
+  }
+  
+  llvm_unreachable("Invalid template argument kind");
+  return TemplateArgument();
+}
+
+bool ASTNodeImporter::ImportTemplateArguments(const TemplateArgument *FromArgs,
+                                              unsigned NumFromArgs,
+                              llvm::SmallVectorImpl<TemplateArgument> &ToArgs) {
+  for (unsigned I = 0; I != NumFromArgs; ++I) {
+    TemplateArgument To = ImportTemplateArgument(FromArgs[I]);
+    if (To.isNull() && !FromArgs[I].isNull())
+      return true;
+    
+    ToArgs.push_back(To);
+  }
+  
+  return false;
+}
+
 bool ASTNodeImporter::IsStructuralMatch(RecordDecl *FromRecord, 
                                         RecordDecl *ToRecord) {
   StructuralEquivalenceContext Ctx(Importer.getFromContext(),
                                    Importer.getToContext(),
-                                   Importer.getDiags(),
                                    Importer.getNonEquivalentDecls());
   return Ctx.IsStructurallyEquivalent(FromRecord, ToRecord);
 }
@@ -1440,11 +1905,18 @@ bool ASTNodeImporter::IsStructuralMatch(RecordDecl *FromRecord,
 bool ASTNodeImporter::IsStructuralMatch(EnumDecl *FromEnum, EnumDecl *ToEnum) {
   StructuralEquivalenceContext Ctx(Importer.getFromContext(),
                                    Importer.getToContext(),
-                                   Importer.getDiags(),
                                    Importer.getNonEquivalentDecls());
   return Ctx.IsStructurallyEquivalent(FromEnum, ToEnum);
 }
 
+bool ASTNodeImporter::IsStructuralMatch(ClassTemplateDecl *From, 
+                                        ClassTemplateDecl *To) {
+  StructuralEquivalenceContext Ctx(Importer.getFromContext(),
+                                   Importer.getToContext(),
+                                   Importer.getNonEquivalentDecls());
+  return Ctx.IsStructurallyEquivalent(From, To);  
+}
+
 Decl *ASTNodeImporter::VisitDecl(Decl *D) {
   Importer.FromDiag(D->getLocation(), diag::err_unsupported_ast_node)
     << D->getDeclKindName();
@@ -1620,9 +2092,10 @@ Decl *ASTNodeImporter::VisitEnumDecl(EnumDecl *D) {
   
   // Create the enum declaration.
   EnumDecl *D2 = EnumDecl::Create(Importer.getToContext(), DC, Loc,
-                                      Name.getAsIdentifierInfo(),
-                                      Importer.Import(D->getTagKeywordLoc()),
-                                      0);
+                                  Name.getAsIdentifierInfo(),
+                                  Importer.Import(D->getTagKeywordLoc()), 0,
+                                  D->isScoped(), D->isScopedUsingClassTag(),
+                                  D->isFixed());
   // Import the qualifier, if any.
   if (D->getQualifier()) {
     NestedNameSpecifier *NNS = Importer.Import(D->getQualifier());
@@ -1764,38 +2237,8 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
   
   Importer.Imported(D, D2);
 
-  if (D->isDefinition()) {
-    D2->startDefinition();
-
-    // Add base classes.
-    if (CXXRecordDecl *D2CXX = dyn_cast<CXXRecordDecl>(D2)) {
-      CXXRecordDecl *D1CXX = cast<CXXRecordDecl>(D);
-
-      llvm::SmallVector<CXXBaseSpecifier *, 4> Bases;
-      for (CXXRecordDecl::base_class_iterator 
-                Base1 = D1CXX->bases_begin(),
-             FromBaseEnd = D1CXX->bases_end();
-           Base1 != FromBaseEnd;
-           ++Base1) {
-        QualType T = Importer.Import(Base1->getType());
-        if (T.isNull())
-          return 0;
-          
-        Bases.push_back(
-          new (Importer.getToContext()) 
-                CXXBaseSpecifier(Importer.Import(Base1->getSourceRange()),
-                                 Base1->isVirtual(),
-                                 Base1->isBaseOfClass(),
-                                 Base1->getAccessSpecifierAsWritten(),
-                                 Importer.Import(Base1->getTypeSourceInfo())));
-      }
-      if (!Bases.empty())
-        D2CXX->setBases(Bases.data(), Bases.size());
-    }
-
-    ImportDeclContext(D);
-    D2->completeDefinition();
-  }
+  if (D->isDefinition() && ImportDefinition(D, D2))
+    return 0;
   
   return D2;
 }
@@ -1939,7 +2382,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   } else if (isa<CXXDestructorDecl>(D)) {
     ToFunction = CXXDestructorDecl::Create(Importer.getToContext(),
                                            cast<CXXRecordDecl>(DC),
-                                           NameInfo, T, 
+                                           NameInfo, T, TInfo,
                                            D->isInlineSpecified(),
                                            D->isImplicit());
   } else if (CXXConversionDecl *FromConversion
@@ -1949,6 +2392,13 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
                                            NameInfo, T, TInfo,
                                            D->isInlineSpecified(),
                                            FromConversion->isExplicit());
+  } else if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
+    ToFunction = CXXMethodDecl::Create(Importer.getToContext(), 
+                                       cast<CXXRecordDecl>(DC),
+                                       NameInfo, T, TInfo,
+                                       Method->isStatic(),
+                                       Method->getStorageClassAsWritten(),
+                                       Method->isInlineSpecified());
   } else {
     ToFunction = FunctionDecl::Create(Importer.getToContext(), DC,
                                       NameInfo, T, TInfo, D->getStorageClass(),
@@ -1965,8 +2415,10 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   }
   ToFunction->setAccess(D->getAccess());
   ToFunction->setLexicalDeclContext(LexicalDC);
+  ToFunction->setVirtualAsWritten(D->isVirtualAsWritten());
+  ToFunction->setTrivial(D->isTrivial());
+  ToFunction->setPure(D->isPure());
   Importer.Imported(D, ToFunction);
-  LexicalDC->addDecl(ToFunction);
 
   // Set the parameters.
   for (unsigned I = 0, N = Parameters.size(); I != N; ++I) {
@@ -1976,7 +2428,10 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   ToFunction->setParams(Parameters.data(), Parameters.size());
 
   // FIXME: Other bits to merge?
-  
+
+  // Add this function to the lexical context.
+  LexicalDC->addDecl(ToFunction);
+
   return ToFunction;
 }
 
@@ -2024,6 +2479,42 @@ Decl *ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   return ToField;
 }
 
+Decl *ASTNodeImporter::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
+  // Import the major distinguishing characteristics of a variable.
+  DeclContext *DC, *LexicalDC;
+  DeclarationName Name;
+  SourceLocation Loc;
+  if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
+    return 0;
+
+  // Import the type.
+  QualType T = Importer.Import(D->getType());
+  if (T.isNull())
+    return 0;
+
+  NamedDecl **NamedChain =
+    new (Importer.getToContext())NamedDecl*[D->getChainingSize()];
+
+  unsigned i = 0;
+  for (IndirectFieldDecl::chain_iterator PI = D->chain_begin(),
+       PE = D->chain_end(); PI != PE; ++PI) {
+    Decl* D = Importer.Import(*PI);
+    if (!D)
+      return 0;
+    NamedChain[i++] = cast<NamedDecl>(D);
+  }
+
+  IndirectFieldDecl *ToIndirectField = IndirectFieldDecl::Create(
+                                         Importer.getToContext(), DC,
+                                         Loc, Name.getAsIdentifierInfo(), T,
+                                         NamedChain, D->getChainingSize());
+  ToIndirectField->setAccess(D->getAccess());
+  ToIndirectField->setLexicalDeclContext(LexicalDC);
+  Importer.Imported(D, ToIndirectField);
+  LexicalDC->addDecl(ToIndirectField);
+  return ToIndirectField;
+}
+
 Decl *ASTNodeImporter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
   // Import the major distinguishing characteristics of an ivar.
   DeclContext *DC, *LexicalDC;
@@ -2434,7 +2925,8 @@ Decl *ASTNodeImporter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
   // If we have an implementation, import it as well.
   if (D->getImplementation()) {
     ObjCCategoryImplDecl *Impl
-      = cast<ObjCCategoryImplDecl>(Importer.Import(D->getImplementation()));
+      = cast_or_null<ObjCCategoryImplDecl>(
+                                       Importer.Import(D->getImplementation()));
     if (!Impl)
       return 0;
     
@@ -2615,8 +3107,8 @@ Decl *ASTNodeImporter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
   
   // If we have an @implementation, import it as well.
   if (D->getImplementation()) {
-    ObjCImplementationDecl *Impl
-      = cast<ObjCImplementationDecl>(Importer.Import(D->getImplementation()));
+    ObjCImplementationDecl *Impl = cast_or_null<ObjCImplementationDecl>(
+                                       Importer.Import(D->getImplementation()));
     if (!Impl)
       return 0;
     
@@ -2626,6 +3118,114 @@ Decl *ASTNodeImporter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
   return ToIface;
 }
 
+Decl *ASTNodeImporter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
+  ObjCCategoryDecl *Category = cast_or_null<ObjCCategoryDecl>(
+                                        Importer.Import(D->getCategoryDecl()));
+  if (!Category)
+    return 0;
+  
+  ObjCCategoryImplDecl *ToImpl = Category->getImplementation();
+  if (!ToImpl) {
+    DeclContext *DC = Importer.ImportContext(D->getDeclContext());
+    if (!DC)
+      return 0;
+    
+    ToImpl = ObjCCategoryImplDecl::Create(Importer.getToContext(), DC,
+                                          Importer.Import(D->getLocation()),
+                                          Importer.Import(D->getIdentifier()),
+                                          Category->getClassInterface());
+    
+    DeclContext *LexicalDC = DC;
+    if (D->getDeclContext() != D->getLexicalDeclContext()) {
+      LexicalDC = Importer.ImportContext(D->getLexicalDeclContext());
+      if (!LexicalDC)
+        return 0;
+      
+      ToImpl->setLexicalDeclContext(LexicalDC);
+    }
+    
+    LexicalDC->addDecl(ToImpl);
+    Category->setImplementation(ToImpl);
+  }
+  
+  Importer.Imported(D, ToImpl);
+  ImportDeclContext(D);
+  return ToImpl;
+}
+
+Decl *ASTNodeImporter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
+  // Find the corresponding interface.
+  ObjCInterfaceDecl *Iface = cast_or_null<ObjCInterfaceDecl>(
+                                       Importer.Import(D->getClassInterface()));
+  if (!Iface)
+    return 0;
+
+  // Import the superclass, if any.
+  ObjCInterfaceDecl *Super = 0;
+  if (D->getSuperClass()) {
+    Super = cast_or_null<ObjCInterfaceDecl>(
+                                          Importer.Import(D->getSuperClass()));
+    if (!Super)
+      return 0;
+  }
+
+  ObjCImplementationDecl *Impl = Iface->getImplementation();
+  if (!Impl) {
+    // We haven't imported an implementation yet. Create a new @implementation
+    // now.
+    Impl = ObjCImplementationDecl::Create(Importer.getToContext(),
+                                  Importer.ImportContext(D->getDeclContext()),
+                                          Importer.Import(D->getLocation()),
+                                          Iface, Super);
+    
+    if (D->getDeclContext() != D->getLexicalDeclContext()) {
+      DeclContext *LexicalDC
+        = Importer.ImportContext(D->getLexicalDeclContext());
+      if (!LexicalDC)
+        return 0;
+      Impl->setLexicalDeclContext(LexicalDC);
+    }
+    
+    // Associate the implementation with the class it implements.
+    Iface->setImplementation(Impl);
+    Importer.Imported(D, Iface->getImplementation());
+  } else {
+    Importer.Imported(D, Iface->getImplementation());
+
+    // Verify that the existing @implementation has the same superclass.
+    if ((Super && !Impl->getSuperClass()) ||
+        (!Super && Impl->getSuperClass()) ||
+        (Super && Impl->getSuperClass() && 
+         Super->getCanonicalDecl() != Impl->getSuperClass())) {
+        Importer.ToDiag(Impl->getLocation(), 
+                        diag::err_odr_objc_superclass_inconsistent)
+          << Iface->getDeclName();
+        // FIXME: It would be nice to have the location of the superclass
+        // below.
+        if (Impl->getSuperClass())
+          Importer.ToDiag(Impl->getLocation(), 
+                          diag::note_odr_objc_superclass)
+          << Impl->getSuperClass()->getDeclName();
+        else
+          Importer.ToDiag(Impl->getLocation(), 
+                          diag::note_odr_objc_missing_superclass);
+        if (D->getSuperClass())
+          Importer.FromDiag(D->getLocation(), 
+                            diag::note_odr_objc_superclass)
+          << D->getSuperClass()->getDeclName();
+        else
+          Importer.FromDiag(D->getLocation(), 
+                            diag::note_odr_objc_missing_superclass);
+      return 0;
+    }
+  }
+    
+  // Import all of the members of this @implementation.
+  ImportDeclContext(D);
+
+  return Impl;
+}
+
 Decl *ASTNodeImporter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   // Import the major distinguishing characteristics of an @property.
   DeclContext *DC, *LexicalDC;
@@ -2688,6 +3288,87 @@ Decl *ASTNodeImporter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   return ToProperty;
 }
 
+Decl *ASTNodeImporter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
+  ObjCPropertyDecl *Property = cast_or_null<ObjCPropertyDecl>(
+                                        Importer.Import(D->getPropertyDecl()));
+  if (!Property)
+    return 0;
+
+  DeclContext *DC = Importer.ImportContext(D->getDeclContext());
+  if (!DC)
+    return 0;
+  
+  // Import the lexical declaration context.
+  DeclContext *LexicalDC = DC;
+  if (D->getDeclContext() != D->getLexicalDeclContext()) {
+    LexicalDC = Importer.ImportContext(D->getLexicalDeclContext());
+    if (!LexicalDC)
+      return 0;
+  }
+
+  ObjCImplDecl *InImpl = dyn_cast<ObjCImplDecl>(LexicalDC);
+  if (!InImpl)
+    return 0;
+
+  // Import the ivar (for an @synthesize).
+  ObjCIvarDecl *Ivar = 0;
+  if (D->getPropertyIvarDecl()) {
+    Ivar = cast_or_null<ObjCIvarDecl>(
+                                    Importer.Import(D->getPropertyIvarDecl()));
+    if (!Ivar)
+      return 0;
+  }
+
+  ObjCPropertyImplDecl *ToImpl
+    = InImpl->FindPropertyImplDecl(Property->getIdentifier());
+  if (!ToImpl) {    
+    ToImpl = ObjCPropertyImplDecl::Create(Importer.getToContext(), DC,
+                                          Importer.Import(D->getLocStart()),
+                                          Importer.Import(D->getLocation()),
+                                          Property,
+                                          D->getPropertyImplementation(),
+                                          Ivar, 
+                                  Importer.Import(D->getPropertyIvarDeclLoc()));
+    ToImpl->setLexicalDeclContext(LexicalDC);
+    Importer.Imported(D, ToImpl);
+    LexicalDC->addDecl(ToImpl);
+  } else {
+    // Check that we have the same kind of property implementation (@synthesize
+    // vs. @dynamic).
+    if (D->getPropertyImplementation() != ToImpl->getPropertyImplementation()) {
+      Importer.ToDiag(ToImpl->getLocation(), 
+                      diag::err_odr_objc_property_impl_kind_inconsistent)
+        << Property->getDeclName() 
+        << (ToImpl->getPropertyImplementation() 
+                                              == ObjCPropertyImplDecl::Dynamic);
+      Importer.FromDiag(D->getLocation(),
+                        diag::note_odr_objc_property_impl_kind)
+        << D->getPropertyDecl()->getDeclName()
+        << (D->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic);
+      return 0;
+    }
+    
+    // For @synthesize, check that we have the same 
+    if (D->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize &&
+        Ivar != ToImpl->getPropertyIvarDecl()) {
+      Importer.ToDiag(ToImpl->getPropertyIvarDeclLoc(), 
+                      diag::err_odr_objc_synthesize_ivar_inconsistent)
+        << Property->getDeclName()
+        << ToImpl->getPropertyIvarDecl()->getDeclName()
+        << Ivar->getDeclName();
+      Importer.FromDiag(D->getPropertyIvarDeclLoc(), 
+                        diag::note_odr_objc_synthesize_ivar_here)
+        << D->getPropertyIvarDecl()->getDeclName();
+      return 0;
+    }
+    
+    // Merge the existing implementation with the new implementation.
+    Importer.Imported(D, ToImpl);
+  }
+  
+  return ToImpl;
+}
+
 Decl *
 ASTNodeImporter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
   // Import the context of this declaration.
@@ -2772,6 +3453,275 @@ Decl *ASTNodeImporter::VisitObjCClassDecl(ObjCClassDecl *D) {
   return ToClass;
 }
 
+Decl *ASTNodeImporter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
+  // For template arguments, we adopt the translation unit as our declaration
+  // context. This context will be fixed when the actual template declaration
+  // is created.
+  
+  // FIXME: Import default argument.
+  return TemplateTypeParmDecl::Create(Importer.getToContext(),
+                              Importer.getToContext().getTranslationUnitDecl(),
+                                      Importer.Import(D->getLocation()),
+                                      D->getDepth(),
+                                      D->getIndex(), 
+                                      Importer.Import(D->getIdentifier()),
+                                      D->wasDeclaredWithTypename(),
+                                      D->isParameterPack());
+}
+
+Decl *
+ASTNodeImporter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
+  // Import the name of this declaration.
+  DeclarationName Name = Importer.Import(D->getDeclName());
+  if (D->getDeclName() && !Name)
+    return 0;
+  
+  // Import the location of this declaration.
+  SourceLocation Loc = Importer.Import(D->getLocation());
+
+  // Import the type of this declaration.
+  QualType T = Importer.Import(D->getType());
+  if (T.isNull())
+    return 0;
+  
+  // Import type-source information.
+  TypeSourceInfo *TInfo = Importer.Import(D->getTypeSourceInfo());
+  if (D->getTypeSourceInfo() && !TInfo)
+    return 0;
+  
+  // FIXME: Import default argument.
+  
+  return NonTypeTemplateParmDecl::Create(Importer.getToContext(),
+                               Importer.getToContext().getTranslationUnitDecl(),
+                                         Loc, D->getDepth(), D->getPosition(),
+                                         Name.getAsIdentifierInfo(),
+                                         T, D->isParameterPack(), TInfo);
+}
+
+Decl *
+ASTNodeImporter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
+  // Import the name of this declaration.
+  DeclarationName Name = Importer.Import(D->getDeclName());
+  if (D->getDeclName() && !Name)
+    return 0;
+  
+  // Import the location of this declaration.
+  SourceLocation Loc = Importer.Import(D->getLocation());
+  
+  // Import template parameters.
+  TemplateParameterList *TemplateParams
+    = ImportTemplateParameterList(D->getTemplateParameters());
+  if (!TemplateParams)
+    return 0;
+  
+  // FIXME: Import default argument.
+  
+  return TemplateTemplateParmDecl::Create(Importer.getToContext(), 
+                              Importer.getToContext().getTranslationUnitDecl(), 
+                                          Loc, D->getDepth(), D->getPosition(),
+                                          D->isParameterPack(),
+                                          Name.getAsIdentifierInfo(), 
+                                          TemplateParams);
+}
+
+Decl *ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+  // If this record has a definition in the translation unit we're coming from,
+  // but this particular declaration is not that definition, import the
+  // definition and map to that.
+  CXXRecordDecl *Definition 
+    = cast_or_null<CXXRecordDecl>(D->getTemplatedDecl()->getDefinition());
+  if (Definition && Definition != D->getTemplatedDecl()) {
+    Decl *ImportedDef
+      = Importer.Import(Definition->getDescribedClassTemplate());
+    if (!ImportedDef)
+      return 0;
+    
+    return Importer.Imported(D, ImportedDef);
+  }
+  
+  // Import the major distinguishing characteristics of this class template.
+  DeclContext *DC, *LexicalDC;
+  DeclarationName Name;
+  SourceLocation Loc;
+  if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
+    return 0;
+  
+  // We may already have a template of the same name; try to find and match it.
+  if (!DC->isFunctionOrMethod()) {
+    llvm::SmallVector<NamedDecl *, 4> ConflictingDecls;
+    for (DeclContext::lookup_result Lookup = DC->lookup(Name);
+         Lookup.first != Lookup.second; 
+         ++Lookup.first) {
+      if (!(*Lookup.first)->isInIdentifierNamespace(Decl::IDNS_Ordinary))
+        continue;
+      
+      Decl *Found = *Lookup.first;
+      if (ClassTemplateDecl *FoundTemplate 
+                                        = dyn_cast<ClassTemplateDecl>(Found)) {
+        if (IsStructuralMatch(D, FoundTemplate)) {
+          // The class templates structurally match; call it the same template.
+          // FIXME: We may be filling in a forward declaration here. Handle
+          // this case!
+          Importer.Imported(D->getTemplatedDecl(), 
+                            FoundTemplate->getTemplatedDecl());
+          return Importer.Imported(D, FoundTemplate);
+        }         
+      }
+      
+      ConflictingDecls.push_back(*Lookup.first);
+    }
+    
+    if (!ConflictingDecls.empty()) {
+      Name = Importer.HandleNameConflict(Name, DC, Decl::IDNS_Ordinary,
+                                         ConflictingDecls.data(), 
+                                         ConflictingDecls.size());
+    }
+    
+    if (!Name)
+      return 0;
+  }
+
+  CXXRecordDecl *DTemplated = D->getTemplatedDecl();
+  
+  // Create the declaration that is being templated.
+  CXXRecordDecl *D2Templated = CXXRecordDecl::Create(Importer.getToContext(),
+                                                     DTemplated->getTagKind(),
+                                                     DC, 
+                                     Importer.Import(DTemplated->getLocation()),
+                                                     Name.getAsIdentifierInfo(),                                                       
+                               Importer.Import(DTemplated->getTagKeywordLoc()));
+  D2Templated->setAccess(DTemplated->getAccess());
+  
+  
+  // Import the qualifier, if any.
+  if (DTemplated->getQualifier()) {
+    NestedNameSpecifier *NNS = Importer.Import(DTemplated->getQualifier());
+    SourceRange NNSRange = Importer.Import(DTemplated->getQualifierRange());
+    D2Templated->setQualifierInfo(NNS, NNSRange);
+  }
+  D2Templated->setLexicalDeclContext(LexicalDC);
+  
+  // Create the class template declaration itself.
+  TemplateParameterList *TemplateParams
+    = ImportTemplateParameterList(D->getTemplateParameters());
+  if (!TemplateParams)
+    return 0;
+  
+  ClassTemplateDecl *D2 = ClassTemplateDecl::Create(Importer.getToContext(), DC, 
+                                                    Loc, Name, TemplateParams, 
+                                                    D2Templated, 
+  /*PrevDecl=*/0);
+  D2Templated->setDescribedClassTemplate(D2);    
+  
+  D2->setAccess(D->getAccess());
+  D2->setLexicalDeclContext(LexicalDC);
+  LexicalDC->addDecl(D2);
+  
+  // Note the relationship between the class templates.
+  Importer.Imported(D, D2);
+  Importer.Imported(DTemplated, D2Templated);
+
+  if (DTemplated->isDefinition() && !D2Templated->isDefinition()) {
+    // FIXME: Import definition!
+  }
+  
+  return D2;
+}
+
+Decl *ASTNodeImporter::VisitClassTemplateSpecializationDecl(
+                                          ClassTemplateSpecializationDecl *D) {
+  // If this record has a definition in the translation unit we're coming from,
+  // but this particular declaration is not that definition, import the
+  // definition and map to that.
+  TagDecl *Definition = D->getDefinition();
+  if (Definition && Definition != D) {
+    Decl *ImportedDef = Importer.Import(Definition);
+    if (!ImportedDef)
+      return 0;
+    
+    return Importer.Imported(D, ImportedDef);
+  }
+
+  ClassTemplateDecl *ClassTemplate
+    = cast_or_null<ClassTemplateDecl>(Importer.Import(
+                                                 D->getSpecializedTemplate()));
+  if (!ClassTemplate)
+    return 0;
+  
+  // Import the context of this declaration.
+  DeclContext *DC = ClassTemplate->getDeclContext();
+  if (!DC)
+    return 0;
+  
+  DeclContext *LexicalDC = DC;
+  if (D->getDeclContext() != D->getLexicalDeclContext()) {
+    LexicalDC = Importer.ImportContext(D->getLexicalDeclContext());
+    if (!LexicalDC)
+      return 0;
+  }
+  
+  // Import the location of this declaration.
+  SourceLocation Loc = Importer.Import(D->getLocation());
+
+  // Import template arguments.
+  llvm::SmallVector<TemplateArgument, 2> TemplateArgs;
+  if (ImportTemplateArguments(D->getTemplateArgs().data(), 
+                              D->getTemplateArgs().size(),
+                              TemplateArgs))
+    return 0;
+  
+  // Try to find an existing specialization with these template arguments.
+  void *InsertPos = 0;
+  ClassTemplateSpecializationDecl *D2
+    = ClassTemplate->findSpecialization(TemplateArgs.data(), 
+                                        TemplateArgs.size(), InsertPos);
+  if (D2) {
+    // We already have a class template specialization with these template
+    // arguments.
+    
+    // FIXME: Check for specialization vs. instantiation errors.
+    
+    if (RecordDecl *FoundDef = D2->getDefinition()) {
+      if (!D->isDefinition() || IsStructuralMatch(D, FoundDef)) {
+        // The record types structurally match, or the "from" translation
+        // unit only had a forward declaration anyway; call it the same
+        // function.
+        return Importer.Imported(D, FoundDef);
+      }
+    }
+  } else {
+    // Create a new specialization.
+    D2 = ClassTemplateSpecializationDecl::Create(Importer.getToContext(), 
+                                                 D->getTagKind(), DC, 
+                                                 Loc, ClassTemplate,
+                                                 TemplateArgs.data(), 
+                                                 TemplateArgs.size(), 
+                                                 /*PrevDecl=*/0);
+    D2->setSpecializationKind(D->getSpecializationKind());
+
+    // Add this specialization to the class template.
+    ClassTemplate->AddSpecialization(D2, InsertPos);
+    
+    // Import the qualifier, if any.
+    if (D->getQualifier()) {
+      NestedNameSpecifier *NNS = Importer.Import(D->getQualifier());
+      SourceRange NNSRange = Importer.Import(D->getQualifierRange());
+      D2->setQualifierInfo(NNS, NNSRange);
+    }
+
+    
+    // Add the specialization to this context.
+    D2->setLexicalDeclContext(LexicalDC);
+    LexicalDC->addDecl(D2);
+  }
+  Importer.Imported(D, D2);
+  
+  if (D->isDefinition() && ImportDefinition(D, D2))
+    return 0;
+  
+  return D2;
+}
+
 //----------------------------------------------------------------------------
 // Import Statements
 //----------------------------------------------------------------------------
@@ -2811,7 +3761,7 @@ Expr *ASTNodeImporter::VisitDeclRefExpr(DeclRefExpr *E) {
                              Importer.Import(E->getQualifierRange()),
                              ToD,
                              Importer.Import(E->getLocation()),
-                             T,
+                             T, E->getValueKind(),
                              /*FIXME:TemplateArgs=*/0);
 }
 
@@ -2856,7 +3806,8 @@ Expr *ASTNodeImporter::VisitUnaryOperator(UnaryOperator *E) {
     return 0;
   
   return new (Importer.getToContext()) UnaryOperator(SubExpr, E->getOpcode(),
-                                                     T,
+                                                     T, E->getValueKind(),
+                                                     E->getObjectKind(),
                                          Importer.Import(E->getOperatorLoc()));                                        
 }
 
@@ -2898,7 +3849,8 @@ Expr *ASTNodeImporter::VisitBinaryOperator(BinaryOperator *E) {
     return 0;
   
   return new (Importer.getToContext()) BinaryOperator(LHS, RHS, E->getOpcode(),
-                                                      T,
+                                                      T, E->getValueKind(),
+                                                      E->getObjectKind(),
                                           Importer.Import(E->getOperatorLoc()));
 }
 
@@ -2925,7 +3877,9 @@ Expr *ASTNodeImporter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
   
   return new (Importer.getToContext()) 
                         CompoundAssignOperator(LHS, RHS, E->getOpcode(),
-                                               T, CompLHSType, CompResultType,
+                                               T, E->getValueKind(),
+                                               E->getObjectKind(),
+                                               CompLHSType, CompResultType,
                                           Importer.Import(E->getOperatorLoc()));
 }
 
@@ -2970,18 +3924,20 @@ Expr *ASTNodeImporter::VisitCStyleCastExpr(CStyleCastExpr *E) {
   if (ImportCastPath(E, BasePath))
     return 0;
 
-  return CStyleCastExpr::Create(Importer.getToContext(), T, E->getCastKind(),
+  return CStyleCastExpr::Create(Importer.getToContext(), T,
+                                E->getValueKind(), E->getCastKind(),
                                 SubExpr, &BasePath, TInfo,
                                 Importer.Import(E->getLParenLoc()),
                                 Importer.Import(E->getRParenLoc()));
 }
 
-ASTImporter::ASTImporter(Diagnostic &Diags,
-                         ASTContext &ToContext, FileManager &ToFileManager,
-                         ASTContext &FromContext, FileManager &FromFileManager)
+ASTImporter::ASTImporter(ASTContext &ToContext, FileManager &ToFileManager,
+                         ASTContext &FromContext, FileManager &FromFileManager,
+                         bool MinimalImport)
   : ToContext(ToContext), FromContext(FromContext),
     ToFileManager(ToFileManager), FromFileManager(FromFileManager),
-    Diags(Diags) {
+    Minimal(MinimalImport) 
+{
   ImportedDecls[FromContext.getTranslationUnitDecl()]
     = ToContext.getTranslationUnitDecl();
 }
@@ -2991,23 +3947,25 @@ ASTImporter::~ASTImporter() { }
 QualType ASTImporter::Import(QualType FromT) {
   if (FromT.isNull())
     return QualType();
+
+  const Type *fromTy = FromT.getTypePtr();
   
   // Check whether we've already imported this type.  
-  llvm::DenseMap<Type *, Type *>::iterator Pos
-    = ImportedTypes.find(FromT.getTypePtr());
+  llvm::DenseMap<const Type *, const Type *>::iterator Pos
+    = ImportedTypes.find(fromTy);
   if (Pos != ImportedTypes.end())
-    return ToContext.getQualifiedType(Pos->second, FromT.getQualifiers());
+    return ToContext.getQualifiedType(Pos->second, FromT.getLocalQualifiers());
   
   // Import the type
   ASTNodeImporter Importer(*this);
-  QualType ToT = Importer.Visit(FromT.getTypePtr());
+  QualType ToT = Importer.Visit(fromTy);
   if (ToT.isNull())
     return ToT;
   
   // Record the imported type.
-  ImportedTypes[FromT.getTypePtr()] = ToT.getTypePtr();
+  ImportedTypes[fromTy] = ToT.getTypePtr();
   
-  return ToContext.getQualifiedType(ToT, FromT.getQualifiers());
+  return ToContext.getQualifiedType(ToT, FromT.getLocalQualifiers());
 }
 
 TypeSourceInfo *ASTImporter::Import(TypeSourceInfo *FromTSI) {
@@ -3109,6 +4067,82 @@ NestedNameSpecifier *ASTImporter::Import(NestedNameSpecifier *FromNNS) {
   return 0;
 }
 
+TemplateName ASTImporter::Import(TemplateName From) {
+  switch (From.getKind()) {
+  case TemplateName::Template:
+    if (TemplateDecl *ToTemplate
+                = cast_or_null<TemplateDecl>(Import(From.getAsTemplateDecl())))
+      return TemplateName(ToTemplate);
+      
+    return TemplateName();
+      
+  case TemplateName::OverloadedTemplate: {
+    OverloadedTemplateStorage *FromStorage = From.getAsOverloadedTemplate();
+    UnresolvedSet<2> ToTemplates;
+    for (OverloadedTemplateStorage::iterator I = FromStorage->begin(),
+                                             E = FromStorage->end();
+         I != E; ++I) {
+      if (NamedDecl *To = cast_or_null<NamedDecl>(Import(*I))) 
+        ToTemplates.addDecl(To);
+      else
+        return TemplateName();
+    }
+    return ToContext.getOverloadedTemplateName(ToTemplates.begin(), 
+                                               ToTemplates.end());
+  }
+      
+  case TemplateName::QualifiedTemplate: {
+    QualifiedTemplateName *QTN = From.getAsQualifiedTemplateName();
+    NestedNameSpecifier *Qualifier = Import(QTN->getQualifier());
+    if (!Qualifier)
+      return TemplateName();
+    
+    if (TemplateDecl *ToTemplate
+        = cast_or_null<TemplateDecl>(Import(From.getAsTemplateDecl())))
+      return ToContext.getQualifiedTemplateName(Qualifier, 
+                                                QTN->hasTemplateKeyword(), 
+                                                ToTemplate);
+    
+    return TemplateName();
+  }
+  
+  case TemplateName::DependentTemplate: {
+    DependentTemplateName *DTN = From.getAsDependentTemplateName();
+    NestedNameSpecifier *Qualifier = Import(DTN->getQualifier());
+    if (!Qualifier)
+      return TemplateName();
+    
+    if (DTN->isIdentifier()) {
+      return ToContext.getDependentTemplateName(Qualifier, 
+                                                Import(DTN->getIdentifier()));
+    }
+    
+    return ToContext.getDependentTemplateName(Qualifier, DTN->getOperator());
+  }
+      
+  case TemplateName::SubstTemplateTemplateParmPack: {
+    SubstTemplateTemplateParmPackStorage *SubstPack
+      = From.getAsSubstTemplateTemplateParmPack();
+    TemplateTemplateParmDecl *Param
+      = cast_or_null<TemplateTemplateParmDecl>(
+                                        Import(SubstPack->getParameterPack()));
+    if (!Param)
+      return TemplateName();
+    
+    ASTNodeImporter Importer(*this);
+    TemplateArgument ArgPack 
+      = Importer.ImportTemplateArgument(SubstPack->getArgumentPack());
+    if (ArgPack.isNull())
+      return TemplateName();
+    
+    return ToContext.getSubstTemplateTemplateParmPack(Param, ArgPack);
+  }
+  }
+  
+  llvm_unreachable("Invalid template name kind");
+  return TemplateName();
+}
+
 SourceLocation ASTImporter::Import(SourceLocation FromLoc) {
   if (FromLoc.isInvalid())
     return SourceLocation();
@@ -3130,8 +4164,8 @@ SourceRange ASTImporter::Import(SourceRange FromRange) {
 }
 
 FileID ASTImporter::Import(FileID FromID) {
-  llvm::DenseMap<unsigned, FileID>::iterator Pos
-    = ImportedFileIDs.find(FromID.getHashValue());
+  llvm::DenseMap<FileID, FileID>::iterator Pos
+    = ImportedFileIDs.find(FromID);
   if (Pos != ImportedFileIDs.end())
     return Pos->second;
   
@@ -3156,7 +4190,8 @@ FileID ASTImporter::Import(FileID FromID) {
                              FromSLoc.getFile().getFileCharacteristic());
   } else {
     // FIXME: We want to re-use the existing MemoryBuffer!
-    const llvm::MemoryBuffer *FromBuf = Cache->getBuffer(getDiags(), FromSM);
+    const llvm::MemoryBuffer *
+        FromBuf = Cache->getBuffer(FromContext.getDiagnostics(), FromSM);
     llvm::MemoryBuffer *ToBuf
       = llvm::MemoryBuffer::getMemBufferCopy(FromBuf->getBuffer(),
                                              FromBuf->getBufferIdentifier());
@@ -3164,10 +4199,21 @@ FileID ASTImporter::Import(FileID FromID) {
   }
   
   
-  ImportedFileIDs[FromID.getHashValue()] = ToID;
+  ImportedFileIDs[FromID] = ToID;
   return ToID;
 }
 
+void ASTImporter::ImportDefinition(Decl *From) {
+  Decl *To = Import(From);
+  if (!To)
+    return;
+  
+  if (DeclContext *FromDC = cast<DeclContext>(From)) {
+    ASTNodeImporter Importer(*this);
+    Importer.ImportDeclContext(FromDC, true);
+  }
+}
+
 DeclarationName ASTImporter::Import(DeclarationName FromName) {
   if (!FromName)
     return DeclarationName();
@@ -3225,7 +4271,7 @@ DeclarationName ASTImporter::Import(DeclarationName FromName) {
   return DeclarationName();
 }
 
-IdentifierInfo *ASTImporter::Import(IdentifierInfo *FromId) {
+IdentifierInfo *ASTImporter::Import(const IdentifierInfo *FromId) {
   if (!FromId)
     return 0;
 
@@ -3252,13 +4298,11 @@ DeclarationName ASTImporter::HandleNameConflict(DeclarationName Name,
 }
 
 DiagnosticBuilder ASTImporter::ToDiag(SourceLocation Loc, unsigned DiagID) {
-  return Diags.Report(FullSourceLoc(Loc, ToContext.getSourceManager()), 
-                      DiagID);
+  return ToContext.getDiagnostics().Report(Loc, DiagID);
 }
 
 DiagnosticBuilder ASTImporter::FromDiag(SourceLocation Loc, unsigned DiagID) {
-  return Diags.Report(FullSourceLoc(Loc, FromContext.getSourceManager()), 
-                      DiagID);
+  return FromContext.getDiagnostics().Report(Loc, DiagID);
 }
 
 Decl *ASTImporter::Imported(Decl *From, Decl *To) {
@@ -3267,12 +4311,11 @@ Decl *ASTImporter::Imported(Decl *From, Decl *To) {
 }
 
 bool ASTImporter::IsStructurallyEquivalent(QualType From, QualType To) {
-  llvm::DenseMap<Type *, Type *>::iterator Pos
+  llvm::DenseMap<const Type *, const Type *>::iterator Pos
    = ImportedTypes.find(From.getTypePtr());
   if (Pos != ImportedTypes.end() && ToContext.hasSameType(Import(From), To))
     return true;
       
-  StructuralEquivalenceContext Ctx(FromContext, ToContext, Diags, 
-                                   NonEquivalentDecls);
+  StructuralEquivalenceContext Ctx(FromContext, ToContext, NonEquivalentDecls);
   return Ctx.IsStructurallyEquivalent(From, To);
 }
diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt
index 82a81ec42411..9fe18407a839 100644
--- a/lib/AST/CMakeLists.txt
+++ b/lib/AST/CMakeLists.txt
@@ -1,4 +1,6 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_LINK_COMPONENTS support)
+
+set(LLVM_USED_LIBS clangBasic)
 
 add_clang_library(clangAST
   APValue.cpp
@@ -17,14 +19,17 @@ add_clang_library(clangAST
   DeclObjC.cpp
   DeclPrinter.cpp
   DeclTemplate.cpp
+  DumpXML.cpp
   Expr.cpp
   ExprClassification.cpp
   ExprConstant.cpp
   ExprCXX.cpp
-  FullExpr.cpp
   InheritViz.cpp
   ItaniumCXXABI.cpp
+  ItaniumMangle.cpp
+  Mangle.cpp
   MicrosoftCXXABI.cpp
+  MicrosoftMangle.cpp
   NestedNameSpecifier.cpp
   ParentMap.cpp
   RecordLayout.cpp
diff --git a/lib/AST/CXXABI.h b/lib/AST/CXXABI.h
index 4b38d7afb6a4..943c43e79198 100644
--- a/lib/AST/CXXABI.h
+++ b/lib/AST/CXXABI.h
@@ -15,6 +15,8 @@
 #ifndef LLVM_CLANG_AST_CXXABI_H
 #define LLVM_CLANG_AST_CXXABI_H
 
+#include "clang/AST/Type.h"
+
 namespace clang {
 
 class ASTContext;
@@ -28,6 +30,13 @@ public:
   /// Returns the size of a member pointer in multiples of the target
   /// pointer size.
   virtual unsigned getMemberPointerSize(const MemberPointerType *MPT) const = 0;
+
+  /// Returns the default calling convention for C++ methods.
+  virtual CallingConv getDefaultMethodCallConv() const = 0;
+
+  // Returns whether the given class is nearly empty, with just virtual pointers
+  // and no data except possibly virtual bases.
+  virtual bool isNearlyEmpty(const CXXRecordDecl *RD) const = 0;
 };
 
 /// Creates an instance of a C++ ABI class.
diff --git a/lib/AST/CXXInheritance.cpp b/lib/AST/CXXInheritance.cpp
index c563c37d58f4..ca9ec18b3997 100644
--- a/lib/AST/CXXInheritance.cpp
+++ b/lib/AST/CXXInheritance.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/RecordLayout.h"
 #include "clang/AST/DeclCXX.h"
 #include <algorithm>
 #include <set>
@@ -75,18 +76,21 @@ void CXXBasePaths::swap(CXXBasePaths &Other) {
   std::swap(DetectedVirtual, Other.DetectedVirtual);
 }
 
-bool CXXRecordDecl::isDerivedFrom(CXXRecordDecl *Base) const {
+bool CXXRecordDecl::isDerivedFrom(const CXXRecordDecl *Base) const {
   CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/false,
                      /*DetectVirtual=*/false);
   return isDerivedFrom(Base, Paths);
 }
 
-bool CXXRecordDecl::isDerivedFrom(CXXRecordDecl *Base, CXXBasePaths &Paths) const {
+bool CXXRecordDecl::isDerivedFrom(const CXXRecordDecl *Base,
+                                  CXXBasePaths &Paths) const {
   if (getCanonicalDecl() == Base->getCanonicalDecl())
     return false;
   
   Paths.setOrigin(const_cast<CXXRecordDecl*>(this));
-  return lookupInBases(&FindBaseClass, Base->getCanonicalDecl(), Paths);
+  return lookupInBases(&FindBaseClass,
+                       const_cast<CXXRecordDecl*>(Base->getCanonicalDecl()),
+                       Paths);
 }
 
 bool CXXRecordDecl::isVirtuallyDerivedFrom(CXXRecordDecl *Base) const {
@@ -662,3 +666,50 @@ CXXRecordDecl::getFinalOverriders(CXXFinalOverriderMap &FinalOverriders) const {
     }
   }
 }
+
+static void 
+AddIndirectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
+                        CXXIndirectPrimaryBaseSet& Bases) {
+  // If the record has a virtual primary base class, add it to our set.
+  const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+  if (Layout.isPrimaryBaseVirtual())
+    Bases.insert(Layout.getPrimaryBase());
+
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    assert(!I->getType()->isDependentType() &&
+           "Cannot get indirect primary bases for class with dependent bases.");
+
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Only bases with virtual bases participate in computing the
+    // indirect primary virtual base classes.
+    if (BaseDecl->getNumVBases())
+      AddIndirectPrimaryBases(BaseDecl, Context, Bases);
+  }
+
+}
+
+void 
+CXXRecordDecl::getIndirectPrimaryBases(CXXIndirectPrimaryBaseSet& Bases) const {
+  ASTContext &Context = getASTContext();
+
+  if (!getNumVBases())
+    return;
+
+  for (CXXRecordDecl::base_class_const_iterator I = bases_begin(),
+       E = bases_end(); I != E; ++I) {
+    assert(!I->getType()->isDependentType() &&
+           "Cannot get indirect primary bases for class with dependent bases.");
+
+    const CXXRecordDecl *BaseDecl =
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // Only bases with virtual bases participate in computing the
+    // indirect primary virtual base classes.
+    if (BaseDecl->getNumVBases())
+      AddIndirectPrimaryBases(BaseDecl, Context, Bases);
+  }
+}
+
diff --git a/lib/AST/Decl.cpp b/lib/AST/Decl.cpp
index b7be02d74533..56db8c7e330b 100644
--- a/lib/AST/Decl.cpp
+++ b/lib/AST/Decl.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyPrinter.h"
+#include "clang/AST/ASTMutationListener.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/Specifiers.h"
@@ -32,35 +33,140 @@ using namespace clang;
 // NamedDecl Implementation
 //===----------------------------------------------------------------------===//
 
+static const VisibilityAttr *GetExplicitVisibility(const Decl *d) {
+  // Use the most recent declaration of a variable.
+  if (const VarDecl *var = dyn_cast<VarDecl>(d))
+    return var->getMostRecentDeclaration()->getAttr<VisibilityAttr>();
+
+  // Use the most recent declaration of a function, and also handle
+  // function template specializations.
+  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(d)) {
+    if (const VisibilityAttr *attr
+          = fn->getMostRecentDeclaration()->getAttr<VisibilityAttr>())
+      return attr;
+
+    // If the function is a specialization of a template with an
+    // explicit visibility attribute, use that.
+    if (FunctionTemplateSpecializationInfo *templateInfo
+          = fn->getTemplateSpecializationInfo())
+      return templateInfo->getTemplate()->getTemplatedDecl()
+        ->getAttr<VisibilityAttr>();
+
+    return 0;
+  }
+
+  // Otherwise, just check the declaration itself first.
+  if (const VisibilityAttr *attr = d->getAttr<VisibilityAttr>())
+    return attr;
+
+  // If there wasn't explicit visibility there, and this is a
+  // specialization of a class template, check for visibility
+  // on the pattern.
+  if (const ClassTemplateSpecializationDecl *spec
+        = dyn_cast<ClassTemplateSpecializationDecl>(d))
+    return spec->getSpecializedTemplate()->getTemplatedDecl()
+      ->getAttr<VisibilityAttr>();
+
+  return 0;
+}
+
+static Visibility GetVisibilityFromAttr(const VisibilityAttr *A) {
+  switch (A->getVisibility()) {
+  case VisibilityAttr::Default:
+    return DefaultVisibility;
+  case VisibilityAttr::Hidden:
+    return HiddenVisibility;
+  case VisibilityAttr::Protected:
+    return ProtectedVisibility;
+  }
+  return DefaultVisibility;
+}
+
+typedef NamedDecl::LinkageInfo LinkageInfo;
+typedef std::pair<Linkage,Visibility> LVPair;
+
+static LVPair merge(LVPair L, LVPair R) {
+  return LVPair(minLinkage(L.first, R.first),
+                minVisibility(L.second, R.second));
+}
+
+static LVPair merge(LVPair L, LinkageInfo R) {
+  return LVPair(minLinkage(L.first, R.linkage()),
+                minVisibility(L.second, R.visibility()));
+}
+
+namespace {
+/// Flags controlling the computation of linkage and visibility.
+struct LVFlags {
+  bool ConsiderGlobalVisibility;
+  bool ConsiderVisibilityAttributes;
+
+  LVFlags() : ConsiderGlobalVisibility(true), 
+              ConsiderVisibilityAttributes(true) {
+  }
+
+  /// \brief Returns a set of flags that is only useful for computing the 
+  /// linkage, not the visibility, of a declaration.
+  static LVFlags CreateOnlyDeclLinkage() {
+    LVFlags F;
+    F.ConsiderGlobalVisibility = false;
+    F.ConsiderVisibilityAttributes = false;
+    return F;
+  }
+  
+  /// Returns a set of flags, otherwise based on these, which ignores
+  /// off all sources of visibility except template arguments.
+  LVFlags onlyTemplateVisibility() const {
+    LVFlags F = *this;
+    F.ConsiderGlobalVisibility = false;
+    F.ConsiderVisibilityAttributes = false;
+    return F;
+  }
+}; 
+} // end anonymous namespace
+
 /// \brief Get the most restrictive linkage for the types in the given
 /// template parameter list.
-static Linkage 
-getLinkageForTemplateParameterList(const TemplateParameterList *Params) {
-  Linkage L = ExternalLinkage;
+static LVPair 
+getLVForTemplateParameterList(const TemplateParameterList *Params) {
+  LVPair LV(ExternalLinkage, DefaultVisibility);
   for (TemplateParameterList::const_iterator P = Params->begin(),
                                           PEnd = Params->end();
        P != PEnd; ++P) {
-    if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P))
+    if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
+      if (NTTP->isExpandedParameterPack()) {
+        for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I) {
+          QualType T = NTTP->getExpansionType(I);
+          if (!T->isDependentType())
+            LV = merge(LV, T->getLinkageAndVisibility());
+        }
+        continue;
+      }
+      
       if (!NTTP->getType()->isDependentType()) {
-        L = minLinkage(L, NTTP->getType()->getLinkage());
+        LV = merge(LV, NTTP->getType()->getLinkageAndVisibility());
         continue;
       }
+    }
 
     if (TemplateTemplateParmDecl *TTP
                                    = dyn_cast<TemplateTemplateParmDecl>(*P)) {
-      L = minLinkage(L, 
-            getLinkageForTemplateParameterList(TTP->getTemplateParameters()));
+      LV = merge(LV, getLVForTemplateParameterList(TTP->getTemplateParameters()));
     }
   }
 
-  return L;
+  return LV;
 }
 
+/// getLVForDecl - Get the linkage and visibility for the given declaration.
+static LinkageInfo getLVForDecl(const NamedDecl *D, LVFlags F);
+
 /// \brief Get the most restrictive linkage for the types and
 /// declarations in the given template argument list.
-static Linkage getLinkageForTemplateArgumentList(const TemplateArgument *Args,
-                                                 unsigned NumArgs) {
-  Linkage L = ExternalLinkage;
+static LVPair getLVForTemplateArgumentList(const TemplateArgument *Args,
+                                           unsigned NumArgs,
+                                           LVFlags &F) {
+  LVPair LV(ExternalLinkage, DefaultVisibility);
 
   for (unsigned I = 0; I != NumArgs; ++I) {
     switch (Args[I].getKind()) {
@@ -70,40 +176,43 @@ static Linkage getLinkageForTemplateArgumentList(const TemplateArgument *Args,
       break;
       
     case TemplateArgument::Type:
-      L = minLinkage(L, Args[I].getAsType()->getLinkage());
+      LV = merge(LV, Args[I].getAsType()->getLinkageAndVisibility());
       break;
 
     case TemplateArgument::Declaration:
-      if (NamedDecl *ND = dyn_cast<NamedDecl>(Args[I].getAsDecl()))
-        L = minLinkage(L, ND->getLinkage());
-      if (ValueDecl *VD = dyn_cast<ValueDecl>(Args[I].getAsDecl()))
-        L = minLinkage(L, VD->getType()->getLinkage());
+      // The decl can validly be null as the representation of nullptr
+      // arguments, valid only in C++0x.
+      if (Decl *D = Args[I].getAsDecl()) {
+        if (NamedDecl *ND = dyn_cast<NamedDecl>(D))
+          LV = merge(LV, getLVForDecl(ND, F));
+      }
       break;
 
     case TemplateArgument::Template:
-      if (TemplateDecl *Template
-                                = Args[I].getAsTemplate().getAsTemplateDecl())
-        L = minLinkage(L, Template->getLinkage());
+    case TemplateArgument::TemplateExpansion:
+      if (TemplateDecl *Template 
+                = Args[I].getAsTemplateOrTemplatePattern().getAsTemplateDecl())
+        LV = merge(LV, getLVForDecl(Template, F));
       break;
 
     case TemplateArgument::Pack:
-      L = minLinkage(L, 
-                     getLinkageForTemplateArgumentList(Args[I].pack_begin(),
-                                                       Args[I].pack_size()));
+      LV = merge(LV, getLVForTemplateArgumentList(Args[I].pack_begin(),
+                                                  Args[I].pack_size(),
+                                                  F));
       break;
     }
   }
 
-  return L;
+  return LV;
 }
 
-static Linkage
-getLinkageForTemplateArgumentList(const TemplateArgumentList &TArgs) {
-  return getLinkageForTemplateArgumentList(TArgs.getFlatArgumentList(), 
-                                           TArgs.flat_size());
+static LVPair
+getLVForTemplateArgumentList(const TemplateArgumentList &TArgs,
+                             LVFlags &F) {
+  return getLVForTemplateArgumentList(TArgs.data(), TArgs.size(), F);
 }
 
-static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
+static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D, LVFlags F) {
   assert(D->getDeclContext()->getRedeclContext()->isFileContext() &&
          "Not a name having namespace scope");
   ASTContext &Context = D->getASTContext();
@@ -117,7 +226,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
   if (const VarDecl *Var = dyn_cast<VarDecl>(D)) {
     // Explicitly declared static.
     if (Var->getStorageClass() == SC_Static)
-      return InternalLinkage;
+      return LinkageInfo::internal();
 
     // - an object or reference that is explicitly declared const
     //   and neither explicitly declared extern nor previously
@@ -135,7 +244,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
           FoundExtern = true;
       
       if (!FoundExtern)
-        return InternalLinkage;
+        return LinkageInfo::internal();
     }
   } else if (isa<FunctionDecl>(D) || isa<FunctionTemplateDecl>(D)) {
     // C++ [temp]p4:
@@ -150,23 +259,88 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
 
     // Explicitly declared static.
     if (Function->getStorageClass() == SC_Static)
-      return InternalLinkage;
+      return LinkageInfo(InternalLinkage, DefaultVisibility, false);
   } else if (const FieldDecl *Field = dyn_cast<FieldDecl>(D)) {
     //   - a data member of an anonymous union.
     if (cast<RecordDecl>(Field->getDeclContext())->isAnonymousStructOrUnion())
-      return InternalLinkage;
+      return LinkageInfo::internal();
+  }
+
+  if (D->isInAnonymousNamespace())
+    return LinkageInfo::uniqueExternal();
+
+  // Set up the defaults.
+
+  // C99 6.2.2p5:
+  //   If the declaration of an identifier for an object has file
+  //   scope and no storage-class specifier, its linkage is
+  //   external.
+  LinkageInfo LV;
+
+  if (F.ConsiderVisibilityAttributes) {
+    if (const VisibilityAttr *VA = GetExplicitVisibility(D)) {
+      LV.setVisibility(GetVisibilityFromAttr(VA), true);
+      F.ConsiderGlobalVisibility = false;
+    } else {
+      // If we're declared in a namespace with a visibility attribute,
+      // use that namespace's visibility, but don't call it explicit.
+      for (const DeclContext *DC = D->getDeclContext();
+           !isa<TranslationUnitDecl>(DC);
+           DC = DC->getParent()) {
+        if (!isa<NamespaceDecl>(DC)) continue;
+        if (const VisibilityAttr *VA =
+              cast<NamespaceDecl>(DC)->getAttr<VisibilityAttr>()) {
+          LV.setVisibility(GetVisibilityFromAttr(VA), false);
+          F.ConsiderGlobalVisibility = false;
+          break;
+        }
+      }
+    }
   }
 
   // C++ [basic.link]p4:
-  
+
   //   A name having namespace scope has external linkage if it is the
   //   name of
   //
   //     - an object or reference, unless it has internal linkage; or
   if (const VarDecl *Var = dyn_cast<VarDecl>(D)) {
+    // GCC applies the following optimization to variables and static
+    // data members, but not to functions:
+    //
+    // Modify the variable's LV by the LV of its type unless this is
+    // C or extern "C".  This follows from [basic.link]p9:
+    //   A type without linkage shall not be used as the type of a
+    //   variable or function with external linkage unless
+    //    - the entity has C language linkage, or
+    //    - the entity is declared within an unnamed namespace, or
+    //    - the entity is not used or is defined in the same
+    //      translation unit.
+    // and [basic.link]p10:
+    //   ...the types specified by all declarations referring to a
+    //   given variable or function shall be identical...
+    // C does not have an equivalent rule.
+    //
+    // Ignore this if we've got an explicit attribute;  the user
+    // probably knows what they're doing.
+    //
+    // Note that we don't want to make the variable non-external
+    // because of this, but unique-external linkage suits us.
+    if (Context.getLangOptions().CPlusPlus && !Var->isExternC()) {
+      LVPair TypeLV = Var->getType()->getLinkageAndVisibility();
+      if (TypeLV.first != ExternalLinkage)
+        return LinkageInfo::uniqueExternal();
+      if (!LV.visibilityExplicit())
+        LV.mergeVisibility(TypeLV.second);
+    }
+
+    if (Var->getStorageClass() == SC_PrivateExtern)
+      LV.setVisibility(HiddenVisibility, true);
+
     if (!Context.getLangOptions().CPlusPlus &&
         (Var->getStorageClass() == SC_Extern ||
          Var->getStorageClass() == SC_PrivateExtern)) {
+
       // C99 6.2.2p4:
       //   For an identifier declared with the storage-class specifier
       //   extern in a scope in which a prior declaration of that
@@ -177,23 +351,22 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
       //   is visible, or if the prior declaration specifies no
       //   linkage, then the identifier has external linkage.
       if (const VarDecl *PrevVar = Var->getPreviousDeclaration()) {
-        if (Linkage L = PrevVar->getLinkage())
-          return L;
+        LinkageInfo PrevLV = getLVForDecl(PrevVar, F);
+        if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
+        LV.mergeVisibility(PrevLV);
       }
     }
 
-    // C99 6.2.2p5:
-    //   If the declaration of an identifier for an object has file
-    //   scope and no storage-class specifier, its linkage is
-    //   external.
-    if (Var->isInAnonymousNamespace())
-      return UniqueExternalLinkage;
+  //     - a function, unless it has internal linkage; or
+  } else if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
+    // In theory, we can modify the function's LV by the LV of its
+    // type unless it has C linkage (see comment above about variables
+    // for justification).  In practice, GCC doesn't do this, so it's
+    // just too painful to make work.
 
-    return ExternalLinkage;
-  }
+    if (Function->getStorageClass() == SC_PrivateExtern)
+      LV.setVisibility(HiddenVisibility, true);
 
-  //     - a function, unless it has internal linkage; or
-  if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
     // C99 6.2.2p5:
     //   If the declaration of an identifier for a function has no
     //   storage-class specifier, its linkage is determined exactly
@@ -213,141 +386,300 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
       //   is visible, or if the prior declaration specifies no
       //   linkage, then the identifier has external linkage.
       if (const FunctionDecl *PrevFunc = Function->getPreviousDeclaration()) {
-        if (Linkage L = PrevFunc->getLinkage())
-          return L;
+        LinkageInfo PrevLV = getLVForDecl(PrevFunc, F);
+        if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
+        LV.mergeVisibility(PrevLV);
       }
     }
 
-    if (Function->isInAnonymousNamespace())
-      return UniqueExternalLinkage;
+    // In C++, then if the type of the function uses a type with
+    // unique-external linkage, it's not legally usable from outside
+    // this translation unit.  However, we should use the C linkage
+    // rules instead for extern "C" declarations.
+    if (Context.getLangOptions().CPlusPlus && !Function->isExternC() &&
+        Function->getType()->getLinkage() == UniqueExternalLinkage)
+      return LinkageInfo::uniqueExternal();
 
     if (FunctionTemplateSpecializationInfo *SpecInfo
                                = Function->getTemplateSpecializationInfo()) {
-      Linkage L = SpecInfo->getTemplate()->getLinkage();
+      LV.merge(getLVForDecl(SpecInfo->getTemplate(),
+                            F.onlyTemplateVisibility()));
       const TemplateArgumentList &TemplateArgs = *SpecInfo->TemplateArguments;
-      L = minLinkage(L, getLinkageForTemplateArgumentList(TemplateArgs));
-      return L;
+      LV.merge(getLVForTemplateArgumentList(TemplateArgs, F));
     }
 
-    return ExternalLinkage;
-  }
-
   //     - a named class (Clause 9), or an unnamed class defined in a
   //       typedef declaration in which the class has the typedef name
   //       for linkage purposes (7.1.3); or
   //     - a named enumeration (7.2), or an unnamed enumeration
   //       defined in a typedef declaration in which the enumeration
   //       has the typedef name for linkage purposes (7.1.3); or
-  if (const TagDecl *Tag = dyn_cast<TagDecl>(D))
-    if (Tag->getDeclName() || Tag->getTypedefForAnonDecl()) {
-      if (Tag->isInAnonymousNamespace())
-        return UniqueExternalLinkage;
-
-      // If this is a class template specialization, consider the
-      // linkage of the template and template arguments.
-      if (const ClassTemplateSpecializationDecl *Spec
-            = dyn_cast<ClassTemplateSpecializationDecl>(Tag)) {
-        const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-        Linkage L = getLinkageForTemplateArgumentList(TemplateArgs);
-        return minLinkage(L, Spec->getSpecializedTemplate()->getLinkage());
-      }
+  } else if (const TagDecl *Tag = dyn_cast<TagDecl>(D)) {
+    // Unnamed tags have no linkage.
+    if (!Tag->getDeclName() && !Tag->getTypedefForAnonDecl())
+      return LinkageInfo::none();
+
+    // If this is a class template specialization, consider the
+    // linkage of the template and template arguments.
+    if (const ClassTemplateSpecializationDecl *Spec
+          = dyn_cast<ClassTemplateSpecializationDecl>(Tag)) {
+      // From the template.
+      LV.merge(getLVForDecl(Spec->getSpecializedTemplate(),
+                            F.onlyTemplateVisibility()));
 
-      return ExternalLinkage;
+      // The arguments at which the template was instantiated.
+      const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
+      LV.merge(getLVForTemplateArgumentList(TemplateArgs, F));
     }
 
+    // Consider -fvisibility unless the type has C linkage.
+    if (F.ConsiderGlobalVisibility)
+      F.ConsiderGlobalVisibility =
+        (Context.getLangOptions().CPlusPlus &&
+         !Tag->getDeclContext()->isExternCContext());
+
   //     - an enumerator belonging to an enumeration with external linkage;
-  if (isa<EnumConstantDecl>(D)) {
-    Linkage L = cast<NamedDecl>(D->getDeclContext())->getLinkage();
-    if (isExternalLinkage(L))
-      return L;
-  }
+  } else if (isa<EnumConstantDecl>(D)) {
+    LinkageInfo EnumLV = getLVForDecl(cast<NamedDecl>(D->getDeclContext()), F);
+    if (!isExternalLinkage(EnumLV.linkage()))
+      return LinkageInfo::none();
+    LV.merge(EnumLV);
 
   //     - a template, unless it is a function template that has
   //       internal linkage (Clause 14);
-  if (const TemplateDecl *Template = dyn_cast<TemplateDecl>(D)) {
-    if (D->isInAnonymousNamespace())
-      return UniqueExternalLinkage;
-
-    return getLinkageForTemplateParameterList(
-                                         Template->getTemplateParameters());
-  }
+  } else if (const TemplateDecl *Template = dyn_cast<TemplateDecl>(D)) {
+    LV.merge(getLVForTemplateParameterList(Template->getTemplateParameters()));
 
   //     - a namespace (7.3), unless it is declared within an unnamed
   //       namespace.
-  if (isa<NamespaceDecl>(D) && !D->isInAnonymousNamespace())
-    return ExternalLinkage;
+  } else if (isa<NamespaceDecl>(D) && !D->isInAnonymousNamespace()) {
+    return LV;
+
+  // By extension, we assign external linkage to Objective-C
+  // interfaces.
+  } else if (isa<ObjCInterfaceDecl>(D)) {
+    // fallout
 
-  return NoLinkage;
+  // Everything not covered here has no linkage.
+  } else {
+    return LinkageInfo::none();
+  }
+
+  // If we ended up with non-external linkage, visibility should
+  // always be default.
+  if (LV.linkage() != ExternalLinkage)
+    return LinkageInfo(LV.linkage(), DefaultVisibility, false);
+
+  // If we didn't end up with hidden visibility, consider attributes
+  // and -fvisibility.
+  if (F.ConsiderGlobalVisibility)
+    LV.mergeVisibility(Context.getLangOptions().getVisibilityMode());
+
+  return LV;
 }
 
-static Linkage getLinkageForClassMember(const NamedDecl *D) {
+static LinkageInfo getLVForClassMember(const NamedDecl *D, LVFlags F) {
+  // Only certain class members have linkage.  Note that fields don't
+  // really have linkage, but it's convenient to say they do for the
+  // purposes of calculating linkage of pointer-to-data-member
+  // template arguments.
   if (!(isa<CXXMethodDecl>(D) ||
         isa<VarDecl>(D) ||
+        isa<FieldDecl>(D) ||
         (isa<TagDecl>(D) &&
          (D->getDeclName() || cast<TagDecl>(D)->getTypedefForAnonDecl()))))
-    return NoLinkage;
+    return LinkageInfo::none();
+
+  LinkageInfo LV;
 
-  // Class members only have linkage if their class has external linkage.
-  Linkage L = cast<RecordDecl>(D->getDeclContext())->getLinkage();
-  if (!isExternalLinkage(L)) return NoLinkage;
+  // The flags we're going to use to compute the class's visibility.
+  LVFlags ClassF = F;
+
+  // If we have an explicit visibility attribute, merge that in.
+  if (F.ConsiderVisibilityAttributes) {
+    if (const VisibilityAttr *VA = GetExplicitVisibility(D)) {
+      LV.mergeVisibility(GetVisibilityFromAttr(VA), true);
+
+      // Ignore global visibility later, but not this attribute.
+      F.ConsiderGlobalVisibility = false;
+
+      // Ignore both global visibility and attributes when computing our
+      // parent's visibility.
+      ClassF = F.onlyTemplateVisibility();
+    }
+  }
+
+  // Class members only have linkage if their class has external
+  // linkage.
+  LV.merge(getLVForDecl(cast<RecordDecl>(D->getDeclContext()), ClassF));
+  if (!isExternalLinkage(LV.linkage()))
+    return LinkageInfo::none();
 
   // If the class already has unique-external linkage, we can't improve.
-  if (L == UniqueExternalLinkage) return UniqueExternalLinkage;
+  if (LV.linkage() == UniqueExternalLinkage)
+    return LinkageInfo::uniqueExternal();
 
-  // If this is a method template specialization, use the linkage for
-  // the template parameters and arguments.
   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D)) {
-    if (FunctionTemplateSpecializationInfo *SpecInfo
+    // If the type of the function uses a type with unique-external
+    // linkage, it's not legally usable from outside this translation unit.
+    if (MD->getType()->getLinkage() == UniqueExternalLinkage)
+      return LinkageInfo::uniqueExternal();
+
+    TemplateSpecializationKind TSK = TSK_Undeclared;
+
+    // If this is a method template specialization, use the linkage for
+    // the template parameters and arguments.
+    if (FunctionTemplateSpecializationInfo *Spec
            = MD->getTemplateSpecializationInfo()) {
-      Linkage ArgLinkage =
-        getLinkageForTemplateArgumentList(*SpecInfo->TemplateArguments);
-      Linkage ParamLinkage =
-        getLinkageForTemplateParameterList(
-                           SpecInfo->getTemplate()->getTemplateParameters());
-      return minLinkage(ArgLinkage, ParamLinkage);
+      LV.merge(getLVForTemplateArgumentList(*Spec->TemplateArguments, F));
+      LV.merge(getLVForTemplateParameterList(
+                              Spec->getTemplate()->getTemplateParameters()));
+
+      TSK = Spec->getTemplateSpecializationKind();
+    } else if (MemberSpecializationInfo *MSI =
+                 MD->getMemberSpecializationInfo()) {
+      TSK = MSI->getTemplateSpecializationKind();
+    }
+
+    // If we're paying attention to global visibility, apply
+    // -finline-visibility-hidden if this is an inline method.
+    //
+    // Note that ConsiderGlobalVisibility doesn't yet have information
+    // about whether containing classes have visibility attributes,
+    // and that's intentional.
+    if (TSK != TSK_ExplicitInstantiationDeclaration &&
+        F.ConsiderGlobalVisibility &&
+        MD->getASTContext().getLangOptions().InlineVisibilityHidden) {
+      // InlineVisibilityHidden only applies to definitions, and
+      // isInlined() only gives meaningful answers on definitions
+      // anyway.
+      const FunctionDecl *Def = 0;
+      if (MD->hasBody(Def) && Def->isInlined())
+        LV.setVisibility(HiddenVisibility);
+    }
+
+    // Note that in contrast to basically every other situation, we
+    // *do* apply -fvisibility to method declarations.
+
+  } else if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
+    if (const ClassTemplateSpecializationDecl *Spec
+        = dyn_cast<ClassTemplateSpecializationDecl>(RD)) {
+      // Merge template argument/parameter information for member
+      // class template specializations.
+      LV.merge(getLVForTemplateArgumentList(Spec->getTemplateArgs(), F));
+      LV.merge(getLVForTemplateParameterList(
+                    Spec->getSpecializedTemplate()->getTemplateParameters()));
     }
 
-  // Similarly for member class template specializations.
-  } else if (const ClassTemplateSpecializationDecl *Spec
-               = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-    Linkage ArgLinkage =
-      getLinkageForTemplateArgumentList(Spec->getTemplateArgs());
-    Linkage ParamLinkage =
-      getLinkageForTemplateParameterList(
-                    Spec->getSpecializedTemplate()->getTemplateParameters());
-    return minLinkage(ArgLinkage, ParamLinkage);
+  // Static data members.
+  } else if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    // Modify the variable's linkage by its type, but ignore the
+    // type's visibility unless it's a definition.
+    LVPair TypeLV = VD->getType()->getLinkageAndVisibility();
+    if (TypeLV.first != ExternalLinkage)
+      LV.mergeLinkage(UniqueExternalLinkage);
+    if (!LV.visibilityExplicit())
+      LV.mergeVisibility(TypeLV.second);
   }
 
-  return ExternalLinkage;
+  F.ConsiderGlobalVisibility &= !LV.visibilityExplicit();
+
+  // Apply -fvisibility if desired.
+  if (F.ConsiderGlobalVisibility && LV.visibility() != HiddenVisibility) {
+    LV.mergeVisibility(D->getASTContext().getLangOptions().getVisibilityMode());
+  }
+
+  return LV;
+}
+
+static void clearLinkageForClass(const CXXRecordDecl *record) {
+  for (CXXRecordDecl::decl_iterator
+         i = record->decls_begin(), e = record->decls_end(); i != e; ++i) {
+    Decl *child = *i;
+    if (isa<NamedDecl>(child))
+      cast<NamedDecl>(child)->ClearLinkageCache();
+  }
+}
+
+void NamedDecl::ClearLinkageCache() {
+  // Note that we can't skip clearing the linkage of children just
+  // because the parent doesn't have cached linkage:  we don't cache
+  // when computing linkage for parent contexts.
+
+  HasCachedLinkage = 0;
+
+  // If we're changing the linkage of a class, we need to reset the
+  // linkage of child declarations, too.
+  if (const CXXRecordDecl *record = dyn_cast<CXXRecordDecl>(this))
+    clearLinkageForClass(record);
+
+  if (ClassTemplateDecl *temp =
+        dyn_cast<ClassTemplateDecl>(const_cast<NamedDecl*>(this))) {
+    // Clear linkage for the template pattern.
+    CXXRecordDecl *record = temp->getTemplatedDecl();
+    record->HasCachedLinkage = 0;
+    clearLinkageForClass(record);
+
+    // We need to clear linkage for specializations, too.
+    for (ClassTemplateDecl::spec_iterator
+           i = temp->spec_begin(), e = temp->spec_end(); i != e; ++i)
+      i->ClearLinkageCache();
+  }
+
+  // Clear cached linkage for function template decls, too.
+  if (FunctionTemplateDecl *temp =
+        dyn_cast<FunctionTemplateDecl>(const_cast<NamedDecl*>(this)))
+    for (FunctionTemplateDecl::spec_iterator
+           i = temp->spec_begin(), e = temp->spec_end(); i != e; ++i)
+      i->ClearLinkageCache();
+    
 }
 
 Linkage NamedDecl::getLinkage() const {
+  if (HasCachedLinkage) {
+    assert(Linkage(CachedLinkage) ==
+             getLVForDecl(this, LVFlags::CreateOnlyDeclLinkage()).linkage());
+    return Linkage(CachedLinkage);
+  }
+
+  CachedLinkage = getLVForDecl(this, 
+                               LVFlags::CreateOnlyDeclLinkage()).linkage();
+  HasCachedLinkage = 1;
+  return Linkage(CachedLinkage);
+}
 
+LinkageInfo NamedDecl::getLinkageAndVisibility() const {
+  LinkageInfo LI = getLVForDecl(this, LVFlags());
+  assert(!HasCachedLinkage || Linkage(CachedLinkage) == LI.linkage());
+  HasCachedLinkage = 1;
+  CachedLinkage = LI.linkage();
+  return LI;
+}
+
+static LinkageInfo getLVForDecl(const NamedDecl *D, LVFlags Flags) {
   // Objective-C: treat all Objective-C declarations as having external
   // linkage.
-  switch (getKind()) {
+  switch (D->getKind()) {
     default:
       break;
+    case Decl::TemplateTemplateParm: // count these as external
+    case Decl::NonTypeTemplateParm:
     case Decl::ObjCAtDefsField:
     case Decl::ObjCCategory:
     case Decl::ObjCCategoryImpl:
-    case Decl::ObjCClass:
     case Decl::ObjCCompatibleAlias:
     case Decl::ObjCForwardProtocol:
     case Decl::ObjCImplementation:
-    case Decl::ObjCInterface:
-    case Decl::ObjCIvar:
     case Decl::ObjCMethod:
     case Decl::ObjCProperty:
     case Decl::ObjCPropertyImpl:
     case Decl::ObjCProtocol:
-      return ExternalLinkage;
+      return LinkageInfo::external();
   }
 
   // Handle linkage for namespace-scope names.
-  if (getDeclContext()->getRedeclContext()->isFileContext())
-    if (Linkage L = getLinkageForNamespaceScopeDecl(this))
-      return L;
+  if (D->getDeclContext()->getRedeclContext()->isFileContext())
+    return getLVForNamespaceScopeDecl(D, Flags);
   
   // C++ [basic.link]p5:
   //   In addition, a member function, static data member, a named
@@ -356,8 +688,8 @@ Linkage NamedDecl::getLinkage() const {
   //   that the class or enumeration has the typedef name for linkage
   //   purposes (7.1.3), has external linkage if the name of the class
   //   has external linkage.
-  if (getDeclContext()->isRecord())
-    return getLinkageForClassMember(this);
+  if (D->getDeclContext()->isRecord())
+    return getLVForClassMember(D, Flags);
 
   // C++ [basic.link]p6:
   //   The name of a function declared in block scope and the name of
@@ -370,36 +702,54 @@ Linkage NamedDecl::getLinkage() const {
   //   one such matching entity, the program is ill-formed. Otherwise,
   //   if no matching entity is found, the block scope entity receives
   //   external linkage.
-  if (getLexicalDeclContext()->isFunctionOrMethod()) {
-    if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(this)) {
-      if (Function->getPreviousDeclaration())
-        if (Linkage L = Function->getPreviousDeclaration()->getLinkage())
-          return L;
-
+  if (D->getLexicalDeclContext()->isFunctionOrMethod()) {
+    if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
       if (Function->isInAnonymousNamespace())
-        return UniqueExternalLinkage;
+        return LinkageInfo::uniqueExternal();
 
-      return ExternalLinkage;
+      LinkageInfo LV;
+      if (Flags.ConsiderVisibilityAttributes) {
+        if (const VisibilityAttr *VA = GetExplicitVisibility(Function))
+          LV.setVisibility(GetVisibilityFromAttr(VA));
+      }
+      
+      if (const FunctionDecl *Prev = Function->getPreviousDeclaration()) {
+        LinkageInfo PrevLV = getLVForDecl(Prev, Flags);
+        if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
+        LV.mergeVisibility(PrevLV);
+      }
+
+      return LV;
     }
 
-    if (const VarDecl *Var = dyn_cast<VarDecl>(this))
+    if (const VarDecl *Var = dyn_cast<VarDecl>(D))
       if (Var->getStorageClass() == SC_Extern ||
           Var->getStorageClass() == SC_PrivateExtern) {
-        if (Var->getPreviousDeclaration())
-          if (Linkage L = Var->getPreviousDeclaration()->getLinkage())
-            return L;
-
         if (Var->isInAnonymousNamespace())
-          return UniqueExternalLinkage;
+          return LinkageInfo::uniqueExternal();
+
+        LinkageInfo LV;
+        if (Var->getStorageClass() == SC_PrivateExtern)
+          LV.setVisibility(HiddenVisibility);
+        else if (Flags.ConsiderVisibilityAttributes) {
+          if (const VisibilityAttr *VA = GetExplicitVisibility(Var))
+            LV.setVisibility(GetVisibilityFromAttr(VA));
+        }
+        
+        if (const VarDecl *Prev = Var->getPreviousDeclaration()) {
+          LinkageInfo PrevLV = getLVForDecl(Prev, Flags);
+          if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
+          LV.mergeVisibility(PrevLV);
+        }
 
-        return ExternalLinkage;
+        return LV;
       }
   }
 
   // C++ [basic.link]p6:
   //   Names not covered by these rules have no linkage.
-  return NoLinkage;
-  }
+  return LinkageInfo::none();
+}
 
 std::string NamedDecl::getQualifiedNameAsString() const {
   return getQualifiedNameAsString(getASTContext().getLangOptions());
@@ -430,8 +780,8 @@ std::string NamedDecl::getQualifiedNameAsString(const PrintingPolicy &P) const {
       const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
       std::string TemplateArgsStr
         = TemplateSpecializationType::PrintTemplateArgumentList(
-                                           TemplateArgs.getFlatArgumentList(),
-                                           TemplateArgs.flat_size(),
+                                           TemplateArgs.data(),
+                                           TemplateArgs.size(),
                                            P);
       OS << Spec->getName() << TemplateArgsStr;
     } else if (const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(*I)) {
@@ -514,6 +864,10 @@ bool NamedDecl::declarationReplaces(NamedDecl *OldD) const {
     return cast<UsingShadowDecl>(this)->getTargetDecl() ==
            cast<UsingShadowDecl>(OldD)->getTargetDecl();
 
+  if (isa<UsingDecl>(this) && isa<UsingDecl>(OldD))
+    return cast<UsingDecl>(this)->getTargetNestedNameDecl() ==
+           cast<UsingDecl>(OldD)->getTargetNestedNameDecl();
+
   // For non-function declarations, if the declarations are of the
   // same kind then this must be a redeclaration, or semantic analysis
   // would not have given us the new declaration.
@@ -545,7 +899,7 @@ bool NamedDecl::isCXXInstanceMember() const {
   if (isa<UsingShadowDecl>(D))
     D = cast<UsingShadowDecl>(D)->getTargetDecl();
 
-  if (isa<FieldDecl>(D))
+  if (isa<FieldDecl>(D) || isa<IndirectFieldDecl>(D))
     return true;
   if (isa<CXXMethodDecl>(D))
     return cast<CXXMethodDecl>(D)->isInstance();
@@ -655,6 +1009,14 @@ VarDecl *VarDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
   return new (C) VarDecl(Var, DC, L, Id, T, TInfo, S, SCAsWritten);
 }
 
+void VarDecl::setStorageClass(StorageClass SC) {
+  assert(isLegalForVariable(SC));
+  if (getStorageClass() != SC)
+    ClearLinkageCache();
+  
+  SClass = SC;
+}
+
 SourceLocation VarDecl::getInnerLocStart() const {
   SourceLocation Start = getTypeSpecStartLoc();
   if (Start.isInvalid())
@@ -785,6 +1147,17 @@ VarDecl *VarDecl::getDefinition() {
   return 0;
 }
 
+VarDecl::DefinitionKind VarDecl::hasDefinition() const {
+  DefinitionKind Kind = DeclarationOnly;
+  
+  const VarDecl *First = getFirstDeclaration();
+  for (redecl_iterator I = First->redecls_begin(), E = First->redecls_end();
+       I != E; ++I)
+    Kind = std::max(Kind, (*I)->isThisDeclarationADefinition());
+
+  return Kind;
+}
+
 const Expr *VarDecl::getAnyInitializer(const VarDecl *&D) const {
   redecl_iterator I = redecls_begin(), E = redecls_end();
   while (I != E && !I->getInit())
@@ -883,15 +1256,14 @@ Expr *ParmVarDecl::getDefaultArg() {
          "Default argument is not yet instantiated!");
   
   Expr *Arg = getInit();
-  if (CXXExprWithTemporaries *E = dyn_cast_or_null<CXXExprWithTemporaries>(Arg))
+  if (ExprWithCleanups *E = dyn_cast_or_null<ExprWithCleanups>(Arg))
     return E->getSubExpr();
 
   return Arg;
 }
 
 unsigned ParmVarDecl::getNumDefaultArgTemporaries() const {
-  if (const CXXExprWithTemporaries *E = 
-        dyn_cast<CXXExprWithTemporaries>(getInit()))
+  if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(getInit()))
     return E->getNumTemporaries();
 
   return 0;
@@ -901,7 +1273,7 @@ CXXTemporary *ParmVarDecl::getDefaultArgTemporary(unsigned i) {
   assert(getNumDefaultArgTemporaries() && 
          "Default arguments does not have any temporaries!");
 
-  CXXExprWithTemporaries *E = cast<CXXExprWithTemporaries>(getInit());
+  ExprWithCleanups *E = cast<ExprWithCleanups>(getInit());
   return E->getTemporary(i);
 }
 
@@ -915,6 +1287,10 @@ SourceRange ParmVarDecl::getDefaultArgRange() const {
   return SourceRange();
 }
 
+bool ParmVarDecl::isParameterPack() const {
+  return isa<PackExpansionType>(getType());
+}
+
 //===----------------------------------------------------------------------===//
 // FunctionDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -926,8 +1302,8 @@ void FunctionDecl::getNameForDiagnostic(std::string &S,
   const TemplateArgumentList *TemplateArgs = getTemplateSpecializationArgs();
   if (TemplateArgs)
     S += TemplateSpecializationType::PrintTemplateArgumentList(
-                                         TemplateArgs->getFlatArgumentList(),
-                                         TemplateArgs->flat_size(),
+                                                         TemplateArgs->data(),
+                                                         TemplateArgs->size(),
                                                                Policy);
     
 }
@@ -966,6 +1342,13 @@ void FunctionDecl::setBody(Stmt *B) {
     EndRangeLoc = B->getLocEnd();
 }
 
+void FunctionDecl::setPure(bool P) {
+  IsPure = P;
+  if (P)
+    if (CXXRecordDecl *Parent = dyn_cast<CXXRecordDecl>(getDeclContext()))
+      Parent->markedVirtualFunctionPure();
+}
+
 bool FunctionDecl::isMain() const {
   ASTContext &Context = getASTContext();
   return !Context.getLangOptions().Freestanding &&
@@ -994,7 +1377,7 @@ bool FunctionDecl::isExternC() const {
       break;
   }
 
-  return false;
+  return isMain();
 }
 
 bool FunctionDecl::isGlobal() const {
@@ -1027,6 +1410,9 @@ FunctionDecl::setPreviousDeclaration(FunctionDecl *PrevDecl) {
     assert((!PrevDecl || PrevFunTmpl) && "Function/function template mismatch");
     FunTmpl->setPreviousDeclaration(PrevFunTmpl);
   }
+  
+  if (PrevDecl->IsInline)
+    IsInline = true;
 }
 
 const FunctionDecl *FunctionDecl::getCanonicalDecl() const {
@@ -1037,6 +1423,14 @@ FunctionDecl *FunctionDecl::getCanonicalDecl() {
   return getFirstDeclaration();
 }
 
+void FunctionDecl::setStorageClass(StorageClass SC) {
+  assert(isLegalForFunction(SC));
+  if (getStorageClass() != SC)
+    ClearLinkageCache();
+  
+  SClass = SC;
+}
+
 /// \brief Returns a value indicating whether this function
 /// corresponds to a builtin function.
 ///
@@ -1083,7 +1477,7 @@ unsigned FunctionDecl::getBuiltinID() const {
 
 
 /// getNumParams - Return the number of parameters this function must have
-/// based on its FunctionType.  This is the length of the PararmInfo array
+/// based on its FunctionType.  This is the length of the ParamInfo array
 /// after it has been created.
 unsigned FunctionDecl::getNumParams() const {
   const FunctionType *FT = getType()->getAs<FunctionType>();
@@ -1093,13 +1487,14 @@ unsigned FunctionDecl::getNumParams() const {
 
 }
 
-void FunctionDecl::setParams(ParmVarDecl **NewParamInfo, unsigned NumParams) {
+void FunctionDecl::setParams(ASTContext &C,
+                             ParmVarDecl **NewParamInfo, unsigned NumParams) {
   assert(ParamInfo == 0 && "Already has param info!");
   assert(NumParams == getNumParams() && "Parameter count mismatch!");
 
   // Zero params -> null pointer.
   if (NumParams) {
-    void *Mem = getASTContext().Allocate(sizeof(ParmVarDecl*)*NumParams);
+    void *Mem = C.Allocate(sizeof(ParmVarDecl*)*NumParams);
     ParamInfo = new (Mem) ParmVarDecl*[NumParams];
     memcpy(ParamInfo, NewParamInfo, sizeof(ParmVarDecl*)*NumParams);
 
@@ -1113,25 +1508,40 @@ void FunctionDecl::setParams(ParmVarDecl **NewParamInfo, unsigned NumParams) {
 /// getMinRequiredArguments - Returns the minimum number of arguments
 /// needed to call this function. This may be fewer than the number of
 /// function parameters, if some of the parameters have default
-/// arguments (in C++).
+/// arguments (in C++) or the last parameter is a parameter pack.
 unsigned FunctionDecl::getMinRequiredArguments() const {
-  unsigned NumRequiredArgs = getNumParams();
-  while (NumRequiredArgs > 0
-         && getParamDecl(NumRequiredArgs-1)->hasDefaultArg())
+  if (!getASTContext().getLangOptions().CPlusPlus)
+    return getNumParams();
+  
+  unsigned NumRequiredArgs = getNumParams();  
+  
+  // If the last parameter is a parameter pack, we don't need an argument for 
+  // it.
+  if (NumRequiredArgs > 0 &&
+      getParamDecl(NumRequiredArgs - 1)->isParameterPack())
+    --NumRequiredArgs;
+      
+  // If this parameter has a default argument, we don't need an argument for
+  // it.
+  while (NumRequiredArgs > 0 &&
+         getParamDecl(NumRequiredArgs-1)->hasDefaultArg())
     --NumRequiredArgs;
 
+  // We might have parameter packs before the end. These can't be deduced,
+  // but they can still handle multiple arguments.
+  unsigned ArgIdx = NumRequiredArgs;
+  while (ArgIdx > 0) {
+    if (getParamDecl(ArgIdx - 1)->isParameterPack())
+      NumRequiredArgs = ArgIdx;
+    
+    --ArgIdx;
+  }
+  
   return NumRequiredArgs;
 }
 
 bool FunctionDecl::isInlined() const {
-  // FIXME: This is not enough. Consider:
-  //
-  // inline void f();
-  // void f() { }
-  //
-  // f is inlined, but does not have inline specified.
-  // To fix this we should add an 'inline' flag to FunctionDecl.
-  if (isInlineSpecified())
+  if (IsInline)
     return true;
   
   if (isa<CXXMethodDecl>(this)) {
@@ -1185,20 +1595,22 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
   ASTContext &Context = getASTContext();
   
   if (!Context.getLangOptions().C99 || hasAttr<GNUInlineAttr>()) {
-    // GNU inline semantics. Based on a number of examples, we came up with the
-    // following heuristic: if the "inline" keyword is present on a
-    // declaration of the function but "extern" is not present on that
-    // declaration, then the symbol is externally visible. Otherwise, the GNU
-    // "extern inline" semantics applies and the symbol is not externally
-    // visible.
+    // If it's not the case that both 'inline' and 'extern' are
+    // specified on the definition, then this inline definition is
+    // externally visible.
+    if (!(isInlineSpecified() && getStorageClassAsWritten() == SC_Extern))
+      return true;
+    
+    // If any declaration is 'inline' but not 'extern', then this definition
+    // is externally visible.
     for (redecl_iterator Redecl = redecls_begin(), RedeclEnd = redecls_end();
          Redecl != RedeclEnd;
          ++Redecl) {
-      if (Redecl->isInlineSpecified() && Redecl->getStorageClass() != SC_Extern)
+      if (Redecl->isInlineSpecified() && 
+          Redecl->getStorageClassAsWritten() != SC_Extern)
         return true;
-    }
+    }    
     
-    // GNU "extern inline" semantics; no externally visible symbol.
     return false;
   }
   
@@ -1271,12 +1683,13 @@ MemberSpecializationInfo *FunctionDecl::getMemberSpecializationInfo() const {
 }
 
 void 
-FunctionDecl::setInstantiationOfMemberFunction(FunctionDecl *FD,
+FunctionDecl::setInstantiationOfMemberFunction(ASTContext &C,
+                                               FunctionDecl *FD,
                                                TemplateSpecializationKind TSK) {
   assert(TemplateOrSpecialization.isNull() && 
          "Member function is already a specialization");
   MemberSpecializationInfo *Info 
-    = new (getASTContext()) MemberSpecializationInfo(FD, TSK);
+    = new (C) MemberSpecializationInfo(FD, TSK);
   TemplateOrSpecialization = Info;
 }
 
@@ -1362,7 +1775,8 @@ FunctionDecl::getTemplateSpecializationArgsAsWritten() const {
 }
 
 void
-FunctionDecl::setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
+FunctionDecl::setFunctionTemplateSpecialization(ASTContext &C,
+                                                FunctionTemplateDecl *Template,
                                      const TemplateArgumentList *TemplateArgs,
                                                 void *InsertPos,
                                                 TemplateSpecializationKind TSK,
@@ -1373,14 +1787,10 @@ FunctionDecl::setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
   FunctionTemplateSpecializationInfo *Info
     = TemplateOrSpecialization.dyn_cast<FunctionTemplateSpecializationInfo*>();
   if (!Info)
-    Info = new (getASTContext()) FunctionTemplateSpecializationInfo;
-
-  Info->Function = this;
-  Info->Template.setPointer(Template);
-  Info->Template.setInt(TSK - 1);
-  Info->TemplateArguments = TemplateArgs;
-  Info->TemplateArgumentsAsWritten = TemplateArgsAsWritten;
-  Info->PointOfInstantiation = PointOfInstantiation;
+    Info = FunctionTemplateSpecializationInfo::Create(C, this, Template, TSK,
+                                                      TemplateArgs,
+                                                      TemplateArgsAsWritten,
+                                                      PointOfInstantiation);
   TemplateOrSpecialization = Info;
 
   // Insert this function template specialization into the set of known
@@ -1401,28 +1811,6 @@ FunctionDecl::setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
 }
 
 void
-FunctionDecl::setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
-                                                unsigned NumTemplateArgs,
-                                           const TemplateArgument *TemplateArgs,
-                                                 TemplateSpecializationKind TSK,
-                                              unsigned NumTemplateArgsAsWritten,
-                                   TemplateArgumentLoc *TemplateArgsAsWritten,
-                                                SourceLocation LAngleLoc,
-                                                SourceLocation RAngleLoc,
-                                          SourceLocation PointOfInstantiation) {
-  ASTContext &Ctx = getASTContext();
-  TemplateArgumentList *TemplArgs
-    = new (Ctx) TemplateArgumentList(Ctx, TemplateArgs, NumTemplateArgs);
-  TemplateArgumentListInfo *TemplArgsInfo
-    = new (Ctx) TemplateArgumentListInfo(LAngleLoc, RAngleLoc);
-  for (unsigned i=0; i != NumTemplateArgsAsWritten; ++i)
-    TemplArgsInfo->addArgument(TemplateArgsAsWritten[i]);
-
-  setFunctionTemplateSpecialization(Template, TemplArgs, /*InsertPos=*/0, TSK,
-                                    TemplArgsInfo, PointOfInstantiation);
-}
-
-void
 FunctionDecl::setDependentTemplateSpecialization(ASTContext &Context,
                                     const UnresolvedSetImpl &Templates,
                              const TemplateArgumentListInfo &TemplateArgs) {
@@ -1533,8 +1921,8 @@ bool FunctionDecl::isOutOfLine() const {
 // FieldDecl Implementation
 //===----------------------------------------------------------------------===//
 
-FieldDecl *FieldDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
-                             IdentifierInfo *Id, QualType T,
+FieldDecl *FieldDecl::Create(const ASTContext &C, DeclContext *DC,
+                             SourceLocation L, IdentifierInfo *Id, QualType T,
                              TypeSourceInfo *TInfo, Expr *BW, bool Mutable) {
   return new (C) FieldDecl(Decl::Field, DC, L, Id, T, TInfo, BW, Mutable);
 }
@@ -1549,6 +1937,25 @@ bool FieldDecl::isAnonymousStructOrUnion() const {
   return false;
 }
 
+unsigned FieldDecl::getFieldIndex() const {
+  if (CachedFieldIndex) return CachedFieldIndex - 1;
+
+  unsigned index = 0;
+  RecordDecl::field_iterator
+    i = getParent()->field_begin(), e = getParent()->field_end();
+  while (true) {
+    assert(i != e && "failed to find field in parent!");
+    if (*i == this)
+      break;
+
+    ++i;
+    ++index;
+  }
+
+  CachedFieldIndex = index + 1;
+  return index;
+}
+
 //===----------------------------------------------------------------------===//
 // TagDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -1569,7 +1976,8 @@ TagDecl* TagDecl::getCanonicalDecl() {
 void TagDecl::setTypedefForAnonDecl(TypedefDecl *TDD) { 
   TypedefDeclOrQualifier = TDD; 
   if (TypeForDecl)
-    TypeForDecl->ClearLinkageCache();
+    const_cast<Type*>(TypeForDecl)->ClearLinkageCache();
+  ClearLinkageCache();
 }
 
 void TagDecl::startDefinition() {
@@ -1591,11 +1999,16 @@ void TagDecl::completeDefinition() {
 
   IsDefinition = true;
   IsBeingDefined = false;
+
+  if (ASTMutationListener *L = getASTMutationListener())
+    L->CompletedTagDefinition(this);
 }
 
 TagDecl* TagDecl::getDefinition() const {
   if (isDefinition())
     return const_cast<TagDecl *>(this);
+  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(this))
+    return CXXRD->getDefinition();
 
   for (redecl_iterator R = redecls_begin(), REnd = redecls_end();
        R != REnd; ++R)
@@ -1631,14 +2044,17 @@ void TagDecl::setQualifierInfo(NestedNameSpecifier *Qualifier,
 
 EnumDecl *EnumDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
                            IdentifierInfo *Id, SourceLocation TKL,
-                           EnumDecl *PrevDecl) {
-  EnumDecl *Enum = new (C) EnumDecl(DC, L, Id, PrevDecl, TKL);
+                           EnumDecl *PrevDecl, bool IsScoped,
+                           bool IsScopedUsingClassTag, bool IsFixed) {
+  EnumDecl *Enum = new (C) EnumDecl(DC, L, Id, PrevDecl, TKL,
+                                    IsScoped, IsScopedUsingClassTag, IsFixed);
   C.getTypeDeclType(Enum, PrevDecl);
   return Enum;
 }
 
 EnumDecl *EnumDecl::Create(ASTContext &C, EmptyShell Empty) {
-  return new (C) EnumDecl(0, SourceLocation(), 0, 0, SourceLocation());
+  return new (C) EnumDecl(0, SourceLocation(), 0, 0, SourceLocation(),
+                          false, false, false);
 }
 
 void EnumDecl::completeDefinition(QualType NewType,
@@ -1646,7 +2062,8 @@ void EnumDecl::completeDefinition(QualType NewType,
                                   unsigned NumPositiveBits,
                                   unsigned NumNegativeBits) {
   assert(!isDefinition() && "Cannot redefine enums!");
-  IntegerType = NewType;
+  if (!IntegerType)
+    IntegerType = NewType.getTypePtr();
   PromotionType = NewPromotionType;
   setNumPositiveBits(NumPositiveBits);
   setNumNegativeBits(NumNegativeBits);
@@ -1664,10 +2081,11 @@ RecordDecl::RecordDecl(Kind DK, TagKind TK, DeclContext *DC, SourceLocation L,
   HasFlexibleArrayMember = false;
   AnonymousStructOrUnion = false;
   HasObjectMember = false;
+  LoadedFieldsFromExternalStorage = false;
   assert(classof(static_cast<Decl*>(this)) && "Invalid Kind!");
 }
 
-RecordDecl *RecordDecl::Create(ASTContext &C, TagKind TK, DeclContext *DC,
+RecordDecl *RecordDecl::Create(const ASTContext &C, TagKind TK, DeclContext *DC,
                                SourceLocation L, IdentifierInfo *Id,
                                SourceLocation TKL, RecordDecl* PrevDecl) {
 
@@ -1676,7 +2094,7 @@ RecordDecl *RecordDecl::Create(ASTContext &C, TagKind TK, DeclContext *DC,
   return R;
 }
 
-RecordDecl *RecordDecl::Create(ASTContext &C, EmptyShell Empty) {
+RecordDecl *RecordDecl::Create(const ASTContext &C, EmptyShell Empty) {
   return new (C) RecordDecl(Record, TTK_Struct, 0, SourceLocation(), 0, 0,
                             SourceLocation());
 }
@@ -1686,6 +2104,13 @@ bool RecordDecl::isInjectedClassName() const {
     cast<RecordDecl>(getDeclContext())->getDeclName() == getDeclName();
 }
 
+RecordDecl::field_iterator RecordDecl::field_begin() const {
+  if (hasExternalLexicalStorage() && !LoadedFieldsFromExternalStorage)
+    LoadFieldsFromExternalStorage();
+
+  return field_iterator(decl_iterator(FirstDecl));
+}
+
 /// completeDefinition - Notes that the definition of this type is now
 /// complete.
 void RecordDecl::completeDefinition() {
@@ -1693,15 +2118,29 @@ void RecordDecl::completeDefinition() {
   TagDecl::completeDefinition();
 }
 
-ValueDecl *RecordDecl::getAnonymousStructOrUnionObject() {
-  // Force the decl chain to come into existence properly.
-  if (!getNextDeclInContext()) getParent()->decls_begin();
+void RecordDecl::LoadFieldsFromExternalStorage() const {
+  ExternalASTSource *Source = getASTContext().getExternalSource();
+  assert(hasExternalLexicalStorage() && Source && "No external storage?");
+
+  // Notify that we have a RecordDecl doing some initialization.
+  ExternalASTSource::Deserializing TheFields(Source);
 
-  assert(isAnonymousStructOrUnion());
-  ValueDecl *D = cast<ValueDecl>(getNextDeclInContext());
-  assert(D->getType()->isRecordType());
-  assert(D->getType()->getAs<RecordType>()->getDecl() == this);
-  return D;
+  llvm::SmallVector<Decl*, 64> Decls;
+  if (Source->FindExternalLexicalDeclsBy<FieldDecl>(this, Decls))
+    return;
+
+#ifndef NDEBUG
+  // Check that all decls we got were FieldDecls.
+  for (unsigned i=0, e=Decls.size(); i != e; ++i)
+    assert(isa<FieldDecl>(Decls[i]));
+#endif
+
+  LoadedFieldsFromExternalStorage = true;
+
+  if (Decls.empty())
+    return;
+
+  llvm::tie(FirstDecl, LastDecl) = BuildDeclChain(Decls);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1721,10 +2160,31 @@ void BlockDecl::setParams(ParmVarDecl **NewParamInfo,
   }
 }
 
-unsigned BlockDecl::getNumParams() const {
-  return NumParams;
+void BlockDecl::setCaptures(ASTContext &Context,
+                            const Capture *begin,
+                            const Capture *end,
+                            bool capturesCXXThis) {
+  CapturesCXXThis = capturesCXXThis;
+
+  if (begin == end) {
+    NumCaptures = 0;
+    Captures = 0;
+    return;
+  }
+
+  NumCaptures = end - begin;
+
+  // Avoid new Capture[] because we don't want to provide a default
+  // constructor.
+  size_t allocationSize = NumCaptures * sizeof(Capture);
+  void *buffer = Context.Allocate(allocationSize, /*alignment*/sizeof(void*));
+  memcpy(buffer, begin, allocationSize);
+  Captures = static_cast<Capture*>(buffer);
 }
 
+SourceRange BlockDecl::getSourceRange() const {
+  return SourceRange(getLocation(), Body? Body->getLocEnd() : getLocation());
+}
 
 //===----------------------------------------------------------------------===//
 // Other Decl Allocation/Deallocation Method Implementations
@@ -1734,11 +2194,22 @@ TranslationUnitDecl *TranslationUnitDecl::Create(ASTContext &C) {
   return new (C) TranslationUnitDecl(C);
 }
 
+LabelDecl *LabelDecl::Create(ASTContext &C, DeclContext *DC,
+                             SourceLocation L, IdentifierInfo *II) {
+  return new (C) LabelDecl(DC, L, II, 0);
+}
+
+
 NamespaceDecl *NamespaceDecl::Create(ASTContext &C, DeclContext *DC,
                                      SourceLocation L, IdentifierInfo *Id) {
   return new (C) NamespaceDecl(DC, L, Id);
 }
 
+NamespaceDecl *NamespaceDecl::getNextNamespace() {
+  return dyn_cast_or_null<NamespaceDecl>(
+                       NextNamespace.get(getASTContext().getExternalSource()));
+}
+
 ImplicitParamDecl *ImplicitParamDecl::Create(ASTContext &C, DeclContext *DC,
     SourceLocation L, IdentifierInfo *Id, QualType T) {
   return new (C) ImplicitParamDecl(ImplicitParam, DC, L, Id, T);
@@ -1748,9 +2219,10 @@ FunctionDecl *FunctionDecl::Create(ASTContext &C, DeclContext *DC,
                                    const DeclarationNameInfo &NameInfo,
                                    QualType T, TypeSourceInfo *TInfo,
                                    StorageClass S, StorageClass SCAsWritten,
-                                   bool isInline, bool hasWrittenPrototype) {
+                                   bool isInlineSpecified, 
+                                   bool hasWrittenPrototype) {
   FunctionDecl *New = new (C) FunctionDecl(Function, DC, NameInfo, T, TInfo,
-                                           S, SCAsWritten, isInline);
+                                           S, SCAsWritten, isInlineSpecified);
   New->HasWrittenPrototype = hasWrittenPrototype;
   return New;
 }
@@ -1766,6 +2238,13 @@ EnumConstantDecl *EnumConstantDecl::Create(ASTContext &C, EnumDecl *CD,
   return new (C) EnumConstantDecl(CD, L, Id, T, E, V);
 }
 
+IndirectFieldDecl *
+IndirectFieldDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
+                          IdentifierInfo *Id, QualType T, NamedDecl **CH,
+                          unsigned CHS) {
+  return new (C) IndirectFieldDecl(DC, L, Id, T, CH, CHS);
+}
+
 SourceRange EnumConstantDecl::getSourceRange() const {
   SourceLocation End = getLocation();
   if (Init)
diff --git a/lib/AST/DeclBase.cpp b/lib/AST/DeclBase.cpp
index 0b958fe82b09..be379d522dd4 100644
--- a/lib/AST/DeclBase.cpp
+++ b/lib/AST/DeclBase.cpp
@@ -24,6 +24,7 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/ASTMutationListener.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -109,10 +110,22 @@ void Decl::add(Kind k) {
 bool Decl::isTemplateParameterPack() const {
   if (const TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(this))
     return TTP->isParameterPack();
-
+  if (const NonTypeTemplateParmDecl *NTTP
+                                = dyn_cast<NonTypeTemplateParmDecl>(this))
+    return NTTP->isParameterPack();
+  if (const TemplateTemplateParmDecl *TTP
+                                    = dyn_cast<TemplateTemplateParmDecl>(this))
+    return TTP->isParameterPack();
   return false;
 }
 
+bool Decl::isParameterPack() const {
+  if (const ParmVarDecl *Parm = dyn_cast<ParmVarDecl>(this))
+    return Parm->isParameterPack();
+  
+  return isTemplateParameterPack();
+}
+
 bool Decl::isFunctionOrFunctionTemplate() const {
   if (const UsingShadowDecl *UD = dyn_cast<UsingShadowDecl>(this))
     return UD->getTargetDecl()->isFunctionOrFunctionTemplate();
@@ -210,6 +223,10 @@ ASTContext &Decl::getASTContext() const {
   return getTranslationUnitDecl()->getASTContext();
 }
 
+ASTMutationListener *Decl::getASTMutationListener() const {
+  return getASTContext().getASTMutationListener();
+}
+
 bool Decl::isUsed(bool CheckUsedAttr) const { 
   if (Used)
     return true;
@@ -243,6 +260,10 @@ unsigned Decl::getIdentifierNamespaceForKind(Kind DeclKind) {
     case ObjCMethod:
     case ObjCProperty:
       return IDNS_Ordinary;
+    case Label:
+      return IDNS_Label;
+    case IndirectField:
+      return IDNS_Ordinary | IDNS_Member;
 
     case ObjCCompatibleAlias:
     case ObjCInterface:
@@ -416,27 +437,34 @@ SourceLocation Decl::getBodyRBrace() const {
   return SourceLocation();
 }
 
-#ifndef NDEBUG
 void Decl::CheckAccessDeclContext() const {
-  // FIXME: Disable this until rdar://8146294 "access specifier for inner class
-  // templates is not set or checked" is fixed.
-  return;
+#ifndef NDEBUG
   // Suppress this check if any of the following hold:
   // 1. this is the translation unit (and thus has no parent)
   // 2. this is a template parameter (and thus doesn't belong to its context)
-  // 3. the context is not a record
-  // 4. it's invalid
+  // 3. this is a non-type template parameter
+  // 4. the context is not a record
+  // 5. it's invalid
+  // 6. it's a C++0x static_assert.
   if (isa<TranslationUnitDecl>(this) ||
       isa<TemplateTypeParmDecl>(this) ||
+      isa<NonTypeTemplateParmDecl>(this) ||
       !isa<CXXRecordDecl>(getDeclContext()) ||
-      isInvalidDecl())
+      isInvalidDecl() ||
+      isa<StaticAssertDecl>(this) ||
+      // FIXME: a ParmVarDecl can have ClassTemplateSpecialization
+      // as DeclContext (?).
+      isa<ParmVarDecl>(this) ||
+      // FIXME: a ClassTemplateSpecialization or CXXRecordDecl can have
+      // AS_none as access specifier.
+      isa<CXXRecordDecl>(this))
     return;
 
   assert(Access != AS_none &&
          "Access specifier is AS_none inside a record decl");
+#endif
 }
 
-#endif
 
 //===----------------------------------------------------------------------===//
 // DeclContext Implementation
@@ -509,15 +537,24 @@ bool DeclContext::isDependentContext() const {
 
 bool DeclContext::isTransparentContext() const {
   if (DeclKind == Decl::Enum)
-    return true; // FIXME: Check for C++0x scoped enums
+    return !cast<EnumDecl>(this)->isScoped();
   else if (DeclKind == Decl::LinkageSpec)
     return true;
-  else if (DeclKind >= Decl::firstRecord && DeclKind <= Decl::lastRecord)
-    return cast<RecordDecl>(this)->isAnonymousStructOrUnion();
 
   return false;
 }
 
+bool DeclContext::isExternCContext() const {
+  const DeclContext *DC = this;
+  while (DC->DeclKind != Decl::TranslationUnit) {
+    if (DC->DeclKind == Decl::LinkageSpec)
+      return cast<LinkageSpecDecl>(DC)->getLanguage()
+        == LinkageSpecDecl::lang_c;
+    DC = DC->getParent();
+  }
+  return false;
+}
+
 bool DeclContext::Encloses(const DeclContext *DC) const {
   if (getPrimaryContext() != this)
     return getPrimaryContext()->Encloses(DC);
@@ -592,6 +629,24 @@ DeclContext *DeclContext::getNextContext() {
   }
 }
 
+std::pair<Decl *, Decl *>
+DeclContext::BuildDeclChain(const llvm::SmallVectorImpl<Decl*> &Decls) {
+  // Build up a chain of declarations via the Decl::NextDeclInContext field.
+  Decl *FirstNewDecl = 0;
+  Decl *PrevDecl = 0;
+  for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
+    Decl *D = Decls[I];
+    if (PrevDecl)
+      PrevDecl->NextDeclInContext = D;
+    else
+      FirstNewDecl = D;
+
+    PrevDecl = D;
+  }
+
+  return std::make_pair(FirstNewDecl, PrevDecl);
+}
+
 /// \brief Load the declarations within this lexical storage from an
 /// external source.
 void
@@ -612,26 +667,22 @@ DeclContext::LoadLexicalDeclsFromExternalStorage() const {
   if (Decls.empty())
     return;
 
-  // Resolve all of the declaration IDs into declarations, building up
-  // a chain of declarations via the Decl::NextDeclInContext field.
-  Decl *FirstNewDecl = 0;
-  Decl *PrevDecl = 0;
-  for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
-    Decl *D = Decls[I];
-    if (PrevDecl)
-      PrevDecl->NextDeclInContext = D;
-    else
-      FirstNewDecl = D;
-
-    PrevDecl = D;
-  }
+  // We may have already loaded just the fields of this record, in which case
+  // don't add the decls, just replace the FirstDecl/LastDecl chain.
+  if (const RecordDecl *RD = dyn_cast<RecordDecl>(this))
+    if (RD->LoadedFieldsFromExternalStorage) {
+      llvm::tie(FirstDecl, LastDecl) = BuildDeclChain(Decls);
+      return;
+    }
 
   // Splice the newly-read declarations into the beginning of the list
   // of declarations.
-  PrevDecl->NextDeclInContext = FirstDecl;
-  FirstDecl = FirstNewDecl;
+  Decl *ExternalFirst, *ExternalLast;
+  llvm::tie(ExternalFirst, ExternalLast) = BuildDeclChain(Decls);
+  ExternalLast->NextDeclInContext = FirstDecl;
+  FirstDecl = ExternalFirst;
   if (!LastDecl)
-    LastDecl = PrevDecl;
+    LastDecl = ExternalLast;
 }
 
 DeclContext::lookup_result
@@ -771,6 +822,11 @@ void DeclContext::addHiddenDecl(Decl *D) {
   } else {
     FirstDecl = LastDecl = D;
   }
+
+  // Notify a C++ record declaration that we've added a member, so it can
+  // update it's class-specific state.
+  if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(this))
+    Record->addedMember(D);
 }
 
 void DeclContext::addDecl(Decl *D) {
@@ -911,6 +967,12 @@ void DeclContext::makeDeclVisibleInContext(NamedDecl *D, bool Recoverable) {
   // parent context, too. This operation is recursive.
   if (isTransparentContext() || isInlineNamespace())
     getParent()->makeDeclVisibleInContext(D, Recoverable);
+
+  Decl *DCAsDecl = cast<Decl>(this);
+  // Notify that a decl was made visible unless it's a Tag being defined. 
+  if (!(isa<TagDecl>(DCAsDecl) && cast<TagDecl>(DCAsDecl)->isBeingDefined()))
+    if (ASTMutationListener *L = DCAsDecl->getASTMutationListener())
+      L->AddedVisibleDecl(this, D);
 }
 
 void DeclContext::makeDeclVisibleInContextImpl(NamedDecl *D) {
diff --git a/lib/AST/DeclCXX.cpp b/lib/AST/DeclCXX.cpp
index f2f0694826c6..fba73f59d563 100644
--- a/lib/AST/DeclCXX.cpp
+++ b/lib/AST/DeclCXX.cpp
@@ -14,6 +14,8 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTMutationListener.h"
+#include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/IdentifierTable.h"
@@ -34,8 +36,8 @@ CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
     HasTrivialDestructor(true), ComputedVisibleConversions(false),
     DeclaredDefaultConstructor(false), DeclaredCopyConstructor(false), 
     DeclaredCopyAssignment(false), DeclaredDestructor(false),
-    Bases(0), NumBases(0), VBases(0), NumVBases(0),
-    Definition(D), FirstFriend(0) {
+    NumBases(0), NumVBases(0), Bases(), VBases(),
+  Definition(D), FirstFriend(0) {
 }
 
 CXXRecordDecl::CXXRecordDecl(Kind K, TagKind TK, DeclContext *DC,
@@ -46,9 +48,9 @@ CXXRecordDecl::CXXRecordDecl(Kind K, TagKind TK, DeclContext *DC,
     DefinitionData(PrevDecl ? PrevDecl->DefinitionData : 0),
     TemplateOrInstantiation() { }
 
-CXXRecordDecl *CXXRecordDecl::Create(ASTContext &C, TagKind TK, DeclContext *DC,
-                                     SourceLocation L, IdentifierInfo *Id,
-                                     SourceLocation TKL,
+CXXRecordDecl *CXXRecordDecl::Create(const ASTContext &C, TagKind TK,
+                                     DeclContext *DC, SourceLocation L,
+                                     IdentifierInfo *Id, SourceLocation TKL,
                                      CXXRecordDecl* PrevDecl,
                                      bool DelayTypeCreation) {
   CXXRecordDecl* R = new (C) CXXRecordDecl(CXXRecord, TK, DC, L, Id,
@@ -60,7 +62,7 @@ CXXRecordDecl *CXXRecordDecl::Create(ASTContext &C, TagKind TK, DeclContext *DC,
   return R;
 }
 
-CXXRecordDecl *CXXRecordDecl::Create(ASTContext &C, EmptyShell Empty) {
+CXXRecordDecl *CXXRecordDecl::Create(const ASTContext &C, EmptyShell Empty) {
   return new (C) CXXRecordDecl(CXXRecord, TTK_Struct, 0, SourceLocation(), 0, 0,
                                SourceLocation());
 }
@@ -75,8 +77,8 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
   //   no base classes [...].
   data().Aggregate = false;
 
-  if (data().Bases)
-    C.Deallocate(data().Bases);
+  if (!data().Bases.isOffset() && data().NumBases > 0)
+    C.Deallocate(data().getBases());
 
   // The set of seen virtual base types.
   llvm::SmallPtrSet<CanQualType, 8> SeenVBaseTypes;
@@ -87,7 +89,7 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
   data().Bases = new(C) CXXBaseSpecifier [NumBases];
   data().NumBases = NumBases;
   for (unsigned i = 0; i < NumBases; ++i) {
-    data().Bases[i] = *Bases[i];
+    data().getBases()[i] = *Bases[i];
     // Keep track of inherited vbases for this base class.
     const CXXBaseSpecifier *Base = Bases[i];
     QualType BaseType = Base->getType();
@@ -97,6 +99,25 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
     CXXRecordDecl *BaseClassDecl
       = cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl());
 
+    // C++ [dcl.init.aggr]p1:
+    //   An aggregate is [...] a class with [...] no base classes [...].
+    data().Aggregate = false;    
+    
+    // C++ [class]p4:
+    //   A POD-struct is an aggregate class...
+    data().PlainOldData = false;
+    
+    // A class with a non-empty base class is not empty.
+    // FIXME: Standard ref?
+    if (!BaseClassDecl->isEmpty())
+      data().Empty = false;
+    
+    // C++ [class.virtual]p1:
+    //   A class that declares or inherits a virtual function is called a 
+    //   polymorphic class.
+    if (BaseClassDecl->isPolymorphic())
+      data().Polymorphic = true;
+    
     // Now go through all virtual bases of this base and add them.
     for (CXXRecordDecl::base_class_iterator VBase =
           BaseClassDecl->vbases_begin(),
@@ -110,8 +131,50 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
       // Add this base if it's not already in the list.
       if (SeenVBaseTypes.insert(C.getCanonicalType(BaseType)))
           VBases.push_back(Base);
+      
+      // C++0x [meta.unary.prop] is_empty:
+      //    T is a class type, but not a union type, with ... no virtual base
+      //    classes
+      data().Empty = false;
+      
+      // C++ [class.ctor]p5:
+      //   A constructor is trivial if its class has no virtual base classes.
+      data().HasTrivialConstructor = false;
+      
+      // C++ [class.copy]p6:
+      //   A copy constructor is trivial if its class has no virtual base 
+      //   classes.
+      data().HasTrivialCopyConstructor = false;
+      
+      // C++ [class.copy]p11:
+      //   A copy assignment operator is trivial if its class has no virtual
+      //   base classes.
+      data().HasTrivialCopyAssignment = false;
+    } else {
+      // C++ [class.ctor]p5:
+      //   A constructor is trivial if all the direct base classes of its
+      //   class have trivial constructors.
+      if (!BaseClassDecl->hasTrivialConstructor())
+        data().HasTrivialConstructor = false;
+      
+      // C++ [class.copy]p6:
+      //   A copy constructor is trivial if all the direct base classes of its
+      //   class have trivial copy constructors.
+      if (!BaseClassDecl->hasTrivialCopyConstructor())
+        data().HasTrivialCopyConstructor = false;
+      
+      // C++ [class.copy]p11:
+      //   A copy assignment operator is trivial if all the direct base classes
+      //   of its class have trivial copy assignment operators.
+      if (!BaseClassDecl->hasTrivialCopyAssignment())
+        data().HasTrivialCopyAssignment = false;
     }
-
+    
+    // C++ [class.ctor]p3:
+    //   A destructor is trivial if all the direct base classes of its class
+    //   have trivial destructors.
+    if (!BaseClassDecl->hasTrivialDestructor())
+      data().HasTrivialDestructor = false;
   }
   
   if (VBases.empty())
@@ -130,10 +193,11 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
     CXXRecordDecl *VBaseClassDecl = cast<CXXRecordDecl>(
       VBaseTypeInfo->getType()->getAs<RecordType>()->getDecl());
 
-    data().VBases[I] =
+    data().getVBases()[I] =
       CXXBaseSpecifier(VBaseClassDecl->getSourceRange(), true,
                        VBaseClassDecl->getTagKind() == TTK_Class,
-                       VBases[I]->getAccessSpecifier(), VBaseTypeInfo);
+                       VBases[I]->getAccessSpecifier(), VBaseTypeInfo,
+                       SourceLocation());
   }
 }
 
@@ -150,7 +214,7 @@ bool CXXRecordDecl::hasAnyDependentBases() const {
   return !forallBases(SawBase, 0);
 }
 
-bool CXXRecordDecl::hasConstCopyConstructor(ASTContext &Context) const {
+bool CXXRecordDecl::hasConstCopyConstructor(const ASTContext &Context) const {
   return getCopyConstructor(Context, Qualifiers::Const) != 0;
 }
 
@@ -177,7 +241,7 @@ GetBestOverloadCandidateSimple(
   return Cands[Best].first;
 }
 
-CXXConstructorDecl *CXXRecordDecl::getCopyConstructor(ASTContext &Context,
+CXXConstructorDecl *CXXRecordDecl::getCopyConstructor(const ASTContext &Context,
                                                       unsigned TypeQuals) const{
   QualType ClassType
     = Context.getTypeDeclType(const_cast<CXXRecordDecl*>(this));
@@ -258,84 +322,268 @@ CXXMethodDecl *CXXRecordDecl::getCopyAssignmentOperator(bool ArgIsConst) const {
   return GetBestOverloadCandidateSimple(Found);
 }
 
-void
-CXXRecordDecl::addedConstructor(ASTContext &Context,
-                                CXXConstructorDecl *ConDecl) {
-  assert(!ConDecl->isImplicit() && "addedConstructor - not for implicit decl");
-  // Note that we have a user-declared constructor.
-  data().UserDeclaredConstructor = true;
-
-  // Note that we have no need of an implicitly-declared default constructor.
-  data().DeclaredDefaultConstructor = true;
+void CXXRecordDecl::markedVirtualFunctionPure() {
+  // C++ [class.abstract]p2: 
+  //   A class is abstract if it has at least one pure virtual function.
+  data().Abstract = true;
+}
+
+void CXXRecordDecl::addedMember(Decl *D) {
+  // Ignore friends and invalid declarations.
+  if (D->getFriendObjectKind() || D->isInvalidDecl())
+    return;
   
-  // C++ [dcl.init.aggr]p1:
-  //   An aggregate is an array or a class (clause 9) with no
-  //   user-declared constructors (12.1) [...].
-  data().Aggregate = false;
+  FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(D);
+  if (FunTmpl)
+    D = FunTmpl->getTemplatedDecl();
+  
+  if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
+    if (Method->isVirtual()) {
+      // C++ [dcl.init.aggr]p1:
+      //   An aggregate is an array or a class with [...] no virtual functions.
+      data().Aggregate = false;
+      
+      // C++ [class]p4:
+      //   A POD-struct is an aggregate class...
+      data().PlainOldData = false;
+      
+      // Virtual functions make the class non-empty.
+      // FIXME: Standard ref?
+      data().Empty = false;
+
+      // C++ [class.virtual]p1:
+      //   A class that declares or inherits a virtual function is called a 
+      //   polymorphic class.
+      data().Polymorphic = true;
+      
+      // None of the special member functions are trivial.
+      data().HasTrivialConstructor = false;
+      data().HasTrivialCopyConstructor = false;
+      data().HasTrivialCopyAssignment = false;
+      // FIXME: Destructor?
+    }
+  }
+  
+  if (D->isImplicit()) {
+    // Notify that an implicit member was added after the definition
+    // was completed.
+    if (!isBeingDefined())
+      if (ASTMutationListener *L = getASTMutationListener())
+        L->AddedCXXImplicitMember(data().Definition, D);
+
+    if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(D)) {
+      // If this is the implicit default constructor, note that we have now
+      // declared it.
+      if (Constructor->isDefaultConstructor())
+        data().DeclaredDefaultConstructor = true;
+      // If this is the implicit copy constructor, note that we have now
+      // declared it.
+      else if (Constructor->isCopyConstructor())
+        data().DeclaredCopyConstructor = true;
+      return;
+    } 
 
-  // C++ [class]p4:
-  //   A POD-struct is an aggregate class [...]
-  data().PlainOldData = false;
+    if (isa<CXXDestructorDecl>(D)) {
+      data().DeclaredDestructor = true;
+      return;
+    } 
+
+    if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
+      // If this is the implicit copy constructor, note that we have now
+      // declared it.
+      // FIXME: Move constructors
+      if (Method->getOverloadedOperator() == OO_Equal)
+        data().DeclaredCopyAssignment = true;
+      return;
+    }
 
-  // C++ [class.ctor]p5:
-  //   A constructor is trivial if it is an implicitly-declared default
-  //   constructor.
-  // FIXME: C++0x: don't do this for "= default" default constructors.
-  data().HasTrivialConstructor = false;
-
-  // Note when we have a user-declared copy constructor, which will
-  // suppress the implicit declaration of a copy constructor.
-  if (ConDecl->isCopyConstructor()) {
-    data().UserDeclaredCopyConstructor = true;
-    data().DeclaredCopyConstructor = true;
+    // Any other implicit declarations are handled like normal declarations.
+  }
+  
+  // Handle (user-declared) constructors.
+  if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(D)) {
+    // Note that we have a user-declared constructor.
+    data().UserDeclaredConstructor = true;
+
+    // Note that we have no need of an implicitly-declared default constructor.
+    data().DeclaredDefaultConstructor = true;
     
-    // C++ [class.copy]p6:
-    //   A copy constructor is trivial if it is implicitly declared.
-    // FIXME: C++0x: don't do this for "= default" copy constructors.
-    data().HasTrivialCopyConstructor = false;
+    // C++ [dcl.init.aggr]p1:
+    //   An aggregate is an array or a class (clause 9) with no
+    //   user-declared constructors (12.1) [...].
+    data().Aggregate = false;
+
+    // C++ [class]p4:
+    //   A POD-struct is an aggregate class [...]
+    data().PlainOldData = false;
+
+    // C++ [class.ctor]p5:
+    //   A constructor is trivial if it is an implicitly-declared default
+    //   constructor.
+    // FIXME: C++0x: don't do this for "= default" default constructors.
+    data().HasTrivialConstructor = false;
+
+    // Note when we have a user-declared copy constructor, which will
+    // suppress the implicit declaration of a copy constructor.
+    if (!FunTmpl && Constructor->isCopyConstructor()) {
+      data().UserDeclaredCopyConstructor = true;
+      data().DeclaredCopyConstructor = true;
+      
+      // C++ [class.copy]p6:
+      //   A copy constructor is trivial if it is implicitly declared.
+      // FIXME: C++0x: don't do this for "= default" copy constructors.
+      data().HasTrivialCopyConstructor = false;
+    }
     
+    return;
   }
-}
 
-void CXXRecordDecl::addedAssignmentOperator(ASTContext &Context,
-                                            CXXMethodDecl *OpDecl) {
-  // We're interested specifically in copy assignment operators.
-  const FunctionProtoType *FnType = OpDecl->getType()->getAs<FunctionProtoType>();
-  assert(FnType && "Overloaded operator has no proto function type.");
-  assert(FnType->getNumArgs() == 1 && !FnType->isVariadic());
-  
-  // Copy assignment operators must be non-templates.
-  if (OpDecl->getPrimaryTemplate() || OpDecl->getDescribedFunctionTemplate())
+  // Handle (user-declared) destructors.
+  if (isa<CXXDestructorDecl>(D)) {
+    data().DeclaredDestructor = true;
+    data().UserDeclaredDestructor = true;
+    
+    // C++ [class]p4: 
+    //   A POD-struct is an aggregate class that has [...] no user-defined 
+    //   destructor.
+    data().PlainOldData = false;
+    
+    // C++ [class.dtor]p3: 
+    //   A destructor is trivial if it is an implicitly-declared destructor and
+    //   [...].
+    //
+    // FIXME: C++0x: don't do this for "= default" destructors
+    data().HasTrivialDestructor = false;
+    
     return;
+  }
   
-  QualType ArgType = FnType->getArgType(0);
-  if (const LValueReferenceType *Ref = ArgType->getAs<LValueReferenceType>())
-    ArgType = Ref->getPointeeType();
-
-  ArgType = ArgType.getUnqualifiedType();
-  QualType ClassType = Context.getCanonicalType(Context.getTypeDeclType(
-    const_cast<CXXRecordDecl*>(this)));
+  // Handle (user-declared) member functions.
+  if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
+    if (Method->getOverloadedOperator() == OO_Equal) {
+      // We're interested specifically in copy assignment operators.
+      const FunctionProtoType *FnType 
+        = Method->getType()->getAs<FunctionProtoType>();
+      assert(FnType && "Overloaded operator has no proto function type.");
+      assert(FnType->getNumArgs() == 1 && !FnType->isVariadic());
+      
+      // Copy assignment operators must be non-templates.
+      if (Method->getPrimaryTemplate() || FunTmpl)
+        return;
+      
+      ASTContext &Context = getASTContext();
+      QualType ArgType = FnType->getArgType(0);
+      if (const LValueReferenceType *Ref =ArgType->getAs<LValueReferenceType>())
+        ArgType = Ref->getPointeeType();
+      
+      ArgType = ArgType.getUnqualifiedType();
+      QualType ClassType = Context.getCanonicalType(Context.getTypeDeclType(
+                                             const_cast<CXXRecordDecl*>(this)));
+      
+      if (!Context.hasSameUnqualifiedType(ClassType, ArgType))
+        return;
+      
+      // This is a copy assignment operator.
+      // FIXME: Move assignment operators.
+      
+      // Suppress the implicit declaration of a copy constructor.
+      data().UserDeclaredCopyAssignment = true;
+      data().DeclaredCopyAssignment = true;
+      
+      // C++ [class.copy]p11:
+      //   A copy assignment operator is trivial if it is implicitly declared.
+      // FIXME: C++0x: don't do this for "= default" copy operators.
+      data().HasTrivialCopyAssignment = false;
+      
+      // C++ [class]p4:
+      //   A POD-struct is an aggregate class that [...] has no user-defined copy
+      //   assignment operator [...].
+      data().PlainOldData = false;
+    }
+    
+    // Keep the list of conversion functions up-to-date.
+    if (CXXConversionDecl *Conversion = dyn_cast<CXXConversionDecl>(D)) {
+      // We don't record specializations.
+      if (Conversion->getPrimaryTemplate())
+        return;
+      
+      // FIXME: We intentionally don't use the decl's access here because it
+      // hasn't been set yet.  That's really just a misdesign in Sema.
 
-  if (!Context.hasSameUnqualifiedType(ClassType, ArgType))
+      if (FunTmpl) {
+        if (FunTmpl->getPreviousDeclaration())
+          data().Conversions.replace(FunTmpl->getPreviousDeclaration(),
+                                     FunTmpl);
+        else
+          data().Conversions.addDecl(FunTmpl);
+      } else {
+        if (Conversion->getPreviousDeclaration())
+          data().Conversions.replace(Conversion->getPreviousDeclaration(),
+                                     Conversion);
+        else
+          data().Conversions.addDecl(Conversion);        
+      }
+    }
+    
     return;
-
-  // This is a copy assignment operator.
-  // Note on the decl that it is a copy assignment operator.
-  OpDecl->setCopyAssignment(true);
-
-  // Suppress the implicit declaration of a copy constructor.
-  data().UserDeclaredCopyAssignment = true;
-  data().DeclaredCopyAssignment = true;
+  }
   
-  // C++ [class.copy]p11:
-  //   A copy assignment operator is trivial if it is implicitly declared.
-  // FIXME: C++0x: don't do this for "= default" copy operators.
-  data().HasTrivialCopyAssignment = false;
-
-  // C++ [class]p4:
-  //   A POD-struct is an aggregate class that [...] has no user-defined copy
-  //   assignment operator [...].
-  data().PlainOldData = false;
+  // Handle non-static data members.
+  if (FieldDecl *Field = dyn_cast<FieldDecl>(D)) {
+    // C++ [dcl.init.aggr]p1:
+    //   An aggregate is an array or a class (clause 9) with [...] no
+    //   private or protected non-static data members (clause 11).
+    //
+    // A POD must be an aggregate.    
+    if (D->getAccess() == AS_private || D->getAccess() == AS_protected) {
+      data().Aggregate = false;
+      data().PlainOldData = false;
+    }
+    
+    // C++ [class]p9:
+    //   A POD struct is a class that is both a trivial class and a 
+    //   standard-layout class, and has no non-static data members of type 
+    //   non-POD struct, non-POD union (or array of such types).
+    ASTContext &Context = getASTContext();
+    QualType T = Context.getBaseElementType(Field->getType());
+    if (!T->isPODType())
+      data().PlainOldData = false;
+    if (T->isReferenceType())
+      data().HasTrivialConstructor = false;
+    
+    if (const RecordType *RecordTy = T->getAs<RecordType>()) {
+      CXXRecordDecl* FieldRec = cast<CXXRecordDecl>(RecordTy->getDecl());
+      if (FieldRec->getDefinition()) {
+        if (!FieldRec->hasTrivialConstructor())
+          data().HasTrivialConstructor = false;
+        if (!FieldRec->hasTrivialCopyConstructor())
+          data().HasTrivialCopyConstructor = false;
+        if (!FieldRec->hasTrivialCopyAssignment())
+          data().HasTrivialCopyAssignment = false;
+        if (!FieldRec->hasTrivialDestructor())
+          data().HasTrivialDestructor = false;
+      }
+    }
+    
+    // If this is not a zero-length bit-field, then the class is not empty.
+    if (data().Empty) {
+      if (!Field->getBitWidth())
+        data().Empty = false;
+      else if (!Field->getBitWidth()->isTypeDependent() &&
+               !Field->getBitWidth()->isValueDependent()) {
+        llvm::APSInt Bits;
+        if (Field->getBitWidth()->isIntegerConstantExpr(Bits, Context))
+          if (!!Bits)
+            data().Empty = false;
+      } 
+    }
+  }
+  
+  // Handle using declarations of conversion functions.
+  if (UsingShadowDecl *Shadow = dyn_cast<UsingShadowDecl>(D))
+    if (Shadow->getDeclName().getNameKind()
+          == DeclarationName::CXXConversionFunctionName)
+      data().Conversions.addDecl(Shadow, Shadow->getAccess());
 }
 
 static CanQualType GetConversionType(ASTContext &Context, NamedDecl *Conv) {
@@ -479,20 +727,6 @@ const UnresolvedSetImpl *CXXRecordDecl::getVisibleConversionFunctions() {
   return &data().VisibleConversions;
 }
 
-#ifndef NDEBUG
-void CXXRecordDecl::CheckConversionFunction(NamedDecl *ConvDecl) {
-  assert(ConvDecl->getDeclContext() == this &&
-         "conversion function does not belong to this record");
-
-  ConvDecl = ConvDecl->getUnderlyingDecl();
-  if (FunctionTemplateDecl *Temp = dyn_cast<FunctionTemplateDecl>(ConvDecl)) {
-    assert(isa<CXXConversionDecl>(Temp->getTemplatedDecl()));
-  } else {
-    assert(isa<CXXConversionDecl>(ConvDecl));
-  }
-}
-#endif
-
 void CXXRecordDecl::removeConversion(const NamedDecl *ConvDecl) {
   // This operation is O(N) but extremely rare.  Sema only uses it to
   // remove UsingShadowDecls in a class that were followed by a direct
@@ -518,17 +752,6 @@ void CXXRecordDecl::removeConversion(const NamedDecl *ConvDecl) {
   llvm_unreachable("conversion not found in set!");
 }
 
-void CXXRecordDecl::setMethodAsVirtual(FunctionDecl *Method) {
-  Method->setVirtualAsWritten(true);
-  setAggregate(false);
-  setPOD(false);
-  setEmpty(false);
-  setPolymorphic(true);
-  setHasTrivialConstructor(false);
-  setHasTrivialCopyConstructor(false);
-  setHasTrivialCopyAssignment(false);
-}
-
 CXXRecordDecl *CXXRecordDecl::getInstantiatedFromMemberClass() const {
   if (MemberSpecializationInfo *MSInfo = getMemberSpecializationInfo())
     return cast<CXXRecordDecl>(MSInfo->getInstantiatedFrom());
@@ -577,28 +800,6 @@ CXXRecordDecl::setTemplateSpecializationKind(TemplateSpecializationKind TSK) {
   assert(false && "Not a class template or member class specialization");
 }
 
-CXXConstructorDecl *
-CXXRecordDecl::getDefaultConstructor() {
-  ASTContext &Context = getASTContext();
-  QualType ClassType = Context.getTypeDeclType(this);
-  DeclarationName ConstructorName
-    = Context.DeclarationNames.getCXXConstructorName(
-                      Context.getCanonicalType(ClassType.getUnqualifiedType()));
-
-  DeclContext::lookup_const_iterator Con, ConEnd;
-  for (llvm::tie(Con, ConEnd) = lookup(ConstructorName);
-       Con != ConEnd; ++Con) {
-    // FIXME: In C++0x, a constructor template can be a default constructor.
-    if (isa<FunctionTemplateDecl>(*Con))
-      continue;
-
-    CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
-    if (Constructor->isDefaultConstructor())
-      return Constructor;
-  }
-  return 0;
-}
-
 CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
   ASTContext &Context = getASTContext();
   QualType ClassType = Context.getTypeDeclType(this);
@@ -618,6 +819,69 @@ CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
   return Dtor;
 }
 
+void CXXRecordDecl::completeDefinition() {
+  completeDefinition(0);
+}
+
+void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
+  RecordDecl::completeDefinition();
+  
+  // If the class may be abstract (but hasn't been marked as such), check for
+  // any pure final overriders.
+  if (mayBeAbstract()) {
+    CXXFinalOverriderMap MyFinalOverriders;
+    if (!FinalOverriders) {
+      getFinalOverriders(MyFinalOverriders);
+      FinalOverriders = &MyFinalOverriders;
+    }
+    
+    bool Done = false;
+    for (CXXFinalOverriderMap::iterator M = FinalOverriders->begin(), 
+                                     MEnd = FinalOverriders->end();
+         M != MEnd && !Done; ++M) {
+      for (OverridingMethods::iterator SO = M->second.begin(), 
+                                    SOEnd = M->second.end();
+           SO != SOEnd && !Done; ++SO) {
+        assert(SO->second.size() > 0 && 
+               "All virtual functions have overridding virtual functions");
+        
+        // C++ [class.abstract]p4:
+        //   A class is abstract if it contains or inherits at least one
+        //   pure virtual function for which the final overrider is pure
+        //   virtual.
+        if (SO->second.front().Method->isPure()) {
+          data().Abstract = true;
+          Done = true;
+          break;
+        }
+      }
+    }
+  }
+  
+  // Set access bits correctly on the directly-declared conversions.
+  for (UnresolvedSetIterator I = data().Conversions.begin(), 
+                             E = data().Conversions.end(); 
+       I != E; ++I)
+    data().Conversions.setAccess(I, (*I)->getAccess());
+}
+
+bool CXXRecordDecl::mayBeAbstract() const {
+  if (data().Abstract || isInvalidDecl() || !data().Polymorphic ||
+      isDependentContext())
+    return false;
+  
+  for (CXXRecordDecl::base_class_const_iterator B = bases_begin(),
+                                             BEnd = bases_end();
+       B != BEnd; ++B) {
+    CXXRecordDecl *BaseDecl 
+      = cast<CXXRecordDecl>(B->getType()->getAs<RecordType>()->getDecl());
+    if (BaseDecl->isAbstract())
+      return true;
+  }
+  
+  return false;
+}
+
 CXXMethodDecl *
 CXXMethodDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                       const DeclarationNameInfo &NameInfo,
@@ -735,85 +999,90 @@ bool CXXMethodDecl::hasInlineBody() const {
   return CheckFn->hasBody(fn) && !fn->isOutOfLine();
 }
 
-CXXBaseOrMemberInitializer::
-CXXBaseOrMemberInitializer(ASTContext &Context,
-                           TypeSourceInfo *TInfo, bool IsVirtual,
-                           SourceLocation L, Expr *Init, SourceLocation R)
-  : BaseOrMember(TInfo), Init(Init), AnonUnionMember(0),
+CXXCtorInitializer::CXXCtorInitializer(ASTContext &Context,
+                                       TypeSourceInfo *TInfo, bool IsVirtual,
+                                       SourceLocation L, Expr *Init,
+                                       SourceLocation R,
+                                       SourceLocation EllipsisLoc)
+  : Initializee(TInfo), MemberOrEllipsisLocation(EllipsisLoc), Init(Init), 
     LParenLoc(L), RParenLoc(R), IsVirtual(IsVirtual), IsWritten(false),
     SourceOrderOrNumArrayIndices(0)
 {
 }
 
-CXXBaseOrMemberInitializer::
-CXXBaseOrMemberInitializer(ASTContext &Context,
-                           FieldDecl *Member, SourceLocation MemberLoc,
-                           SourceLocation L, Expr *Init, SourceLocation R)
-  : BaseOrMember(Member), MemberLocation(MemberLoc), Init(Init),
-    AnonUnionMember(0), LParenLoc(L), RParenLoc(R), IsVirtual(false),
+CXXCtorInitializer::CXXCtorInitializer(ASTContext &Context,
+                                       FieldDecl *Member,
+                                       SourceLocation MemberLoc,
+                                       SourceLocation L, Expr *Init,
+                                       SourceLocation R)
+  : Initializee(Member), MemberOrEllipsisLocation(MemberLoc), Init(Init),
+    LParenLoc(L), RParenLoc(R), IsVirtual(false),
+    IsWritten(false), SourceOrderOrNumArrayIndices(0)
+{
+}
+
+CXXCtorInitializer::CXXCtorInitializer(ASTContext &Context,
+                                       IndirectFieldDecl *Member,
+                                       SourceLocation MemberLoc,
+                                       SourceLocation L, Expr *Init,
+                                       SourceLocation R)
+  : Initializee(Member), MemberOrEllipsisLocation(MemberLoc), Init(Init),
+    LParenLoc(L), RParenLoc(R), IsVirtual(false),
     IsWritten(false), SourceOrderOrNumArrayIndices(0)
 {
 }
 
-CXXBaseOrMemberInitializer::
-CXXBaseOrMemberInitializer(ASTContext &Context,
-                           FieldDecl *Member, SourceLocation MemberLoc,
-                           SourceLocation L, Expr *Init, SourceLocation R,
-                           VarDecl **Indices,
-                           unsigned NumIndices)
-  : BaseOrMember(Member), MemberLocation(MemberLoc), Init(Init), 
-    AnonUnionMember(0), LParenLoc(L), RParenLoc(R), IsVirtual(false),
+CXXCtorInitializer::CXXCtorInitializer(ASTContext &Context,
+                                       FieldDecl *Member,
+                                       SourceLocation MemberLoc,
+                                       SourceLocation L, Expr *Init,
+                                       SourceLocation R,
+                                       VarDecl **Indices,
+                                       unsigned NumIndices)
+  : Initializee(Member), MemberOrEllipsisLocation(MemberLoc), Init(Init), 
+    LParenLoc(L), RParenLoc(R), IsVirtual(false),
     IsWritten(false), SourceOrderOrNumArrayIndices(NumIndices)
 {
   VarDecl **MyIndices = reinterpret_cast<VarDecl **> (this + 1);
   memcpy(MyIndices, Indices, NumIndices * sizeof(VarDecl *));
 }
 
-CXXBaseOrMemberInitializer *
-CXXBaseOrMemberInitializer::Create(ASTContext &Context,
-                                   FieldDecl *Member, 
-                                   SourceLocation MemberLoc,
-                                   SourceLocation L,
-                                   Expr *Init,
-                                   SourceLocation R,
-                                   VarDecl **Indices,
-                                   unsigned NumIndices) {
-  void *Mem = Context.Allocate(sizeof(CXXBaseOrMemberInitializer) +
+CXXCtorInitializer *CXXCtorInitializer::Create(ASTContext &Context,
+                                               FieldDecl *Member, 
+                                               SourceLocation MemberLoc,
+                                               SourceLocation L, Expr *Init,
+                                               SourceLocation R,
+                                               VarDecl **Indices,
+                                               unsigned NumIndices) {
+  void *Mem = Context.Allocate(sizeof(CXXCtorInitializer) +
                                sizeof(VarDecl *) * NumIndices,
-                               llvm::alignof<CXXBaseOrMemberInitializer>());
-  return new (Mem) CXXBaseOrMemberInitializer(Context, Member, MemberLoc,
-                                              L, Init, R, Indices, NumIndices);
+                               llvm::alignOf<CXXCtorInitializer>());
+  return new (Mem) CXXCtorInitializer(Context, Member, MemberLoc, L, Init, R,
+                                      Indices, NumIndices);
 }
 
-TypeLoc CXXBaseOrMemberInitializer::getBaseClassLoc() const {
+TypeLoc CXXCtorInitializer::getBaseClassLoc() const {
   if (isBaseInitializer())
-    return BaseOrMember.get<TypeSourceInfo*>()->getTypeLoc();
+    return Initializee.get<TypeSourceInfo*>()->getTypeLoc();
   else
     return TypeLoc();
 }
 
-Type *CXXBaseOrMemberInitializer::getBaseClass() {
-  if (isBaseInitializer())
-    return BaseOrMember.get<TypeSourceInfo*>()->getType().getTypePtr();
-  else
-    return 0;
-}
-
-const Type *CXXBaseOrMemberInitializer::getBaseClass() const {
+const Type *CXXCtorInitializer::getBaseClass() const {
   if (isBaseInitializer())
-    return BaseOrMember.get<TypeSourceInfo*>()->getType().getTypePtr();
+    return Initializee.get<TypeSourceInfo*>()->getType().getTypePtr();
   else
     return 0;
 }
 
-SourceLocation CXXBaseOrMemberInitializer::getSourceLocation() const {
-  if (isMemberInitializer())
+SourceLocation CXXCtorInitializer::getSourceLocation() const {
+  if (isAnyMemberInitializer())
     return getMemberLocation();
   
   return getBaseClassLoc().getLocalSourceRange().getBegin();
 }
 
-SourceRange CXXBaseOrMemberInitializer::getSourceRange() const {
+SourceRange CXXCtorInitializer::getSourceRange() const {
   return SourceRange(getSourceLocation(), getRParenLoc());
 }
 
@@ -847,25 +1116,40 @@ bool CXXConstructorDecl::isDefaultConstructor() const {
 
 bool
 CXXConstructorDecl::isCopyConstructor(unsigned &TypeQuals) const {
+  return isCopyOrMoveConstructor(TypeQuals) &&
+         getParamDecl(0)->getType()->isLValueReferenceType();
+}
+
+bool CXXConstructorDecl::isMoveConstructor(unsigned &TypeQuals) const {
+  return isCopyOrMoveConstructor(TypeQuals) &&
+    getParamDecl(0)->getType()->isRValueReferenceType();
+}
+
+/// \brief Determine whether this is a copy or move constructor.
+bool CXXConstructorDecl::isCopyOrMoveConstructor(unsigned &TypeQuals) const {
   // C++ [class.copy]p2:
   //   A non-template constructor for class X is a copy constructor
   //   if its first parameter is of type X&, const X&, volatile X& or
   //   const volatile X&, and either there are no other parameters
   //   or else all other parameters have default arguments (8.3.6).
+  // C++0x [class.copy]p3:
+  //   A non-template constructor for class X is a move constructor if its
+  //   first parameter is of type X&&, const X&&, volatile X&&, or 
+  //   const volatile X&&, and either there are no other parameters or else 
+  //   all other parameters have default arguments.
   if ((getNumParams() < 1) ||
       (getNumParams() > 1 && !getParamDecl(1)->hasDefaultArg()) ||
       (getPrimaryTemplate() != 0) ||
       (getDescribedFunctionTemplate() != 0))
     return false;
-
+  
   const ParmVarDecl *Param = getParamDecl(0);
-
-  // Do we have a reference type? Rvalue references don't count.
-  const LValueReferenceType *ParamRefType =
-    Param->getType()->getAs<LValueReferenceType>();
+  
+  // Do we have a reference type? 
+  const ReferenceType *ParamRefType = Param->getType()->getAs<ReferenceType>();
   if (!ParamRefType)
     return false;
-
+  
   // Is it a reference to our class type?
   ASTContext &Context = getASTContext();
   
@@ -875,12 +1159,12 @@ CXXConstructorDecl::isCopyConstructor(unsigned &TypeQuals) const {
     = Context.getCanonicalType(Context.getTagDeclType(getParent()));
   if (PointeeType.getUnqualifiedType() != ClassTy)
     return false;
-
+  
   // FIXME: other qualifiers?
-
-  // We have a copy constructor.
+  
+  // We have a copy or move constructor.
   TypeQuals = PointeeType.getCVRQualifiers();
-  return true;
+  return true;  
 }
 
 bool CXXConstructorDecl::isConvertingConstructor(bool AllowExplicit) const {
@@ -899,7 +1183,7 @@ bool CXXConstructorDecl::isConvertingConstructor(bool AllowExplicit) const {
          (getNumParams() > 1 && getParamDecl(1)->hasDefaultArg());
 }
 
-bool CXXConstructorDecl::isCopyConstructorLikeSpecialization() const {
+bool CXXConstructorDecl::isSpecializationCopyingObject() const {
   if ((getNumParams() < 1) ||
       (getNumParams() > 1 && !getParamDecl(1)->hasDefaultArg()) ||
       (getPrimaryTemplate() == 0) ||
@@ -911,12 +1195,6 @@ bool CXXConstructorDecl::isCopyConstructorLikeSpecialization() const {
   ASTContext &Context = getASTContext();
   CanQualType ParamType = Context.getCanonicalType(Param->getType());
   
-  // Strip off the lvalue reference, if any.
-  if (CanQual<LValueReferenceType> ParamRefType
-                                    = ParamType->getAs<LValueReferenceType>())
-    ParamType = ParamRefType->getPointeeType();
-
-  
   // Is it the same as our our class type?
   CanQualType ClassTy 
     = Context.getCanonicalType(Context.getTagDeclType(getParent()));
@@ -926,21 +1204,38 @@ bool CXXConstructorDecl::isCopyConstructorLikeSpecialization() const {
   return true;  
 }
 
+const CXXConstructorDecl *CXXConstructorDecl::getInheritedConstructor() const {
+  // Hack: we store the inherited constructor in the overridden method table
+  method_iterator It = begin_overridden_methods();
+  if (It == end_overridden_methods())
+    return 0;
+
+  return cast<CXXConstructorDecl>(*It);
+}
+
+void
+CXXConstructorDecl::setInheritedConstructor(const CXXConstructorDecl *BaseCtor){
+  // Hack: we store the inherited constructor in the overridden method table
+  assert(size_overridden_methods() == 0 && "Base ctor already set.");
+  addOverriddenMethod(BaseCtor);
+}
+
 CXXDestructorDecl *
 CXXDestructorDecl::Create(ASTContext &C, EmptyShell Empty) {
   return new (C) CXXDestructorDecl(0, DeclarationNameInfo(),
-                                   QualType(), false, false);
+                                   QualType(), 0, false, false);
 }
 
 CXXDestructorDecl *
 CXXDestructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
                           const DeclarationNameInfo &NameInfo,
-                          QualType T, bool isInline,
+                          QualType T, TypeSourceInfo *TInfo,
+                          bool isInline,
                           bool isImplicitlyDeclared) {
   assert(NameInfo.getName().getNameKind()
          == DeclarationName::CXXDestructorName &&
          "Name must refer to a destructor");
-  return new (C) CXXDestructorDecl(RD, NameInfo, T, isInline,
+  return new (C) CXXDestructorDecl(RD, NameInfo, T, TInfo, isInline,
                                    isImplicitlyDeclared);
 }
 
@@ -1004,6 +1299,44 @@ NamespaceAliasDecl *NamespaceAliasDecl::Create(ASTContext &C, DeclContext *DC,
                                     Qualifier, IdentLoc, Namespace);
 }
 
+UsingDecl *UsingShadowDecl::getUsingDecl() const {
+  const UsingShadowDecl *Shadow = this;
+  while (const UsingShadowDecl *NextShadow =
+         dyn_cast<UsingShadowDecl>(Shadow->UsingOrNextShadow))
+    Shadow = NextShadow;
+  return cast<UsingDecl>(Shadow->UsingOrNextShadow);
+}
+
+void UsingDecl::addShadowDecl(UsingShadowDecl *S) {
+  assert(std::find(shadow_begin(), shadow_end(), S) == shadow_end() &&
+         "declaration already in set");
+  assert(S->getUsingDecl() == this);
+
+  if (FirstUsingShadow)
+    S->UsingOrNextShadow = FirstUsingShadow;
+  FirstUsingShadow = S;
+}
+
+void UsingDecl::removeShadowDecl(UsingShadowDecl *S) {
+  assert(std::find(shadow_begin(), shadow_end(), S) != shadow_end() &&
+         "declaration not in set");
+  assert(S->getUsingDecl() == this);
+
+  // Remove S from the shadow decl chain. This is O(n) but hopefully rare.
+
+  if (FirstUsingShadow == S) {
+    FirstUsingShadow = dyn_cast<UsingShadowDecl>(S->UsingOrNextShadow);
+    S->UsingOrNextShadow = this;
+    return;
+  }
+
+  UsingShadowDecl *Prev = FirstUsingShadow;
+  while (Prev->UsingOrNextShadow != S)
+    Prev = cast<UsingShadowDecl>(Prev->UsingOrNextShadow);
+  Prev->UsingOrNextShadow = S->UsingOrNextShadow;
+  S->UsingOrNextShadow = this;
+}
+
 UsingDecl *UsingDecl::Create(ASTContext &C, DeclContext *DC,
                              SourceRange NNR, SourceLocation UL,
                              NestedNameSpecifier* TargetNNS,
diff --git a/lib/AST/DeclObjC.cpp b/lib/AST/DeclObjC.cpp
index d952cc36ef1d..45f5188d4043 100644
--- a/lib/AST/DeclObjC.cpp
+++ b/lib/AST/DeclObjC.cpp
@@ -153,6 +153,9 @@ ObjCContainerDecl::FindPropertyDeclaration(IdentifierInfo *PropertyId) const {
 ObjCPropertyDecl *
 ObjCInterfaceDecl::FindPropertyVisibleInPrimaryClass(
                                             IdentifierInfo *PropertyId) const {
+  if (ExternallyCompleted)
+    LoadExternalDefinition();
+
   if (ObjCPropertyDecl *PD =
       ObjCPropertyDecl::findPropertyDecl(cast<DeclContext>(this), PropertyId))
     return PD;
@@ -171,6 +174,9 @@ void ObjCInterfaceDecl::mergeClassExtensionProtocolList(
                               ObjCProtocolDecl *const* ExtList, unsigned ExtNum,
                               ASTContext &C)
 {
+  if (ExternallyCompleted)
+    LoadExternalDefinition();
+
   if (AllReferencedProtocols.empty() && ReferencedProtocols.empty()) {
     AllReferencedProtocols.set(ExtList, ExtNum, C);
     return;
@@ -270,6 +276,9 @@ ObjCMethodDecl *ObjCInterfaceDecl::lookupMethod(Selector Sel,
   const ObjCInterfaceDecl* ClassDecl = this;
   ObjCMethodDecl *MethodDecl = 0;
 
+  if (ExternallyCompleted)
+    LoadExternalDefinition();
+
   while (ClassDecl != NULL) {
     if ((MethodDecl = ClassDecl->getMethod(Sel, isInstance)))
       return MethodDecl;
@@ -302,14 +311,16 @@ ObjCMethodDecl *ObjCInterfaceDecl::lookupMethod(Selector Sel,
   return NULL;
 }
 
-ObjCMethodDecl *ObjCInterfaceDecl::lookupPrivateInstanceMethod(
-                                   const Selector &Sel) {
+ObjCMethodDecl *ObjCInterfaceDecl::lookupPrivateMethod(
+                                   const Selector &Sel,
+                                   bool Instance) {
   ObjCMethodDecl *Method = 0;
   if (ObjCImplementationDecl *ImpDecl = getImplementation())
-    Method = ImpDecl->getInstanceMethod(Sel);
+    Method = Instance ? ImpDecl->getInstanceMethod(Sel) 
+                      : ImpDecl->getClassMethod(Sel);
   
   if (!Method && getSuperClass())
-    return getSuperClass()->lookupPrivateInstanceMethod(Sel);
+    return getSuperClass()->lookupPrivateMethod(Sel, Instance);
   return Method;
 }
 
@@ -443,11 +454,29 @@ ObjCInterfaceDecl(DeclContext *DC, SourceLocation atLoc, IdentifierInfo *Id,
   : ObjCContainerDecl(ObjCInterface, DC, atLoc, Id),
     TypeForDecl(0), SuperClass(0),
     CategoryList(0), IvarList(0), 
-    ForwardDecl(FD), InternalInterface(isInternal),
+    ForwardDecl(FD), InternalInterface(isInternal), ExternallyCompleted(false),
     ClassLoc(CLoc) {
 }
 
+void ObjCInterfaceDecl::LoadExternalDefinition() const {
+  assert(ExternallyCompleted && "Class is not externally completed");
+  ExternallyCompleted = false;
+  getASTContext().getExternalSource()->CompleteType(
+                                        const_cast<ObjCInterfaceDecl *>(this));
+}
+
+void ObjCInterfaceDecl::setExternallyCompleted() {
+  assert(getASTContext().getExternalSource() && 
+         "Class can't be externally completed without an external source");
+  assert(!ForwardDecl && 
+         "Forward declarations can't be externally completed");
+  ExternallyCompleted = true;
+}
+
 ObjCImplementationDecl *ObjCInterfaceDecl::getImplementation() const {
+  if (ExternallyCompleted)
+    LoadExternalDefinition();
+
   return getASTContext().getObjCImplementation(
                                           const_cast<ObjCInterfaceDecl*>(this));
 }
@@ -506,6 +535,9 @@ ObjCIvarDecl *ObjCInterfaceDecl::all_declared_ivar_begin() {
 ///
 ObjCCategoryDecl *
 ObjCInterfaceDecl::FindCategoryDeclaration(IdentifierInfo *CategoryId) const {
+  if (ExternallyCompleted)
+    LoadExternalDefinition();
+
   for (ObjCCategoryDecl *Category = getCategoryList();
        Category; Category = Category->getNextClassCategory())
     if (Category->getIdentifier() == CategoryId)
@@ -711,7 +743,7 @@ ObjCClassDecl::ObjCClassDecl(DeclContext *DC, SourceLocation L,
 void ObjCClassDecl::setClassList(ASTContext &C, ObjCInterfaceDecl*const*List,
                                  const SourceLocation *Locs, unsigned Num) {
   ForwardDecls = (ObjCClassRef*) C.Allocate(sizeof(ObjCClassRef)*Num,
-                                            llvm::alignof<ObjCClassRef>());
+                                            llvm::alignOf<ObjCClassRef>());
   for (unsigned i = 0; i < Num; ++i)
     new (&ForwardDecls[i]) ObjCClassRef(List[i], Locs[i]);
   
@@ -896,7 +928,6 @@ ObjCPropertyDecl *ObjCPropertyDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) ObjCPropertyDecl(DC, L, Id, AtLoc, T);
 }
 
-
 //===----------------------------------------------------------------------===//
 // ObjCPropertyImplDecl
 //===----------------------------------------------------------------------===//
@@ -907,8 +938,16 @@ ObjCPropertyImplDecl *ObjCPropertyImplDecl::Create(ASTContext &C,
                                                    SourceLocation L,
                                                    ObjCPropertyDecl *property,
                                                    Kind PK,
-                                                   ObjCIvarDecl *ivar) {
-  return new (C) ObjCPropertyImplDecl(DC, atLoc, L, property, PK, ivar);
+                                                   ObjCIvarDecl *ivar,
+                                                   SourceLocation ivarLoc) {
+  return new (C) ObjCPropertyImplDecl(DC, atLoc, L, property, PK, ivar,
+                                      ivarLoc);
 }
 
+SourceRange ObjCPropertyImplDecl::getSourceRange() const {
+  SourceLocation EndLoc = getLocation();
+  if (IvarLoc.isValid())
+    EndLoc = IvarLoc;
 
+  return SourceRange(AtLoc, EndLoc);
+}
diff --git a/lib/AST/DeclPrinter.cpp b/lib/AST/DeclPrinter.cpp
index f18d2f0215dd..77b42574709f 100644
--- a/lib/AST/DeclPrinter.cpp
+++ b/lib/AST/DeclPrinter.cpp
@@ -51,6 +51,7 @@ namespace {
     void VisitFunctionDecl(FunctionDecl *D);
     void VisitFieldDecl(FieldDecl *D);
     void VisitVarDecl(VarDecl *D);
+    void VisitLabelDecl(LabelDecl *D);
     void VisitParmVarDecl(ParmVarDecl *D);
     void VisitFileScopeAsmDecl(FileScopeAsmDecl *D);
     void VisitNamespaceDecl(NamespaceDecl *D);
@@ -307,9 +308,26 @@ void DeclPrinter::VisitTypedefDecl(TypedefDecl *D) {
 }
 
 void DeclPrinter::VisitEnumDecl(EnumDecl *D) {
-  Out << "enum " << D << " {\n";
-  VisitDeclContext(D);
-  Indent() << "}";
+  Out << "enum ";
+  if (D->isScoped()) {
+    if (D->isScopedUsingClassTag())
+      Out << "class ";
+    else
+      Out << "struct ";
+  }
+  Out << D;
+
+  if (D->isFixed()) {
+    std::string Underlying;
+    D->getIntegerType().getAsStringInternal(Underlying, Policy);
+    Out << " : " << Underlying;
+  }
+
+  if (D->isDefinition()) {
+    Out << " {\n";
+    VisitDeclContext(D);
+    Indent() << "}";
+  }
 }
 
 void DeclPrinter::VisitRecordDecl(RecordDecl *D) {
@@ -349,9 +367,15 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
   PrintingPolicy SubPolicy(Policy);
   SubPolicy.SuppressSpecifiers = false;
   std::string Proto = D->getNameInfo().getAsString();
-  if (isa<FunctionType>(D->getType().getTypePtr())) {
-    const FunctionType *AFT = D->getType()->getAs<FunctionType>();
 
+  QualType Ty = D->getType();
+  while (const ParenType *PT = dyn_cast<ParenType>(Ty)) {
+    Proto = '(' + Proto + ')';
+    Ty = PT->getInnerType();
+  }
+
+  if (isa<FunctionType>(Ty)) {
+    const FunctionType *AFT = Ty->getAs<FunctionType>();
     const FunctionProtoType *FT = 0;
     if (D->hasWrittenPrototype())
       FT = dyn_cast<FunctionProtoType>(AFT);
@@ -379,6 +403,16 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
     Proto += ")";
     
+    if (FT && FT->getTypeQuals()) {
+      unsigned TypeQuals = FT->getTypeQuals();
+      if (TypeQuals & Qualifiers::Const)
+        Proto += " const";
+      if (TypeQuals & Qualifiers::Volatile) 
+        Proto += " volatile";
+      if (TypeQuals & Qualifiers::Restrict)
+        Proto += " restrict";
+    }
+    
     if (FT && FT->hasExceptionSpec()) {
       Proto += " throw(";
       if (FT->hasAnyExceptionSpec())
@@ -399,18 +433,18 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     if (D->hasAttr<NoReturnAttr>())
       Proto += " __attribute((noreturn))";
     if (CXXConstructorDecl *CDecl = dyn_cast<CXXConstructorDecl>(D)) {
-      if (CDecl->getNumBaseOrMemberInitializers() > 0) {
+      if (CDecl->getNumCtorInitializers() > 0) {
         Proto += " : ";
         Out << Proto;
         Proto.clear();
         for (CXXConstructorDecl::init_const_iterator B = CDecl->init_begin(),
              E = CDecl->init_end();
              B != E; ++B) {
-          CXXBaseOrMemberInitializer * BMInitializer = (*B);
+          CXXCtorInitializer * BMInitializer = (*B);
           if (B != CDecl->init_begin())
             Out << ", ";
-          if (BMInitializer->isMemberInitializer()) {
-            FieldDecl *FD = BMInitializer->getMember();
+          if (BMInitializer->isAnyMemberInitializer()) {
+            FieldDecl *FD = BMInitializer->getAnyMember();
             Out << FD;
           } else {
             Out << QualType(BMInitializer->getBaseClass(),
@@ -422,8 +456,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
             // Nothing to print
           } else {
             Expr *Init = BMInitializer->getInit();
-            if (CXXExprWithTemporaries *Tmp
-                  = dyn_cast<CXXExprWithTemporaries>(Init))
+            if (ExprWithCleanups *Tmp = dyn_cast<ExprWithCleanups>(Init))
               Init = Tmp->getSubExpr();
             
             Init = Init->IgnoreParens();
@@ -461,7 +494,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     else
       AFT->getResultType().getAsStringInternal(Proto, Policy);
   } else {
-    D->getType().getAsStringInternal(Proto, Policy);
+    Ty.getAsStringInternal(Proto, Policy);
   }
 
   Out << Proto;
@@ -505,6 +538,11 @@ void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
   }
 }
 
+void DeclPrinter::VisitLabelDecl(LabelDecl *D) {
+  Out << D->getNameAsString() << ":";
+}
+
+
 void DeclPrinter::VisitVarDecl(VarDecl *D) {
   if (!Policy.SuppressSpecifiers && D->getStorageClass() != SC_None)
     Out << VarDecl::getStorageClassSpecifierString(D->getStorageClass()) << " ";
@@ -518,12 +556,15 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
     T = Parm->getOriginalType();
   T.getAsStringInternal(Name, Policy);
   Out << Name;
-  if (D->getInit()) {
+  if (Expr *Init = D->getInit()) {
     if (D->hasCXXDirectInitializer())
       Out << "(";
-    else
-      Out << " = ";
-    D->getInit()->printPretty(Out, Context, 0, Policy, Indentation);
+    else {
+        CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(Init);
+        if (!CCE || CCE->getConstructor()->isCopyConstructor())
+          Out << " = ";
+    }
+    Init->printPretty(Out, Context, 0, Policy, Indentation);
     if (D->hasCXXDirectInitializer())
       Out << ")";
   }
@@ -646,6 +687,9 @@ void DeclPrinter::VisitTemplateDecl(TemplateDecl *D) {
                  dyn_cast<NonTypeTemplateParmDecl>(Param)) {
       Out << NTTP->getType().getAsString(Policy);
 
+      if (NTTP->isParameterPack() && !isa<PackExpansionType>(NTTP->getType()))
+        Out << "...";
+        
       if (IdentifierInfo *Name = NTTP->getIdentifier()) {
         Out << ' ';
         Out << Name->getName();
@@ -661,8 +705,11 @@ void DeclPrinter::VisitTemplateDecl(TemplateDecl *D) {
 
   Out << "> ";
 
-  if (isa<TemplateTemplateParmDecl>(D)) {
-    Out << "class " << D->getName();
+  if (TemplateTemplateParmDecl *TTP = dyn_cast<TemplateTemplateParmDecl>(D)) {
+    Out << "class ";
+    if (TTP->isParameterPack())
+      Out << "...";
+    Out << D->getName();
   } else {
     Visit(D->getTemplatedDecl());
   }
diff --git a/lib/AST/DeclTemplate.cpp b/lib/AST/DeclTemplate.cpp
index e69338a7730d..a73deeab3a61 100644
--- a/lib/AST/DeclTemplate.cpp
+++ b/lib/AST/DeclTemplate.cpp
@@ -14,10 +14,13 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/TypeLoc.h"
+#include "clang/AST/ASTMutationListener.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/STLExtras.h"
+#include <memory>
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -35,7 +38,7 @@ TemplateParameterList::TemplateParameterList(SourceLocation TemplateLoc,
 }
 
 TemplateParameterList *
-TemplateParameterList::Create(ASTContext &C, SourceLocation TemplateLoc,
+TemplateParameterList::Create(const ASTContext &C, SourceLocation TemplateLoc,
                               SourceLocation LAngleLoc, NamedDecl **Params,
                               unsigned NumParams, SourceLocation RAngleLoc) {
   unsigned Size = sizeof(TemplateParameterList) 
@@ -47,24 +50,33 @@ TemplateParameterList::Create(ASTContext &C, SourceLocation TemplateLoc,
 }
 
 unsigned TemplateParameterList::getMinRequiredArguments() const {
-  unsigned NumRequiredArgs = size();
-  iterator Param = const_cast<TemplateParameterList *>(this)->end(),
-      ParamBegin = const_cast<TemplateParameterList *>(this)->begin();
-  while (Param != ParamBegin) {
-    --Param;
-
-    if (!(*Param)->isTemplateParameterPack() &&
-        !(isa<TemplateTypeParmDecl>(*Param) &&
-          cast<TemplateTypeParmDecl>(*Param)->hasDefaultArgument()) &&
-        !(isa<NonTypeTemplateParmDecl>(*Param) &&
-          cast<NonTypeTemplateParmDecl>(*Param)->hasDefaultArgument()) &&
-        !(isa<TemplateTemplateParmDecl>(*Param) &&
-          cast<TemplateTemplateParmDecl>(*Param)->hasDefaultArgument()))
+  unsigned NumRequiredArgs = 0;
+  for (iterator P = const_cast<TemplateParameterList *>(this)->begin(), 
+             PEnd = const_cast<TemplateParameterList *>(this)->end(); 
+       P != PEnd; ++P) {
+    if ((*P)->isTemplateParameterPack()) {
+      if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P))
+        if (NTTP->isExpandedParameterPack()) {
+          NumRequiredArgs += NTTP->getNumExpansionTypes();
+          continue;
+        }
+      
       break;
-
-    --NumRequiredArgs;
+    }
+  
+    if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*P)) {
+      if (TTP->hasDefaultArgument())
+        break;
+    } else if (NonTypeTemplateParmDecl *NTTP 
+                                    = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
+      if (NTTP->hasDefaultArgument())
+        break;
+    } else if (cast<TemplateTemplateParmDecl>(*P)->hasDefaultArgument())
+      break;
+    
+    ++NumRequiredArgs;
   }
-
+  
   return NumRequiredArgs;
 }
 
@@ -92,7 +104,7 @@ RedeclarableTemplateDecl::CommonBase *RedeclarableTemplateDecl::getCommonPtr() {
   RedeclarableTemplateDecl *First = getCanonicalDecl();
 
   if (First->CommonOrPrev.isNull()) {
-    CommonBase *CommonPtr = First->newCommon();
+    CommonBase *CommonPtr = First->newCommon(getASTContext());
     First->CommonOrPrev = CommonPtr;
     CommonPtr->Latest = First;
   }
@@ -156,9 +168,10 @@ FunctionTemplateDecl *FunctionTemplateDecl::Create(ASTContext &C,
   return new (C) FunctionTemplateDecl(DC, L, Name, Params, Decl);
 }
 
-RedeclarableTemplateDecl::CommonBase *FunctionTemplateDecl::newCommon() {
-  Common *CommonPtr = new (getASTContext()) Common;
-  getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
+RedeclarableTemplateDecl::CommonBase *
+FunctionTemplateDecl::newCommon(ASTContext &C) {
+  Common *CommonPtr = new (C) Common;
+  C.AddDeallocation(DeallocateCommon, CommonPtr);
   return CommonPtr;
 }
 
@@ -188,9 +201,33 @@ ClassTemplateDecl *ClassTemplateDecl::Create(ASTContext &C,
   return New;
 }
 
-RedeclarableTemplateDecl::CommonBase *ClassTemplateDecl::newCommon() {
-  Common *CommonPtr = new (getASTContext()) Common;
-  getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
+void ClassTemplateDecl::LoadLazySpecializations() {
+  Common *CommonPtr = getCommonPtr();
+  if (CommonPtr->LazySpecializations) {
+    ASTContext &Context = getASTContext();
+    uint32_t *Specs = CommonPtr->LazySpecializations;
+    CommonPtr->LazySpecializations = 0;
+    for (uint32_t I = 0, N = *Specs++; I != N; ++I)
+      (void)Context.getExternalSource()->GetExternalDecl(Specs[I]);
+  }
+}
+
+llvm::FoldingSet<ClassTemplateSpecializationDecl> &
+ClassTemplateDecl::getSpecializations() {
+  LoadLazySpecializations();
+  return getCommonPtr()->Specializations;
+}  
+
+llvm::FoldingSet<ClassTemplatePartialSpecializationDecl> &
+ClassTemplateDecl::getPartialSpecializations() {
+  LoadLazySpecializations();
+  return getCommonPtr()->PartialSpecializations;
+}  
+
+RedeclarableTemplateDecl::CommonBase *
+ClassTemplateDecl::newCommon(ASTContext &C) {
+  Common *CommonPtr = new (C) Common;
+  C.AddDeallocation(DeallocateCommon, CommonPtr);
   return CommonPtr;
 }
 
@@ -200,6 +237,13 @@ ClassTemplateDecl::findSpecialization(const TemplateArgument *Args,
   return findSpecializationImpl(getSpecializations(), Args, NumArgs, InsertPos);
 }
 
+void ClassTemplateDecl::AddSpecialization(ClassTemplateSpecializationDecl *D,
+                                          void *InsertPos) {
+  getSpecializations().InsertNode(D, InsertPos);
+  if (ASTMutationListener *L = getASTMutationListener())
+    L->AddedCXXTemplateSpecialization(this, D);
+}
+
 ClassTemplatePartialSpecializationDecl *
 ClassTemplateDecl::findPartialSpecialization(const TemplateArgument *Args,
                                              unsigned NumArgs,
@@ -208,6 +252,14 @@ ClassTemplateDecl::findPartialSpecialization(const TemplateArgument *Args,
                                 InsertPos);
 }
 
+void ClassTemplateDecl::AddPartialSpecialization(
+                                      ClassTemplatePartialSpecializationDecl *D,
+                                      void *InsertPos) {
+  getPartialSpecializations().InsertNode(D, InsertPos);
+  if (ASTMutationListener *L = getASTMutationListener())
+    L->AddedCXXTemplateSpecialization(this, D);
+}
+
 void ClassTemplateDecl::getPartialSpecializations(
           llvm::SmallVectorImpl<ClassTemplatePartialSpecializationDecl *> &PS) {
   llvm::FoldingSet<ClassTemplatePartialSpecializationDecl> &PartialSpecs
@@ -258,10 +310,13 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   if (!CommonPtr->InjectedClassNameType.isNull())
     return CommonPtr->InjectedClassNameType;
 
-  // FIXME: n2800 14.6.1p1 should say how the template arguments
-  // corresponding to template parameter packs should be pack
-  // expansions. We already say that in 14.6.2.1p2, so it would be
-  // better to fix that redundancy.
+  // C++0x [temp.dep.type]p2:
+  //  The template argument list of a primary template is a template argument 
+  //  list in which the nth template argument has the value of the nth template
+  //  parameter of the class template. If the nth template parameter is a 
+  //  template parameter pack (14.5.3), the nth template argument is a pack 
+  //  expansion (14.5.3) whose pattern is the name of the template parameter 
+  //  pack.
   ASTContext &Context = getASTContext();
   TemplateParameterList *Params = getTemplateParameters();
   llvm::SmallVector<TemplateArgument, 16> TemplateArgs;
@@ -269,19 +324,38 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   for (TemplateParameterList::iterator Param = Params->begin(),
                                     ParamEnd = Params->end();
        Param != ParamEnd; ++Param) {
-    if (isa<TemplateTypeParmDecl>(*Param)) {
-      QualType ParamType = Context.getTypeDeclType(cast<TypeDecl>(*Param));
-      TemplateArgs.push_back(TemplateArgument(ParamType));
+    TemplateArgument Arg;
+    if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*Param)) {
+      QualType ArgType = Context.getTypeDeclType(TTP);
+      if (TTP->isParameterPack())
+        ArgType = Context.getPackExpansionType(ArgType, 
+                                               llvm::Optional<unsigned>());
+      
+      Arg = TemplateArgument(ArgType);
     } else if (NonTypeTemplateParmDecl *NTTP =
                  dyn_cast<NonTypeTemplateParmDecl>(*Param)) {
       Expr *E = new (Context) DeclRefExpr(NTTP,
                                   NTTP->getType().getNonLValueExprType(Context),
+                                  Expr::getValueKindForType(NTTP->getType()),
                                           NTTP->getLocation());
-      TemplateArgs.push_back(TemplateArgument(E));
+
+      if (NTTP->isParameterPack())
+        E = new (Context) PackExpansionExpr(Context.DependentTy, E,
+                                            NTTP->getLocation(),
+                                            llvm::Optional<unsigned>());
+      Arg = TemplateArgument(E);
     } else {
       TemplateTemplateParmDecl *TTP = cast<TemplateTemplateParmDecl>(*Param);
-      TemplateArgs.push_back(TemplateArgument(TemplateName(TTP)));
+      if (TTP->isParameterPack())
+        Arg = TemplateArgument(TemplateName(TTP), llvm::Optional<unsigned>());
+      else
+        Arg = TemplateArgument(TemplateName(TTP));
     }
+    
+    if ((*Param)->isTemplateParameterPack())
+      Arg = TemplateArgument::CreatePackCopy(Context, &Arg, 1);
+    
+    TemplateArgs.push_back(Arg);
   }
 
   CommonPtr->InjectedClassNameType
@@ -296,7 +370,7 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
 //===----------------------------------------------------------------------===//
 
 TemplateTypeParmDecl *
-TemplateTypeParmDecl::Create(ASTContext &C, DeclContext *DC,
+TemplateTypeParmDecl::Create(const ASTContext &C, DeclContext *DC,
                              SourceLocation L, unsigned D, unsigned P,
                              IdentifierInfo *Id, bool Typename,
                              bool ParameterPack) {
@@ -305,7 +379,7 @@ TemplateTypeParmDecl::Create(ASTContext &C, DeclContext *DC,
 }
 
 TemplateTypeParmDecl *
-TemplateTypeParmDecl::Create(ASTContext &C, EmptyShell Empty) {
+TemplateTypeParmDecl::Create(const ASTContext &C, EmptyShell Empty) {
   return new (C) TemplateTypeParmDecl(0, SourceLocation(), 0, false,
                                       QualType(), false);
 }
@@ -326,12 +400,62 @@ unsigned TemplateTypeParmDecl::getIndex() const {
 // NonTypeTemplateParmDecl Method Implementations
 //===----------------------------------------------------------------------===//
 
+NonTypeTemplateParmDecl::NonTypeTemplateParmDecl(DeclContext *DC, 
+                                                 SourceLocation L, unsigned D,
+                                                 unsigned P, IdentifierInfo *Id, 
+                                                 QualType T, 
+                                                 TypeSourceInfo *TInfo,
+                                                 const QualType *ExpandedTypes,
+                                                 unsigned NumExpandedTypes,
+                                                TypeSourceInfo **ExpandedTInfos)
+  : DeclaratorDecl(NonTypeTemplateParm, DC, L, Id, T, TInfo),
+    TemplateParmPosition(D, P), DefaultArgumentAndInherited(0, false),
+    ParameterPack(true), ExpandedParameterPack(true),
+    NumExpandedTypes(NumExpandedTypes)
+{
+  if (ExpandedTypes && ExpandedTInfos) {
+    void **TypesAndInfos = reinterpret_cast<void **>(this + 1);
+    for (unsigned I = 0; I != NumExpandedTypes; ++I) {
+      TypesAndInfos[2*I] = ExpandedTypes[I].getAsOpaquePtr();
+      TypesAndInfos[2*I + 1] = ExpandedTInfos[I];
+    }
+  }
+}
+
 NonTypeTemplateParmDecl *
-NonTypeTemplateParmDecl::Create(ASTContext &C, DeclContext *DC,
+NonTypeTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC,
                                 SourceLocation L, unsigned D, unsigned P,
                                 IdentifierInfo *Id, QualType T,
-                                TypeSourceInfo *TInfo) {
-  return new (C) NonTypeTemplateParmDecl(DC, L, D, P, Id, T, TInfo);
+                                bool ParameterPack, TypeSourceInfo *TInfo) {
+  return new (C) NonTypeTemplateParmDecl(DC, L, D, P, Id, T, ParameterPack,
+                                         TInfo);
+}
+
+NonTypeTemplateParmDecl *
+NonTypeTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC, 
+                                SourceLocation L, unsigned D, unsigned P, 
+                                IdentifierInfo *Id, QualType T, 
+                                TypeSourceInfo *TInfo,
+                                const QualType *ExpandedTypes, 
+                                unsigned NumExpandedTypes,
+                                TypeSourceInfo **ExpandedTInfos) {
+  unsigned Size = sizeof(NonTypeTemplateParmDecl) 
+                + NumExpandedTypes * 2 * sizeof(void*);
+  void *Mem = C.Allocate(Size);
+  return new (Mem) NonTypeTemplateParmDecl(DC, L, D, P, Id, T, TInfo, 
+                                           ExpandedTypes, NumExpandedTypes, 
+                                           ExpandedTInfos);
+}
+
+SourceLocation NonTypeTemplateParmDecl::getInnerLocStart() const {
+  SourceLocation Start = getTypeSpecStartLoc();
+  if (Start.isInvalid())
+    Start = getLocation();
+  return Start;
+}
+
+SourceRange NonTypeTemplateParmDecl::getSourceRange() const {
+  return SourceRange(getOuterLocStart(), getLocation());
 }
 
 SourceLocation NonTypeTemplateParmDecl::getDefaultArgumentLoc() const {
@@ -345,128 +469,29 @@ SourceLocation NonTypeTemplateParmDecl::getDefaultArgumentLoc() const {
 //===----------------------------------------------------------------------===//
 
 TemplateTemplateParmDecl *
-TemplateTemplateParmDecl::Create(ASTContext &C, DeclContext *DC,
+TemplateTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC,
                                  SourceLocation L, unsigned D, unsigned P,
-                                 IdentifierInfo *Id,
+                                 bool ParameterPack, IdentifierInfo *Id,
                                  TemplateParameterList *Params) {
-  return new (C) TemplateTemplateParmDecl(DC, L, D, P, Id, Params);
-}
-
-//===----------------------------------------------------------------------===//
-// TemplateArgumentListBuilder Implementation
-//===----------------------------------------------------------------------===//
-
-void TemplateArgumentListBuilder::Append(const TemplateArgument &Arg) {
-  assert((Arg.getKind() != TemplateArgument::Type ||
-          Arg.getAsType().isCanonical()) && "Type must be canonical!");
-  assert(FlatArgs.size() < MaxFlatArgs && "Argument list builder is full!");
-  assert(!StructuredArgs &&
-         "Can't append arguments when an argument pack has been added!");
-
-  FlatArgs.push_back(Arg);
-}
-
-void TemplateArgumentListBuilder::BeginPack() {
-  assert(!AddingToPack && "Already adding to pack!");
-  assert(!StructuredArgs && "Argument list already contains a pack!");
-
-  AddingToPack = true;
-  PackBeginIndex = FlatArgs.size();
-}
-
-void TemplateArgumentListBuilder::EndPack() {
-  assert(AddingToPack && "Not adding to pack!");
-  assert(!StructuredArgs && "Argument list already contains a pack!");
-
-  AddingToPack = false;
-
-  // FIXME: This is a memory leak!
-  StructuredArgs = new TemplateArgument[MaxStructuredArgs];
-
-  // First copy the flat entries over to the list  (if any)
-  for (unsigned I = 0; I != PackBeginIndex; ++I) {
-    NumStructuredArgs++;
-    StructuredArgs[I] = FlatArgs[I];
-  }
-
-  // Next, set the pack.
-  TemplateArgument *PackArgs = 0;
-  unsigned NumPackArgs = NumFlatArgs - PackBeginIndex;
-  // FIXME: NumPackArgs shouldn't be negative here???
-  if (NumPackArgs)
-    PackArgs = FlatArgs.data()+PackBeginIndex;
-
-  StructuredArgs[NumStructuredArgs++].setArgumentPack(PackArgs, NumPackArgs,
-                                                      /*CopyArgs=*/false);
+  return new (C) TemplateTemplateParmDecl(DC, L, D, P, ParameterPack, Id, 
+                                          Params);
 }
 
 //===----------------------------------------------------------------------===//
 // TemplateArgumentList Implementation
 //===----------------------------------------------------------------------===//
-TemplateArgumentList::TemplateArgumentList(ASTContext &Context,
-                                           TemplateArgumentListBuilder &Builder,
-                                           bool TakeArgs)
-  : FlatArguments(Builder.getFlatArguments(), TakeArgs),
-    NumFlatArguments(Builder.flatSize()),
-    StructuredArguments(Builder.getStructuredArguments(), TakeArgs),
-    NumStructuredArguments(Builder.structuredSize()) {
-
-  if (!TakeArgs)
-    return;
-
-  // If this does take ownership of the arguments, then we have to new them
-  // and copy over.
-  TemplateArgument *NewArgs =
-    new (Context) TemplateArgument[Builder.flatSize()];
-  std::copy(Builder.getFlatArguments(),
-            Builder.getFlatArguments()+Builder.flatSize(), NewArgs);
-  FlatArguments.setPointer(NewArgs);
-      
-  // Just reuse the structured and flat arguments array if possible.
-  if (Builder.getStructuredArguments() == Builder.getFlatArguments()) {
-    StructuredArguments.setPointer(NewArgs);    
-    StructuredArguments.setInt(0);
-  } else {
-    TemplateArgument *NewSArgs =
-      new (Context) TemplateArgument[Builder.flatSize()];
-    std::copy(Builder.getFlatArguments(),
-              Builder.getFlatArguments()+Builder.flatSize(), NewSArgs);
-    StructuredArguments.setPointer(NewSArgs);
-  }
-}
-
-TemplateArgumentList::TemplateArgumentList(ASTContext &Context,
-                                           const TemplateArgument *Args,
-                                           unsigned NumArgs)
-  : NumFlatArguments(0), NumStructuredArguments(0) {
-  init(Context, Args, NumArgs);
-}
-
-/// Produces a shallow copy of the given template argument list.  This
-/// assumes that the input argument list outlives it.  This takes the list as
-/// a pointer to avoid looking like a copy constructor, since this really
-/// really isn't safe to use that way.
-TemplateArgumentList::TemplateArgumentList(const TemplateArgumentList *Other)
-  : FlatArguments(Other->FlatArguments.getPointer(), false),
-    NumFlatArguments(Other->flat_size()),
-    StructuredArguments(Other->StructuredArguments.getPointer(), false),
-    NumStructuredArguments(Other->NumStructuredArguments) { }
-
-void TemplateArgumentList::init(ASTContext &Context,
-                           const TemplateArgument *Args,
-                           unsigned NumArgs) {
-assert(NumFlatArguments == 0 && NumStructuredArguments == 0 &&
-       "Already initialized!");
-
-NumFlatArguments = NumStructuredArguments = NumArgs;
-TemplateArgument *NewArgs = new (Context) TemplateArgument[NumArgs];
-std::copy(Args, Args+NumArgs, NewArgs);
-FlatArguments.setPointer(NewArgs);
-FlatArguments.setInt(1); // Owns the pointer.
-
-// Just reuse the flat arguments array.
-StructuredArguments.setPointer(NewArgs);    
-StructuredArguments.setInt(0); // Doesn't own the pointer.
+TemplateArgumentList *
+TemplateArgumentList::CreateCopy(ASTContext &Context,
+                                 const TemplateArgument *Args,
+                                 unsigned NumArgs) {
+  std::size_t Size = sizeof(TemplateArgumentList)
+                   + NumArgs * sizeof(TemplateArgument);
+  void *Mem = Context.Allocate(Size);
+  TemplateArgument *StoredArgs 
+    = reinterpret_cast<TemplateArgument *>(
+                                static_cast<TemplateArgumentList *>(Mem) + 1);
+  std::uninitialized_copy(Args, Args + NumArgs, StoredArgs);
+  return new (Mem) TemplateArgumentList(StoredArgs, NumArgs, true);
 }
 
 //===----------------------------------------------------------------------===//
@@ -476,14 +501,15 @@ ClassTemplateSpecializationDecl::
 ClassTemplateSpecializationDecl(ASTContext &Context, Kind DK, TagKind TK,
                                 DeclContext *DC, SourceLocation L,
                                 ClassTemplateDecl *SpecializedTemplate,
-                                TemplateArgumentListBuilder &Builder,
+                                const TemplateArgument *Args,
+                                unsigned NumArgs,
                                 ClassTemplateSpecializationDecl *PrevDecl)
   : CXXRecordDecl(DK, TK, DC, L,
                   SpecializedTemplate->getIdentifier(),
                   PrevDecl),
     SpecializedTemplate(SpecializedTemplate),
     ExplicitInfo(0),
-    TemplateArgs(Context, Builder, /*TakeArgs=*/true),
+    TemplateArgs(TemplateArgumentList::CreateCopy(Context, Args, NumArgs)),
     SpecializationKind(TSK_Undeclared) {
 }
 
@@ -497,14 +523,15 @@ ClassTemplateSpecializationDecl *
 ClassTemplateSpecializationDecl::Create(ASTContext &Context, TagKind TK,
                                         DeclContext *DC, SourceLocation L,
                                         ClassTemplateDecl *SpecializedTemplate,
-                                        TemplateArgumentListBuilder &Builder,
+                                        const TemplateArgument *Args,
+                                        unsigned NumArgs,
                                    ClassTemplateSpecializationDecl *PrevDecl) {
   ClassTemplateSpecializationDecl *Result
     = new (Context)ClassTemplateSpecializationDecl(Context,
                                                    ClassTemplateSpecialization,
                                                    TK, DC, L,
                                                    SpecializedTemplate,
-                                                   Builder,
+                                                   Args, NumArgs,
                                                    PrevDecl);
   Context.getTypeDeclType(Result, PrevDecl);
   return Result;
@@ -524,8 +551,8 @@ ClassTemplateSpecializationDecl::getNameForDiagnostic(std::string &S,
 
   const TemplateArgumentList &TemplateArgs = getTemplateArgs();
   S += TemplateSpecializationType::PrintTemplateArgumentList(
-                                       TemplateArgs.getFlatArgumentList(),
-                                       TemplateArgs.flat_size(),
+                                                          TemplateArgs.data(),
+                                                          TemplateArgs.size(),
                                                              Policy);
 }
 
@@ -545,7 +572,8 @@ ClassTemplatePartialSpecializationDecl::
 Create(ASTContext &Context, TagKind TK,DeclContext *DC, SourceLocation L,
        TemplateParameterList *Params,
        ClassTemplateDecl *SpecializedTemplate,
-       TemplateArgumentListBuilder &Builder,
+       const TemplateArgument *Args,
+       unsigned NumArgs,
        const TemplateArgumentListInfo &ArgInfos,
        QualType CanonInjectedType,
        ClassTemplatePartialSpecializationDecl *PrevDecl,
@@ -559,7 +587,7 @@ Create(ASTContext &Context, TagKind TK,DeclContext *DC, SourceLocation L,
     = new (Context)ClassTemplatePartialSpecializationDecl(Context, TK,
                                                           DC, L, Params,
                                                           SpecializedTemplate,
-                                                          Builder,
+                                                          Args, NumArgs,
                                                           ClonedArgs, N,
                                                           PrevDecl,
                                                           SequenceNumber);
@@ -575,19 +603,6 @@ ClassTemplatePartialSpecializationDecl::Create(ASTContext &Context,
   return new (Context)ClassTemplatePartialSpecializationDecl();
 }
 
-void ClassTemplatePartialSpecializationDecl::
-initTemplateArgsAsWritten(const TemplateArgumentListInfo &ArgInfos) {
-  assert(ArgsAsWritten == 0 && "ArgsAsWritten already set");
-  unsigned N = ArgInfos.size();
-  TemplateArgumentLoc *ClonedArgs
-    = new (getASTContext()) TemplateArgumentLoc[N];
-  for (unsigned I = 0; I != N; ++I)
-    ClonedArgs[I] = ArgInfos[I];
-  
-  ArgsAsWritten = ClonedArgs;
-  NumArgsAsWritten = N;
-}
-
 //===----------------------------------------------------------------------===//
 // FriendTemplateDecl Implementation
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/DeclarationName.cpp b/lib/AST/DeclarationName.cpp
index 860a0b276a79..cef54e97c93a 100644
--- a/lib/AST/DeclarationName.cpp
+++ b/lib/AST/DeclarationName.cpp
@@ -20,6 +20,7 @@
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
@@ -93,10 +94,8 @@ int DeclarationName::compare(DeclarationName LHS, DeclarationName RHS) {
     Selector RHSSelector = RHS.getObjCSelector();
     unsigned LN = LHSSelector.getNumArgs(), RN = RHSSelector.getNumArgs();
     for (unsigned I = 0, N = std::min(LN, RN); I != N; ++I) {
-      IdentifierInfo *LHSId = LHSSelector.getIdentifierInfoForSlot(I);
-      IdentifierInfo *RHSId = RHSSelector.getIdentifierInfoForSlot(I);
-        
-      switch (LHSId->getName().compare(RHSId->getName())) {
+      switch (LHSSelector.getNameForSlot(I).compare(
+                                               RHSSelector.getNameForSlot(I))) {
       case -1: return true;
       case 1: return false;
       default: break;
@@ -385,7 +384,7 @@ void DeclarationName::dump() const {
   llvm::errs() << '\n';
 }
 
-DeclarationNameTable::DeclarationNameTable(ASTContext &C) : Ctx(C) {
+DeclarationNameTable::DeclarationNameTable(const ASTContext &C) : Ctx(C) {
   CXXSpecialNamesImpl = new llvm::FoldingSet<CXXSpecialName>;
   CXXLiteralOperatorNames = new llvm::FoldingSet<CXXLiteralOperatorIdName>;
 
@@ -512,6 +511,28 @@ DeclarationNameLoc::DeclarationNameLoc(DeclarationName Name) {
   }
 }
 
+bool DeclarationNameInfo::containsUnexpandedParameterPack() const {
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXOperatorName:
+  case DeclarationName::CXXLiteralOperatorName:
+  case DeclarationName::CXXUsingDirective:
+    return false;
+
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    if (TypeSourceInfo *TInfo = LocInfo.NamedType.TInfo)
+      return TInfo->getType()->containsUnexpandedParameterPack();
+
+    return Name.getCXXNameType()->containsUnexpandedParameterPack();
+  }
+  llvm_unreachable("All name kinds handled.");
+}
+
 std::string DeclarationNameInfo::getAsString() const {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
diff --git a/lib/AST/DumpXML.cpp b/lib/AST/DumpXML.cpp
new file mode 100644
index 000000000000..9d828fcfb85a
--- /dev/null
+++ b/lib/AST/DumpXML.cpp
@@ -0,0 +1,1028 @@
+//===--- DumpXML.cpp - Detailed XML dumping ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the Decl::dumpXML() method, a debugging tool to
+//  print a detailed graph of an AST in an unspecified XML format.
+//
+//  There is no guarantee of stability for this format.
+//
+//===----------------------------------------------------------------------===//
+
+// Only pay for this in code size in assertions-enabled builds.
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclFriend.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/DeclVisitor.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/StmtCXX.h"
+#include "clang/AST/StmtObjC.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/AST/TemplateBase.h"
+#include "clang/AST/TemplateName.h"
+#include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
+#include "clang/AST/TypeLocVisitor.h"
+#include "clang/AST/TypeVisitor.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+using namespace clang;
+
+#ifndef NDEBUG
+
+namespace {
+
+enum NodeState {
+  NS_Attrs, NS_LazyChildren, NS_Children
+};
+
+struct Node {
+  llvm::StringRef Name;
+  NodeState State;
+  Node(llvm::StringRef name) : Name(name), State(NS_Attrs) {}
+
+  bool isDoneWithAttrs() const { return State != NS_Attrs; }
+};
+
+template <class Impl> struct XMLDeclVisitor {
+#define DISPATCH(NAME, CLASS) \
+  static_cast<Impl*>(this)->NAME(static_cast<CLASS*>(D))
+
+  void dispatch(Decl *D) {
+    switch (D->getKind()) {
+      default: llvm_unreachable("Decl that isn't part of DeclNodes.inc!");
+#define DECL(DERIVED, BASE) \
+      case Decl::DERIVED: \
+        DISPATCH(dispatch##DERIVED##DeclAttrs, DERIVED##Decl); \
+        static_cast<Impl*>(this)->completeAttrs(); \
+        DISPATCH(dispatch##DERIVED##DeclChildren, DERIVED##Decl); \
+        DISPATCH(dispatch##DERIVED##DeclAsContext, DERIVED##Decl); \
+        break;
+#define ABSTRACT_DECL(DECL)
+#include "clang/AST/DeclNodes.inc"
+    }
+  }
+
+#define DECL(DERIVED, BASE) \
+  void dispatch##DERIVED##DeclAttrs(DERIVED##Decl *D) { \
+    DISPATCH(dispatch##BASE##Attrs, BASE); \
+    DISPATCH(visit##DERIVED##DeclAttrs, DERIVED##Decl); \
+  } \
+  void visit##DERIVED##DeclAttrs(DERIVED##Decl *D) {} \
+  void dispatch##DERIVED##DeclChildren(DERIVED##Decl *D) { \
+    DISPATCH(dispatch##BASE##Children, BASE); \
+    DISPATCH(visit##DERIVED##DeclChildren, DERIVED##Decl); \
+  } \
+  void visit##DERIVED##DeclChildren(DERIVED##Decl *D) {} \
+  void dispatch##DERIVED##DeclAsContext(DERIVED##Decl *D) { \
+    DISPATCH(dispatch##BASE##AsContext, BASE); \
+    DISPATCH(visit##DERIVED##DeclAsContext, DERIVED##Decl); \
+  } \
+  void visit##DERIVED##DeclAsContext(DERIVED##Decl *D) {}
+#include "clang/AST/DeclNodes.inc"
+
+  void dispatchDeclAttrs(Decl *D) {
+    DISPATCH(visitDeclAttrs, Decl);
+  }
+  void visitDeclAttrs(Decl *D) {}
+
+  void dispatchDeclChildren(Decl *D) {
+    DISPATCH(visitDeclChildren, Decl);
+  }
+  void visitDeclChildren(Decl *D) {}
+
+  void dispatchDeclAsContext(Decl *D) {
+    DISPATCH(visitDeclAsContext, Decl);
+  }
+  void visitDeclAsContext(Decl *D) {}
+
+#undef DISPATCH  
+};
+
+template <class Impl> struct XMLTypeVisitor {
+#define DISPATCH(NAME, CLASS) \
+  static_cast<Impl*>(this)->NAME(static_cast<CLASS*>(T))
+
+  void dispatch(Type *T) {
+    switch (T->getTypeClass()) {
+      default: llvm_unreachable("Type that isn't part of TypeNodes.inc!");
+#define TYPE(DERIVED, BASE) \
+      case Type::DERIVED: \
+        DISPATCH(dispatch##DERIVED##TypeAttrs, DERIVED##Type); \
+        static_cast<Impl*>(this)->completeAttrs(); \
+        DISPATCH(dispatch##DERIVED##TypeChildren, DERIVED##Type); \
+        break;
+#define ABSTRACT_TYPE(DERIVED, BASE)
+#include "clang/AST/TypeNodes.def"
+    }
+  }
+
+#define TYPE(DERIVED, BASE) \
+  void dispatch##DERIVED##TypeAttrs(DERIVED##Type *T) { \
+    DISPATCH(dispatch##BASE##Attrs, BASE); \
+    DISPATCH(visit##DERIVED##TypeAttrs, DERIVED##Type); \
+  } \
+  void visit##DERIVED##TypeAttrs(DERIVED##Type *T) {} \
+  void dispatch##DERIVED##TypeChildren(DERIVED##Type *T) { \
+    DISPATCH(dispatch##BASE##Children, BASE); \
+    DISPATCH(visit##DERIVED##TypeChildren, DERIVED##Type); \
+  } \
+  void visit##DERIVED##TypeChildren(DERIVED##Type *T) {}
+#include "clang/AST/TypeNodes.def"
+
+  void dispatchTypeAttrs(Type *T) {
+    DISPATCH(visitTypeAttrs, Type);
+  }
+  void visitTypeAttrs(Type *T) {}
+
+  void dispatchTypeChildren(Type *T) {
+    DISPATCH(visitTypeChildren, Type);
+  }
+  void visitTypeChildren(Type *T) {}
+
+#undef DISPATCH  
+};
+
+static llvm::StringRef getTypeKindName(Type *T) {
+  switch (T->getTypeClass()) {
+#define TYPE(DERIVED, BASE) case Type::DERIVED: return #DERIVED "Type";
+#define ABSTRACT_TYPE(DERIVED, BASE)
+#include "clang/AST/TypeNodes.def"
+  }
+
+  llvm_unreachable("unknown type kind!");
+  return "unknown_type";
+}
+
+struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
+                   public XMLTypeVisitor<XMLDumper> {
+  llvm::raw_ostream &out;
+  ASTContext &Context;
+  llvm::SmallVector<Node, 16> Stack;
+  unsigned Indent;
+  explicit XMLDumper(llvm::raw_ostream &OS, ASTContext &context)
+    : out(OS), Context(context), Indent(0) {}
+
+  void indent() {
+    for (unsigned I = Indent; I; --I)
+      out << ' ';
+  }
+
+  /// Push a new node on the stack.
+  void push(llvm::StringRef name) {
+    if (!Stack.empty()) {
+      assert(Stack.back().isDoneWithAttrs());
+      if (Stack.back().State == NS_LazyChildren) {
+        Stack.back().State = NS_Children;
+        out << ">\n";
+      }
+      Indent++;
+      indent();
+    }
+    Stack.push_back(Node(name));
+    out << '<' << name;
+  }
+
+  /// Set the given attribute to the given value.
+  void set(llvm::StringRef attr, llvm::StringRef value) {
+    assert(!Stack.empty() && !Stack.back().isDoneWithAttrs());
+    out << ' ' << attr << '=' << '"' << value << '"'; // TODO: quotation
+  }
+
+  /// Finish attributes.
+  void completeAttrs() {
+    assert(!Stack.empty() && !Stack.back().isDoneWithAttrs());
+    Stack.back().State = NS_LazyChildren;
+  }
+
+  /// Pop a node.
+  void pop() {
+    assert(!Stack.empty() && Stack.back().isDoneWithAttrs());
+    if (Stack.back().State == NS_LazyChildren) {
+      out << "/>\n";
+    } else {
+      indent();
+      out << "</" << Stack.back().Name << ">\n";
+    }
+    if (Stack.size() > 1) Indent--;
+    Stack.pop_back();
+  }
+
+  //---- General utilities -------------------------------------------//
+
+  void setPointer(llvm::StringRef prop, const void *p) {
+    llvm::SmallString<10> buffer;
+    llvm::raw_svector_ostream os(buffer);
+    os << p;
+    os.flush();
+    set(prop, buffer);
+  }
+
+  void setPointer(void *p) {
+    setPointer("ptr", p);
+  }
+
+  void setInteger(llvm::StringRef prop, const llvm::APSInt &v) {
+    set(prop, v.toString(10));
+  }
+
+  void setInteger(llvm::StringRef prop, unsigned n) {
+    llvm::SmallString<10> buffer;
+    llvm::raw_svector_ostream os(buffer);
+    os << n;
+    os.flush();
+    set(prop, buffer);
+  }
+
+  void setFlag(llvm::StringRef prop, bool flag) {
+    if (flag) set(prop, "true");
+  }
+
+  void setName(DeclarationName Name) {
+    if (!Name)
+      return set("name", "");
+
+    // Common case.
+    if (Name.isIdentifier())
+      return set("name", Name.getAsIdentifierInfo()->getName());
+
+    set("name", Name.getAsString());
+  }
+
+  class TemporaryContainer {
+    XMLDumper &Dumper;
+  public:
+    TemporaryContainer(XMLDumper &dumper, llvm::StringRef name)
+      : Dumper(dumper) {
+      Dumper.push(name);
+      Dumper.completeAttrs();
+    }
+
+    ~TemporaryContainer() {
+      Dumper.pop();
+    }
+  };
+
+  void visitTemplateParameters(TemplateParameterList *L) {
+    push("template_parameters");
+    completeAttrs();
+    for (TemplateParameterList::iterator
+           I = L->begin(), E = L->end(); I != E; ++I)
+      dispatch(*I);
+    pop();
+  }
+
+  void visitTemplateArguments(const TemplateArgumentList &L) {
+    push("template_arguments");
+    completeAttrs();
+    for (unsigned I = 0, E = L.size(); I != E; ++I)
+      dispatch(L[I]);
+    pop();
+  }
+
+  /// Visits a reference to the given declaration.
+  void visitDeclRef(Decl *D) {
+    push(D->getDeclKindName());
+    setPointer("ref", D);
+    completeAttrs();
+    pop();
+  }
+  void visitDeclRef(llvm::StringRef Name, Decl *D) {
+    TemporaryContainer C(*this, Name);
+    if (D) visitDeclRef(D);
+  }
+
+  void dispatch(const TemplateArgument &A) {
+    switch (A.getKind()) {
+    case TemplateArgument::Null: {
+      TemporaryContainer C(*this, "null");
+      break;
+    }
+    case TemplateArgument::Type: {
+      dispatch(A.getAsType());
+      break;
+    }
+    case TemplateArgument::Template:
+    case TemplateArgument::TemplateExpansion:
+      // FIXME: Implement!
+      break;
+        
+    case TemplateArgument::Declaration: {
+      visitDeclRef(A.getAsDecl());
+      break;
+    }
+    case TemplateArgument::Integral: {
+      push("integer");
+      setInteger("value", *A.getAsIntegral());
+      completeAttrs();
+      pop();
+      break;
+    }
+    case TemplateArgument::Expression: {
+      dispatch(A.getAsExpr());
+      break;
+    }
+    case TemplateArgument::Pack: {
+      for (TemplateArgument::pack_iterator P = A.pack_begin(), 
+                                        PEnd = A.pack_end();
+           P != PEnd; ++P)
+        dispatch(*P);
+      break;
+    }
+    }
+  }
+
+  void dispatch(const TemplateArgumentLoc &A) {
+    dispatch(A.getArgument());
+  }
+
+  //---- Declarations ------------------------------------------------//
+  // Calls are made in this order:
+  //   # Enter a new node.
+  //   push("FieldDecl")
+  //
+  //   # In this phase, attributes are set on the node.
+  //   visitDeclAttrs(D)
+  //   visitNamedDeclAttrs(D)
+  //   ...
+  //   visitFieldDeclAttrs(D)
+  //
+  //   # No more attributes after this point.
+  //   completeAttrs()
+  //
+  //   # Create "header" child nodes, i.e. those which logically
+  //   # belong to the declaration itself.
+  //   visitDeclChildren(D)
+  //   visitNamedDeclChildren(D)
+  //   ...
+  //   visitFieldDeclChildren(D)
+  //
+  //   # Create nodes for the lexical children.
+  //   visitDeclAsContext(D)
+  //   visitNamedDeclAsContext(D)
+  //   ...
+  //   visitFieldDeclAsContext(D)
+  //
+  //   # Finish the node.
+  //   pop();
+  void dispatch(Decl *D) {
+    push(D->getDeclKindName());
+    XMLDeclVisitor<XMLDumper>::dispatch(D);
+    pop();
+  }
+  void visitDeclAttrs(Decl *D) {
+    setPointer(D);
+  }
+
+  /// Visit all the lexical decls in the given context.
+  void visitDeclContext(DeclContext *DC) {
+    for (DeclContext::decl_iterator
+           I = DC->decls_begin(), E = DC->decls_end(); I != E; ++I)
+      dispatch(*I);
+
+    // FIXME: point out visible declarations not in lexical context?
+  }
+
+  /// Set the "access" attribute on the current node according to the
+  /// given specifier.
+  void setAccess(AccessSpecifier AS) {
+    switch (AS) {
+    case AS_public: return set("access", "public");
+    case AS_protected: return set("access", "protected");
+    case AS_private: return set("access", "private");
+    case AS_none: llvm_unreachable("explicit forbidden access");
+    }
+  }
+
+  template <class T> void visitRedeclarableAttrs(T *D) {
+    if (T *Prev = D->getPreviousDeclaration())
+      setPointer("previous", Prev);
+  }
+
+
+  // TranslationUnitDecl
+  void visitTranslationUnitDeclAsContext(TranslationUnitDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // LinkageSpecDecl
+  void visitLinkageSpecDeclAttrs(LinkageSpecDecl *D) {
+    llvm::StringRef lang = "";
+    switch (D->getLanguage()) {
+    case LinkageSpecDecl::lang_c: lang = "C"; break;
+    case LinkageSpecDecl::lang_cxx: lang = "C++"; break;
+    }
+    set("lang", lang);
+  }
+  void visitLinkageSpecDeclAsContext(LinkageSpecDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // NamespaceDecl
+  void visitNamespaceDeclAttrs(NamespaceDecl *D) {
+    setFlag("inline", D->isInline());
+    if (!D->isOriginalNamespace())
+      setPointer("original", D->getOriginalNamespace());
+  }
+  void visitNamespaceDeclAsContext(NamespaceDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // NamedDecl
+  void visitNamedDeclAttrs(NamedDecl *D) {
+    setName(D->getDeclName());
+  }
+
+  // ValueDecl
+  void visitValueDeclChildren(ValueDecl *D) {
+    dispatch(D->getType());
+  }
+
+  // DeclaratorDecl
+  void visitDeclaratorDeclChildren(DeclaratorDecl *D) {
+    //dispatch(D->getTypeSourceInfo()->getTypeLoc());
+  }
+
+  // VarDecl
+  void visitVarDeclAttrs(VarDecl *D) {
+    visitRedeclarableAttrs(D);
+    if (D->getStorageClass() != SC_None)
+      set("storage",
+          VarDecl::getStorageClassSpecifierString(D->getStorageClass()));
+    setFlag("directinit", D->hasCXXDirectInitializer());
+    setFlag("nrvo", D->isNRVOVariable());
+    // TODO: instantiation, etc.
+  }
+  void visitVarDeclChildren(VarDecl *D) {
+    if (D->hasInit()) dispatch(D->getInit());
+  }
+
+  // ParmVarDecl?
+
+  // FunctionDecl
+  void visitFunctionDeclAttrs(FunctionDecl *D) {
+    visitRedeclarableAttrs(D);
+    setFlag("pure", D->isPure());
+    setFlag("trivial", D->isTrivial());
+    setFlag("returnzero", D->hasImplicitReturnZero());
+    setFlag("prototype", D->hasWrittenPrototype());
+    setFlag("deleted", D->isDeleted());
+    if (D->getStorageClass() != SC_None)
+      set("storage",
+          VarDecl::getStorageClassSpecifierString(D->getStorageClass()));
+    setFlag("inline", D->isInlineSpecified());
+    // TODO: instantiation, etc.
+  }
+  void visitFunctionDeclChildren(FunctionDecl *D) {
+    for (FunctionDecl::param_iterator
+           I = D->param_begin(), E = D->param_end(); I != E; ++I)
+      dispatch(*I);
+    if (D->isThisDeclarationADefinition())
+      dispatch(D->getBody());
+  }
+
+  // CXXMethodDecl ?
+  // CXXConstructorDecl ?
+  // CXXDestructorDecl ?
+  // CXXConversionDecl ?
+
+  void dispatch(CXXCtorInitializer *Init) {
+    // TODO
+  }
+
+  // FieldDecl
+  void visitFieldDeclAttrs(FieldDecl *D) {
+    setFlag("mutable", D->isMutable());
+  }
+  void visitFieldDeclChildren(FieldDecl *D) {
+    if (D->isBitField()) {
+      TemporaryContainer C(*this, "bitwidth");
+      dispatch(D->getBitWidth());
+    }
+    // TODO: C++0x member initializer
+  }
+
+  // EnumConstantDecl
+  void visitEnumConstantDeclChildren(EnumConstantDecl *D) {
+    // value in any case?
+    if (D->getInitExpr()) dispatch(D->getInitExpr());
+  }
+
+  // IndirectFieldDecl
+  void visitIndirectFieldDeclChildren(IndirectFieldDecl *D) {
+    for (IndirectFieldDecl::chain_iterator
+           I = D->chain_begin(), E = D->chain_end(); I != E; ++I) {
+      NamedDecl *VD = const_cast<NamedDecl*>(*I);
+      push(isa<VarDecl>(VD) ? "variable" : "field");
+      setPointer("ptr", VD);
+      completeAttrs();
+      pop();
+    }
+  }
+
+  // TypeDecl
+  void visitTypeDeclAttrs(TypeDecl *D) {
+    setPointer("typeptr", D->getTypeForDecl());
+  }
+
+  // TypedefDecl
+  void visitTypedefDeclAttrs(TypedefDecl *D) {
+    visitRedeclarableAttrs(D);
+  }
+  void visitTypedefDeclChildren(TypedefDecl *D) {
+    dispatch(D->getTypeSourceInfo()->getTypeLoc());
+  }
+
+  // TagDecl
+  void visitTagDeclAttrs(TagDecl *D) {
+    visitRedeclarableAttrs(D);
+  }
+  void visitTagDeclAsContext(TagDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // EnumDecl
+  void visitEnumDeclAttrs(EnumDecl *D) {
+    setFlag("scoped", D->isScoped());
+    setFlag("fixed", D->isFixed());
+  }
+  void visitEnumDeclChildren(EnumDecl *D) {
+    {
+      TemporaryContainer C(*this, "promotion_type");
+      dispatch(D->getPromotionType());
+    }
+    {
+      TemporaryContainer C(*this, "integer_type");
+      dispatch(D->getIntegerType());
+    }
+  }
+
+  // RecordDecl ?
+
+  void visitCXXRecordDeclChildren(CXXRecordDecl *D) {
+    if (!D->isThisDeclarationADefinition()) return;
+
+    for (CXXRecordDecl::base_class_iterator
+           I = D->bases_begin(), E = D->bases_end(); I != E; ++I) {
+      push("base");
+      setAccess(I->getAccessSpecifier());
+      completeAttrs();
+      dispatch(I->getTypeSourceInfo()->getTypeLoc());
+      pop();
+    }
+  }
+
+  // ClassTemplateSpecializationDecl ?
+
+  // FileScopeAsmDecl ?
+
+  // BlockDecl
+  void visitBlockDeclAttrs(BlockDecl *D) {
+    setFlag("variadic", D->isVariadic());
+  }
+  void visitBlockDeclChildren(BlockDecl *D) {
+    for (FunctionDecl::param_iterator
+           I = D->param_begin(), E = D->param_end(); I != E; ++I)
+      dispatch(*I);
+    dispatch(D->getBody());
+  }
+
+  // AccessSpecDecl
+  void visitAccessSpecDeclAttrs(AccessSpecDecl *D) {
+    setAccess(D->getAccess());
+  }
+
+  // TemplateDecl
+  void visitTemplateDeclChildren(TemplateDecl *D) {
+    visitTemplateParameters(D->getTemplateParameters());
+    dispatch(D->getTemplatedDecl());
+  }
+
+  // FunctionTemplateDecl
+  void visitFunctionTemplateDeclAttrs(FunctionTemplateDecl *D) {
+    visitRedeclarableAttrs(D);
+  }
+  void visitFunctionTemplateDeclChildren(FunctionTemplateDecl *D) {
+    // Mention all the specializations which don't have explicit
+    // declarations elsewhere.
+    for (FunctionTemplateDecl::spec_iterator
+           I = D->spec_begin(), E = D->spec_end(); I != E; ++I) {
+      FunctionTemplateSpecializationInfo *Info
+        = I->getTemplateSpecializationInfo();
+
+      bool Unknown = false;
+      switch (Info->getTemplateSpecializationKind()) {
+      case TSK_ImplicitInstantiation: Unknown = false; break;
+      case TSK_Undeclared: Unknown = true; break;
+
+      // These will be covered at their respective sites.
+      case TSK_ExplicitSpecialization: continue;
+      case TSK_ExplicitInstantiationDeclaration: continue;
+      case TSK_ExplicitInstantiationDefinition: continue;
+      }
+
+      TemporaryContainer C(*this,
+                           Unknown ? "uninstantiated" : "instantiation");
+      visitTemplateArguments(*Info->TemplateArguments);
+      dispatch(Info->Function);
+    }
+  }
+
+  // ClasTemplateDecl
+  void visitClassTemplateDeclAttrs(ClassTemplateDecl *D) {
+    visitRedeclarableAttrs(D);
+  }
+  void visitClassTemplateDeclChildren(ClassTemplateDecl *D) {
+    // Mention all the specializations which don't have explicit
+    // declarations elsewhere.
+    for (ClassTemplateDecl::spec_iterator
+           I = D->spec_begin(), E = D->spec_end(); I != E; ++I) {
+
+      bool Unknown = false;
+      switch (I->getTemplateSpecializationKind()) {
+      case TSK_ImplicitInstantiation: Unknown = false; break;
+      case TSK_Undeclared: Unknown = true; break;
+
+      // These will be covered at their respective sites.
+      case TSK_ExplicitSpecialization: continue;
+      case TSK_ExplicitInstantiationDeclaration: continue;
+      case TSK_ExplicitInstantiationDefinition: continue;
+      }
+
+      TemporaryContainer C(*this,
+                           Unknown ? "uninstantiated" : "instantiation");
+      visitTemplateArguments(I->getTemplateArgs());
+      dispatch(*I);
+    }
+  }
+
+  // TemplateTypeParmDecl
+  void visitTemplateTypeParmDeclAttrs(TemplateTypeParmDecl *D) {
+    setInteger("depth", D->getDepth());
+    setInteger("index", D->getIndex());
+  }
+  void visitTemplateTypeParmDeclChildren(TemplateTypeParmDecl *D) {
+    if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited())
+      dispatch(D->getDefaultArgumentInfo()->getTypeLoc());
+    // parameter pack?
+  }
+
+  // NonTypeTemplateParmDecl
+  void visitNonTypeTemplateParmDeclAttrs(NonTypeTemplateParmDecl *D) {
+    setInteger("depth", D->getDepth());
+    setInteger("index", D->getIndex());
+  }
+  void visitNonTypeTemplateParmDeclChildren(NonTypeTemplateParmDecl *D) {
+    if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited())
+      dispatch(D->getDefaultArgument());
+    // parameter pack?
+  }
+
+  // TemplateTemplateParmDecl
+  void visitTemplateTemplateParmDeclAttrs(TemplateTemplateParmDecl *D) {
+    setInteger("depth", D->getDepth());
+    setInteger("index", D->getIndex());
+  }
+  void visitTemplateTemplateParmDeclChildren(TemplateTemplateParmDecl *D) {
+    if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited())
+      dispatch(D->getDefaultArgument());
+    // parameter pack?
+  }
+
+  // FriendDecl
+  void visitFriendDeclChildren(FriendDecl *D) {
+    if (TypeSourceInfo *T = D->getFriendType())
+      dispatch(T->getTypeLoc());
+    else
+      dispatch(D->getFriendDecl());
+  }
+
+  // UsingDirectiveDecl ?
+  // UsingDecl ?
+  // UsingShadowDecl ?
+  // NamespaceAliasDecl ?
+  // UnresolvedUsingValueDecl ?
+  // UnresolvedUsingTypenameDecl ?
+  // StaticAssertDecl ?
+
+  // ObjCImplDecl
+  void visitObjCImplDeclChildren(ObjCImplDecl *D) {
+    visitDeclRef(D->getClassInterface());
+  }
+  void visitObjCImplDeclAsContext(ObjCImplDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // ObjCClassDecl
+  void visitObjCClassDeclChildren(ObjCClassDecl *D) {
+    for (ObjCClassDecl::iterator I = D->begin(), E = D->end(); I != E; ++I)
+      visitDeclRef(I->getInterface());
+  }
+
+  // ObjCInterfaceDecl
+  void visitCategoryList(ObjCCategoryDecl *D) {
+    if (!D) return;
+
+    TemporaryContainer C(*this, "categories");
+    for (; D; D = D->getNextClassCategory())
+      visitDeclRef(D);
+  }
+  void visitObjCInterfaceDeclAttrs(ObjCInterfaceDecl *D) {
+    setPointer("typeptr", D->getTypeForDecl());
+    setFlag("forward_decl", D->isForwardDecl());
+    setFlag("implicit_interface", D->isImplicitInterfaceDecl());
+  }
+  void visitObjCInterfaceDeclChildren(ObjCInterfaceDecl *D) {
+    visitDeclRef("super", D->getSuperClass());
+    visitDeclRef("implementation", D->getImplementation());
+    if (D->protocol_begin() != D->protocol_end()) {
+      TemporaryContainer C(*this, "protocols");
+      for (ObjCInterfaceDecl::protocol_iterator
+             I = D->protocol_begin(), E = D->protocol_end(); I != E; ++I)
+        visitDeclRef(*I);
+    }
+    visitCategoryList(D->getCategoryList());
+  }
+  void visitObjCInterfaceDeclAsContext(ObjCInterfaceDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // ObjCCategoryDecl
+  void visitObjCCategoryDeclAttrs(ObjCCategoryDecl *D) {
+    setFlag("extension", D->IsClassExtension());
+    setFlag("synth_bitfield", D->hasSynthBitfield());
+  }
+  void visitObjCCategoryDeclChildren(ObjCCategoryDecl *D) {
+    visitDeclRef("interface", D->getClassInterface());
+    visitDeclRef("implementation", D->getImplementation());
+    if (D->protocol_begin() != D->protocol_end()) {
+      TemporaryContainer C(*this, "protocols");
+      for (ObjCCategoryDecl::protocol_iterator
+             I = D->protocol_begin(), E = D->protocol_end(); I != E; ++I)
+        visitDeclRef(*I);
+    }
+  }
+  void visitObjCCategoryDeclAsContext(ObjCCategoryDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // ObjCCategoryImplDecl
+  void visitObjCCategoryImplDeclAttrs(ObjCCategoryImplDecl *D) {
+    set("identifier", D->getName());
+  }
+  void visitObjCCategoryImplDeclChildren(ObjCCategoryImplDecl *D) {
+    visitDeclRef(D->getCategoryDecl());
+  }
+
+  // ObjCImplementationDecl
+  void visitObjCImplementationDeclAttrs(ObjCImplementationDecl *D) {
+    setFlag("synth_bitfield", D->hasSynthBitfield());
+    set("identifier", D->getName());
+  }
+  void visitObjCImplementationDeclChildren(ObjCImplementationDecl *D) {
+    visitDeclRef("super", D->getSuperClass());
+    if (D->init_begin() != D->init_end()) {
+      TemporaryContainer C(*this, "initializers");
+      for (ObjCImplementationDecl::init_iterator
+             I = D->init_begin(), E = D->init_end(); I != E; ++I)
+        dispatch(*I);
+    }
+  }
+
+  // ObjCForwardProtocolDecl
+  void visitObjCForwardProtocolDeclChildren(ObjCForwardProtocolDecl *D) {
+    for (ObjCForwardProtocolDecl::protocol_iterator
+           I = D->protocol_begin(), E = D->protocol_end(); I != E; ++I)
+      visitDeclRef(*I);
+  }
+
+  // ObjCProtocolDecl
+  void visitObjCProtocolDeclAttrs(ObjCProtocolDecl *D) {
+    setFlag("forward_decl", D->isForwardDecl());
+  }
+  void visitObjCProtocolDeclChildren(ObjCProtocolDecl *D) {
+    if (D->protocol_begin() != D->protocol_end()) {
+      TemporaryContainer C(*this, "protocols");
+      for (ObjCInterfaceDecl::protocol_iterator
+             I = D->protocol_begin(), E = D->protocol_end(); I != E; ++I)
+        visitDeclRef(*I);
+    }
+  }
+  void visitObjCProtocolDeclAsContext(ObjCProtocolDecl *D) {
+    visitDeclContext(D);
+  }
+
+  // ObjCMethodDecl
+  void visitObjCMethodDeclAttrs(ObjCMethodDecl *D) {
+    // decl qualifier?
+    // implementation control?
+
+    setFlag("instance", D->isInstanceMethod());
+    setFlag("variadic", D->isVariadic());
+    setFlag("synthesized", D->isSynthesized());
+    setFlag("defined", D->isDefined());
+  }
+  void visitObjCMethodDeclChildren(ObjCMethodDecl *D) {
+    dispatch(D->getResultType());
+    for (ObjCMethodDecl::param_iterator
+           I = D->param_begin(), E = D->param_end(); I != E; ++I)
+      dispatch(*I);
+    if (D->isThisDeclarationADefinition())
+      dispatch(D->getBody());
+  }
+
+  // ObjCIvarDecl
+  void setAccessControl(llvm::StringRef prop, ObjCIvarDecl::AccessControl AC) {
+    switch (AC) {
+    case ObjCIvarDecl::None: return set(prop, "none");
+    case ObjCIvarDecl::Private: return set(prop, "private");
+    case ObjCIvarDecl::Protected: return set(prop, "protected");
+    case ObjCIvarDecl::Public: return set(prop, "public");
+    case ObjCIvarDecl::Package: return set(prop, "package");
+    }
+  }
+  void visitObjCIvarDeclAttrs(ObjCIvarDecl *D) {
+    setFlag("synthesize", D->getSynthesize());
+    setAccessControl("access", D->getAccessControl());
+  }
+
+  // ObjCCompatibleAliasDecl
+  void visitObjCCompatibleAliasDeclChildren(ObjCCompatibleAliasDecl *D) {
+    visitDeclRef(D->getClassInterface());
+  }
+
+  // FIXME: ObjCPropertyDecl
+  // FIXME: ObjCPropertyImplDecl
+
+  //---- Types -----------------------------------------------------//
+  void dispatch(TypeLoc TL) {
+    dispatch(TL.getType()); // for now
+  }
+
+  void dispatch(QualType T) {
+    if (T.hasLocalQualifiers()) {
+      push("QualType");
+      Qualifiers Qs = T.getLocalQualifiers();
+      setFlag("const", Qs.hasConst());
+      setFlag("volatile", Qs.hasVolatile());
+      setFlag("restrict", Qs.hasRestrict());
+      if (Qs.hasAddressSpace()) setInteger("addrspace", Qs.getAddressSpace());
+      if (Qs.hasObjCGCAttr()) {
+        switch (Qs.getObjCGCAttr()) {
+        case Qualifiers::Weak: set("gc", "weak"); break;
+        case Qualifiers::Strong: set("gc", "strong"); break;
+        case Qualifiers::GCNone: llvm_unreachable("explicit none");
+        }
+      }
+      
+      completeAttrs();
+      dispatch(QualType(T.getTypePtr(), 0));
+      pop();
+      return;
+    }
+
+    Type *Ty = const_cast<Type*>(T.getTypePtr());
+    push(getTypeKindName(Ty));
+    XMLTypeVisitor<XMLDumper>::dispatch(const_cast<Type*>(T.getTypePtr()));
+    pop();
+  }
+
+  void setCallingConv(CallingConv CC) {
+    switch (CC) {
+    case CC_Default: return;
+    case CC_C: return set("cc", "cdecl");
+    case CC_X86FastCall: return set("cc", "x86_fastcall");
+    case CC_X86StdCall: return set("cc", "x86_stdcall");
+    case CC_X86ThisCall: return set("cc", "x86_thiscall");
+    case CC_X86Pascal: return set("cc", "x86_pascal");
+    }
+  }
+
+  void visitTypeAttrs(Type *D) {
+    setPointer(D);
+    setFlag("dependent", D->isDependentType());
+    setFlag("variably_modified", D->isVariablyModifiedType());
+
+    setPointer("canonical", D->getCanonicalTypeInternal().getAsOpaquePtr());
+  }
+
+  void visitPointerTypeChildren(PointerType *T) {
+    dispatch(T->getPointeeType());
+  }
+  void visitReferenceTypeChildren(ReferenceType *T) {
+    dispatch(T->getPointeeType());
+  }
+  void visitObjCObjectPointerTypeChildren(ObjCObjectPointerType *T) {
+    dispatch(T->getPointeeType());
+  }
+  void visitBlockPointerTypeChildren(BlockPointerType *T) {
+    dispatch(T->getPointeeType());
+  }
+
+  // Types that just wrap declarations.
+  void visitTagTypeChildren(TagType *T) {
+    visitDeclRef(T->getDecl());
+  }
+  void visitTypedefTypeChildren(TypedefType *T) {
+    visitDeclRef(T->getDecl());
+  }
+  void visitObjCInterfaceTypeChildren(ObjCInterfaceType *T) {
+    visitDeclRef(T->getDecl());
+  }
+  void visitUnresolvedUsingTypeChildren(UnresolvedUsingType *T) {
+    visitDeclRef(T->getDecl());
+  }
+  void visitInjectedClassNameTypeChildren(InjectedClassNameType *T) {
+    visitDeclRef(T->getDecl());
+  }
+
+  void visitFunctionTypeAttrs(FunctionType *T) {
+    setFlag("noreturn", T->getNoReturnAttr());
+    setCallingConv(T->getCallConv());
+    if (T->getRegParmType()) setInteger("regparm", T->getRegParmType());
+  }
+  void visitFunctionTypeChildren(FunctionType *T) {
+    dispatch(T->getResultType());
+  }
+
+  void visitFunctionProtoTypeAttrs(FunctionProtoType *T) {
+    setFlag("const", T->getTypeQuals() & Qualifiers::Const);
+    setFlag("volatile", T->getTypeQuals() & Qualifiers::Volatile);
+    setFlag("restrict", T->getTypeQuals() & Qualifiers::Restrict);
+  }
+  void visitFunctionProtoTypeChildren(FunctionProtoType *T) {
+    push("parameters");
+    setFlag("variadic", T->isVariadic());
+    completeAttrs();
+    for (FunctionProtoType::arg_type_iterator
+           I = T->arg_type_begin(), E = T->arg_type_end(); I != E; ++I)
+      dispatch(*I);
+    pop();
+
+    if (T->hasExceptionSpec()) {
+      push("exception_specifiers");
+      setFlag("any", T->hasAnyExceptionSpec());
+      completeAttrs();
+      for (FunctionProtoType::exception_iterator
+             I = T->exception_begin(), E = T->exception_end(); I != E; ++I)
+        dispatch(*I);
+      pop();
+    }
+  }
+
+  void visitTemplateSpecializationTypeChildren(TemplateSpecializationType *T) {
+    if (const RecordType *RT = T->getAs<RecordType>())
+      visitDeclRef(RT->getDecl());
+
+    // TODO: TemplateName
+
+    push("template_arguments");
+    completeAttrs();
+    for (unsigned I = 0, E = T->getNumArgs(); I != E; ++I)
+      dispatch(T->getArg(I));
+    pop();
+  }
+
+  //---- Statements ------------------------------------------------//
+  void dispatch(Stmt *S) {
+    // FIXME: this is not really XML at all
+    push("Stmt");
+    out << ">\n";
+    Stack.back().State = NS_Children; // explicitly become non-lazy
+    S->dump(out, Context.getSourceManager());
+    out << '\n';
+    pop();
+  }
+};
+}
+
+void Decl::dumpXML() const {
+  dumpXML(llvm::errs());
+}
+
+void Decl::dumpXML(llvm::raw_ostream &out) const {
+  XMLDumper(out, getASTContext()).dispatch(const_cast<Decl*>(this));
+}
+
+#else /* ifndef NDEBUG */
+
+void Decl::dumpXML() const {}
+void Decl::dumpXML(llvm::raw_ostream &out) const {}
+
+#endif
diff --git a/lib/AST/Expr.cpp b/lib/AST/Expr.cpp
index 5feef1c80332..391b26ab48ea 100644
--- a/lib/AST/Expr.cpp
+++ b/lib/AST/Expr.cpp
@@ -20,15 +20,16 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 using namespace clang;
 
-void Expr::ANCHOR() {} // key function for Expr class.
-
 /// isKnownToHaveBooleanValue - Return true if this is an integer expression
 /// that is known to return 0 or 1.  This happens for _Bool/bool expressions
 /// but also int expressions which are produced by things like comparisons in
@@ -90,6 +91,42 @@ bool Expr::isKnownToHaveBooleanValue() const {
   return false;
 }
 
+// Amusing macro metaprogramming hack: check whether a class provides
+// a more specific implementation of getExprLoc().
+namespace {
+  /// This implementation is used when a class provides a custom
+  /// implementation of getExprLoc.
+  template <class E, class T>
+  SourceLocation getExprLocImpl(const Expr *expr,
+                                SourceLocation (T::*v)() const) {
+    return static_cast<const E*>(expr)->getExprLoc();
+  }
+
+  /// This implementation is used when a class doesn't provide
+  /// a custom implementation of getExprLoc.  Overload resolution
+  /// should pick it over the implementation above because it's
+  /// more specialized according to function template partial ordering.
+  template <class E>
+  SourceLocation getExprLocImpl(const Expr *expr,
+                                SourceLocation (Expr::*v)() const) {
+    return static_cast<const E*>(expr)->getSourceRange().getBegin();
+  }
+}
+
+SourceLocation Expr::getExprLoc() const {
+  switch (getStmtClass()) {
+  case Stmt::NoStmtClass: llvm_unreachable("statement without class");
+#define ABSTRACT_STMT(type)
+#define STMT(type, base) \
+  case Stmt::type##Class: llvm_unreachable(#type " is not an Expr"); break;
+#define EXPR(type, base) \
+  case Stmt::type##Class: return getExprLocImpl<type>(this, &type::getExprLoc);
+#include "clang/AST/StmtNodes.inc"
+  }
+  llvm_unreachable("unknown statement kind");
+  return SourceLocation();
+}
+
 //===----------------------------------------------------------------------===//
 // Primary Expressions.
 //===----------------------------------------------------------------------===//
@@ -105,6 +142,25 @@ void ExplicitTemplateArgumentList::initializeFrom(
     new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
 }
 
+void ExplicitTemplateArgumentList::initializeFrom(
+                                   const TemplateArgumentListInfo &Info,
+                                   bool &Dependent, 
+                                   bool &ContainsUnexpandedParameterPack) {
+  LAngleLoc = Info.getLAngleLoc();
+  RAngleLoc = Info.getRAngleLoc();
+  NumTemplateArgs = Info.size();
+
+  TemplateArgumentLoc *ArgBuffer = getTemplateArgs();
+  for (unsigned i = 0; i != NumTemplateArgs; ++i) {
+    Dependent = Dependent || Info[i].getArgument().isDependent();
+    ContainsUnexpandedParameterPack 
+      = ContainsUnexpandedParameterPack || 
+        Info[i].getArgument().containsUnexpandedParameterPack();
+
+    new (&ArgBuffer[i]) TemplateArgumentLoc(Info[i]);
+  }
+}
+
 void ExplicitTemplateArgumentList::copyInto(
                                       TemplateArgumentListInfo &Info) const {
   Info.setLAngleLoc(LAngleLoc);
@@ -123,11 +179,14 @@ std::size_t ExplicitTemplateArgumentList::sizeFor(
   return sizeFor(Info.size());
 }
 
-void DeclRefExpr::computeDependence() {
+/// \brief Compute the type- and value-dependence of a declaration reference
+/// based on the declaration being referenced.
+static void computeDeclRefDependence(NamedDecl *D, QualType T,
+                                     bool &TypeDependent,
+                                     bool &ValueDependent) {
   TypeDependent = false;
   ValueDependent = false;
   
-  NamedDecl *D = getDecl();
 
   // (TD) C++ [temp.dep.expr]p3:
   //   An id-expression is type-dependent if it contains:
@@ -136,63 +195,93 @@ void DeclRefExpr::computeDependence() {
   //
   // (VD) C++ [temp.dep.constexpr]p2:
   //  An identifier is value-dependent if it is:
-
+  
   //  (TD)  - an identifier that was declared with dependent type
   //  (VD)  - a name declared with a dependent type,
-  if (getType()->isDependentType()) {
+  if (T->isDependentType()) {
     TypeDependent = true;
     ValueDependent = true;
+    return;
   }
+  
   //  (TD)  - a conversion-function-id that specifies a dependent type
-  else if (D->getDeclName().getNameKind() 
-                               == DeclarationName::CXXConversionFunctionName &&
+  if (D->getDeclName().getNameKind() 
+           == DeclarationName::CXXConversionFunctionName &&
            D->getDeclName().getCXXNameType()->isDependentType()) {
     TypeDependent = true;
     ValueDependent = true;
-  }
-  //  (TD)  - a template-id that is dependent,
-  else if (hasExplicitTemplateArgs() && 
-           TemplateSpecializationType::anyDependentTemplateArguments(
-                                                       getTemplateArgs(), 
-                                                       getNumTemplateArgs())) {
-    TypeDependent = true;
-    ValueDependent = true;
+    return;
   }
   //  (VD)  - the name of a non-type template parameter,
-  else if (isa<NonTypeTemplateParmDecl>(D))
+  if (isa<NonTypeTemplateParmDecl>(D)) {
     ValueDependent = true;
+    return;
+  }
+  
   //  (VD) - a constant with integral or enumeration type and is
   //         initialized with an expression that is value-dependent.
-  else if (VarDecl *Var = dyn_cast<VarDecl>(D)) {
+  if (VarDecl *Var = dyn_cast<VarDecl>(D)) {
     if (Var->getType()->isIntegralOrEnumerationType() &&
         Var->getType().getCVRQualifiers() == Qualifiers::Const) {
       if (const Expr *Init = Var->getAnyInitializer())
         if (Init->isValueDependent())
           ValueDependent = true;
     } 
+    
     // (VD) - FIXME: Missing from the standard: 
     //      -  a member function or a static data member of the current 
     //         instantiation
     else if (Var->isStaticDataMember() && 
              Var->getDeclContext()->isDependentContext())
       ValueDependent = true;
-  } 
+    
+    return;
+  }
+  
   // (VD) - FIXME: Missing from the standard: 
   //      -  a member function or a static data member of the current 
   //         instantiation
-  else if (isa<CXXMethodDecl>(D) && D->getDeclContext()->isDependentContext())
+  if (isa<CXXMethodDecl>(D) && D->getDeclContext()->isDependentContext()) {
     ValueDependent = true;
-  //  (TD)  - a nested-name-specifier or a qualified-id that names a
-  //          member of an unknown specialization.
-  //        (handled by DependentScopeDeclRefExpr)
+    return;
+  }  
+}
+
+void DeclRefExpr::computeDependence() {
+  bool TypeDependent = false;
+  bool ValueDependent = false;
+  computeDeclRefDependence(getDecl(), getType(), TypeDependent, ValueDependent);
+  
+  // (TD) C++ [temp.dep.expr]p3:
+  //   An id-expression is type-dependent if it contains:
+  //
+  // and 
+  //
+  // (VD) C++ [temp.dep.constexpr]p2:
+  //  An identifier is value-dependent if it is:
+  if (!TypeDependent && !ValueDependent &&
+      hasExplicitTemplateArgs() && 
+      TemplateSpecializationType::anyDependentTemplateArguments(
+                                                            getTemplateArgs(), 
+                                                       getNumTemplateArgs())) {
+    TypeDependent = true;
+    ValueDependent = true;
+  }
+  
+  ExprBits.TypeDependent = TypeDependent;
+  ExprBits.ValueDependent = ValueDependent;
+  
+  // Is the declaration a parameter pack?
+  if (getDecl()->isParameterPack())
+    ExprBits.ContainsUnexpandedParameterPack = true;
 }
 
 DeclRefExpr::DeclRefExpr(NestedNameSpecifier *Qualifier, 
                          SourceRange QualifierRange,
                          ValueDecl *D, SourceLocation NameLoc,
                          const TemplateArgumentListInfo *TemplateArgs,
-                         QualType T)
-  : Expr(DeclRefExprClass, T, false, false),
+                         QualType T, ExprValueKind VK)
+  : Expr(DeclRefExprClass, T, VK, OK_Ordinary, false, false, false),
     DecoratedD(D,
                (Qualifier? HasQualifierFlag : 0) |
                (TemplateArgs ? HasExplicitTemplateArgumentListFlag : 0)),
@@ -213,8 +302,8 @@ DeclRefExpr::DeclRefExpr(NestedNameSpecifier *Qualifier,
                          SourceRange QualifierRange,
                          ValueDecl *D, const DeclarationNameInfo &NameInfo,
                          const TemplateArgumentListInfo *TemplateArgs,
-                         QualType T)
-  : Expr(DeclRefExprClass, T, false, false),
+                         QualType T, ExprValueKind VK)
+  : Expr(DeclRefExprClass, T, VK, OK_Ordinary, false, false, false),
     DecoratedD(D,
                (Qualifier? HasQualifierFlag : 0) |
                (TemplateArgs ? HasExplicitTemplateArgumentListFlag : 0)),
@@ -237,10 +326,11 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
                                  ValueDecl *D,
                                  SourceLocation NameLoc,
                                  QualType T,
+                                 ExprValueKind VK,
                                  const TemplateArgumentListInfo *TemplateArgs) {
   return Create(Context, Qualifier, QualifierRange, D,
                 DeclarationNameInfo(D->getDeclName(), NameLoc),
-                T, TemplateArgs);
+                T, VK, TemplateArgs);
 }
 
 DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
@@ -249,6 +339,7 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
                                  ValueDecl *D,
                                  const DeclarationNameInfo &NameInfo,
                                  QualType T,
+                                 ExprValueKind VK,
                                  const TemplateArgumentListInfo *TemplateArgs) {
   std::size_t Size = sizeof(DeclRefExpr);
   if (Qualifier != 0)
@@ -257,21 +348,23 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
   if (TemplateArgs)
     Size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
   
-  void *Mem = Context.Allocate(Size, llvm::alignof<DeclRefExpr>());
+  void *Mem = Context.Allocate(Size, llvm::alignOf<DeclRefExpr>());
   return new (Mem) DeclRefExpr(Qualifier, QualifierRange, D, NameInfo,
-                               TemplateArgs, T);
+                               TemplateArgs, T, VK);
 }
 
-DeclRefExpr *DeclRefExpr::CreateEmpty(ASTContext &Context, bool HasQualifier,
+DeclRefExpr *DeclRefExpr::CreateEmpty(ASTContext &Context, 
+                                      bool HasQualifier,
+                                      bool HasExplicitTemplateArgs,
                                       unsigned NumTemplateArgs) {
   std::size_t Size = sizeof(DeclRefExpr);
   if (HasQualifier)
     Size += sizeof(NameQualifier);
   
-  if (NumTemplateArgs)
+  if (HasExplicitTemplateArgs)
     Size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
   
-  void *Mem = Context.Allocate(Size, llvm::alignof<DeclRefExpr>());
+  void *Mem = Context.Allocate(Size, llvm::alignOf<DeclRefExpr>());
   return new (Mem) DeclRefExpr(EmptyShell());
 }
 
@@ -432,7 +525,7 @@ StringLiteral *StringLiteral::Create(ASTContext &C, const char *StrData,
   // any concatenated string tokens.
   void *Mem = C.Allocate(sizeof(StringLiteral)+
                          sizeof(SourceLocation)*(NumStrs-1),
-                         llvm::alignof<StringLiteral>());
+                         llvm::alignOf<StringLiteral>());
   StringLiteral *SL = new (Mem) StringLiteral(Ty);
 
   // OPTIMIZE: could allocate this appended to the StringLiteral.
@@ -452,7 +545,7 @@ StringLiteral *StringLiteral::Create(ASTContext &C, const char *StrData,
 StringLiteral *StringLiteral::CreateEmpty(ASTContext &C, unsigned NumStrs) {
   void *Mem = C.Allocate(sizeof(StringLiteral)+
                          sizeof(SourceLocation)*(NumStrs-1),
-                         llvm::alignof<StringLiteral>());
+                         llvm::alignOf<StringLiteral>());
   StringLiteral *SL = new (Mem) StringLiteral(QualType());
   SL->StrData = 0;
   SL->ByteLength = 0;
@@ -467,6 +560,72 @@ void StringLiteral::setString(ASTContext &C, llvm::StringRef Str) {
   ByteLength = Str.size();
 }
 
+/// getLocationOfByte - Return a source location that points to the specified
+/// byte of this string literal.
+///
+/// Strings are amazingly complex.  They can be formed from multiple tokens and
+/// can have escape sequences in them in addition to the usual trigraph and
+/// escaped newline business.  This routine handles this complexity.
+///
+SourceLocation StringLiteral::
+getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
+                  const LangOptions &Features, const TargetInfo &Target) const {
+  assert(!isWide() && "This doesn't work for wide strings yet");
+  
+  // Loop over all of the tokens in this string until we find the one that
+  // contains the byte we're looking for.
+  unsigned TokNo = 0;
+  while (1) {
+    assert(TokNo < getNumConcatenated() && "Invalid byte number!");
+    SourceLocation StrTokLoc = getStrTokenLoc(TokNo);
+    
+    // Get the spelling of the string so that we can get the data that makes up
+    // the string literal, not the identifier for the macro it is potentially
+    // expanded through.
+    SourceLocation StrTokSpellingLoc = SM.getSpellingLoc(StrTokLoc);
+    
+    // Re-lex the token to get its length and original spelling.
+    std::pair<FileID, unsigned> LocInfo =SM.getDecomposedLoc(StrTokSpellingLoc);
+    bool Invalid = false;
+    llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+    if (Invalid)
+      return StrTokSpellingLoc;
+    
+    const char *StrData = Buffer.data()+LocInfo.second;
+    
+    // Create a langops struct and enable trigraphs.  This is sufficient for
+    // relexing tokens.
+    LangOptions LangOpts;
+    LangOpts.Trigraphs = true;
+    
+    // Create a lexer starting at the beginning of this token.
+    Lexer TheLexer(StrTokSpellingLoc, Features, Buffer.begin(), StrData,
+                   Buffer.end());
+    Token TheTok;
+    TheLexer.LexFromRawLexer(TheTok);
+    
+    // Use the StringLiteralParser to compute the length of the string in bytes.
+    StringLiteralParser SLP(&TheTok, 1, SM, Features, Target);
+    unsigned TokNumBytes = SLP.GetStringLength();
+    
+    // If the byte is in this token, return the location of the byte.
+    if (ByteNo < TokNumBytes ||
+        (ByteNo == TokNumBytes && TokNo == getNumConcatenated())) {
+      unsigned Offset = SLP.getOffsetOfStringByte(TheTok, ByteNo); 
+      
+      // Now that we know the offset of the token in the spelling, use the
+      // preprocessor to get the offset in the original source.
+      return Lexer::AdvanceToTokenCharacter(StrTokLoc, Offset, SM, Features);
+    }
+    
+    // Move to the next string token.
+    ++TokNo;
+    ByteNo -= TokNumBytes;
+  }
+}
+
+
+
 /// getOpcodeStr - Turn an Opcode enum value into the punctuation char it
 /// corresponds to, e.g. "sizeof" or "[pre]++".
 const char *UnaryOperator::getOpcodeStr(Opcode Op) {
@@ -522,43 +681,82 @@ OverloadedOperatorKind UnaryOperator::getOverloadedOperator(Opcode Opc) {
 // Postfix Operators.
 //===----------------------------------------------------------------------===//
 
-CallExpr::CallExpr(ASTContext& C, StmtClass SC, Expr *fn, Expr **args,
-                   unsigned numargs, QualType t, SourceLocation rparenloc)
-  : Expr(SC, t,
-         fn->isTypeDependent() || hasAnyTypeDependentArguments(args, numargs),
-         fn->isValueDependent() || hasAnyValueDependentArguments(args,numargs)),
+CallExpr::CallExpr(ASTContext& C, StmtClass SC, Expr *fn, unsigned NumPreArgs,
+                   Expr **args, unsigned numargs, QualType t, ExprValueKind VK,
+                   SourceLocation rparenloc)
+  : Expr(SC, t, VK, OK_Ordinary,
+         fn->isTypeDependent(),
+         fn->isValueDependent(),
+         fn->containsUnexpandedParameterPack()),
     NumArgs(numargs) {
 
-  SubExprs = new (C) Stmt*[numargs+1];
+  SubExprs = new (C) Stmt*[numargs+PREARGS_START+NumPreArgs];
   SubExprs[FN] = fn;
-  for (unsigned i = 0; i != numargs; ++i)
-    SubExprs[i+ARGS_START] = args[i];
+  for (unsigned i = 0; i != numargs; ++i) {
+    if (args[i]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (args[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (args[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
+    SubExprs[i+PREARGS_START+NumPreArgs] = args[i];
+  }
 
+  CallExprBits.NumPreArgs = NumPreArgs;
   RParenLoc = rparenloc;
 }
 
 CallExpr::CallExpr(ASTContext& C, Expr *fn, Expr **args, unsigned numargs,
-                   QualType t, SourceLocation rparenloc)
-  : Expr(CallExprClass, t,
-         fn->isTypeDependent() || hasAnyTypeDependentArguments(args, numargs),
-         fn->isValueDependent() || hasAnyValueDependentArguments(args,numargs)),
+                   QualType t, ExprValueKind VK, SourceLocation rparenloc)
+  : Expr(CallExprClass, t, VK, OK_Ordinary,
+         fn->isTypeDependent(),
+         fn->isValueDependent(),
+         fn->containsUnexpandedParameterPack()),
     NumArgs(numargs) {
 
-  SubExprs = new (C) Stmt*[numargs+1];
+  SubExprs = new (C) Stmt*[numargs+PREARGS_START];
   SubExprs[FN] = fn;
-  for (unsigned i = 0; i != numargs; ++i)
-    SubExprs[i+ARGS_START] = args[i];
+  for (unsigned i = 0; i != numargs; ++i) {
+    if (args[i]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (args[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (args[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
+    SubExprs[i+PREARGS_START] = args[i];
+  }
 
+  CallExprBits.NumPreArgs = 0;
   RParenLoc = rparenloc;
 }
 
 CallExpr::CallExpr(ASTContext &C, StmtClass SC, EmptyShell Empty)
   : Expr(SC, Empty), SubExprs(0), NumArgs(0) {
-  SubExprs = new (C) Stmt*[1];
+  // FIXME: Why do we allocate this?
+  SubExprs = new (C) Stmt*[PREARGS_START];
+  CallExprBits.NumPreArgs = 0;
+}
+
+CallExpr::CallExpr(ASTContext &C, StmtClass SC, unsigned NumPreArgs,
+                   EmptyShell Empty)
+  : Expr(SC, Empty), SubExprs(0), NumArgs(0) {
+  // FIXME: Why do we allocate this?
+  SubExprs = new (C) Stmt*[PREARGS_START+NumPreArgs];
+  CallExprBits.NumPreArgs = NumPreArgs;
 }
 
 Decl *CallExpr::getCalleeDecl() {
   Expr *CEE = getCallee()->IgnoreParenCasts();
+  // If we're calling a dereference, look at the pointer instead.
+  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(CEE)) {
+    if (BO->isPtrMemOp())
+      CEE = BO->getRHS()->IgnoreParenCasts();
+  } else if (UnaryOperator *UO = dyn_cast<UnaryOperator>(CEE)) {
+    if (UO->getOpcode() == UO_Deref)
+      CEE = UO->getSubExpr()->IgnoreParenCasts();
+  }
   if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CEE))
     return DRE->getDecl();
   if (MemberExpr *ME = dyn_cast<MemberExpr>(CEE))
@@ -585,12 +783,14 @@ void CallExpr::setNumArgs(ASTContext& C, unsigned NumArgs) {
   }
 
   // Otherwise, we are growing the # arguments.  New an bigger argument array.
-  Stmt **NewSubExprs = new (C) Stmt*[NumArgs+1];
+  unsigned NumPreArgs = getNumPreArgs();
+  Stmt **NewSubExprs = new (C) Stmt*[NumArgs+PREARGS_START+NumPreArgs];
   // Copy over args.
-  for (unsigned i = 0; i != getNumArgs()+ARGS_START; ++i)
+  for (unsigned i = 0; i != getNumArgs()+PREARGS_START+NumPreArgs; ++i)
     NewSubExprs[i] = SubExprs[i];
   // Null out new args.
-  for (unsigned i = getNumArgs()+ARGS_START; i != NumArgs+ARGS_START; ++i)
+  for (unsigned i = getNumArgs()+PREARGS_START+NumPreArgs;
+       i != NumArgs+PREARGS_START+NumPreArgs; ++i)
     NewSubExprs[i] = 0;
 
   if (SubExprs) C.Deallocate(SubExprs);
@@ -600,7 +800,7 @@ void CallExpr::setNumArgs(ASTContext& C, unsigned NumArgs) {
 
 /// isBuiltinCall - If this is a call to a builtin, return the builtin ID.  If
 /// not, return 0.
-unsigned CallExpr::isBuiltinCall(ASTContext &Context) const {
+unsigned CallExpr::isBuiltinCall(const ASTContext &Context) const {
   // All simple function calls (e.g. func()) are implicitly cast to pointer to
   // function. As a result, we try and obtain the DeclRefExpr from the
   // ImplicitCastExpr.
@@ -663,10 +863,10 @@ OffsetOfExpr::OffsetOfExpr(ASTContext &C, QualType type,
                            OffsetOfNode* compsPtr, unsigned numComps, 
                            Expr** exprsPtr, unsigned numExprs,
                            SourceLocation RParenLoc)
-  : Expr(OffsetOfExprClass, type, /*TypeDependent=*/false, 
-         /*ValueDependent=*/tsi->getType()->isDependentType() ||
-         hasAnyTypeDependentArguments(exprsPtr, numExprs) ||
-         hasAnyValueDependentArguments(exprsPtr, numExprs)),
+  : Expr(OffsetOfExprClass, type, VK_RValue, OK_Ordinary,
+         /*TypeDependent=*/false, 
+         /*ValueDependent=*/tsi->getType()->isDependentType(),
+         tsi->getType()->containsUnexpandedParameterPack()),
     OperatorLoc(OperatorLoc), RParenLoc(RParenLoc), TSInfo(tsi), 
     NumComps(numComps), NumExprs(numExprs) 
 {
@@ -675,6 +875,11 @@ OffsetOfExpr::OffsetOfExpr(ASTContext &C, QualType type,
   }
   
   for(unsigned i = 0; i < numExprs; ++i) {
+    if (exprsPtr[i]->isTypeDependent() || exprsPtr[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (exprsPtr[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
     setIndexExpr(i, exprsPtr[i]);
   }
 }
@@ -694,7 +899,9 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
                                DeclAccessPair founddecl,
                                DeclarationNameInfo nameinfo,
                                const TemplateArgumentListInfo *targs,
-                               QualType ty) {
+                               QualType ty,
+                               ExprValueKind vk,
+                               ExprObjectKind ok) {
   std::size_t Size = sizeof(MemberExpr);
 
   bool hasQualOrFound = (qual != 0 ||
@@ -706,8 +913,9 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
   if (targs)
     Size += ExplicitTemplateArgumentList::sizeFor(*targs);
 
-  void *Mem = C.Allocate(Size, llvm::alignof<MemberExpr>());
-  MemberExpr *E = new (Mem) MemberExpr(base, isarrow, memberdecl, nameinfo, ty);
+  void *Mem = C.Allocate(Size, llvm::alignOf<MemberExpr>());
+  MemberExpr *E = new (Mem) MemberExpr(base, isarrow, memberdecl, nameinfo,
+                                       ty, vk, ok);
 
   if (hasQualOrFound) {
     if (qual && qual->isDependent()) {
@@ -732,12 +940,16 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
 
 const char *CastExpr::getCastKindName() const {
   switch (getCastKind()) {
-  case CK_Unknown:
-    return "Unknown";
+  case CK_Dependent:
+    return "Dependent";
   case CK_BitCast:
     return "BitCast";
   case CK_LValueBitCast:
     return "LValueBitCast";
+  case CK_LValueToRValue:
+    return "LValueToRValue";
+  case CK_GetObjCProperty:
+    return "GetObjCProperty";
   case CK_NoOp:
     return "NoOp";
   case CK_BaseToDerived:
@@ -756,6 +968,8 @@ const char *CastExpr::getCastKindName() const {
     return "FunctionToPointerDecay";
   case CK_NullToMemberPointer:
     return "NullToMemberPointer";
+  case CK_NullToPointer:
+    return "NullToPointer";
   case CK_BaseToDerivedMemberPointer:
     return "BaseToDerivedMemberPointer";
   case CK_DerivedToBaseMemberPointer:
@@ -768,18 +982,24 @@ const char *CastExpr::getCastKindName() const {
     return "IntegralToPointer";
   case CK_PointerToIntegral:
     return "PointerToIntegral";
+  case CK_PointerToBoolean:
+    return "PointerToBoolean";
   case CK_ToVoid:
     return "ToVoid";
   case CK_VectorSplat:
     return "VectorSplat";
   case CK_IntegralCast:
     return "IntegralCast";
+  case CK_IntegralToBoolean:
+    return "IntegralToBoolean";
   case CK_IntegralToFloating:
     return "IntegralToFloating";
   case CK_FloatingToIntegral:
     return "FloatingToIntegral";
   case CK_FloatingCast:
     return "FloatingCast";
+  case CK_FloatingToBoolean:
+    return "FloatingToBoolean";
   case CK_MemberPointerToBoolean:
     return "MemberPointerToBoolean";
   case CK_AnyPointerToObjCPointerCast:
@@ -788,9 +1008,29 @@ const char *CastExpr::getCastKindName() const {
     return "AnyPointerToBlockPointerCast";
   case CK_ObjCObjectLValueCast:
     return "ObjCObjectLValueCast";
+  case CK_FloatingRealToComplex:
+    return "FloatingRealToComplex";
+  case CK_FloatingComplexToReal:
+    return "FloatingComplexToReal";
+  case CK_FloatingComplexToBoolean:
+    return "FloatingComplexToBoolean";
+  case CK_FloatingComplexCast:
+    return "FloatingComplexCast";
+  case CK_FloatingComplexToIntegralComplex:
+    return "FloatingComplexToIntegralComplex";
+  case CK_IntegralRealToComplex:
+    return "IntegralRealToComplex";
+  case CK_IntegralComplexToReal:
+    return "IntegralComplexToReal";
+  case CK_IntegralComplexToBoolean:
+    return "IntegralComplexToBoolean";
+  case CK_IntegralComplexCast:
+    return "IntegralComplexCast";
+  case CK_IntegralComplexToFloatingComplex:
+    return "IntegralComplexToFloatingComplex";
   }
 
-  assert(0 && "Unhandled cast kind!");
+  llvm_unreachable("Unhandled cast kind!");
   return 0;
 }
 
@@ -859,7 +1099,7 @@ ImplicitCastExpr *ImplicitCastExpr::CreateEmpty(ASTContext &C,
 
 
 CStyleCastExpr *CStyleCastExpr::Create(ASTContext &C, QualType T,
-                                       CastKind K, Expr *Op,
+                                       ExprValueKind VK, CastKind K, Expr *Op,
                                        const CXXCastPath *BasePath,
                                        TypeSourceInfo *WrittenTy,
                                        SourceLocation L, SourceLocation R) {
@@ -867,7 +1107,7 @@ CStyleCastExpr *CStyleCastExpr::Create(ASTContext &C, QualType T,
   void *Buffer =
     C.Allocate(sizeof(CStyleCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
   CStyleCastExpr *E =
-    new (Buffer) CStyleCastExpr(T, K, Op, PathSize, WrittenTy, L, R);
+    new (Buffer) CStyleCastExpr(T, VK, K, Op, PathSize, WrittenTy, L, R);
   if (PathSize) E->setCastPath(*BasePath);
   return E;
 }
@@ -984,16 +1224,19 @@ OverloadedOperatorKind BinaryOperator::getOverloadedOperator(Opcode Opc) {
 InitListExpr::InitListExpr(ASTContext &C, SourceLocation lbraceloc,
                            Expr **initExprs, unsigned numInits,
                            SourceLocation rbraceloc)
-  : Expr(InitListExprClass, QualType(), false, false),
+  : Expr(InitListExprClass, QualType(), VK_RValue, OK_Ordinary, false, false,
+         false),
     InitExprs(C, numInits),
     LBraceLoc(lbraceloc), RBraceLoc(rbraceloc), SyntacticForm(0),
     UnionFieldInit(0), HadArrayRangeDesignator(false) 
 {      
   for (unsigned I = 0; I != numInits; ++I) {
     if (initExprs[I]->isTypeDependent())
-      TypeDependent = true;
+      ExprBits.TypeDependent = true;
     if (initExprs[I]->isValueDependent())
-      ValueDependent = true;
+      ExprBits.ValueDependent = true;
+    if (initExprs[I]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
   }
       
   InitExprs.insert(C, InitExprs.end(), initExprs, initExprs+numInits);
@@ -1020,6 +1263,35 @@ Expr *InitListExpr::updateInit(ASTContext &C, unsigned Init, Expr *expr) {
   return Result;
 }
 
+SourceRange InitListExpr::getSourceRange() const {
+  if (SyntacticForm)
+    return SyntacticForm->getSourceRange();
+  SourceLocation Beg = LBraceLoc, End = RBraceLoc;
+  if (Beg.isInvalid()) {
+    // Find the first non-null initializer.
+    for (InitExprsTy::const_iterator I = InitExprs.begin(),
+                                     E = InitExprs.end(); 
+      I != E; ++I) {
+      if (Stmt *S = *I) {
+        Beg = S->getLocStart();
+        break;
+      }  
+    }
+  }
+  if (End.isInvalid()) {
+    // Find the first non-null initializer from the end.
+    for (InitExprsTy::const_reverse_iterator I = InitExprs.rbegin(),
+                                             E = InitExprs.rend();
+      I != E; ++I) {
+      if (Stmt *S = *I) {
+        End = S->getSourceRange().getEnd();
+        break;
+      }  
+    }
+  }
+  return SourceRange(Beg, End);
+}
+
 /// getFunctionType - Return the underlying function type for this block.
 ///
 const FunctionType *BlockExpr::getFunctionType() const {
@@ -1195,21 +1467,11 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
     return false;
   }
 
-  case ObjCImplicitSetterGetterRefExprClass: {   // Dot syntax for message send.
-#if 0
-    const ObjCImplicitSetterGetterRefExpr *Ref =
-      cast<ObjCImplicitSetterGetterRefExpr>(this);
-    // FIXME: We really want the location of the '.' here.
-    Loc = Ref->getLocation();
-    R1 = SourceRange(Ref->getLocation(), Ref->getLocation());
-    if (Ref->getBase())
-      R2 = Ref->getBase()->getSourceRange();
-#else
+  case ObjCPropertyRefExprClass:
     Loc = getExprLoc();
     R1 = getSourceRange();
-#endif
     return true;
-  }
+
   case StmtExprClass: {
     // Statement exprs don't logically have side effects themselves, but are
     // sometimes used in macros in ways that give them a type that is unused.
@@ -1217,9 +1479,13 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
     // however, if the result of the stmt expr is dead, we don't want to emit a
     // warning.
     const CompoundStmt *CS = cast<StmtExpr>(this)->getSubStmt();
-    if (!CS->body_empty())
+    if (!CS->body_empty()) {
       if (const Expr *E = dyn_cast<Expr>(CS->body_back()))
         return E->isUnusedResultAWarning(Loc, R1, R2, Ctx);
+      if (const LabelStmt *Label = dyn_cast<LabelStmt>(CS->body_back()))
+        if (const Expr *E = dyn_cast<Expr>(Label->getSubStmt()))
+          return E->isUnusedResultAWarning(Loc, R1, R2, Ctx);
+    }
 
     if (getType()->isVoidType())
       return false;
@@ -1268,8 +1534,8 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
   case CXXBindTemporaryExprClass:
     return (cast<CXXBindTemporaryExpr>(this)
             ->getSubExpr()->isUnusedResultAWarning(Loc, R1, R2, Ctx));
-  case CXXExprWithTemporariesClass:
-    return (cast<CXXExprWithTemporaries>(this)
+  case ExprWithCleanupsClass:
+    return (cast<ExprWithCleanups>(this)
             ->getSubExpr()->isUnusedResultAWarning(Loc, R1, R2, Ctx));
   }
 }
@@ -1311,12 +1577,254 @@ bool Expr::isOBJCGCCandidate(ASTContext &Ctx) const {
     return cast<ArraySubscriptExpr>(this)->getBase()->isOBJCGCCandidate(Ctx);
   }
 }
+
+bool Expr::isBoundMemberFunction(ASTContext &Ctx) const {
+  if (isTypeDependent())
+    return false;
+  return ClassifyLValue(Ctx) == Expr::LV_MemberFunction;
+}
+
+static Expr::CanThrowResult MergeCanThrow(Expr::CanThrowResult CT1,
+                                          Expr::CanThrowResult CT2) {
+  // CanThrowResult constants are ordered so that the maximum is the correct
+  // merge result.
+  return CT1 > CT2 ? CT1 : CT2;
+}
+
+static Expr::CanThrowResult CanSubExprsThrow(ASTContext &C, const Expr *CE) {
+  Expr *E = const_cast<Expr*>(CE);
+  Expr::CanThrowResult R = Expr::CT_Cannot;
+  for (Expr::child_range I = E->children(); I && R != Expr::CT_Can; ++I) {
+    R = MergeCanThrow(R, cast<Expr>(*I)->CanThrow(C));
+  }
+  return R;
+}
+
+static Expr::CanThrowResult CanCalleeThrow(const Decl *D,
+                                           bool NullThrows = true) {
+  if (!D)
+    return NullThrows ? Expr::CT_Can : Expr::CT_Cannot;
+
+  // See if we can get a function type from the decl somehow.
+  const ValueDecl *VD = dyn_cast<ValueDecl>(D);
+  if (!VD) // If we have no clue what we're calling, assume the worst.
+    return Expr::CT_Can;
+
+  // As an extension, we assume that __attribute__((nothrow)) functions don't
+  // throw.
+  if (isa<FunctionDecl>(D) && D->hasAttr<NoThrowAttr>())
+    return Expr::CT_Cannot;
+
+  QualType T = VD->getType();
+  const FunctionProtoType *FT;
+  if ((FT = T->getAs<FunctionProtoType>())) {
+  } else if (const PointerType *PT = T->getAs<PointerType>())
+    FT = PT->getPointeeType()->getAs<FunctionProtoType>();
+  else if (const ReferenceType *RT = T->getAs<ReferenceType>())
+    FT = RT->getPointeeType()->getAs<FunctionProtoType>();
+  else if (const MemberPointerType *MT = T->getAs<MemberPointerType>())
+    FT = MT->getPointeeType()->getAs<FunctionProtoType>();
+  else if (const BlockPointerType *BT = T->getAs<BlockPointerType>())
+    FT = BT->getPointeeType()->getAs<FunctionProtoType>();
+
+  if (!FT)
+    return Expr::CT_Can;
+
+  return FT->hasEmptyExceptionSpec() ? Expr::CT_Cannot : Expr::CT_Can;
+}
+
+static Expr::CanThrowResult CanDynamicCastThrow(const CXXDynamicCastExpr *DC) {
+  if (DC->isTypeDependent())
+    return Expr::CT_Dependent;
+
+  if (!DC->getTypeAsWritten()->isReferenceType())
+    return Expr::CT_Cannot;
+
+  return DC->getCastKind() == clang::CK_Dynamic? Expr::CT_Can : Expr::CT_Cannot;
+}
+
+static Expr::CanThrowResult CanTypeidThrow(ASTContext &C,
+                                           const CXXTypeidExpr *DC) {
+  if (DC->isTypeOperand())
+    return Expr::CT_Cannot;
+
+  Expr *Op = DC->getExprOperand();
+  if (Op->isTypeDependent())
+    return Expr::CT_Dependent;
+
+  const RecordType *RT = Op->getType()->getAs<RecordType>();
+  if (!RT)
+    return Expr::CT_Cannot;
+
+  if (!cast<CXXRecordDecl>(RT->getDecl())->isPolymorphic())
+    return Expr::CT_Cannot;
+
+  if (Op->Classify(C).isPRValue())
+    return Expr::CT_Cannot;
+
+  return Expr::CT_Can;
+}
+
+Expr::CanThrowResult Expr::CanThrow(ASTContext &C) const {
+  // C++ [expr.unary.noexcept]p3:
+  //   [Can throw] if in a potentially-evaluated context the expression would
+  //   contain:
+  switch (getStmtClass()) {
+  case CXXThrowExprClass:
+    //   - a potentially evaluated throw-expression
+    return CT_Can;
+
+  case CXXDynamicCastExprClass: {
+    //   - a potentially evaluated dynamic_cast expression dynamic_cast<T>(v),
+    //     where T is a reference type, that requires a run-time check
+    CanThrowResult CT = CanDynamicCastThrow(cast<CXXDynamicCastExpr>(this));
+    if (CT == CT_Can)
+      return CT;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+  case CXXTypeidExprClass:
+    //   - a potentially evaluated typeid expression applied to a glvalue
+    //     expression whose type is a polymorphic class type
+    return CanTypeidThrow(C, cast<CXXTypeidExpr>(this));
+
+    //   - a potentially evaluated call to a function, member function, function
+    //     pointer, or member function pointer that does not have a non-throwing
+    //     exception-specification
+  case CallExprClass:
+  case CXXOperatorCallExprClass:
+  case CXXMemberCallExprClass: {
+    CanThrowResult CT = CanCalleeThrow(cast<CallExpr>(this)->getCalleeDecl());
+    if (CT == CT_Can)
+      return CT;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+  case CXXConstructExprClass:
+  case CXXTemporaryObjectExprClass: {
+    CanThrowResult CT = CanCalleeThrow(
+        cast<CXXConstructExpr>(this)->getConstructor());
+    if (CT == CT_Can)
+      return CT;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+  case CXXNewExprClass: {
+    CanThrowResult CT = MergeCanThrow(
+        CanCalleeThrow(cast<CXXNewExpr>(this)->getOperatorNew()),
+        CanCalleeThrow(cast<CXXNewExpr>(this)->getConstructor(),
+                       /*NullThrows*/false));
+    if (CT == CT_Can)
+      return CT;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+  case CXXDeleteExprClass: {
+    CanThrowResult CT = CanCalleeThrow(
+        cast<CXXDeleteExpr>(this)->getOperatorDelete());
+    if (CT == CT_Can)
+      return CT;
+    const Expr *Arg = cast<CXXDeleteExpr>(this)->getArgument();
+    // Unwrap exactly one implicit cast, which converts all pointers to void*.
+    if (const ImplicitCastExpr *Cast = dyn_cast<ImplicitCastExpr>(Arg))
+      Arg = Cast->getSubExpr();
+    if (const PointerType *PT = Arg->getType()->getAs<PointerType>()) {
+      if (const RecordType *RT = PT->getPointeeType()->getAs<RecordType>()) {
+        CanThrowResult CT2 = CanCalleeThrow(
+            cast<CXXRecordDecl>(RT->getDecl())->getDestructor());
+        if (CT2 == CT_Can)
+          return CT2;
+        CT = MergeCanThrow(CT, CT2);
+      }
+    }
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+  case CXXBindTemporaryExprClass: {
+    // The bound temporary has to be destroyed again, which might throw.
+    CanThrowResult CT = CanCalleeThrow(
+      cast<CXXBindTemporaryExpr>(this)->getTemporary()->getDestructor());
+    if (CT == CT_Can)
+      return CT;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+    // ObjC message sends are like function calls, but never have exception
+    // specs.
+  case ObjCMessageExprClass:
+  case ObjCPropertyRefExprClass:
+    return CT_Can;
+
+    // Many other things have subexpressions, so we have to test those.
+    // Some are simple:
+  case ParenExprClass:
+  case MemberExprClass:
+  case CXXReinterpretCastExprClass:
+  case CXXConstCastExprClass:
+  case ConditionalOperatorClass:
+  case CompoundLiteralExprClass:
+  case ExtVectorElementExprClass:
+  case InitListExprClass:
+  case DesignatedInitExprClass:
+  case ParenListExprClass:
+  case VAArgExprClass:
+  case CXXDefaultArgExprClass:
+  case ExprWithCleanupsClass:
+  case ObjCIvarRefExprClass:
+  case ObjCIsaExprClass:
+  case ShuffleVectorExprClass:
+    return CanSubExprsThrow(C, this);
+
+    // Some might be dependent for other reasons.
+  case UnaryOperatorClass:
+  case ArraySubscriptExprClass:
+  case ImplicitCastExprClass:
+  case CStyleCastExprClass:
+  case CXXStaticCastExprClass:
+  case CXXFunctionalCastExprClass:
+  case BinaryOperatorClass:
+  case CompoundAssignOperatorClass: {
+    CanThrowResult CT = isTypeDependent() ? CT_Dependent : CT_Cannot;
+    return MergeCanThrow(CT, CanSubExprsThrow(C, this));
+  }
+
+    // FIXME: We should handle StmtExpr, but that opens a MASSIVE can of worms.
+  case StmtExprClass:
+    return CT_Can;
+
+  case ChooseExprClass:
+    if (isTypeDependent() || isValueDependent())
+      return CT_Dependent;
+    return cast<ChooseExpr>(this)->getChosenSubExpr(C)->CanThrow(C);
+
+    // Some expressions are always dependent.
+  case DependentScopeDeclRefExprClass:
+  case CXXUnresolvedConstructExprClass:
+  case CXXDependentScopeMemberExprClass:
+    return CT_Dependent;
+
+  default:
+    // All other expressions don't have subexpressions, or else they are
+    // unevaluated.
+    return CT_Cannot;
+  }
+}
+
 Expr* Expr::IgnoreParens() {
   Expr* E = this;
-  while (ParenExpr* P = dyn_cast<ParenExpr>(E))
-    E = P->getSubExpr();
-
-  return E;
+  while (true) {
+    if (ParenExpr* P = dyn_cast<ParenExpr>(E)) {
+      E = P->getSubExpr();
+      continue;
+    }
+    if (UnaryOperator* P = dyn_cast<UnaryOperator>(E)) {
+      if (P->getOpcode() == UO_Extension) {
+        E = P->getSubExpr();
+        continue;
+      }
+    }
+    return E;
+  }
 }
 
 /// IgnoreParenCasts - Ignore parentheses and casts.  Strip off any ParenExpr
@@ -1324,24 +1832,68 @@ Expr* Expr::IgnoreParens() {
 Expr *Expr::IgnoreParenCasts() {
   Expr *E = this;
   while (true) {
-    if (ParenExpr *P = dyn_cast<ParenExpr>(E))
+    if (ParenExpr* P = dyn_cast<ParenExpr>(E)) {
       E = P->getSubExpr();
-    else if (CastExpr *P = dyn_cast<CastExpr>(E))
+      continue;
+    }
+    if (CastExpr *P = dyn_cast<CastExpr>(E)) {
       E = P->getSubExpr();
-    else
-      return E;
+      continue;
+    }
+    if (UnaryOperator* P = dyn_cast<UnaryOperator>(E)) {
+      if (P->getOpcode() == UO_Extension) {
+        E = P->getSubExpr();
+        continue;
+      }
+    }
+    return E;
   }
 }
 
+/// IgnoreParenLValueCasts - Ignore parentheses and lvalue-to-rvalue
+/// casts.  This is intended purely as a temporary workaround for code
+/// that hasn't yet been rewritten to do the right thing about those
+/// casts, and may disappear along with the last internal use.
+Expr *Expr::IgnoreParenLValueCasts() {
+  Expr *E = this;
+  while (true) {
+    if (ParenExpr *P = dyn_cast<ParenExpr>(E)) {
+      E = P->getSubExpr();
+      continue;
+    } else if (CastExpr *P = dyn_cast<CastExpr>(E)) {
+      if (P->getCastKind() == CK_LValueToRValue) {
+        E = P->getSubExpr();
+        continue;
+      }
+    } else if (UnaryOperator* P = dyn_cast<UnaryOperator>(E)) {
+      if (P->getOpcode() == UO_Extension) {
+        E = P->getSubExpr();
+        continue;
+      }
+    }
+    break;
+  }
+  return E;
+}
+  
 Expr *Expr::IgnoreParenImpCasts() {
   Expr *E = this;
   while (true) {
-    if (ParenExpr *P = dyn_cast<ParenExpr>(E))
+    if (ParenExpr *P = dyn_cast<ParenExpr>(E)) {
       E = P->getSubExpr();
-    else if (ImplicitCastExpr *P = dyn_cast<ImplicitCastExpr>(E))
+      continue;
+    }
+    if (ImplicitCastExpr *P = dyn_cast<ImplicitCastExpr>(E)) {
       E = P->getSubExpr();
-    else
-      return E;
+      continue;
+    }
+    if (UnaryOperator* P = dyn_cast<UnaryOperator>(E)) {
+      if (P->getOpcode() == UO_Extension) {
+        E = P->getSubExpr();
+        continue;
+      }
+    }
+    return E;
   }
 }
 
@@ -1366,9 +1918,9 @@ Expr *Expr::IgnoreParenNoopCasts(ASTContext &Ctx) {
         continue;
       }
 
-      if ((E->getType()->isPointerType() || 
+      if ((E->getType()->isPointerType() ||
            E->getType()->isIntegralType(Ctx)) &&
-          (SE->getType()->isPointerType() || 
+          (SE->getType()->isPointerType() ||
            SE->getType()->isIntegralType(Ctx)) &&
           Ctx.getTypeSize(E->getType()) == Ctx.getTypeSize(SE->getType())) {
         E = SE;
@@ -1376,6 +1928,13 @@ Expr *Expr::IgnoreParenNoopCasts(ASTContext &Ctx) {
       }
     }
 
+    if (UnaryOperator* P = dyn_cast<UnaryOperator>(E)) {
+      if (P->getOpcode() == UO_Extension) {
+        E = P->getSubExpr();
+        continue;
+      }
+    }
+
     return E;
   }
 }
@@ -1390,7 +1949,7 @@ bool Expr::isDefaultArgument() const {
 
 /// \brief Skip over any no-op casts and any temporary-binding
 /// expressions.
-static const Expr *skipTemporaryBindingsAndNoOpCasts(const Expr *E) {
+static const Expr *skipTemporaryBindingsNoOpCastsAndParens(const Expr *E) {
   while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
     if (ICE->getCastKind() == CK_NoOp)
       E = ICE->getSubExpr();
@@ -1407,50 +1966,48 @@ static const Expr *skipTemporaryBindingsAndNoOpCasts(const Expr *E) {
     else
       break;
   }
-  
-  return E;
-}
 
-const Expr *Expr::getTemporaryObject() const {
-  const Expr *E = skipTemporaryBindingsAndNoOpCasts(this);
+  return E->IgnoreParens();
+}
 
-  // A cast can produce a temporary object. The object's construction
-  // is represented as a CXXConstructExpr.
-  if (const CastExpr *Cast = dyn_cast<CastExpr>(E)) {
-    // Only user-defined and constructor conversions can produce
-    // temporary objects.
-    if (Cast->getCastKind() != CK_ConstructorConversion &&
-        Cast->getCastKind() != CK_UserDefinedConversion)
-      return 0;
+/// isTemporaryObject - Determines if this expression produces a
+/// temporary of the given class type.
+bool Expr::isTemporaryObject(ASTContext &C, const CXXRecordDecl *TempTy) const {
+  if (!C.hasSameUnqualifiedType(getType(), C.getTypeDeclType(TempTy)))
+    return false;
 
-    // Strip off temporary bindings and no-op casts.
-    const Expr *Sub = skipTemporaryBindingsAndNoOpCasts(Cast->getSubExpr());
+  const Expr *E = skipTemporaryBindingsNoOpCastsAndParens(this);
 
-    // If this is a constructor conversion, see if we have an object
-    // construction.
-    if (Cast->getCastKind() == CK_ConstructorConversion)
-      return dyn_cast<CXXConstructExpr>(Sub);
+  // Temporaries are by definition pr-values of class type.
+  if (!E->Classify(C).isPRValue()) {
+    // In this context, property reference is a message call and is pr-value.
+    if (!isa<ObjCPropertyRefExpr>(E))
+      return false;
+  }
 
-    // If this is a user-defined conversion, see if we have a call to
-    // a function that itself returns a temporary object.
-    if (Cast->getCastKind() == CK_UserDefinedConversion)
-      if (const CallExpr *CE = dyn_cast<CallExpr>(Sub))
-        if (CE->getCallReturnType()->isRecordType())
-          return CE;
+  // Black-list a few cases which yield pr-values of class type that don't
+  // refer to temporaries of that type:
 
-    return 0;
+  // - implicit derived-to-base conversions
+  if (isa<ImplicitCastExpr>(E)) {
+    switch (cast<ImplicitCastExpr>(E)->getCastKind()) {
+    case CK_DerivedToBase:
+    case CK_UncheckedDerivedToBase:
+      return false;
+    default:
+      break;
+    }
   }
 
-  // A call returning a class type returns a temporary.
-  if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
-    if (CE->getCallReturnType()->isRecordType())
-      return CE;
+  // - member expressions (all)
+  if (isa<MemberExpr>(E))
+    return false;
 
-    return 0;
-  }
+  // - opaque values (all)
+  if (isa<OpaqueValueExpr>(E))
+    return false;
 
-  // Explicit temporary object constructors create temporaries.
-  return dyn_cast<CXXTemporaryObjectExpr>(E);
+  return true;
 }
 
 /// hasAnyTypeDependentArguments - Determines if any of the expressions
@@ -1533,6 +2090,9 @@ bool Expr::isConstantInitializer(ASTContext &Ctx, bool IsForRef) const {
   case ParenExprClass:
     return cast<ParenExpr>(this)->getSubExpr()
       ->isConstantInitializer(Ctx, IsForRef);
+  case ChooseExprClass:
+    return cast<ChooseExpr>(this)->getChosenSubExpr(Ctx)
+      ->isConstantInitializer(Ctx, IsForRef);
   case UnaryOperatorClass: {
     const UnaryOperator* Exp = cast<UnaryOperator>(this);
     if (Exp->getOpcode() == UO_Extension)
@@ -1572,11 +2132,14 @@ bool Expr::isConstantInitializer(ASTContext &Ctx, bool IsForRef) const {
   return isEvaluatable(Ctx);
 }
 
-/// isNullPointerConstant - C99 6.3.2.3p3 -  Return true if this is either an
-/// integer constant expression with the value zero, or if this is one that is
-/// cast to void*.
-bool Expr::isNullPointerConstant(ASTContext &Ctx,
-                                 NullPointerConstantValueDependence NPC) const {
+/// isNullPointerConstant - C99 6.3.2.3p3 - Return whether this is a null 
+/// pointer constant or not, as well as the specific kind of constant detected.
+/// Null pointer constants can be integer constant expressions with the
+/// value zero, casts of zero to void*, nullptr (C++0X), or __null
+/// (a GNU extension).
+Expr::NullPointerConstantKind
+Expr::isNullPointerConstant(ASTContext &Ctx,
+                            NullPointerConstantValueDependence NPC) const {
   if (isValueDependent()) {
     switch (NPC) {
     case NPC_NeverValueDependent:
@@ -1584,10 +2147,13 @@ bool Expr::isNullPointerConstant(ASTContext &Ctx,
       // If the unthinkable happens, fall through to the safest alternative.
         
     case NPC_ValueDependentIsNull:
-      return isTypeDependent() || getType()->isIntegralType(Ctx);
+      if (isTypeDependent() || getType()->isIntegralType(Ctx))
+        return NPCK_ZeroInteger;
+      else
+        return NPCK_NotNull;
         
     case NPC_ValueDependentIsNotNull:
-      return false;
+      return NPCK_NotNull;
     }
   }
 
@@ -1616,30 +2182,61 @@ bool Expr::isNullPointerConstant(ASTContext &Ctx,
     return DefaultArg->getExpr()->isNullPointerConstant(Ctx, NPC);
   } else if (isa<GNUNullExpr>(this)) {
     // The GNU __null extension is always a null pointer constant.
-    return true;
+    return NPCK_GNUNull;
   }
 
   // C++0x nullptr_t is always a null pointer constant.
   if (getType()->isNullPtrType())
-    return true;
-
+    return NPCK_CXX0X_nullptr;
+
+  if (const RecordType *UT = getType()->getAsUnionType())
+    if (UT && UT->getDecl()->hasAttr<TransparentUnionAttr>())
+      if (const CompoundLiteralExpr *CLE = dyn_cast<CompoundLiteralExpr>(this)){
+        const Expr *InitExpr = CLE->getInitializer();
+        if (const InitListExpr *ILE = dyn_cast<InitListExpr>(InitExpr))
+          return ILE->getInit(0)->isNullPointerConstant(Ctx, NPC);
+      }
   // This expression must be an integer type.
   if (!getType()->isIntegerType() || 
       (Ctx.getLangOptions().CPlusPlus && getType()->isEnumeralType()))
-    return false;
+    return NPCK_NotNull;
 
   // If we have an integer constant expression, we need to *evaluate* it and
   // test for the value 0.
   llvm::APSInt Result;
-  return isIntegerConstantExpr(Result, Ctx) && Result == 0;
+  bool IsNull = isIntegerConstantExpr(Result, Ctx) && Result == 0;
+
+  return (IsNull ? NPCK_ZeroInteger : NPCK_NotNull);
+}
+
+/// \brief If this expression is an l-value for an Objective C
+/// property, find the underlying property reference expression.
+const ObjCPropertyRefExpr *Expr::getObjCProperty() const {
+  const Expr *E = this;
+  while (true) {
+    assert((E->getValueKind() == VK_LValue &&
+            E->getObjectKind() == OK_ObjCProperty) &&
+           "expression is not a property reference");
+    E = E->IgnoreParenCasts();
+    if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
+      if (BO->getOpcode() == BO_Comma) {
+        E = BO->getRHS();
+        continue;
+      }
+    }
+
+    break;
+  }
+
+  return cast<ObjCPropertyRefExpr>(E);
 }
 
 FieldDecl *Expr::getBitField() {
   Expr *E = this->IgnoreParens();
 
   while (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (ICE->getValueKind() != VK_RValue &&
-        ICE->getCastKind() == CK_NoOp)
+    if (ICE->getCastKind() == CK_LValueToRValue ||
+        (ICE->getValueKind() != VK_RValue && ICE->getCastKind() == CK_NoOp))
       E = ICE->getSubExpr()->IgnoreParens();
     else
       break;
@@ -1650,6 +2247,11 @@ FieldDecl *Expr::getBitField() {
       if (Field->isBitField())
         return Field;
 
+  if (DeclRefExpr *DeclRef = dyn_cast<DeclRefExpr>(E))
+    if (FieldDecl *Field = dyn_cast<FieldDecl>(DeclRef->getDecl()))
+      if (Field->isBitField())
+        return Field;
+
   if (BinaryOperator *BinOp = dyn_cast<BinaryOperator>(E))
     if (BinOp->isAssignmentOp() && BinOp->getLHS())
       return BinOp->getLHS()->getBitField();
@@ -1741,21 +2343,24 @@ void ExtVectorElementExpr::getEncodedElementAccess(
 }
 
 ObjCMessageExpr::ObjCMessageExpr(QualType T,
+                                 ExprValueKind VK,
                                  SourceLocation LBracLoc,
                                  SourceLocation SuperLoc,
                                  bool IsInstanceSuper,
                                  QualType SuperType,
                                  Selector Sel, 
+                                 SourceLocation SelLoc,
                                  ObjCMethodDecl *Method,
                                  Expr **Args, unsigned NumArgs,
                                  SourceLocation RBracLoc)
-  : Expr(ObjCMessageExprClass, T, /*TypeDependent=*/false,
-         /*ValueDependent=*/false),
+  : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary,
+         /*TypeDependent=*/false, /*ValueDependent=*/false,
+         /*ContainsUnexpandedParameterPack=*/false),
     NumArgs(NumArgs), Kind(IsInstanceSuper? SuperInstance : SuperClass),
     HasMethod(Method != 0), SuperLoc(SuperLoc),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
   setReceiverPointer(SuperType.getAsOpaquePtr());
   if (NumArgs)
@@ -1763,88 +2368,115 @@ ObjCMessageExpr::ObjCMessageExpr(QualType T,
 }
 
 ObjCMessageExpr::ObjCMessageExpr(QualType T,
+                                 ExprValueKind VK,
                                  SourceLocation LBracLoc,
                                  TypeSourceInfo *Receiver,
-                                 Selector Sel, 
+                                 Selector Sel,
+                                 SourceLocation SelLoc,
                                  ObjCMethodDecl *Method,
                                  Expr **Args, unsigned NumArgs,
                                  SourceLocation RBracLoc)
-  : Expr(ObjCMessageExprClass, T, T->isDependentType(),
-         (T->isDependentType() || 
-          hasAnyValueDependentArguments(Args, NumArgs))),
+  : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary, T->isDependentType(),
+         T->isDependentType(), T->containsUnexpandedParameterPack()),
     NumArgs(NumArgs), Kind(Class), HasMethod(Method != 0),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
   setReceiverPointer(Receiver);
-  if (NumArgs)
-    memcpy(getArgs(), Args, NumArgs * sizeof(Expr *));
+  Expr **MyArgs = getArgs();
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    if (Args[I]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (Args[I]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (Args[I]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+  
+    MyArgs[I] = Args[I];
+  }
 }
 
 ObjCMessageExpr::ObjCMessageExpr(QualType T,
+                                 ExprValueKind VK,
                                  SourceLocation LBracLoc,
                                  Expr *Receiver,
                                  Selector Sel, 
+                                 SourceLocation SelLoc,
                                  ObjCMethodDecl *Method,
                                  Expr **Args, unsigned NumArgs,
                                  SourceLocation RBracLoc)
-  : Expr(ObjCMessageExprClass, T, Receiver->isTypeDependent(),
-         (Receiver->isTypeDependent() || 
-          hasAnyValueDependentArguments(Args, NumArgs))),
+  : Expr(ObjCMessageExprClass, T, VK, OK_Ordinary, Receiver->isTypeDependent(),
+         Receiver->isTypeDependent(),
+         Receiver->containsUnexpandedParameterPack()),
     NumArgs(NumArgs), Kind(Instance), HasMethod(Method != 0),
     SelectorOrMethod(reinterpret_cast<uintptr_t>(Method? Method
                                                        : Sel.getAsOpaquePtr())),
-    LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
+    SelectorLoc(SelLoc), LBracLoc(LBracLoc), RBracLoc(RBracLoc) 
 {
   setReceiverPointer(Receiver);
-  if (NumArgs)
-    memcpy(getArgs(), Args, NumArgs * sizeof(Expr *));
+  Expr **MyArgs = getArgs();
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    if (Args[I]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (Args[I]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (Args[I]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+  
+    MyArgs[I] = Args[I];
+  }
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
+                                         ExprValueKind VK,
                                          SourceLocation LBracLoc,
                                          SourceLocation SuperLoc,
                                          bool IsInstanceSuper,
                                          QualType SuperType,
                                          Selector Sel, 
+                                         SourceLocation SelLoc,
                                          ObjCMethodDecl *Method,
                                          Expr **Args, unsigned NumArgs,
                                          SourceLocation RBracLoc) {
   unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
     NumArgs * sizeof(Expr *);
   void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
-  return new (Mem) ObjCMessageExpr(T, LBracLoc, SuperLoc, IsInstanceSuper,
-                                   SuperType, Sel, Method, Args, NumArgs, 
+  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, SuperLoc, IsInstanceSuper,
+                                   SuperType, Sel, SelLoc, Method, Args,NumArgs, 
                                    RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
+                                         ExprValueKind VK,
                                          SourceLocation LBracLoc,
                                          TypeSourceInfo *Receiver,
                                          Selector Sel, 
+                                         SourceLocation SelLoc,
                                          ObjCMethodDecl *Method,
                                          Expr **Args, unsigned NumArgs,
                                          SourceLocation RBracLoc) {
   unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
     NumArgs * sizeof(Expr *);
   void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
-  return new (Mem) ObjCMessageExpr(T, LBracLoc, Receiver, Sel, Method, Args, 
-                                   NumArgs, RBracLoc);
+  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel, SelLoc,
+                                   Method, Args, NumArgs, RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::Create(ASTContext &Context, QualType T,
+                                         ExprValueKind VK,
                                          SourceLocation LBracLoc,
                                          Expr *Receiver,
-                                         Selector Sel, 
+                                         Selector Sel,
+                                         SourceLocation SelLoc,
                                          ObjCMethodDecl *Method,
                                          Expr **Args, unsigned NumArgs,
                                          SourceLocation RBracLoc) {
   unsigned Size = sizeof(ObjCMessageExpr) + sizeof(void *) + 
     NumArgs * sizeof(Expr *);
   void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
-  return new (Mem) ObjCMessageExpr(T, LBracLoc, Receiver, Sel, Method, Args, 
-                                   NumArgs, RBracLoc);
+  return new (Mem) ObjCMessageExpr(T, VK, LBracLoc, Receiver, Sel, SelLoc,
+                                   Method, Args, NumArgs, RBracLoc);
 }
 
 ObjCMessageExpr *ObjCMessageExpr::CreateEmpty(ASTContext &Context, 
@@ -1854,7 +2486,23 @@ ObjCMessageExpr *ObjCMessageExpr::CreateEmpty(ASTContext &Context,
   void *Mem = Context.Allocate(Size, llvm::AlignOf<ObjCMessageExpr>::Alignment);
   return new (Mem) ObjCMessageExpr(EmptyShell(), NumArgs);
 }
-         
+
+SourceRange ObjCMessageExpr::getReceiverRange() const {
+  switch (getReceiverKind()) {
+  case Instance:
+    return getInstanceReceiver()->getSourceRange();
+
+  case Class:
+    return getClassReceiverTypeInfo()->getTypeLoc().getSourceRange();
+
+  case SuperInstance:
+  case SuperClass:
+    return getSuperLoc();
+  }
+
+  return SourceLocation();
+}
+
 Selector ObjCMessageExpr::getSelector() const {
   if (HasMethod)
     return reinterpret_cast<const ObjCMethodDecl *>(SelectorOrMethod)
@@ -1883,19 +2531,40 @@ ObjCInterfaceDecl *ObjCMessageExpr::getReceiverInterface() const {
     break;
 
   case SuperClass:
-    if (const ObjCObjectPointerType *Iface
-                       = getSuperType()->getAs<ObjCObjectPointerType>())
-      return Iface->getInterfaceDecl();
+    if (const ObjCObjectType *Iface
+          = getSuperType()->getAs<ObjCObjectType>())
+      return Iface->getInterface();
     break;
   }
 
   return 0;
 }
 
-bool ChooseExpr::isConditionTrue(ASTContext &C) const {
+bool ChooseExpr::isConditionTrue(const ASTContext &C) const {
   return getCond()->EvaluateAsInt(C) != 0;
 }
 
+ShuffleVectorExpr::ShuffleVectorExpr(ASTContext &C, Expr **args, unsigned nexpr,
+                                     QualType Type, SourceLocation BLoc,
+                                     SourceLocation RP) 
+   : Expr(ShuffleVectorExprClass, Type, VK_RValue, OK_Ordinary,
+          Type->isDependentType(), Type->isDependentType(),
+          Type->containsUnexpandedParameterPack()),
+     BuiltinLoc(BLoc), RParenLoc(RP), NumExprs(nexpr) 
+{
+  SubExprs = new (C) Stmt*[nexpr];
+  for (unsigned i = 0; i < nexpr; i++) {
+    if (args[i]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (args[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (args[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
+    SubExprs[i] = args[i];
+  }
+}
+
 void ShuffleVectorExpr::setExprs(ASTContext &C, Expr ** Exprs,
                                  unsigned NumExprs) {
   if (SubExprs) C.Deallocate(SubExprs);
@@ -1926,13 +2595,15 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
                                        unsigned NumIndexExprs,
                                        Expr *Init)
   : Expr(DesignatedInitExprClass, Ty,
-         Init->isTypeDependent(), Init->isValueDependent()),
+         Init->getValueKind(), Init->getObjectKind(),
+         Init->isTypeDependent(), Init->isValueDependent(),
+         Init->containsUnexpandedParameterPack()),
     EqualOrColonLoc(EqualOrColonLoc), GNUSyntax(GNUSyntax),
     NumDesignators(NumDesignators), NumSubExprs(NumIndexExprs + 1) {
   this->Designators = new (C) Designator[NumDesignators];
 
   // Record the initializer itself.
-  child_iterator Child = child_begin();
+  child_range Child = children();
   *Child++ = Init;
 
   // Copy the designators and their subexpressions, computing
@@ -1944,8 +2615,12 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
     if (this->Designators[I].isArrayDesignator()) {
       // Compute type- and value-dependence.
       Expr *Index = IndexExprs[IndexIdx];
-      ValueDependent = ValueDependent ||
-        Index->isTypeDependent() || Index->isValueDependent();
+      if (Index->isTypeDependent() || Index->isValueDependent())
+        ExprBits.ValueDependent = true;
+
+      // Propagate unexpanded parameter packs.
+      if (Index->containsUnexpandedParameterPack())
+        ExprBits.ContainsUnexpandedParameterPack = true;
 
       // Copy the index expressions into permanent storage.
       *Child++ = IndexExprs[IndexIdx++];
@@ -1953,9 +2628,14 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
       // Compute type- and value-dependence.
       Expr *Start = IndexExprs[IndexIdx];
       Expr *End = IndexExprs[IndexIdx + 1];
-      ValueDependent = ValueDependent ||
-        Start->isTypeDependent() || Start->isValueDependent() ||
-        End->isTypeDependent() || End->isValueDependent();
+      if (Start->isTypeDependent() || Start->isValueDependent() ||
+          End->isTypeDependent() || End->isValueDependent())
+        ExprBits.ValueDependent = true;
+
+      // Propagate unexpanded parameter packs.
+      if (Start->containsUnexpandedParameterPack() ||
+          End->containsUnexpandedParameterPack())
+        ExprBits.ContainsUnexpandedParameterPack = true;
 
       // Copy the start/end expressions into permanent storage.
       *Child++ = IndexExprs[IndexIdx++];
@@ -2066,14 +2746,30 @@ void DesignatedInitExpr::ExpandDesignator(ASTContext &C, unsigned Idx,
 ParenListExpr::ParenListExpr(ASTContext& C, SourceLocation lparenloc,
                              Expr **exprs, unsigned nexprs,
                              SourceLocation rparenloc)
-: Expr(ParenListExprClass, QualType(),
-       hasAnyTypeDependentArguments(exprs, nexprs),
-       hasAnyValueDependentArguments(exprs, nexprs)),
-  NumExprs(nexprs), LParenLoc(lparenloc), RParenLoc(rparenloc) {
+  : Expr(ParenListExprClass, QualType(), VK_RValue, OK_Ordinary,
+         false, false, false),
+    NumExprs(nexprs), LParenLoc(lparenloc), RParenLoc(rparenloc) {
 
   Exprs = new (C) Stmt*[nexprs];
-  for (unsigned i = 0; i != nexprs; ++i)
+  for (unsigned i = 0; i != nexprs; ++i) {
+    if (exprs[i]->isTypeDependent())
+      ExprBits.TypeDependent = true;
+    if (exprs[i]->isValueDependent())
+      ExprBits.ValueDependent = true;
+    if (exprs[i]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
     Exprs[i] = exprs[i];
+  }
+}
+
+const OpaqueValueExpr *OpaqueValueExpr::findInCopyConstruct(const Expr *e) {
+  if (const ExprWithCleanups *ewc = dyn_cast<ExprWithCleanups>(e))
+    e = ewc->getSubExpr();
+  e = cast<CXXConstructExpr>(e)->getArg(0);
+  while (const ImplicitCastExpr *ice = dyn_cast<ImplicitCastExpr>(e))
+    e = ice->getSubExpr();
+  return cast<OpaqueValueExpr>(e);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2093,257 +2789,43 @@ const Expr* ConstExprIterator::operator->() const { return cast<Expr>(*I); }
 //  Child Iterators for iterating over subexpressions/substatements
 //===----------------------------------------------------------------------===//
 
-// DeclRefExpr
-Stmt::child_iterator DeclRefExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator DeclRefExpr::child_end() { return child_iterator(); }
-
-// ObjCIvarRefExpr
-Stmt::child_iterator ObjCIvarRefExpr::child_begin() { return &Base; }
-Stmt::child_iterator ObjCIvarRefExpr::child_end() { return &Base+1; }
-
-// ObjCPropertyRefExpr
-Stmt::child_iterator ObjCPropertyRefExpr::child_begin() { return &Base; }
-Stmt::child_iterator ObjCPropertyRefExpr::child_end() { return &Base+1; }
-
-// ObjCImplicitSetterGetterRefExpr
-Stmt::child_iterator ObjCImplicitSetterGetterRefExpr::child_begin() {
-  // If this is accessing a class member, skip that entry.
-  if (Base) return &Base;
-  return &Base+1;
-}
-Stmt::child_iterator ObjCImplicitSetterGetterRefExpr::child_end() {
-  return &Base+1;
-}
-
-// ObjCSuperExpr
-Stmt::child_iterator ObjCSuperExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator ObjCSuperExpr::child_end() { return child_iterator(); }
-
-// ObjCIsaExpr
-Stmt::child_iterator ObjCIsaExpr::child_begin() { return &Base; }
-Stmt::child_iterator ObjCIsaExpr::child_end() { return &Base+1; }
-
-// PredefinedExpr
-Stmt::child_iterator PredefinedExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator PredefinedExpr::child_end() { return child_iterator(); }
-
-// IntegerLiteral
-Stmt::child_iterator IntegerLiteral::child_begin() { return child_iterator(); }
-Stmt::child_iterator IntegerLiteral::child_end() { return child_iterator(); }
-
-// CharacterLiteral
-Stmt::child_iterator CharacterLiteral::child_begin() { return child_iterator();}
-Stmt::child_iterator CharacterLiteral::child_end() { return child_iterator(); }
-
-// FloatingLiteral
-Stmt::child_iterator FloatingLiteral::child_begin() { return child_iterator(); }
-Stmt::child_iterator FloatingLiteral::child_end() { return child_iterator(); }
-
-// ImaginaryLiteral
-Stmt::child_iterator ImaginaryLiteral::child_begin() { return &Val; }
-Stmt::child_iterator ImaginaryLiteral::child_end() { return &Val+1; }
-
-// StringLiteral
-Stmt::child_iterator StringLiteral::child_begin() { return child_iterator(); }
-Stmt::child_iterator StringLiteral::child_end() { return child_iterator(); }
-
-// ParenExpr
-Stmt::child_iterator ParenExpr::child_begin() { return &Val; }
-Stmt::child_iterator ParenExpr::child_end() { return &Val+1; }
-
-// UnaryOperator
-Stmt::child_iterator UnaryOperator::child_begin() { return &Val; }
-Stmt::child_iterator UnaryOperator::child_end() { return &Val+1; }
-
-// OffsetOfExpr
-Stmt::child_iterator OffsetOfExpr::child_begin() {
-  return reinterpret_cast<Stmt **> (reinterpret_cast<OffsetOfNode *> (this + 1)
-                                      + NumComps);
-}
-Stmt::child_iterator OffsetOfExpr::child_end() {
-  return child_iterator(&*child_begin() + NumExprs);
-}
-
 // SizeOfAlignOfExpr
-Stmt::child_iterator SizeOfAlignOfExpr::child_begin() {
+Stmt::child_range SizeOfAlignOfExpr::children() {
   // If this is of a type and the type is a VLA type (and not a typedef), the
   // size expression of the VLA needs to be treated as an executable expression.
   // Why isn't this weirdness documented better in StmtIterator?
   if (isArgumentType()) {
-    if (VariableArrayType* T = dyn_cast<VariableArrayType>(
+    if (const VariableArrayType* T = dyn_cast<VariableArrayType>(
                                    getArgumentType().getTypePtr()))
-      return child_iterator(T);
-    return child_iterator();
+      return child_range(child_iterator(T), child_iterator());
+    return child_range();
   }
-  return child_iterator(&Argument.Ex);
-}
-Stmt::child_iterator SizeOfAlignOfExpr::child_end() {
-  if (isArgumentType())
-    return child_iterator();
-  return child_iterator(&Argument.Ex + 1);
-}
-
-// ArraySubscriptExpr
-Stmt::child_iterator ArraySubscriptExpr::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator ArraySubscriptExpr::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// CallExpr
-Stmt::child_iterator CallExpr::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator CallExpr::child_end() {
-  return &SubExprs[0]+NumArgs+ARGS_START;
-}
-
-// MemberExpr
-Stmt::child_iterator MemberExpr::child_begin() { return &Base; }
-Stmt::child_iterator MemberExpr::child_end() { return &Base+1; }
-
-// ExtVectorElementExpr
-Stmt::child_iterator ExtVectorElementExpr::child_begin() { return &Base; }
-Stmt::child_iterator ExtVectorElementExpr::child_end() { return &Base+1; }
-
-// CompoundLiteralExpr
-Stmt::child_iterator CompoundLiteralExpr::child_begin() { return &Init; }
-Stmt::child_iterator CompoundLiteralExpr::child_end() { return &Init+1; }
-
-// CastExpr
-Stmt::child_iterator CastExpr::child_begin() { return &Op; }
-Stmt::child_iterator CastExpr::child_end() { return &Op+1; }
-
-// BinaryOperator
-Stmt::child_iterator BinaryOperator::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator BinaryOperator::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// ConditionalOperator
-Stmt::child_iterator ConditionalOperator::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator ConditionalOperator::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// AddrLabelExpr
-Stmt::child_iterator AddrLabelExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator AddrLabelExpr::child_end() { return child_iterator(); }
-
-// StmtExpr
-Stmt::child_iterator StmtExpr::child_begin() { return &SubStmt; }
-Stmt::child_iterator StmtExpr::child_end() { return &SubStmt+1; }
-
-// TypesCompatibleExpr
-Stmt::child_iterator TypesCompatibleExpr::child_begin() {
-  return child_iterator();
-}
-
-Stmt::child_iterator TypesCompatibleExpr::child_end() {
-  return child_iterator();
-}
-
-// ChooseExpr
-Stmt::child_iterator ChooseExpr::child_begin() { return &SubExprs[0]; }
-Stmt::child_iterator ChooseExpr::child_end() { return &SubExprs[0]+END_EXPR; }
-
-// GNUNullExpr
-Stmt::child_iterator GNUNullExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator GNUNullExpr::child_end() { return child_iterator(); }
-
-// ShuffleVectorExpr
-Stmt::child_iterator ShuffleVectorExpr::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator ShuffleVectorExpr::child_end() {
-  return &SubExprs[0]+NumExprs;
-}
-
-// VAArgExpr
-Stmt::child_iterator VAArgExpr::child_begin() { return &Val; }
-Stmt::child_iterator VAArgExpr::child_end() { return &Val+1; }
-
-// InitListExpr
-Stmt::child_iterator InitListExpr::child_begin() {
-  return InitExprs.size() ? &InitExprs[0] : 0;
-}
-Stmt::child_iterator InitListExpr::child_end() {
-  return InitExprs.size() ? &InitExprs[0] + InitExprs.size() : 0;
-}
-
-// DesignatedInitExpr
-Stmt::child_iterator DesignatedInitExpr::child_begin() {
-  char* Ptr = static_cast<char*>(static_cast<void *>(this));
-  Ptr += sizeof(DesignatedInitExpr);
-  return reinterpret_cast<Stmt**>(reinterpret_cast<void**>(Ptr));
-}
-Stmt::child_iterator DesignatedInitExpr::child_end() {
-  return child_iterator(&*child_begin() + NumSubExprs);
-}
-
-// ImplicitValueInitExpr
-Stmt::child_iterator ImplicitValueInitExpr::child_begin() {
-  return child_iterator();
-}
-
-Stmt::child_iterator ImplicitValueInitExpr::child_end() {
-  return child_iterator();
-}
-
-// ParenListExpr
-Stmt::child_iterator ParenListExpr::child_begin() {
-  return &Exprs[0];
-}
-Stmt::child_iterator ParenListExpr::child_end() {
-  return &Exprs[0]+NumExprs;
-}
-
-// ObjCStringLiteral
-Stmt::child_iterator ObjCStringLiteral::child_begin() {
-  return &String;
-}
-Stmt::child_iterator ObjCStringLiteral::child_end() {
-  return &String+1;
-}
-
-// ObjCEncodeExpr
-Stmt::child_iterator ObjCEncodeExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator ObjCEncodeExpr::child_end() { return child_iterator(); }
-
-// ObjCSelectorExpr
-Stmt::child_iterator ObjCSelectorExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator ObjCSelectorExpr::child_end() {
-  return child_iterator();
-}
-
-// ObjCProtocolExpr
-Stmt::child_iterator ObjCProtocolExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator ObjCProtocolExpr::child_end() {
-  return child_iterator();
+  return child_range(&Argument.Ex, &Argument.Ex + 1);
 }
 
 // ObjCMessageExpr
-Stmt::child_iterator ObjCMessageExpr::child_begin() {
+Stmt::child_range ObjCMessageExpr::children() {
+  Stmt **begin;
   if (getReceiverKind() == Instance)
-    return reinterpret_cast<Stmt **>(this + 1);
-  return getArgs();
-}
-Stmt::child_iterator ObjCMessageExpr::child_end() {
-  return getArgs() + getNumArgs();
+    begin = reinterpret_cast<Stmt **>(this + 1);
+  else
+    begin = reinterpret_cast<Stmt **>(getArgs());
+  return child_range(begin,
+                     reinterpret_cast<Stmt **>(getArgs() + getNumArgs()));
 }
 
 // Blocks
-Stmt::child_iterator BlockExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator BlockExpr::child_end() { return child_iterator(); }
+BlockDeclRefExpr::BlockDeclRefExpr(VarDecl *d, QualType t, ExprValueKind VK,
+                                   SourceLocation l, bool ByRef, 
+                                   bool constAdded)
+  : Expr(BlockDeclRefExprClass, t, VK, OK_Ordinary, false, false,
+         d->isParameterPack()),
+    D(d), Loc(l), IsByRef(ByRef), ConstQualAdded(constAdded)
+{
+  bool TypeDependent = false;
+  bool ValueDependent = false;
+  computeDeclRefDependence(D, getType(), TypeDependent, ValueDependent);
+  ExprBits.TypeDependent = TypeDependent;
+  ExprBits.ValueDependent = ValueDependent;
+}
 
-Stmt::child_iterator BlockDeclRefExpr::child_begin() { return child_iterator();}
-Stmt::child_iterator BlockDeclRefExpr::child_end() { return child_iterator(); }
diff --git a/lib/AST/ExprCXX.cpp b/lib/AST/ExprCXX.cpp
index 0a101300d8fa..28ff9fb57afc 100644
--- a/lib/AST/ExprCXX.cpp
+++ b/lib/AST/ExprCXX.cpp
@@ -29,57 +29,18 @@ QualType CXXTypeidExpr::getTypeOperand() const {
                                                         .getUnqualifiedType();
 }
 
-// CXXTypeidExpr - has child iterators if the operand is an expression
-Stmt::child_iterator CXXTypeidExpr::child_begin() {
-  return isTypeOperand() ? child_iterator() 
-                         : reinterpret_cast<Stmt **>(&Operand);
-}
-Stmt::child_iterator CXXTypeidExpr::child_end() {
-  return isTypeOperand() ? child_iterator() 
-                         : reinterpret_cast<Stmt **>(&Operand) + 1;
-}
-
-// CXXBoolLiteralExpr
-Stmt::child_iterator CXXBoolLiteralExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator CXXBoolLiteralExpr::child_end() {
-  return child_iterator();
-}
-
-// CXXNullPtrLiteralExpr
-Stmt::child_iterator CXXNullPtrLiteralExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator CXXNullPtrLiteralExpr::child_end() {
-  return child_iterator();
-}
-
-// CXXThisExpr
-Stmt::child_iterator CXXThisExpr::child_begin() { return child_iterator(); }
-Stmt::child_iterator CXXThisExpr::child_end() { return child_iterator(); }
-
-// CXXThrowExpr
-Stmt::child_iterator CXXThrowExpr::child_begin() { return &Op; }
-Stmt::child_iterator CXXThrowExpr::child_end() {
-  // If Op is 0, we are processing throw; which has no children.
-  return Op ? &Op+1 : &Op;
-}
-
-// CXXDefaultArgExpr
-Stmt::child_iterator CXXDefaultArgExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator CXXDefaultArgExpr::child_end() {
-  return child_iterator();
+QualType CXXUuidofExpr::getTypeOperand() const {
+  assert(isTypeOperand() && "Cannot call getTypeOperand for __uuidof(expr)");
+  return Operand.get<TypeSourceInfo *>()->getType().getNonReferenceType()
+                                                        .getUnqualifiedType();
 }
 
 // CXXScalarValueInitExpr
-Stmt::child_iterator CXXScalarValueInitExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator CXXScalarValueInitExpr::child_end() {
-  return child_iterator();
+SourceRange CXXScalarValueInitExpr::getSourceRange() const {
+  SourceLocation Start = RParenLoc;
+  if (TypeInfo)
+    Start = TypeInfo->getTypeLoc().getBeginLoc();
+  return SourceRange(Start, RParenLoc);
 }
 
 // CXXNewExpr
@@ -88,22 +49,44 @@ CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
                        SourceRange TypeIdParens, Expr *arraySize,
                        CXXConstructorDecl *constructor, bool initializer,
                        Expr **constructorArgs, unsigned numConsArgs,
-                       FunctionDecl *operatorDelete, QualType ty,
-                       SourceLocation startLoc, SourceLocation endLoc)
-  : Expr(CXXNewExprClass, ty, ty->isDependentType(), ty->isDependentType()),
-    GlobalNew(globalNew),
-    Initializer(initializer), SubExprs(0), OperatorNew(operatorNew),
+                       FunctionDecl *operatorDelete,
+                       bool usualArrayDeleteWantsSize, QualType ty,
+                       TypeSourceInfo *AllocatedTypeInfo,
+                       SourceLocation startLoc, SourceLocation endLoc,
+                       SourceLocation constructorLParen,
+                       SourceLocation constructorRParen)
+  : Expr(CXXNewExprClass, ty, VK_RValue, OK_Ordinary,
+         ty->isDependentType(), ty->isDependentType(),
+         ty->containsUnexpandedParameterPack()),
+    GlobalNew(globalNew), Initializer(initializer),
+    UsualArrayDeleteWantsSize(usualArrayDeleteWantsSize),
+    SubExprs(0), OperatorNew(operatorNew),
     OperatorDelete(operatorDelete), Constructor(constructor),
-    TypeIdParens(TypeIdParens), StartLoc(startLoc), EndLoc(endLoc) {
-      
+    AllocatedTypeInfo(AllocatedTypeInfo), TypeIdParens(TypeIdParens),
+    StartLoc(startLoc), EndLoc(endLoc), ConstructorLParen(constructorLParen),
+    ConstructorRParen(constructorRParen) {
   AllocateArgsArray(C, arraySize != 0, numPlaceArgs, numConsArgs);
   unsigned i = 0;
-  if (Array)
+  if (Array) {
+    if (arraySize->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
     SubExprs[i++] = arraySize;
-  for (unsigned j = 0; j < NumPlacementArgs; ++j)
+  }
+
+  for (unsigned j = 0; j < NumPlacementArgs; ++j) {
+    if (placementArgs[j]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
     SubExprs[i++] = placementArgs[j];
-  for (unsigned j = 0; j < NumConstructorArgs; ++j)
+  }
+
+  for (unsigned j = 0; j < NumConstructorArgs; ++j) {
+    if (constructorArgs[j]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
     SubExprs[i++] = constructorArgs[j];
+  }
 }
 
 void CXXNewExpr::AllocateArgsArray(ASTContext &C, bool isArray,
@@ -118,27 +101,59 @@ void CXXNewExpr::AllocateArgsArray(ASTContext &C, bool isArray,
 }
 
 
-Stmt::child_iterator CXXNewExpr::child_begin() { return &SubExprs[0]; }
-Stmt::child_iterator CXXNewExpr::child_end() {
-  return &SubExprs[0] + Array + getNumPlacementArgs() + getNumConstructorArgs();
-}
-
 // CXXDeleteExpr
-Stmt::child_iterator CXXDeleteExpr::child_begin() { return &Argument; }
-Stmt::child_iterator CXXDeleteExpr::child_end() { return &Argument+1; }
+QualType CXXDeleteExpr::getDestroyedType() const {
+  const Expr *Arg = getArgument();
+  while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Arg)) {
+    if (ICE->getCastKind() != CK_UserDefinedConversion &&
+        ICE->getType()->isVoidPointerType())
+      Arg = ICE->getSubExpr();
+    else
+      break;
+  }
+  // The type-to-delete may not be a pointer if it's a dependent type.
+  const QualType ArgType = Arg->getType();
 
-// CXXPseudoDestructorExpr
-Stmt::child_iterator CXXPseudoDestructorExpr::child_begin() { return &Base; }
-Stmt::child_iterator CXXPseudoDestructorExpr::child_end() {
-  return &Base + 1;
+  if (ArgType->isDependentType() && !ArgType->isPointerType())
+    return QualType();
+
+  return ArgType->getAs<PointerType>()->getPointeeType();
 }
 
+// CXXPseudoDestructorExpr
 PseudoDestructorTypeStorage::PseudoDestructorTypeStorage(TypeSourceInfo *Info)
  : Type(Info) 
 {
   Location = Info->getTypeLoc().getLocalSourceRange().getBegin();
 }
 
+CXXPseudoDestructorExpr::CXXPseudoDestructorExpr(ASTContext &Context,
+    Expr *Base, bool isArrow, SourceLocation OperatorLoc,
+    NestedNameSpecifier *Qualifier, SourceRange QualifierRange,
+    TypeSourceInfo *ScopeType, SourceLocation ColonColonLoc,
+    SourceLocation TildeLoc, PseudoDestructorTypeStorage DestroyedType)
+  : Expr(CXXPseudoDestructorExprClass,
+         Context.getPointerType(Context.getFunctionType(Context.VoidTy, 0, 0,
+                                         FunctionProtoType::ExtProtoInfo())),
+         VK_RValue, OK_Ordinary,
+         /*isTypeDependent=*/(Base->isTypeDependent() ||
+           (DestroyedType.getTypeSourceInfo() &&
+            DestroyedType.getTypeSourceInfo()->getType()->isDependentType())),
+         /*isValueDependent=*/Base->isValueDependent(),
+         // ContainsUnexpandedParameterPack
+         (Base->containsUnexpandedParameterPack() ||
+          (Qualifier && Qualifier->containsUnexpandedParameterPack()) ||
+          (ScopeType && 
+           ScopeType->getType()->containsUnexpandedParameterPack()) ||
+          (DestroyedType.getTypeSourceInfo() &&
+           DestroyedType.getTypeSourceInfo()->getType()
+                                   ->containsUnexpandedParameterPack()))),
+    Base(static_cast<Stmt *>(Base)), IsArrow(isArrow),
+    OperatorLoc(OperatorLoc), Qualifier(Qualifier),
+    QualifierRange(QualifierRange), 
+    ScopeType(ScopeType), ColonColonLoc(ColonColonLoc), TildeLoc(TildeLoc),
+    DestroyedType(DestroyedType) { }
+
 QualType CXXPseudoDestructorExpr::getDestroyedType() const {
   if (TypeSourceInfo *TInfo = DestroyedType.getTypeSourceInfo())
     return TInfo->getType();
@@ -156,7 +171,7 @@ SourceRange CXXPseudoDestructorExpr::getSourceRange() const {
 
 // UnresolvedLookupExpr
 UnresolvedLookupExpr *
-UnresolvedLookupExpr::Create(ASTContext &C, bool Dependent,
+UnresolvedLookupExpr::Create(ASTContext &C, 
                              CXXRecordDecl *NamingClass,
                              NestedNameSpecifier *Qualifier,
                              SourceRange QualifierRange,
@@ -168,73 +183,94 @@ UnresolvedLookupExpr::Create(ASTContext &C, bool Dependent,
 {
   void *Mem = C.Allocate(sizeof(UnresolvedLookupExpr) + 
                          ExplicitTemplateArgumentList::sizeFor(Args));
-  UnresolvedLookupExpr *ULE
-    = new (Mem) UnresolvedLookupExpr(C, 
-                                     Dependent ? C.DependentTy : C.OverloadTy,
-                                     Dependent, NamingClass,
-                                     Qualifier, QualifierRange, NameInfo,
-                                     ADL,
-                                     /*Overload*/ true,
-                                     /*ExplicitTemplateArgs*/ true,
-                                     Begin, End);
-
-  reinterpret_cast<ExplicitTemplateArgumentList*>(ULE+1)->initializeFrom(Args);
-
-  return ULE;
+  return new (Mem) UnresolvedLookupExpr(C, NamingClass,
+                                        Qualifier, QualifierRange, NameInfo,
+                                        ADL, /*Overload*/ true, &Args,
+                                        Begin, End);
 }
 
 UnresolvedLookupExpr *
-UnresolvedLookupExpr::CreateEmpty(ASTContext &C, unsigned NumTemplateArgs) {
+UnresolvedLookupExpr::CreateEmpty(ASTContext &C, bool HasExplicitTemplateArgs, 
+                                  unsigned NumTemplateArgs) {
   std::size_t size = sizeof(UnresolvedLookupExpr);
-  if (NumTemplateArgs != 0)
+  if (HasExplicitTemplateArgs)
     size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
 
-  void *Mem = C.Allocate(size, llvm::alignof<UnresolvedLookupExpr>());
+  void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedLookupExpr>());
   UnresolvedLookupExpr *E = new (Mem) UnresolvedLookupExpr(EmptyShell());
-  E->HasExplicitTemplateArgs = NumTemplateArgs != 0;
+  E->HasExplicitTemplateArgs = HasExplicitTemplateArgs;
   return E;
 }
 
-OverloadExpr::OverloadExpr(StmtClass K, ASTContext &C, QualType T, 
-                           bool Dependent, NestedNameSpecifier *Qualifier, 
-                           SourceRange QRange,
+OverloadExpr::OverloadExpr(StmtClass K, ASTContext &C, 
+                           NestedNameSpecifier *Qualifier, SourceRange QRange,
                            const DeclarationNameInfo &NameInfo,
-                           bool HasTemplateArgs,
+                           const TemplateArgumentListInfo *TemplateArgs,
                            UnresolvedSetIterator Begin, 
-                           UnresolvedSetIterator End)
-  : Expr(K, T, Dependent, Dependent),
-  Results(0), NumResults(0), NameInfo(NameInfo), Qualifier(Qualifier), 
-  QualifierRange(QRange), HasExplicitTemplateArgs(HasTemplateArgs)
+                           UnresolvedSetIterator End,
+                           bool KnownDependent,
+                           bool KnownContainsUnexpandedParameterPack)
+  : Expr(K, C.OverloadTy, VK_LValue, OK_Ordinary, KnownDependent, 
+         KnownDependent,
+         (KnownContainsUnexpandedParameterPack ||
+          NameInfo.containsUnexpandedParameterPack() ||
+          (Qualifier && Qualifier->containsUnexpandedParameterPack()))),
+    Results(0), NumResults(End - Begin), NameInfo(NameInfo), 
+    Qualifier(Qualifier), QualifierRange(QRange), 
+    HasExplicitTemplateArgs(TemplateArgs != 0)
 {
-  initializeResults(C, Begin, End);
-}
-
-void OverloadExpr::initializeResults(ASTContext &C,
-                                     UnresolvedSetIterator Begin,
-                                     UnresolvedSetIterator End) {
-  assert(Results == 0 && "Results already initialized!");
   NumResults = End - Begin;
   if (NumResults) {
+    // Determine whether this expression is type-dependent.
+    for (UnresolvedSetImpl::const_iterator I = Begin; I != End; ++I) {
+      if ((*I)->getDeclContext()->isDependentContext() ||
+          isa<UnresolvedUsingValueDecl>(*I)) {
+        ExprBits.TypeDependent = true;
+        ExprBits.ValueDependent = true;
+      }
+    }
+
     Results = static_cast<DeclAccessPair *>(
                                 C.Allocate(sizeof(DeclAccessPair) * NumResults, 
-                                           llvm::alignof<DeclAccessPair>()));
+                                           llvm::alignOf<DeclAccessPair>()));
     memcpy(Results, &*Begin.getIterator(), 
            NumResults * sizeof(DeclAccessPair));
   }
-}
-
 
-bool OverloadExpr::ComputeDependence(UnresolvedSetIterator Begin,
-                                     UnresolvedSetIterator End,
-                                     const TemplateArgumentListInfo *Args) {
-  for (UnresolvedSetImpl::const_iterator I = Begin; I != End; ++I)
-    if ((*I)->getDeclContext()->isDependentContext())
-      return true;
+  // If we have explicit template arguments, check for dependent
+  // template arguments and whether they contain any unexpanded pack
+  // expansions.
+  if (TemplateArgs) {
+    bool Dependent = false;
+    bool ContainsUnexpandedParameterPack = false;
+    getExplicitTemplateArgs().initializeFrom(*TemplateArgs, Dependent,
+                                             ContainsUnexpandedParameterPack);
+
+    if (Dependent) {
+        ExprBits.TypeDependent = true;
+        ExprBits.ValueDependent = true;
+    } 
+    if (ContainsUnexpandedParameterPack)
+      ExprBits.ContainsUnexpandedParameterPack = true;
+  }
 
-  if (Args && TemplateSpecializationType::anyDependentTemplateArguments(*Args))
-    return true;
+  if (isTypeDependent())
+    setType(C.DependentTy);
+}
 
-  return false;
+void OverloadExpr::initializeResults(ASTContext &C,
+                                     UnresolvedSetIterator Begin,
+                                     UnresolvedSetIterator End) {
+  assert(Results == 0 && "Results already initialized!");
+  NumResults = End - Begin;
+  if (NumResults) {
+     Results = static_cast<DeclAccessPair *>(
+                               C.Allocate(sizeof(DeclAccessPair) * NumResults,
+ 
+                                          llvm::alignOf<DeclAccessPair>()));
+     memcpy(Results, &*Begin.getIterator(), 
+            NumResults * sizeof(DeclAccessPair));
+  }
 }
 
 CXXRecordDecl *OverloadExpr::getNamingClass() const {
@@ -244,21 +280,30 @@ CXXRecordDecl *OverloadExpr::getNamingClass() const {
     return cast<UnresolvedMemberExpr>(this)->getNamingClass();
 }
 
-Stmt::child_iterator UnresolvedLookupExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator UnresolvedLookupExpr::child_end() {
-  return child_iterator();
-}
-// UnaryTypeTraitExpr
-Stmt::child_iterator UnaryTypeTraitExpr::child_begin() {
-  return child_iterator();
-}
-Stmt::child_iterator UnaryTypeTraitExpr::child_end() {
-  return child_iterator();
+// DependentScopeDeclRefExpr
+DependentScopeDeclRefExpr::DependentScopeDeclRefExpr(QualType T,
+                            NestedNameSpecifier *Qualifier,
+                            SourceRange QualifierRange,
+                            const DeclarationNameInfo &NameInfo,
+                            const TemplateArgumentListInfo *Args)
+  : Expr(DependentScopeDeclRefExprClass, T, VK_LValue, OK_Ordinary,
+         true, true,
+         (NameInfo.containsUnexpandedParameterPack() ||
+          (Qualifier && Qualifier->containsUnexpandedParameterPack()))),
+    NameInfo(NameInfo), QualifierRange(QualifierRange), Qualifier(Qualifier),
+    HasExplicitTemplateArgs(Args != 0)
+{
+  if (Args) {
+    bool Dependent = true;
+    bool ContainsUnexpandedParameterPack
+      = ExprBits.ContainsUnexpandedParameterPack;
+
+    reinterpret_cast<ExplicitTemplateArgumentList*>(this+1)
+      ->initializeFrom(*Args, Dependent, ContainsUnexpandedParameterPack);
+    ExprBits.ContainsUnexpandedParameterPack = ContainsUnexpandedParameterPack;
+  }
 }
 
-// DependentScopeDeclRefExpr
 DependentScopeDeclRefExpr *
 DependentScopeDeclRefExpr::Create(ASTContext &C,
                                   NestedNameSpecifier *Qualifier,
@@ -266,251 +311,46 @@ DependentScopeDeclRefExpr::Create(ASTContext &C,
                                   const DeclarationNameInfo &NameInfo,
                                   const TemplateArgumentListInfo *Args) {
   std::size_t size = sizeof(DependentScopeDeclRefExpr);
-  if (Args) size += ExplicitTemplateArgumentList::sizeFor(*Args);
-  void *Mem = C.Allocate(size);
-
-  DependentScopeDeclRefExpr *DRE
-    = new (Mem) DependentScopeDeclRefExpr(C.DependentTy,
-                                          Qualifier, QualifierRange,
-                                          NameInfo, Args != 0);
-
   if (Args)
-    reinterpret_cast<ExplicitTemplateArgumentList*>(DRE+1)
-      ->initializeFrom(*Args);
-
-  return DRE;
+    size += ExplicitTemplateArgumentList::sizeFor(*Args);
+  void *Mem = C.Allocate(size);
+  return new (Mem) DependentScopeDeclRefExpr(C.DependentTy,
+                                             Qualifier, QualifierRange,
+                                             NameInfo, Args);
 }
 
 DependentScopeDeclRefExpr *
 DependentScopeDeclRefExpr::CreateEmpty(ASTContext &C,
+                                       bool HasExplicitTemplateArgs,
                                        unsigned NumTemplateArgs) {
   std::size_t size = sizeof(DependentScopeDeclRefExpr);
-  if (NumTemplateArgs)
+  if (HasExplicitTemplateArgs)
     size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
   void *Mem = C.Allocate(size);
-
-  return new (Mem) DependentScopeDeclRefExpr(QualType(), 0, SourceRange(),
-                                             DeclarationNameInfo(),
-                                             NumTemplateArgs != 0);
-}
-
-StmtIterator DependentScopeDeclRefExpr::child_begin() {
-  return child_iterator();
+  DependentScopeDeclRefExpr *E 
+    = new (Mem) DependentScopeDeclRefExpr(QualType(), 0, SourceRange(),
+                                          DeclarationNameInfo(), 0);
+  E->HasExplicitTemplateArgs = HasExplicitTemplateArgs;
+  return E;
 }
 
-StmtIterator DependentScopeDeclRefExpr::child_end() {
-  return child_iterator();
-}
+SourceRange CXXConstructExpr::getSourceRange() const {
+  if (ParenRange.isValid())
+    return SourceRange(Loc, ParenRange.getEnd());
 
-bool UnaryTypeTraitExpr::EvaluateTrait(ASTContext& C) const {
-  switch(UTT) {
-  default: assert(false && "Unknown type trait or not implemented");
-  case UTT_IsPOD: return QueriedType->isPODType();
-  case UTT_IsLiteral: return QueriedType->isLiteralType();
-  case UTT_IsClass: // Fallthrough
-  case UTT_IsUnion:
-    if (const RecordType *Record = QueriedType->getAs<RecordType>()) {
-      bool Union = Record->getDecl()->isUnion();
-      return UTT == UTT_IsUnion ? Union : !Union;
-    }
-    return false;
-  case UTT_IsEnum: return QueriedType->isEnumeralType();
-  case UTT_IsPolymorphic:
-    if (const RecordType *Record = QueriedType->getAs<RecordType>()) {
-      // Type traits are only parsed in C++, so we've got CXXRecords.
-      return cast<CXXRecordDecl>(Record->getDecl())->isPolymorphic();
-    }
-    return false;
-  case UTT_IsAbstract:
-    if (const RecordType *RT = QueriedType->getAs<RecordType>())
-      return cast<CXXRecordDecl>(RT->getDecl())->isAbstract();
-    return false;
-  case UTT_IsEmpty:
-    if (const RecordType *Record = QueriedType->getAs<RecordType>()) {
-      return !Record->getDecl()->isUnion()
-          && cast<CXXRecordDecl>(Record->getDecl())->isEmpty();
-    }
-    return false;
-  case UTT_HasTrivialConstructor:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If __is_pod (type) is true then the trait is true, else if type is
-    //   a cv class or union type (or array thereof) with a trivial default
-    //   constructor ([class.ctor]) then the trait is true, else it is false.
-    if (QueriedType->isPODType())
-      return true;
-    if (const RecordType *RT =
-          C.getBaseElementType(QueriedType)->getAs<RecordType>())
-      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialConstructor();
-    return false;
-  case UTT_HasTrivialCopy:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If __is_pod (type) is true or type is a reference type then
-    //   the trait is true, else if type is a cv class or union type
-    //   with a trivial copy constructor ([class.copy]) then the trait
-    //   is true, else it is false.
-    if (QueriedType->isPODType() || QueriedType->isReferenceType())
-      return true;
-    if (const RecordType *RT = QueriedType->getAs<RecordType>())
-      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialCopyConstructor();
-    return false;
-  case UTT_HasTrivialAssign:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If type is const qualified or is a reference type then the
-    //   trait is false. Otherwise if __is_pod (type) is true then the
-    //   trait is true, else if type is a cv class or union type with
-    //   a trivial copy assignment ([class.copy]) then the trait is
-    //   true, else it is false.
-    // Note: the const and reference restrictions are interesting,
-    // given that const and reference members don't prevent a class
-    // from having a trivial copy assignment operator (but do cause
-    // errors if the copy assignment operator is actually used, q.v.
-    // [class.copy]p12).
-
-    if (C.getBaseElementType(QueriedType).isConstQualified())
-      return false;
-    if (QueriedType->isPODType())
-      return true;
-    if (const RecordType *RT = QueriedType->getAs<RecordType>())
-      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialCopyAssignment();
-    return false;
-  case UTT_HasTrivialDestructor:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If __is_pod (type) is true or type is a reference type
-    //   then the trait is true, else if type is a cv class or union
-    //   type (or array thereof) with a trivial destructor
-    //   ([class.dtor]) then the trait is true, else it is
-    //   false.
-    if (QueriedType->isPODType() || QueriedType->isReferenceType())
-      return true;
-    if (const RecordType *RT =
-          C.getBaseElementType(QueriedType)->getAs<RecordType>())
-      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialDestructor();
-    return false;
-  // TODO: Propagate nothrowness for implicitly declared special members.
-  case UTT_HasNothrowAssign:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If type is const qualified or is a reference type then the
-    //   trait is false. Otherwise if __has_trivial_assign (type)
-    //   is true then the trait is true, else if type is a cv class
-    //   or union type with copy assignment operators that are known
-    //   not to throw an exception then the trait is true, else it is
-    //   false.
-    if (C.getBaseElementType(QueriedType).isConstQualified())
-      return false;
-    if (QueriedType->isReferenceType())
-      return false;
-    if (QueriedType->isPODType())
-      return true;
-    if (const RecordType *RT = QueriedType->getAs<RecordType>()) {
-      CXXRecordDecl* RD = cast<CXXRecordDecl>(RT->getDecl());
-      if (RD->hasTrivialCopyAssignment())
-        return true;
-
-      bool FoundAssign = false;
-      bool AllNoThrow = true;
-      DeclarationName Name = C.DeclarationNames.getCXXOperatorName(OO_Equal);
-      DeclContext::lookup_const_iterator Op, OpEnd;
-      for (llvm::tie(Op, OpEnd) = RD->lookup(Name);
-           Op != OpEnd; ++Op) {
-        CXXMethodDecl *Operator = cast<CXXMethodDecl>(*Op);
-        if (Operator->isCopyAssignmentOperator()) {
-          FoundAssign = true;
-          const FunctionProtoType *CPT
-              = Operator->getType()->getAs<FunctionProtoType>();
-          if (!CPT->hasEmptyExceptionSpec()) {
-            AllNoThrow = false;
-            break;
-          }
-        }
+  SourceLocation End = Loc;
+  for (unsigned I = getNumArgs(); I > 0; --I) {
+    const Expr *Arg = getArg(I-1);
+    if (!Arg->isDefaultArgument()) {
+      SourceLocation NewEnd = Arg->getLocEnd();
+      if (NewEnd.isValid()) {
+        End = NewEnd;
+        break;
       }
-
-      return FoundAssign && AllNoThrow;
     }
-    return false;
-  case UTT_HasNothrowCopy:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If __has_trivial_copy (type) is true then the trait is true, else
-    //   if type is a cv class or union type with copy constructors that are
-    //   known not to throw an exception then the trait is true, else it is
-    //   false.
-    if (QueriedType->isPODType() || QueriedType->isReferenceType())
-      return true;
-    if (const RecordType *RT = QueriedType->getAs<RecordType>()) {
-      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-      if (RD->hasTrivialCopyConstructor())
-        return true;
-
-      bool FoundConstructor = false;
-      bool AllNoThrow = true;
-      unsigned FoundTQs;
-      DeclarationName ConstructorName
-          = C.DeclarationNames.getCXXConstructorName(
-                                          C.getCanonicalType(QueriedType));
-      DeclContext::lookup_const_iterator Con, ConEnd;
-      for (llvm::tie(Con, ConEnd) = RD->lookup(ConstructorName);
-           Con != ConEnd; ++Con) {
-        CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
-        if (Constructor->isCopyConstructor(FoundTQs)) {
-          FoundConstructor = true;
-          const FunctionProtoType *CPT
-              = Constructor->getType()->getAs<FunctionProtoType>();
-          if (!CPT->hasEmptyExceptionSpec()) {
-            AllNoThrow = false;
-            break;
-          }
-        }
-      }
-
-      return FoundConstructor && AllNoThrow;
-    }
-    return false;
-  case UTT_HasNothrowConstructor:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If __has_trivial_constructor (type) is true then the trait is
-    //   true, else if type is a cv class or union type (or array
-    //   thereof) with a default constructor that is known not to
-    //   throw an exception then the trait is true, else it is false.
-    if (QueriedType->isPODType())
-      return true;
-    if (const RecordType *RT =
-          C.getBaseElementType(QueriedType)->getAs<RecordType>()) {
-      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-      if (RD->hasTrivialConstructor())
-        return true;
-
-      if (CXXConstructorDecl *Constructor = RD->getDefaultConstructor()) {
-        const FunctionProtoType *CPT
-            = Constructor->getType()->getAs<FunctionProtoType>();
-        // TODO: check whether evaluating default arguments can throw.
-        // For now, we'll be conservative and assume that they can throw.
-        if (CPT->hasEmptyExceptionSpec() && CPT->getNumArgs() == 0)
-          return true;
-      }
-    }
-    return false;
-  case UTT_HasVirtualDestructor:
-    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
-    //   If type is a class type with a virtual destructor ([class.dtor])
-    //   then the trait is true, else it is false.
-    if (const RecordType *Record = QueriedType->getAs<RecordType>()) {
-      CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
-      if (CXXDestructorDecl *Destructor = RD->getDestructor())
-        return Destructor->isVirtual();
-    }
-    return false;
   }
-}
 
-SourceRange CXXConstructExpr::getSourceRange() const { 
-  // FIXME: Should we know where the parentheses are, if there are any?
-  for (std::reverse_iterator<Stmt**> I(&Args[NumArgs]), E(&Args[0]); I!=E;++I) {
-    // Ignore CXXDefaultExprs when computing the range, as they don't
-    // have a range.
-    if (!isa<CXXDefaultArgExpr>(*I))
-      return SourceRange(Loc, (*I)->getLocEnd());
-  }
-  
-  return SourceRange(Loc);
+  return SourceRange(Loc, End);
 }
 
 SourceRange CXXOperatorCallExpr::getSourceRange() const {
@@ -546,6 +386,17 @@ Expr *CXXMemberCallExpr::getImplicitObjectArgument() {
   return 0;
 }
 
+CXXRecordDecl *CXXMemberCallExpr::getRecordDecl() {
+  Expr* ThisArg = getImplicitObjectArgument();
+  if (!ThisArg)
+    return 0;
+
+  if (ThisArg->getType()->isAnyPointerType())
+    return ThisArg->getType()->getPointeeType()->getAsCXXRecordDecl();
+
+  return ThisArg->getType()->getAsCXXRecordDecl();
+}
+
 SourceRange CXXMemberCallExpr::getSourceRange() const {
   SourceLocation LocStart = getCallee()->getLocStart();
   if (LocStart.isInvalid() && getNumArgs() > 0)
@@ -572,15 +423,18 @@ const char *CXXNamedCastExpr::getCastName() const {
 }
 
 CXXStaticCastExpr *CXXStaticCastExpr::Create(ASTContext &C, QualType T,
+                                             ExprValueKind VK,
                                              CastKind K, Expr *Op,
                                              const CXXCastPath *BasePath,
                                              TypeSourceInfo *WrittenTy,
-                                             SourceLocation L) {
+                                             SourceLocation L, 
+                                             SourceLocation RParenLoc) {
   unsigned PathSize = (BasePath ? BasePath->size() : 0);
   void *Buffer = C.Allocate(sizeof(CXXStaticCastExpr)
                             + PathSize * sizeof(CXXBaseSpecifier*));
   CXXStaticCastExpr *E =
-    new (Buffer) CXXStaticCastExpr(T, K, Op, PathSize, WrittenTy, L);
+    new (Buffer) CXXStaticCastExpr(T, VK, K, Op, PathSize, WrittenTy, L,
+                                   RParenLoc);
   if (PathSize) E->setCastPath(*BasePath);
   return E;
 }
@@ -593,15 +447,18 @@ CXXStaticCastExpr *CXXStaticCastExpr::CreateEmpty(ASTContext &C,
 }
 
 CXXDynamicCastExpr *CXXDynamicCastExpr::Create(ASTContext &C, QualType T,
+                                               ExprValueKind VK,
                                                CastKind K, Expr *Op,
                                                const CXXCastPath *BasePath,
                                                TypeSourceInfo *WrittenTy,
-                                               SourceLocation L) {
+                                               SourceLocation L, 
+                                               SourceLocation RParenLoc) {
   unsigned PathSize = (BasePath ? BasePath->size() : 0);
   void *Buffer = C.Allocate(sizeof(CXXDynamicCastExpr)
                             + PathSize * sizeof(CXXBaseSpecifier*));
   CXXDynamicCastExpr *E =
-    new (Buffer) CXXDynamicCastExpr(T, K, Op, PathSize, WrittenTy, L);
+    new (Buffer) CXXDynamicCastExpr(T, VK, K, Op, PathSize, WrittenTy, L,
+                                    RParenLoc);
   if (PathSize) E->setCastPath(*BasePath);
   return E;
 }
@@ -614,14 +471,17 @@ CXXDynamicCastExpr *CXXDynamicCastExpr::CreateEmpty(ASTContext &C,
 }
 
 CXXReinterpretCastExpr *
-CXXReinterpretCastExpr::Create(ASTContext &C, QualType T, CastKind K, Expr *Op,
+CXXReinterpretCastExpr::Create(ASTContext &C, QualType T, ExprValueKind VK,
+                               CastKind K, Expr *Op,
                                const CXXCastPath *BasePath,
-                               TypeSourceInfo *WrittenTy, SourceLocation L) {
+                               TypeSourceInfo *WrittenTy, SourceLocation L, 
+                               SourceLocation RParenLoc) {
   unsigned PathSize = (BasePath ? BasePath->size() : 0);
   void *Buffer =
     C.Allocate(sizeof(CXXReinterpretCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
   CXXReinterpretCastExpr *E =
-    new (Buffer) CXXReinterpretCastExpr(T, K, Op, PathSize, WrittenTy, L);
+    new (Buffer) CXXReinterpretCastExpr(T, VK, K, Op, PathSize, WrittenTy, L,
+                                        RParenLoc);
   if (PathSize) E->setCastPath(*BasePath);
   return E;
 }
@@ -633,10 +493,12 @@ CXXReinterpretCastExpr::CreateEmpty(ASTContext &C, unsigned PathSize) {
   return new (Buffer) CXXReinterpretCastExpr(EmptyShell(), PathSize);
 }
 
-CXXConstCastExpr *CXXConstCastExpr::Create(ASTContext &C, QualType T, Expr *Op,
+CXXConstCastExpr *CXXConstCastExpr::Create(ASTContext &C, QualType T,
+                                           ExprValueKind VK, Expr *Op,
                                            TypeSourceInfo *WrittenTy,
-                                           SourceLocation L) {
-  return new (C) CXXConstCastExpr(T, Op, WrittenTy, L);
+                                           SourceLocation L, 
+                                           SourceLocation RParenLoc) {
+  return new (C) CXXConstCastExpr(T, VK, Op, WrittenTy, L, RParenLoc);
 }
 
 CXXConstCastExpr *CXXConstCastExpr::CreateEmpty(ASTContext &C) {
@@ -644,7 +506,7 @@ CXXConstCastExpr *CXXConstCastExpr::CreateEmpty(ASTContext &C) {
 }
 
 CXXFunctionalCastExpr *
-CXXFunctionalCastExpr::Create(ASTContext &C, QualType T,
+CXXFunctionalCastExpr::Create(ASTContext &C, QualType T, ExprValueKind VK,
                               TypeSourceInfo *Written, SourceLocation L,
                               CastKind K, Expr *Op, const CXXCastPath *BasePath,
                                SourceLocation R) {
@@ -652,7 +514,7 @@ CXXFunctionalCastExpr::Create(ASTContext &C, QualType T,
   void *Buffer = C.Allocate(sizeof(CXXFunctionalCastExpr)
                             + PathSize * sizeof(CXXBaseSpecifier*));
   CXXFunctionalCastExpr *E =
-    new (Buffer) CXXFunctionalCastExpr(T, Written, L, K, Op, PathSize, R);
+    new (Buffer) CXXFunctionalCastExpr(T, VK, Written, L, K, Op, PathSize, R);
   if (PathSize) E->setCastPath(*BasePath);
   return E;
 }
@@ -689,15 +551,22 @@ CXXBindTemporaryExpr *CXXBindTemporaryExpr::Create(ASTContext &C,
 
 CXXTemporaryObjectExpr::CXXTemporaryObjectExpr(ASTContext &C,
                                                CXXConstructorDecl *Cons,
-                                               QualType writtenTy,
-                                               SourceLocation tyBeginLoc,
+                                               TypeSourceInfo *Type,
                                                Expr **Args,
                                                unsigned NumArgs,
-                                               SourceLocation rParenLoc,
+                                               SourceRange parenRange,
                                                bool ZeroInitialization)
-  : CXXConstructExpr(C, CXXTemporaryObjectExprClass, writtenTy, tyBeginLoc,
-                     Cons, false, Args, NumArgs, ZeroInitialization),
-  TyBeginLoc(tyBeginLoc), RParenLoc(rParenLoc) {
+  : CXXConstructExpr(C, CXXTemporaryObjectExprClass, 
+                     Type->getType().getNonReferenceType(), 
+                     Type->getTypeLoc().getBeginLoc(),
+                     Cons, false, Args, NumArgs, ZeroInitialization,
+                     CXXConstructExpr::CK_Complete, parenRange),
+    Type(Type) {
+}
+
+SourceRange CXXTemporaryObjectExpr::getSourceRange() const {
+  return SourceRange(Type->getTypeLoc().getBeginLoc(),
+                     getParenRange().getEnd());
 }
 
 CXXConstructExpr *CXXConstructExpr::Create(ASTContext &C, QualType T,
@@ -705,10 +574,11 @@ CXXConstructExpr *CXXConstructExpr::Create(ASTContext &C, QualType T,
                                            CXXConstructorDecl *D, bool Elidable,
                                            Expr **Args, unsigned NumArgs,
                                            bool ZeroInitialization,
-                                           ConstructionKind ConstructKind) {
+                                           ConstructionKind ConstructKind,
+                                           SourceRange ParenRange) {
   return new (C) CXXConstructExpr(C, CXXConstructExprClass, T, Loc, D, 
                                   Elidable, Args, NumArgs, ZeroInitialization,
-                                  ConstructKind);
+                                  ConstructKind, ParenRange);
 }
 
 CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
@@ -716,31 +586,39 @@ CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
                                    CXXConstructorDecl *D, bool elidable,
                                    Expr **args, unsigned numargs,
                                    bool ZeroInitialization, 
-                                   ConstructionKind ConstructKind)
-: Expr(SC, T,
-       T->isDependentType(),
-       (T->isDependentType() ||
-        CallExpr::hasAnyValueDependentArguments(args, numargs))),
-  Constructor(D), Loc(Loc), Elidable(elidable), 
-  ZeroInitialization(ZeroInitialization), ConstructKind(ConstructKind),
-  Args(0), NumArgs(numargs) 
+                                   ConstructionKind ConstructKind,
+                                   SourceRange ParenRange)
+  : Expr(SC, T, VK_RValue, OK_Ordinary,
+         T->isDependentType(), T->isDependentType(),
+         T->containsUnexpandedParameterPack()),
+    Constructor(D), Loc(Loc), ParenRange(ParenRange), Elidable(elidable),
+    ZeroInitialization(ZeroInitialization), ConstructKind(ConstructKind),
+    Args(0), NumArgs(numargs) 
 {
   if (NumArgs) {
     Args = new (C) Stmt*[NumArgs];
     
     for (unsigned i = 0; i != NumArgs; ++i) {
       assert(args[i] && "NULL argument in CXXConstructExpr");
+
+      if (args[i]->isValueDependent())
+        ExprBits.ValueDependent = true;
+      if (args[i]->containsUnexpandedParameterPack())
+        ExprBits.ContainsUnexpandedParameterPack = true;
+  
       Args[i] = args[i];
     }
   }
 }
 
-CXXExprWithTemporaries::CXXExprWithTemporaries(ASTContext &C,
-                                               Expr *subexpr,
-                                               CXXTemporary **temps,
-                                               unsigned numtemps)
-  : Expr(CXXExprWithTemporariesClass, subexpr->getType(),
-       subexpr->isTypeDependent(), subexpr->isValueDependent()),
+ExprWithCleanups::ExprWithCleanups(ASTContext &C,
+                                   Expr *subexpr,
+                                   CXXTemporary **temps,
+                                   unsigned numtemps)
+  : Expr(ExprWithCleanupsClass, subexpr->getType(),
+         subexpr->getValueKind(), subexpr->getObjectKind(),
+         subexpr->isTypeDependent(), subexpr->isValueDependent(),
+         subexpr->containsUnexpandedParameterPack()),
     SubExpr(subexpr), Temps(0), NumTemps(0) {
   if (numtemps) {
     setNumTemporaries(C, numtemps);
@@ -749,75 +627,53 @@ CXXExprWithTemporaries::CXXExprWithTemporaries(ASTContext &C,
   }
 }
 
-void CXXExprWithTemporaries::setNumTemporaries(ASTContext &C, unsigned N) {
+void ExprWithCleanups::setNumTemporaries(ASTContext &C, unsigned N) {
   assert(Temps == 0 && "Cannot resize with this");
   NumTemps = N;
   Temps = new (C) CXXTemporary*[NumTemps];
 }
 
 
-CXXExprWithTemporaries *CXXExprWithTemporaries::Create(ASTContext &C,
-                                                       Expr *SubExpr,
-                                                       CXXTemporary **Temps,
-                                                       unsigned NumTemps) {
-  return new (C) CXXExprWithTemporaries(C, SubExpr, Temps, NumTemps);
+ExprWithCleanups *ExprWithCleanups::Create(ASTContext &C,
+                                           Expr *SubExpr,
+                                           CXXTemporary **Temps,
+                                           unsigned NumTemps) {
+  return new (C) ExprWithCleanups(C, SubExpr, Temps, NumTemps);
 }
 
-// CXXBindTemporaryExpr
-Stmt::child_iterator CXXBindTemporaryExpr::child_begin() {
-  return &SubExpr;
-}
-
-Stmt::child_iterator CXXBindTemporaryExpr::child_end() {
-  return &SubExpr + 1;
-}
-
-// CXXConstructExpr
-Stmt::child_iterator CXXConstructExpr::child_begin() {
-  return &Args[0];
-}
-Stmt::child_iterator CXXConstructExpr::child_end() {
-  return &Args[0]+NumArgs;
-}
-
-// CXXExprWithTemporaries
-Stmt::child_iterator CXXExprWithTemporaries::child_begin() {
-  return &SubExpr;
-}
-
-Stmt::child_iterator CXXExprWithTemporaries::child_end() {
-  return &SubExpr + 1;
-}
-
-CXXUnresolvedConstructExpr::CXXUnresolvedConstructExpr(
-                                                 SourceLocation TyBeginLoc,
-                                                 QualType T,
+CXXUnresolvedConstructExpr::CXXUnresolvedConstructExpr(TypeSourceInfo *Type,
                                                  SourceLocation LParenLoc,
                                                  Expr **Args,
                                                  unsigned NumArgs,
                                                  SourceLocation RParenLoc)
-  : Expr(CXXUnresolvedConstructExprClass, T.getNonReferenceType(),
-         T->isDependentType(), true),
-    TyBeginLoc(TyBeginLoc),
-    Type(T),
+  : Expr(CXXUnresolvedConstructExprClass, 
+         Type->getType().getNonReferenceType(),
+         VK_LValue, OK_Ordinary,
+         Type->getType()->isDependentType(), true,
+         Type->getType()->containsUnexpandedParameterPack()),
+    Type(Type),
     LParenLoc(LParenLoc),
     RParenLoc(RParenLoc),
     NumArgs(NumArgs) {
   Stmt **StoredArgs = reinterpret_cast<Stmt **>(this + 1);
-  memcpy(StoredArgs, Args, sizeof(Expr *) * NumArgs);
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    if (Args[I]->containsUnexpandedParameterPack())
+      ExprBits.ContainsUnexpandedParameterPack = true;
+
+    StoredArgs[I] = Args[I];
+  }
 }
 
 CXXUnresolvedConstructExpr *
 CXXUnresolvedConstructExpr::Create(ASTContext &C,
-                                   SourceLocation TyBegin,
-                                   QualType T,
+                                   TypeSourceInfo *Type,
                                    SourceLocation LParenLoc,
                                    Expr **Args,
                                    unsigned NumArgs,
                                    SourceLocation RParenLoc) {
   void *Mem = C.Allocate(sizeof(CXXUnresolvedConstructExpr) +
                          sizeof(Expr *) * NumArgs);
-  return new (Mem) CXXUnresolvedConstructExpr(TyBegin, T, LParenLoc,
+  return new (Mem) CXXUnresolvedConstructExpr(Type, LParenLoc,
                                               Args, NumArgs, RParenLoc);
 }
 
@@ -829,12 +685,8 @@ CXXUnresolvedConstructExpr::CreateEmpty(ASTContext &C, unsigned NumArgs) {
   return new (Mem) CXXUnresolvedConstructExpr(Empty, NumArgs);
 }
 
-Stmt::child_iterator CXXUnresolvedConstructExpr::child_begin() {
-  return child_iterator(reinterpret_cast<Stmt **>(this + 1));
-}
-
-Stmt::child_iterator CXXUnresolvedConstructExpr::child_end() {
-  return child_iterator(reinterpret_cast<Stmt **>(this + 1) + NumArgs);
+SourceRange CXXUnresolvedConstructExpr::getSourceRange() const {
+  return SourceRange(Type->getTypeLoc().getBeginLoc(), RParenLoc);
 }
 
 CXXDependentScopeMemberExpr::CXXDependentScopeMemberExpr(ASTContext &C,
@@ -846,17 +698,46 @@ CXXDependentScopeMemberExpr::CXXDependentScopeMemberExpr(ASTContext &C,
                                           NamedDecl *FirstQualifierFoundInScope,
                                           DeclarationNameInfo MemberNameInfo,
                                    const TemplateArgumentListInfo *TemplateArgs)
-  : Expr(CXXDependentScopeMemberExprClass, C.DependentTy, true, true),
+  : Expr(CXXDependentScopeMemberExprClass, C.DependentTy,
+         VK_LValue, OK_Ordinary, true, true,
+         ((Base && Base->containsUnexpandedParameterPack()) ||
+          (Qualifier && Qualifier->containsUnexpandedParameterPack()) ||
+          MemberNameInfo.containsUnexpandedParameterPack())),
     Base(Base), BaseType(BaseType), IsArrow(IsArrow),
     HasExplicitTemplateArgs(TemplateArgs != 0),
     OperatorLoc(OperatorLoc),
     Qualifier(Qualifier), QualifierRange(QualifierRange),
     FirstQualifierFoundInScope(FirstQualifierFoundInScope),
     MemberNameInfo(MemberNameInfo) {
-  if (TemplateArgs)
-    getExplicitTemplateArgs().initializeFrom(*TemplateArgs);
+  if (TemplateArgs) {
+    bool Dependent = true;
+    bool ContainsUnexpandedParameterPack = false;
+    getExplicitTemplateArgs().initializeFrom(*TemplateArgs, Dependent,
+                                             ContainsUnexpandedParameterPack);
+    if (ContainsUnexpandedParameterPack)
+      ExprBits.ContainsUnexpandedParameterPack = true;
+  }
 }
 
+CXXDependentScopeMemberExpr::CXXDependentScopeMemberExpr(ASTContext &C,
+                          Expr *Base, QualType BaseType,
+                          bool IsArrow,
+                          SourceLocation OperatorLoc,
+                          NestedNameSpecifier *Qualifier,
+                          SourceRange QualifierRange,
+                          NamedDecl *FirstQualifierFoundInScope,
+                          DeclarationNameInfo MemberNameInfo)
+  : Expr(CXXDependentScopeMemberExprClass, C.DependentTy,
+         VK_LValue, OK_Ordinary, true, true,
+         ((Base && Base->containsUnexpandedParameterPack()) ||
+          (Qualifier && Qualifier->containsUnexpandedParameterPack()) ||
+          MemberNameInfo.containsUnexpandedParameterPack())),
+    Base(Base), BaseType(BaseType), IsArrow(IsArrow),
+    HasExplicitTemplateArgs(false), OperatorLoc(OperatorLoc),
+    Qualifier(Qualifier), QualifierRange(QualifierRange),
+    FirstQualifierFoundInScope(FirstQualifierFoundInScope),
+    MemberNameInfo(MemberNameInfo) { }
+
 CXXDependentScopeMemberExpr *
 CXXDependentScopeMemberExpr::Create(ASTContext &C,
                                 Expr *Base, QualType BaseType, bool IsArrow,
@@ -877,7 +758,7 @@ CXXDependentScopeMemberExpr::Create(ASTContext &C,
   if (TemplateArgs)
     size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
 
-  void *Mem = C.Allocate(size, llvm::alignof<CXXDependentScopeMemberExpr>());
+  void *Mem = C.Allocate(size, llvm::alignOf<CXXDependentScopeMemberExpr>());
   return new (Mem) CXXDependentScopeMemberExpr(C, Base, BaseType,
                                                IsArrow, OperatorLoc,
                                                Qualifier, QualifierRange,
@@ -887,8 +768,9 @@ CXXDependentScopeMemberExpr::Create(ASTContext &C,
 
 CXXDependentScopeMemberExpr *
 CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
+                                         bool HasExplicitTemplateArgs,
                                          unsigned NumTemplateArgs) {
-  if (NumTemplateArgs == 0)
+  if (!HasExplicitTemplateArgs)
     return new (C) CXXDependentScopeMemberExpr(C, 0, QualType(),
                                                0, SourceLocation(), 0,
                                                SourceRange(), 0,
@@ -896,7 +778,7 @@ CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
 
   std::size_t size = sizeof(CXXDependentScopeMemberExpr) +
                      ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
-  void *Mem = C.Allocate(size, llvm::alignof<CXXDependentScopeMemberExpr>());
+  void *Mem = C.Allocate(size, llvm::alignOf<CXXDependentScopeMemberExpr>());
   CXXDependentScopeMemberExpr *E
     =  new (Mem) CXXDependentScopeMemberExpr(C, 0, QualType(),
                                              0, SourceLocation(), 0,
@@ -906,18 +788,7 @@ CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
   return E;
 }
 
-Stmt::child_iterator CXXDependentScopeMemberExpr::child_begin() {
-  return child_iterator(&Base);
-}
-
-Stmt::child_iterator CXXDependentScopeMemberExpr::child_end() {
-  if (isImplicitAccess())
-    return child_iterator(&Base);
-  return child_iterator(&Base + 1);
-}
-
-UnresolvedMemberExpr::UnresolvedMemberExpr(ASTContext &C, QualType T, 
-                                           bool Dependent, 
+UnresolvedMemberExpr::UnresolvedMemberExpr(ASTContext &C, 
                                            bool HasUnresolvedUsing,
                                            Expr *Base, QualType BaseType,
                                            bool IsArrow,
@@ -928,17 +799,21 @@ UnresolvedMemberExpr::UnresolvedMemberExpr(ASTContext &C, QualType T,
                                    const TemplateArgumentListInfo *TemplateArgs,
                                            UnresolvedSetIterator Begin, 
                                            UnresolvedSetIterator End)
-  : OverloadExpr(UnresolvedMemberExprClass, C, T, Dependent,
+  : OverloadExpr(UnresolvedMemberExprClass, C, 
                  Qualifier, QualifierRange, MemberNameInfo,
-                 TemplateArgs != 0, Begin, End),
+                 TemplateArgs, Begin, End,
+                 // Dependent
+                 ((Base && Base->isTypeDependent()) ||
+                  BaseType->isDependentType()),
+                 // Contains unexpanded parameter pack
+                 ((Base && Base->containsUnexpandedParameterPack()) ||
+                  BaseType->containsUnexpandedParameterPack())),
     IsArrow(IsArrow), HasUnresolvedUsing(HasUnresolvedUsing),
     Base(Base), BaseType(BaseType), OperatorLoc(OperatorLoc) {
-  if (TemplateArgs)
-    getExplicitTemplateArgs().initializeFrom(*TemplateArgs);
 }
 
 UnresolvedMemberExpr *
-UnresolvedMemberExpr::Create(ASTContext &C, bool Dependent,
+UnresolvedMemberExpr::Create(ASTContext &C, 
                              bool HasUnresolvedUsing,
                              Expr *Base, QualType BaseType, bool IsArrow,
                              SourceLocation OperatorLoc,
@@ -952,23 +827,23 @@ UnresolvedMemberExpr::Create(ASTContext &C, bool Dependent,
   if (TemplateArgs)
     size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
 
-  void *Mem = C.Allocate(size, llvm::alignof<UnresolvedMemberExpr>());
+  void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedMemberExpr>());
   return new (Mem) UnresolvedMemberExpr(C, 
-                             Dependent ? C.DependentTy : C.OverloadTy,
-                             Dependent, HasUnresolvedUsing, Base, BaseType,
+                             HasUnresolvedUsing, Base, BaseType,
                              IsArrow, OperatorLoc, Qualifier, QualifierRange,
                              MemberNameInfo, TemplateArgs, Begin, End);
 }
 
 UnresolvedMemberExpr *
-UnresolvedMemberExpr::CreateEmpty(ASTContext &C, unsigned NumTemplateArgs) {
+UnresolvedMemberExpr::CreateEmpty(ASTContext &C, bool HasExplicitTemplateArgs,
+                                  unsigned NumTemplateArgs) {
   std::size_t size = sizeof(UnresolvedMemberExpr);
-  if (NumTemplateArgs != 0)
+  if (HasExplicitTemplateArgs)
     size += ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
 
-  void *Mem = C.Allocate(size, llvm::alignof<UnresolvedMemberExpr>());
+  void *Mem = C.Allocate(size, llvm::alignOf<UnresolvedMemberExpr>());
   UnresolvedMemberExpr *E = new (Mem) UnresolvedMemberExpr(EmptyShell());
-  E->HasExplicitTemplateArgs = NumTemplateArgs != 0;
+  E->HasExplicitTemplateArgs = HasExplicitTemplateArgs;
   return E;
 }
 
@@ -980,7 +855,7 @@ CXXRecordDecl *UnresolvedMemberExpr::getNamingClass() const {
   // lookup.
   CXXRecordDecl *Record = 0;
   if (getQualifier()) {
-    Type *T = getQualifier()->getAsType();
+    const Type *T = getQualifier()->getAsType();
     assert(T && "qualifier in member expression does not name type");
     Record = T->getAsCXXRecordDecl();
     assert(Record && "qualifier in member expression does not name record");
@@ -1001,12 +876,18 @@ CXXRecordDecl *UnresolvedMemberExpr::getNamingClass() const {
   return Record;
 }
 
-Stmt::child_iterator UnresolvedMemberExpr::child_begin() {
-  return child_iterator(&Base);
-}
+SubstNonTypeTemplateParmPackExpr::
+SubstNonTypeTemplateParmPackExpr(QualType T, 
+                                 NonTypeTemplateParmDecl *Param,
+                                 SourceLocation NameLoc,
+                                 const TemplateArgument &ArgPack)
+  : Expr(SubstNonTypeTemplateParmPackExprClass, T, VK_RValue, OK_Ordinary, 
+         true, false, true),
+    Param(Param), Arguments(ArgPack.pack_begin()), 
+    NumArguments(ArgPack.pack_size()), NameLoc(NameLoc) { }
 
-Stmt::child_iterator UnresolvedMemberExpr::child_end() {
-  if (isImplicitAccess())
-    return child_iterator(&Base);
-  return child_iterator(&Base + 1);
+TemplateArgument SubstNonTypeTemplateParmPackExpr::getArgumentPack() const {
+  return TemplateArgument(Arguments, NumArguments);
 }
+
+
diff --git a/lib/AST/ExprClassification.cpp b/lib/AST/ExprClassification.cpp
index d7e38ebbf5c9..890898a985e8 100644
--- a/lib/AST/ExprClassification.cpp
+++ b/lib/AST/ExprClassification.cpp
@@ -29,10 +29,27 @@ static Cl::Kinds ClassifyUnnamed(ASTContext &Ctx, QualType T);
 static Cl::Kinds ClassifyMemberExpr(ASTContext &Ctx, const MemberExpr *E);
 static Cl::Kinds ClassifyBinaryOp(ASTContext &Ctx, const BinaryOperator *E);
 static Cl::Kinds ClassifyConditional(ASTContext &Ctx,
-                                    const ConditionalOperator *E);
+                                     const Expr *trueExpr,
+                                     const Expr *falseExpr);
 static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
                                        Cl::Kinds Kind, SourceLocation &Loc);
 
+static Cl::Kinds ClassifyExprValueKind(const LangOptions &Lang,
+                                       const Expr *E,
+                                       ExprValueKind Kind) {
+  switch (Kind) {
+  case VK_RValue:
+    return Lang.CPlusPlus && E->getType()->isRecordType() ?
+      Cl::CL_ClassTemporary : Cl::CL_PRValue;
+  case VK_LValue:
+    return Cl::CL_LValue;
+  case VK_XValue:
+    return Cl::CL_XValue;
+  }
+  llvm_unreachable("Invalid value category of implicit cast.");
+  return Cl::CL_PRValue;
+}
+
 Cl Expr::ClassifyImpl(ASTContext &Ctx, SourceLocation *Loc) const {
   assert(!TR->isReferenceType() && "Expressions can't have reference type.");
 
@@ -48,6 +65,19 @@ Cl Expr::ClassifyImpl(ASTContext &Ctx, SourceLocation *Loc) const {
       kind = Cl::CL_Void;
   }
 
+  // Enable this assertion for testing.
+  switch (kind) {
+  case Cl::CL_LValue: assert(getValueKind() == VK_LValue); break;
+  case Cl::CL_XValue: assert(getValueKind() == VK_XValue); break;
+  case Cl::CL_Function:
+  case Cl::CL_Void:
+  case Cl::CL_DuplicateVectorComponents:
+  case Cl::CL_MemberFunction:
+  case Cl::CL_SubObjCPropertySetting:
+  case Cl::CL_ClassTemporary:
+  case Cl::CL_PRValue: assert(getValueKind() == VK_RValue); break;
+  }
+
   Cl::ModifiableType modifiable = Cl::CM_Untested;
   if (Loc)
     modifiable = IsModifiable(Ctx, this, kind, *Loc);
@@ -60,7 +90,13 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
 
   switch (E->getStmtClass()) {
     // First come the expressions that are always lvalues, unconditionally.
-
+  case Stmt::NoStmtClass:
+#define ABSTRACT_STMT(Kind)
+#define STMT(Kind, Base) case Expr::Kind##Class:
+#define EXPR(Kind, Base)
+#include "clang/AST/StmtNodes.inc"
+    llvm_unreachable("cannot classify a statement");
+    break;
   case Expr::ObjCIsaExprClass:
     // C++ [expr.prim.general]p1: A string literal is an lvalue.
   case Expr::StringLiteralClass:
@@ -70,20 +106,56 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::PredefinedExprClass:
     // Property references are lvalues
   case Expr::ObjCPropertyRefExprClass:
-  case Expr::ObjCImplicitSetterGetterRefExprClass:
     // C++ [expr.typeid]p1: The result of a typeid expression is an lvalue of...
   case Expr::CXXTypeidExprClass:
     // Unresolved lookups get classified as lvalues.
     // FIXME: Is this wise? Should they get their own kind?
   case Expr::UnresolvedLookupExprClass:
   case Expr::UnresolvedMemberExprClass:
+  case Expr::CXXDependentScopeMemberExprClass:
+  case Expr::CXXUnresolvedConstructExprClass:
+  case Expr::DependentScopeDeclRefExprClass:
     // ObjC instance variables are lvalues
     // FIXME: ObjC++0x might have different rules
   case Expr::ObjCIvarRefExprClass:
+    return Cl::CL_LValue;
     // C99 6.5.2.5p5 says that compound literals are lvalues.
-    // FIXME: C++ might have a different opinion.
+    // In C++, they're class temporaries.
   case Expr::CompoundLiteralExprClass:
-    return Cl::CL_LValue;
+    return Ctx.getLangOptions().CPlusPlus? Cl::CL_ClassTemporary 
+                                         : Cl::CL_LValue;
+
+    // Expressions that are prvalues.
+  case Expr::CXXBoolLiteralExprClass:
+  case Expr::CXXPseudoDestructorExprClass:
+  case Expr::SizeOfAlignOfExprClass:
+  case Expr::CXXNewExprClass:
+  case Expr::CXXThisExprClass:
+  case Expr::CXXNullPtrLiteralExprClass:
+  case Expr::ImaginaryLiteralClass:
+  case Expr::GNUNullExprClass:
+  case Expr::OffsetOfExprClass:
+  case Expr::CXXThrowExprClass:
+  case Expr::ShuffleVectorExprClass:
+  case Expr::IntegerLiteralClass:
+  case Expr::CharacterLiteralClass:
+  case Expr::AddrLabelExprClass:
+  case Expr::CXXDeleteExprClass:
+  case Expr::ImplicitValueInitExprClass:
+  case Expr::BlockExprClass:
+  case Expr::FloatingLiteralClass:
+  case Expr::CXXNoexceptExprClass:
+  case Expr::CXXScalarValueInitExprClass:
+  case Expr::UnaryTypeTraitExprClass:
+  case Expr::BinaryTypeTraitExprClass:
+  case Expr::ObjCSelectorExprClass:
+  case Expr::ObjCProtocolExprClass:
+  case Expr::ObjCStringLiteralClass:
+  case Expr::ParenListExprClass:
+  case Expr::InitListExprClass:
+  case Expr::SizeOfPackExprClass:
+  case Expr::SubstNonTypeTemplateParmPackExprClass:
+    return Cl::CL_PRValue;
 
     // Next come the complicated cases.
 
@@ -115,11 +187,22 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
       return Cl::CL_LValue;
 
       // GNU extensions, simply look through them.
-    case UO_Real:
-    case UO_Imag:
     case UO_Extension:
       return ClassifyInternal(Ctx, cast<UnaryOperator>(E)->getSubExpr());
 
+    // Treat _Real and _Imag basically as if they were member
+    // expressions:  l-value only if the operand is a true l-value.
+    case UO_Real:
+    case UO_Imag: {
+      const Expr *Op = cast<UnaryOperator>(E)->getSubExpr()->IgnoreParens();
+      Cl::Kinds K = ClassifyInternal(Ctx, Op);
+      if (K != Cl::CL_LValue) return K;
+
+      if (isa<ObjCPropertyRefExpr>(Op))
+        return Cl::CL_SubObjCPropertySetting;
+      return Cl::CL_LValue;
+    }
+
       // C++ [expr.pre.incr]p1: The result is the updated operand; it is an
       //   lvalue, [...]
       // Not so in C.
@@ -131,19 +214,15 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
       return Cl::CL_PRValue;
     }
 
+  case Expr::OpaqueValueExprClass:
+    return ClassifyExprValueKind(Lang, E,
+                                 cast<OpaqueValueExpr>(E)->getValueKind());
+
     // Implicit casts are lvalues if they're lvalue casts. Other than that, we
     // only specifically record class temporaries.
   case Expr::ImplicitCastExprClass:
-    switch (cast<ImplicitCastExpr>(E)->getValueKind()) {
-    case VK_RValue:
-      return Lang.CPlusPlus && E->getType()->isRecordType() ?
-        Cl::CL_ClassTemporary : Cl::CL_PRValue;
-    case VK_LValue:
-      return Cl::CL_LValue;
-    case VK_XValue:
-      return Cl::CL_XValue;
-    }
-    llvm_unreachable("Invalid value category of implicit cast.");
+    return ClassifyExprValueKind(Lang, E,
+                                 cast<ImplicitCastExpr>(E)->getValueKind());
 
     // C++ [expr.prim.general]p4: The presence of parentheses does not affect
     //   whether the expression is an lvalue.
@@ -160,6 +239,7 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::CallExprClass:
   case Expr::CXXOperatorCallExprClass:
   case Expr::CXXMemberCallExprClass:
+  case Expr::CUDAKernelCallExprClass:
     return ClassifyUnnamed(Ctx, cast<CallExpr>(E)->getCallReturnType());
 
     // __builtin_choose_expr is equivalent to the chosen expression.
@@ -180,9 +260,9 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
   case Expr::CXXBindTemporaryExprClass:
     return ClassifyInternal(Ctx, cast<CXXBindTemporaryExpr>(E)->getSubExpr());
 
-    // And the temporary lifetime guard.
-  case Expr::CXXExprWithTemporariesClass:
-    return ClassifyInternal(Ctx, cast<CXXExprWithTemporaries>(E)->getSubExpr());
+    // And the cleanups guard.
+  case Expr::ExprWithCleanupsClass:
+    return ClassifyInternal(Ctx, cast<ExprWithCleanups>(E)->getSubExpr());
 
     // Casts depend completely on the target type. All casts work the same.
   case Expr::CStyleCastExprClass:
@@ -195,10 +275,18 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
     if (!Lang.CPlusPlus) return Cl::CL_PRValue;
     return ClassifyUnnamed(Ctx, cast<ExplicitCastExpr>(E)->getTypeAsWritten());
 
-  case Expr::ConditionalOperatorClass:
+  case Expr::BinaryConditionalOperatorClass: {
+    if (!Lang.CPlusPlus) return Cl::CL_PRValue;
+    const BinaryConditionalOperator *co = cast<BinaryConditionalOperator>(E);
+    return ClassifyConditional(Ctx, co->getTrueExpr(), co->getFalseExpr());
+  }
+
+  case Expr::ConditionalOperatorClass: {
     // Once again, only C++ is interesting.
     if (!Lang.CPlusPlus) return Cl::CL_PRValue;
-    return ClassifyConditional(Ctx, cast<ConditionalOperator>(E));
+    const ConditionalOperator *co = cast<ConditionalOperator>(E);
+    return ClassifyConditional(Ctx, co->getTrueExpr(), co->getFalseExpr());
+  }
 
     // ObjC message sends are effectively function calls, if the target function
     // is known.
@@ -207,17 +295,35 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
           cast<ObjCMessageExpr>(E)->getMethodDecl()) {
       return ClassifyUnnamed(Ctx, Method->getResultType());
     }
-
+    return Cl::CL_PRValue;
+      
     // Some C++ expressions are always class temporaries.
   case Expr::CXXConstructExprClass:
   case Expr::CXXTemporaryObjectExprClass:
-  case Expr::CXXScalarValueInitExprClass:
     return Cl::CL_ClassTemporary;
 
-    // Everything we haven't handled is a prvalue.
-  default:
+  case Expr::VAArgExprClass:
+    return ClassifyUnnamed(Ctx, E->getType());
+      
+  case Expr::DesignatedInitExprClass:
+    return ClassifyInternal(Ctx, cast<DesignatedInitExpr>(E)->getInit());
+      
+  case Expr::StmtExprClass: {
+    const CompoundStmt *S = cast<StmtExpr>(E)->getSubStmt();
+    if (const Expr *LastExpr = dyn_cast_or_null<Expr>(S->body_back()))
+      return ClassifyUnnamed(Ctx, LastExpr->getType());
     return Cl::CL_PRValue;
   }
+      
+  case Expr::CXXUuidofExprClass:
+    return Cl::CL_LValue;
+      
+  case Expr::PackExpansionExprClass:
+    return ClassifyInternal(Ctx, cast<PackExpansionExpr>(E)->getPattern());
+  }
+  
+  llvm_unreachable("unhandled expression kind in classification");
+  return Cl::CL_LValue;
 }
 
 /// ClassifyDecl - Return the classification of an expression referencing the
@@ -239,6 +345,7 @@ static Cl::Kinds ClassifyDecl(ASTContext &Ctx, const Decl *D) {
     islvalue = NTTParm->getType()->isReferenceType();
   else
     islvalue = isa<VarDecl>(D) || isa<FieldDecl>(D) ||
+	  isa<IndirectFieldDecl>(D) ||
       (Ctx.getLangOptions().CPlusPlus &&
         (isa<FunctionDecl>(D) || isa<FunctionTemplateDecl>(D)));
 
@@ -274,9 +381,8 @@ static Cl::Kinds ClassifyMemberExpr(ASTContext &Ctx, const MemberExpr *E) {
     if (E->isArrow())
       return Cl::CL_LValue;
     // ObjC property accesses are not lvalues, but get special treatment.
-    Expr *Base = E->getBase();
-    if (isa<ObjCPropertyRefExpr>(Base) ||
-        isa<ObjCImplicitSetterGetterRefExpr>(Base))
+    Expr *Base = E->getBase()->IgnoreParens();
+    if (isa<ObjCPropertyRefExpr>(Base))
       return Cl::CL_SubObjCPropertySetting;
     return ClassifyInternal(Ctx, Base);
   }
@@ -301,6 +407,9 @@ static Cl::Kinds ClassifyMemberExpr(ASTContext &Ctx, const MemberExpr *E) {
     // *E1 is an lvalue
     if (E->isArrow())
       return Cl::CL_LValue;
+    Expr *Base = E->getBase()->IgnoreParenImpCasts();
+    if (isa<ObjCPropertyRefExpr>(Base))
+      return Cl::CL_SubObjCPropertySetting;
     return ClassifyInternal(Ctx, E->getBase());
   }
 
@@ -320,8 +429,10 @@ static Cl::Kinds ClassifyBinaryOp(ASTContext &Ctx, const BinaryOperator *E) {
   assert(Ctx.getLangOptions().CPlusPlus &&
          "This is only relevant for C++.");
   // C++ [expr.ass]p1: All [...] return an lvalue referring to the left operand.
+  // Except we override this for writes to ObjC properties.
   if (E->isAssignmentOp())
-    return Cl::CL_LValue;
+    return (E->getLHS()->getObjectKind() == OK_ObjCProperty
+              ? Cl::CL_PRValue : Cl::CL_LValue);
 
   // C++ [expr.comma]p1: the result is of the same value category as its right
   //   operand, [...].
@@ -345,13 +456,11 @@ static Cl::Kinds ClassifyBinaryOp(ASTContext &Ctx, const BinaryOperator *E) {
   return Cl::CL_PRValue;
 }
 
-static Cl::Kinds ClassifyConditional(ASTContext &Ctx,
-                                     const ConditionalOperator *E) {
+static Cl::Kinds ClassifyConditional(ASTContext &Ctx, const Expr *True,
+                                     const Expr *False) {
   assert(Ctx.getLangOptions().CPlusPlus &&
          "This is only relevant for C++.");
 
-  Expr *True = E->getTrueExpr();
-  Expr *False = E->getFalseExpr();
   // C++ [expr.cond]p2
   //   If either the second or the third operand has type (cv) void, [...]
   //   the result [...] is a prvalue.
@@ -375,9 +484,10 @@ static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
   if (Kind == Cl::CL_PRValue) {
     // For the sake of better diagnostics, we want to specifically recognize
     // use of the GCC cast-as-lvalue extension.
-    if (const CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(E->IgnoreParens())){
-      if (CE->getSubExpr()->Classify(Ctx).isLValue()) {
-        Loc = CE->getLParenLoc();
+    if (const ExplicitCastExpr *CE =
+          dyn_cast<ExplicitCastExpr>(E->IgnoreParens())) {
+      if (CE->getSubExpr()->IgnoreParenImpCasts()->isLValue()) {
+        Loc = CE->getExprLoc();
         return Cl::CM_LValueCast;
       }
     }
@@ -401,9 +511,8 @@ static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
 
   // Assignment to a property in ObjC is an implicit setter access. But a
   // setter might not exist.
-  if (const ObjCImplicitSetterGetterRefExpr *Expr =
-        dyn_cast<ObjCImplicitSetterGetterRefExpr>(E)) {
-    if (Expr->getSetterMethod() == 0)
+  if (const ObjCPropertyRefExpr *Expr = dyn_cast<ObjCPropertyRefExpr>(E)) {
+    if (Expr->isImplicitProperty() && Expr->getImplicitPropertySetter() == 0)
       return Cl::CM_NoSetterProperty;
   }
 
@@ -420,7 +529,8 @@ static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
 
   // Records with any const fields (recursively) are not modifiable.
   if (const RecordType *R = CT->getAs<RecordType>()) {
-    assert(!Ctx.getLangOptions().CPlusPlus &&
+    assert((E->getObjectKind() == OK_ObjCProperty ||
+            !Ctx.getLangOptions().CPlusPlus) &&
            "C++ struct assignment should be resolved by the "
            "copy assignment operator.");
     if (R->hasConstFields())
@@ -430,7 +540,7 @@ static Cl::ModifiableType IsModifiable(ASTContext &Ctx, const Expr *E,
   return Cl::CM_Modifiable;
 }
 
-Expr::isLvalueResult Expr::isLvalue(ASTContext &Ctx) const {
+Expr::LValueClassification Expr::ClassifyLValue(ASTContext &Ctx) const {
   Classification VC = Classify(Ctx);
   switch (VC.getKind()) {
   case Cl::CL_LValue: return LV_Valid;
diff --git a/lib/AST/ExprConstant.cpp b/lib/AST/ExprConstant.cpp
index 7347f5a43e17..656bb99df98e 100644
--- a/lib/AST/ExprConstant.cpp
+++ b/lib/AST/ExprConstant.cpp
@@ -43,12 +43,20 @@ using llvm::APFloat;
 /// evaluate the expression regardless of what the RHS is, but C only allows
 /// certain things in certain situations.
 struct EvalInfo {
-  ASTContext &Ctx;
+  const ASTContext &Ctx;
 
   /// EvalResult - Contains information about the evaluation.
   Expr::EvalResult &EvalResult;
 
-  EvalInfo(ASTContext &ctx, Expr::EvalResult& evalresult)
+  llvm::DenseMap<const OpaqueValueExpr*, APValue> OpaqueValues;
+  const APValue *getOpaqueValue(const OpaqueValueExpr *e) {
+    llvm::DenseMap<const OpaqueValueExpr*, APValue>::iterator
+      i = OpaqueValues.find(e);
+    if (i == OpaqueValues.end()) return 0;
+    return &i->second;
+  }
+
+  EvalInfo(const ASTContext &ctx, Expr::EvalResult& evalresult)
     : Ctx(ctx), EvalResult(evalresult) {}
 };
 
@@ -73,12 +81,24 @@ namespace {
     APSInt &getComplexIntReal() { return IntReal; }
     APSInt &getComplexIntImag() { return IntImag; }
 
-    void moveInto(APValue &v) {
+    void moveInto(APValue &v) const {
       if (isComplexFloat())
         v = APValue(FloatReal, FloatImag);
       else
         v = APValue(IntReal, IntImag);
     }
+    void setFrom(const APValue &v) {
+      assert(v.isComplexFloat() || v.isComplexInt());
+      if (v.isComplexFloat()) {
+        makeComplexFloat();
+        FloatReal = v.getComplexFloatReal();
+        FloatImag = v.getComplexFloatImag();
+      } else {
+        makeComplexInt();
+        IntReal = v.getComplexIntReal();
+        IntImag = v.getComplexIntImag();
+      }
+    }
   };
 
   struct LValue {
@@ -88,12 +108,18 @@ namespace {
     Expr *getLValueBase() { return Base; }
     CharUnits getLValueOffset() { return Offset; }
 
-    void moveInto(APValue &v) {
+    void moveInto(APValue &v) const {
       v = APValue(Base, Offset);
     }
+    void setFrom(const APValue &v) {
+      assert(v.isLValue());
+      Base = v.getLValueBase();
+      Offset = v.getLValueOffset();
+    }
   };
 }
 
+static bool Evaluate(EvalInfo &info, const Expr *E);
 static bool EvaluateLValue(const Expr *E, LValue &Result, EvalInfo &Info);
 static bool EvaluatePointer(const Expr *E, LValue &Result, EvalInfo &Info);
 static bool EvaluateInteger(const Expr *E, APSInt  &Result, EvalInfo &Info);
@@ -192,7 +218,7 @@ static bool HandleConversionToBool(const Expr* E, bool& Result,
 }
 
 static APSInt HandleFloatToIntCast(QualType DestType, QualType SrcType,
-                                   APFloat &Value, ASTContext &Ctx) {
+                                   APFloat &Value, const ASTContext &Ctx) {
   unsigned DestWidth = Ctx.getIntWidth(DestType);
   // Determine whether we are converting to unsigned or signed.
   bool DestSigned = DestType->isSignedIntegerType();
@@ -206,7 +232,7 @@ static APSInt HandleFloatToIntCast(QualType DestType, QualType SrcType,
 }
 
 static APFloat HandleFloatToFloatCast(QualType DestType, QualType SrcType,
-                                      APFloat &Value, ASTContext &Ctx) {
+                                      APFloat &Value, const ASTContext &Ctx) {
   bool ignored;
   APFloat Result = Value;
   Result.convert(Ctx.getFloatTypeSemantics(DestType),
@@ -215,18 +241,18 @@ static APFloat HandleFloatToFloatCast(QualType DestType, QualType SrcType,
 }
 
 static APSInt HandleIntToIntCast(QualType DestType, QualType SrcType,
-                                 APSInt &Value, ASTContext &Ctx) {
+                                 APSInt &Value, const ASTContext &Ctx) {
   unsigned DestWidth = Ctx.getIntWidth(DestType);
   APSInt Result = Value;
   // Figure out if this is a truncate, extend or noop cast.
   // If the input is signed, do a sign extend, noop, or truncate.
-  Result.extOrTrunc(DestWidth);
+  Result = Result.extOrTrunc(DestWidth);
   Result.setIsUnsigned(DestType->isUnsignedIntegerType());
   return Result;
 }
 
 static APFloat HandleIntToFloatCast(QualType DestType, QualType SrcType,
-                                    APSInt &Value, ASTContext &Ctx) {
+                                    APSInt &Value, const ASTContext &Ctx) {
 
   APFloat Result(Ctx.getFloatTypeSemantics(DestType), 1);
   Result.convertFromAPInt(Value, Value.isSigned(),
@@ -291,8 +317,34 @@ public:
       if (Visit(E->getInit(i))) return true;
     return false;
   }
+    
+  bool VisitSizeOfPackExpr(SizeOfPackExpr *) { return false; }
 };
 
+class OpaqueValueEvaluation {
+  EvalInfo &info;
+  OpaqueValueExpr *opaqueValue;
+
+public:
+  OpaqueValueEvaluation(EvalInfo &info, OpaqueValueExpr *opaqueValue,
+                        Expr *value)
+    : info(info), opaqueValue(opaqueValue) {
+
+    // If evaluation fails, fail immediately.
+    if (!Evaluate(info, value)) {
+      this->opaqueValue = 0;
+      return;
+    }
+    info.OpaqueValues[opaqueValue] = info.EvalResult.Val;
+  }
+
+  bool hasError() const { return opaqueValue == 0; }
+
+  ~OpaqueValueEvaluation() {
+    if (opaqueValue) info.OpaqueValues.erase(opaqueValue);
+  }
+};
+  
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
@@ -402,7 +454,7 @@ bool LValueExprEvaluator::VisitMemberExpr(MemberExpr *E) {
       break;
   }
 
-  Result.Offset += CharUnits::fromQuantity(RL.getFieldOffset(i) / 8);
+  Result.Offset += Info.Ctx.toCharUnitsFromBits(RL.getFieldOffset(i));
   return true;
 }
 
@@ -460,17 +512,20 @@ public:
       { return Success(E); }
   bool VisitCallExpr(CallExpr *E);
   bool VisitBlockExpr(BlockExpr *E) {
-    if (!E->hasBlockDeclRefExprs())
+    if (!E->getBlockDecl()->hasCaptures())
       return Success(E);
     return false;
   }
   bool VisitImplicitValueInitExpr(ImplicitValueInitExpr *E)
       { return Success((Expr*)0); }
+  bool VisitBinaryConditionalOperator(BinaryConditionalOperator *E);
   bool VisitConditionalOperator(ConditionalOperator *E);
   bool VisitChooseExpr(ChooseExpr *E)
       { return Visit(E->getChosenSubExpr(Info.Ctx)); }
   bool VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E)
       { return Success((Expr*)0); }
+
+  bool VisitOpaqueValueExpr(OpaqueValueExpr *E);
   // FIXME: Missing: @protocol, @selector
 };
 } // end anonymous namespace
@@ -532,35 +587,6 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
   default:
     break;
 
-  case CK_Unknown: {
-    // FIXME: The handling for CK_Unknown is ugly/shouldn't be necessary!
-
-    // Check for pointer->pointer cast
-    if (SubExpr->getType()->isPointerType() ||
-        SubExpr->getType()->isObjCObjectPointerType() ||
-        SubExpr->getType()->isNullPtrType() ||
-        SubExpr->getType()->isBlockPointerType())
-      return Visit(SubExpr);
-
-    if (SubExpr->getType()->isIntegralOrEnumerationType()) {
-      APValue Value;
-      if (!EvaluateIntegerOrLValue(SubExpr, Value, Info))
-        break;
-
-      if (Value.isInt()) {
-        Value.getInt().extOrTrunc((unsigned)Info.Ctx.getTypeSize(E->getType()));
-        Result.Base = 0;
-        Result.Offset = CharUnits::fromQuantity(Value.getInt().getZExtValue());
-        return true;
-      } else {
-        Result.Base = Value.getLValueBase();
-        Result.Offset = Value.getLValueOffset();
-        return true;
-      }
-    }
-    break;
-  }
-
   case CK_NoOp:
   case CK_BitCast:
   case CK_LValueBitCast:
@@ -568,13 +594,54 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
   case CK_AnyPointerToBlockPointerCast:
     return Visit(SubExpr);
 
+  case CK_DerivedToBase:
+  case CK_UncheckedDerivedToBase: {
+    LValue BaseLV;
+    if (!EvaluatePointer(E->getSubExpr(), BaseLV, Info))
+      return false;
+
+    // Now figure out the necessary offset to add to the baseLV to get from
+    // the derived class to the base class.
+    CharUnits Offset = CharUnits::Zero();
+
+    QualType Ty = E->getSubExpr()->getType();
+    const CXXRecordDecl *DerivedDecl = 
+      Ty->getAs<PointerType>()->getPointeeType()->getAsCXXRecordDecl();
+
+    for (CastExpr::path_const_iterator PathI = E->path_begin(), 
+         PathE = E->path_end(); PathI != PathE; ++PathI) {
+      const CXXBaseSpecifier *Base = *PathI;
+
+      // FIXME: If the base is virtual, we'd need to determine the type of the
+      // most derived class and we don't support that right now.
+      if (Base->isVirtual())
+        return false;
+
+      const CXXRecordDecl *BaseDecl = Base->getType()->getAsCXXRecordDecl();
+      const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(DerivedDecl);
+
+      Offset += Layout.getBaseClassOffset(BaseDecl);
+      DerivedDecl = BaseDecl;
+    }
+
+    Result.Base = BaseLV.getLValueBase();
+    Result.Offset = BaseLV.getLValueOffset() + Offset;
+    return true;
+  }
+
+  case CK_NullToPointer: {
+    Result.Base = 0;
+    Result.Offset = CharUnits::Zero();
+    return true;
+  }
+
   case CK_IntegralToPointer: {
     APValue Value;
     if (!EvaluateIntegerOrLValue(SubExpr, Value, Info))
       break;
 
     if (Value.isInt()) {
-      Value.getInt().extOrTrunc((unsigned)Info.Ctx.getTypeSize(E->getType()));
+      Value.getInt() = Value.getInt().extOrTrunc((unsigned)Info.Ctx.getTypeSize(E->getType()));
       Result.Base = 0;
       Result.Offset = CharUnits::fromQuantity(Value.getInt().getZExtValue());
       return true;
@@ -602,6 +669,26 @@ bool PointerExprEvaluator::VisitCallExpr(CallExpr *E) {
   return false;
 }
 
+bool PointerExprEvaluator::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
+  const APValue *value = Info.getOpaqueValue(e);
+  if (!value)
+    return (e->getSourceExpr() ? Visit(e->getSourceExpr()) : false);
+  Result.setFrom(*value);
+  return true;
+}
+
+bool PointerExprEvaluator::
+VisitBinaryConditionalOperator(BinaryConditionalOperator *e) {
+  OpaqueValueEvaluation opaque(Info, e->getOpaqueValue(), e->getCommon());
+  if (opaque.hasError()) return false;
+
+  bool cond;
+  if (!HandleConversionToBool(e->getCond(), cond, Info))
+    return false;
+
+  return Visit(cond ? e->getTrueExpr() : e->getFalseExpr());
+}
+
 bool PointerExprEvaluator::VisitConditionalOperator(ConditionalOperator *E) {
   bool BoolResult;
   if (!HandleConversionToBool(E->getCond(), BoolResult, Info))
@@ -718,8 +805,7 @@ APValue VectorExprEvaluator::VisitCastExpr(const CastExpr* E) {
 
   llvm::SmallVector<APValue, 4> Elts;
   for (unsigned i = 0; i != NElts; ++i) {
-    APSInt Tmp = Init;
-    Tmp.extOrTrunc(EltWidth);
+    APSInt Tmp = Init.extOrTrunc(EltWidth);
 
     if (EltTy->isIntegerType())
       Elts.push_back(APValue(Tmp));
@@ -898,15 +984,14 @@ public:
   bool VisitCharacterLiteral(const CharacterLiteral *E) {
     return Success(E->getValue(), E);
   }
-  bool VisitTypesCompatibleExpr(const TypesCompatibleExpr *E) {
-    // Per gcc docs "this built-in function ignores top level
-    // qualifiers".  We need to use the canonical version to properly
-    // be able to strip CRV qualifiers from the type.
-    QualType T0 = Info.Ctx.getCanonicalType(E->getArgType1());
-    QualType T1 = Info.Ctx.getCanonicalType(E->getArgType2());
-    return Success(Info.Ctx.typesAreCompatible(T0.getUnqualifiedType(),
-                                               T1.getUnqualifiedType()),
-                   E);
+
+  bool VisitOpaqueValueExpr(OpaqueValueExpr *e) {
+    const APValue *value = Info.getOpaqueValue(e);
+    if (!value) {
+      if (e->getSourceExpr()) return Visit(e->getSourceExpr());
+      return Error(e->getExprLoc(), diag::note_invalid_subexpr_in_ice, e);
+    }
+    return Success(value->getInt(), e);
   }
 
   bool CheckReferencedDecl(const Expr *E, const Decl *D);
@@ -927,6 +1012,7 @@ public:
   bool VisitOffsetOfExpr(const OffsetOfExpr *E);
   bool VisitUnaryOperator(const UnaryOperator *E);
   bool VisitConditionalOperator(const ConditionalOperator *E);
+  bool VisitBinaryConditionalOperator(const BinaryConditionalOperator *E);
 
   bool VisitCastExpr(CastExpr* E);
   bool VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E);
@@ -948,7 +1034,11 @@ public:
   }
 
   bool VisitUnaryTypeTraitExpr(const UnaryTypeTraitExpr *E) {
-    return Success(E->EvaluateTrait(Info.Ctx), E);
+    return Success(E->getValue(), E);
+  }
+
+  bool VisitBinaryTypeTraitExpr(const BinaryTypeTraitExpr *E) {
+    return Success(E->getValue(), E);
   }
 
   bool VisitChooseExpr(const ChooseExpr *E) {
@@ -958,6 +1048,9 @@ public:
   bool VisitUnaryReal(const UnaryOperator *E);
   bool VisitUnaryImag(const UnaryOperator *E);
 
+  bool VisitCXXNoexceptExpr(const CXXNoexceptExpr *E);
+  bool VisitSizeOfPackExpr(const SizeOfPackExpr *E);
+    
 private:
   CharUnits GetAlignOfExpr(const Expr *E);
   CharUnits GetAlignOfType(QualType T);
@@ -1018,7 +1111,6 @@ bool IntExprEvaluator::CheckReferencedDecl(const Expr* E, const Decl* D) {
         }
 
         VD->setEvaluatedValue(APValue());
-        return false;
       }
     }
   }
@@ -1160,6 +1252,24 @@ bool IntExprEvaluator::VisitCallExpr(CallExpr *E) {
 
   case Builtin::BI__builtin_expect:
     return Visit(E->getArg(0));
+      
+  case Builtin::BIstrlen:
+  case Builtin::BI__builtin_strlen:
+    // As an extension, we support strlen() and __builtin_strlen() as constant
+    // expressions when the argument is a string literal.
+    if (StringLiteral *S
+               = dyn_cast<StringLiteral>(E->getArg(0)->IgnoreParenImpCasts())) {
+      // The string literal may have embedded null characters. Find the first
+      // one and truncate there.
+      llvm::StringRef Str = S->getString();
+      llvm::StringRef::size_type Pos = Str.find(0);
+      if (Pos != llvm::StringRef::npos)
+        Str = Str.substr(0, Pos);
+      
+      return Success(Str.size(), E);
+    }
+      
+    return Error(E->getLocStart(), diag::note_invalid_subexpr_in_ice, E);
   }
 }
 
@@ -1403,12 +1513,25 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       return Error(E->getOperatorLoc(), diag::note_expr_divide_by_zero, E);
     return Success(Result.getInt() % RHS, E);
   case BO_Shl: {
-    // FIXME: Warn about out of range shift amounts!
-    unsigned SA =
-      (unsigned) RHS.getLimitedValue(Result.getInt().getBitWidth()-1);
+    // During constant-folding, a negative shift is an opposite shift.
+    if (RHS.isSigned() && RHS.isNegative()) {
+      RHS = -RHS;
+      goto shift_right;
+    }
+
+  shift_left:
+    unsigned SA
+      = (unsigned) RHS.getLimitedValue(Result.getInt().getBitWidth()-1);
     return Success(Result.getInt() << SA, E);
   }
   case BO_Shr: {
+    // During constant-folding, a negative shift is an opposite shift.
+    if (RHS.isSigned() && RHS.isNegative()) {
+      RHS = -RHS;
+      goto shift_left;
+    }
+
+  shift_right:
     unsigned SA =
       (unsigned) RHS.getLimitedValue(Result.getInt().getBitWidth()-1);
     return Success(Result.getInt() >> SA, E);
@@ -1423,6 +1546,18 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   }
 }
 
+bool IntExprEvaluator::
+VisitBinaryConditionalOperator(const BinaryConditionalOperator *e) {
+  OpaqueValueEvaluation opaque(Info, e->getOpaqueValue(), e->getCommon());
+  if (opaque.hasError()) return false;
+
+  bool cond;
+  if (!HandleConversionToBool(e->getCond(), cond, Info))
+    return false;
+
+  return Visit(cond ? e->getTrueExpr() : e->getFalseExpr());
+}
+
 bool IntExprEvaluator::VisitConditionalOperator(const ConditionalOperator *E) {
   bool Cond;
   if (!HandleConversionToBool(E->getCond(), Cond, Info))
@@ -1439,12 +1574,9 @@ CharUnits IntExprEvaluator::GetAlignOfType(QualType T) {
   if (const ReferenceType *Ref = T->getAs<ReferenceType>())
     T = Ref->getPointeeType();
 
-  // Get information about the alignment.
-  unsigned CharSize = Info.Ctx.Target.getCharWidth();
-
   // __alignof is defined to return the preferred alignment.
-  return CharUnits::fromQuantity(
-      Info.Ctx.getPreferredTypeAlign(T.getTypePtr()) / CharSize);
+  return Info.Ctx.toCharUnitsFromBits(
+    Info.Ctx.getPreferredTypeAlign(T.getTypePtr()));
 }
 
 CharUnits IntExprEvaluator::GetAlignOfExpr(const Expr *E) {
@@ -1527,17 +1659,9 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *E) {
         return false;
       RecordDecl *RD = RT->getDecl();
       const ASTRecordLayout &RL = Info.Ctx.getASTRecordLayout(RD);
-      unsigned i = 0;
-      // FIXME: It would be nice if we didn't have to loop here!
-      for (RecordDecl::field_iterator Field = RD->field_begin(),
-                                      FieldEnd = RD->field_end();
-           Field != FieldEnd; (void)++Field, ++i) {
-        if (*Field == MemberDecl)
-          break;
-      }
+      unsigned i = MemberDecl->getFieldIndex();
       assert(i < RL.getFieldCount() && "offsetof field in wrong type");
-      Result += CharUnits::fromQuantity(
-                           RL.getFieldOffset(i) / Info.Ctx.getCharWidth());
+      Result += Info.Ctx.toCharUnitsFromBits(RL.getFieldOffset(i));
       CurrentType = MemberDecl->getType().getNonReferenceType();
       break;
     }
@@ -1565,9 +1689,7 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *E) {
         return false;
       
       // Add the offset to the base.
-      Result += CharUnits::fromQuantity(
-                RL.getBaseClassOffset(cast<CXXRecordDecl>(BaseRT->getDecl()))
-                                        / Info.Ctx.getCharWidth());
+      Result += RL.getBaseClassOffset(cast<CXXRecordDecl>(BaseRT->getDecl()));
       break;
     }
     }
@@ -1723,6 +1845,14 @@ bool IntExprEvaluator::VisitUnaryImag(const UnaryOperator *E) {
   return Success(0, E);
 }
 
+bool IntExprEvaluator::VisitSizeOfPackExpr(const SizeOfPackExpr *E) {
+  return Success(E->getPackLength(), E);
+}
+
+bool IntExprEvaluator::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
+  return Success(E->getValue(), E);
+}
+
 //===----------------------------------------------------------------------===//
 // Float Evaluation
 //===----------------------------------------------------------------------===//
@@ -1749,6 +1879,7 @@ public:
   bool VisitCastExpr(CastExpr *E);
   bool VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
   bool VisitConditionalOperator(ConditionalOperator *E);
+  bool VisitBinaryConditionalOperator(BinaryConditionalOperator *E);
 
   bool VisitChooseExpr(const ChooseExpr *E)
     { return Visit(E->getChosenSubExpr(Info.Ctx)); }
@@ -1757,6 +1888,16 @@ public:
   bool VisitUnaryReal(const UnaryOperator *E);
   bool VisitUnaryImag(const UnaryOperator *E);
 
+  bool VisitDeclRefExpr(const DeclRefExpr *E);
+
+  bool VisitOpaqueValueExpr(const OpaqueValueExpr *e) {
+    const APValue *value = Info.getOpaqueValue(e);
+    if (!value)
+      return (e->getSourceExpr() ? Visit(e->getSourceExpr()) : false);
+    Result = value->getFloat();
+    return true;
+  }
+
   // FIXME: Missing: array subscript of vector, member of vector,
   //                 ImplicitValueInitExpr
 };
@@ -1767,7 +1908,7 @@ static bool EvaluateFloat(const Expr* E, APFloat& Result, EvalInfo &Info) {
   return FloatExprEvaluator(Info, Result).Visit(const_cast<Expr*>(E));
 }
 
-static bool TryEvaluateBuiltinNaN(ASTContext &Context,
+static bool TryEvaluateBuiltinNaN(const ASTContext &Context,
                                   QualType ResultTy,
                                   const Expr *Arg,
                                   bool SNaN,
@@ -1844,6 +1985,45 @@ bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) {
   }
 }
 
+bool FloatExprEvaluator::VisitDeclRefExpr(const DeclRefExpr *E) {
+  const Decl *D = E->getDecl();
+  if (!isa<VarDecl>(D) || isa<ParmVarDecl>(D)) return false;
+  const VarDecl *VD = cast<VarDecl>(D);
+
+  // Require the qualifiers to be const and not volatile.
+  CanQualType T = Info.Ctx.getCanonicalType(E->getType());
+  if (!T.isConstQualified() || T.isVolatileQualified())
+    return false;
+
+  const Expr *Init = VD->getAnyInitializer();
+  if (!Init) return false;
+
+  if (APValue *V = VD->getEvaluatedValue()) {
+    if (V->isFloat()) {
+      Result = V->getFloat();
+      return true;
+    }
+    return false;
+  }
+
+  if (VD->isEvaluatingValue())
+    return false;
+
+  VD->setEvaluatingValue();
+
+  Expr::EvalResult InitResult;
+  if (Init->Evaluate(InitResult, Info.Ctx) && !InitResult.HasSideEffects &&
+      InitResult.Val.isFloat()) {
+    // Cache the evaluated value in the variable declaration.
+    Result = InitResult.Val.getFloat();
+    VD->setEvaluatedValue(InitResult.Val);
+    return true;
+  }
+
+  VD->setEvaluatedValue(APValue());
+  return false;
+}
+
 bool FloatExprEvaluator::VisitUnaryReal(const UnaryOperator *E) {
   if (E->getSubExpr()->getType()->isAnyComplexType()) {
     ComplexValue CV;
@@ -1902,6 +2082,10 @@ bool FloatExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
     return true;
   }
 
+  // We can't evaluate pointer-to-member operations.
+  if (E->isPtrMemOp())
+    return false;
+
   // FIXME: Diagnostics?  I really don't understand how the warnings
   // and errors are supposed to work.
   APFloat RHS(0.0);
@@ -1950,7 +2134,14 @@ bool FloatExprEvaluator::VisitCastExpr(CastExpr *E) {
                                     Result, Info.Ctx);
     return true;
   }
-  // FIXME: Handle complex types
+
+  if (E->getCastKind() == CK_FloatingComplexToReal) {
+    ComplexValue V;
+    if (!EvaluateComplex(SubExpr, V, Info))
+      return false;
+    Result = V.getComplexFloatReal();
+    return true;
+  }
 
   return false;
 }
@@ -1960,6 +2151,18 @@ bool FloatExprEvaluator::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E)
   return true;
 }
 
+bool FloatExprEvaluator::
+VisitBinaryConditionalOperator(BinaryConditionalOperator *e) {
+  OpaqueValueEvaluation opaque(Info, e->getOpaqueValue(), e->getCommon());
+  if (opaque.hasError()) return false;
+
+  bool cond;
+  if (!HandleConversionToBool(e->getCond(), cond, Info))
+    return false;
+
+  return Visit(cond ? e->getTrueExpr() : e->getFalseExpr());
+}
+
 bool FloatExprEvaluator::VisitConditionalOperator(ConditionalOperator *E) {
   bool Cond;
   if (!HandleConversionToBool(E->getCond(), Cond, Info))
@@ -1997,12 +2200,21 @@ public:
   bool VisitCastExpr(CastExpr *E);
 
   bool VisitBinaryOperator(const BinaryOperator *E);
+  bool VisitUnaryOperator(const UnaryOperator *E);
+  bool VisitConditionalOperator(const ConditionalOperator *E);
+  bool VisitBinaryConditionalOperator(const BinaryConditionalOperator *E);
   bool VisitChooseExpr(const ChooseExpr *E)
     { return Visit(E->getChosenSubExpr(Info.Ctx)); }
   bool VisitUnaryExtension(const UnaryOperator *E)
     { return Visit(E->getSubExpr()); }
-  // FIXME Missing: unary +/-/~, binary div, ImplicitValueInitExpr,
-  //                conditional ?:, comma
+  bool VisitOpaqueValueExpr(const OpaqueValueExpr *e) {
+    const APValue *value = Info.getOpaqueValue(e);
+    if (!value)
+      return (e->getSourceExpr() ? Visit(e->getSourceExpr()) : false);
+    Result.setFrom(*value);
+    return true;
+  }
+  // FIXME Missing: ImplicitValueInitExpr
 };
 } // end anonymous namespace
 
@@ -2038,99 +2250,143 @@ bool ComplexExprEvaluator::VisitImaginaryLiteral(ImaginaryLiteral *E) {
 }
 
 bool ComplexExprEvaluator::VisitCastExpr(CastExpr *E) {
-  Expr* SubExpr = E->getSubExpr();
-  QualType EltType = E->getType()->getAs<ComplexType>()->getElementType();
-  QualType SubType = SubExpr->getType();
 
-  if (SubType->isRealFloatingType()) {
+  switch (E->getCastKind()) {
+  case CK_BitCast:
+  case CK_LValueBitCast:
+  case CK_BaseToDerived:
+  case CK_DerivedToBase:
+  case CK_UncheckedDerivedToBase:
+  case CK_Dynamic:
+  case CK_ToUnion:
+  case CK_ArrayToPointerDecay:
+  case CK_FunctionToPointerDecay:
+  case CK_NullToPointer:
+  case CK_NullToMemberPointer:
+  case CK_BaseToDerivedMemberPointer:
+  case CK_DerivedToBaseMemberPointer:
+  case CK_MemberPointerToBoolean:
+  case CK_ConstructorConversion:
+  case CK_IntegralToPointer:
+  case CK_PointerToIntegral:
+  case CK_PointerToBoolean:
+  case CK_ToVoid:
+  case CK_VectorSplat:
+  case CK_IntegralCast:
+  case CK_IntegralToBoolean:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingToBoolean:
+  case CK_FloatingCast:
+  case CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
+  case CK_ObjCObjectLValueCast:
+  case CK_FloatingComplexToReal:
+  case CK_FloatingComplexToBoolean:
+  case CK_IntegralComplexToReal:
+  case CK_IntegralComplexToBoolean:
+    llvm_unreachable("invalid cast kind for complex value");
+
+  case CK_LValueToRValue:
+  case CK_NoOp:
+    return Visit(E->getSubExpr());
+
+  case CK_Dependent:
+  case CK_GetObjCProperty:
+  case CK_UserDefinedConversion:
+    return false;
+
+  case CK_FloatingRealToComplex: {
     APFloat &Real = Result.FloatReal;
-    if (!EvaluateFloat(SubExpr, Real, Info))
+    if (!EvaluateFloat(E->getSubExpr(), Real, Info))
       return false;
 
-    if (EltType->isRealFloatingType()) {
-      Result.makeComplexFloat();
-      Real = HandleFloatToFloatCast(EltType, SubType, Real, Info.Ctx);
-      Result.FloatImag = APFloat(Real.getSemantics());
-      return true;
-    } else {
-      Result.makeComplexInt();
-      Result.IntReal = HandleFloatToIntCast(EltType, SubType, Real, Info.Ctx);
-      Result.IntImag = APSInt(Result.IntReal.getBitWidth(),
-                              !Result.IntReal.isSigned());
-      return true;
-    }
-  } else if (SubType->isIntegerType()) {
+    Result.makeComplexFloat();
+    Result.FloatImag = APFloat(Real.getSemantics());
+    return true;
+  }
+
+  case CK_FloatingComplexCast: {
+    if (!Visit(E->getSubExpr()))
+      return false;
+
+    QualType To = E->getType()->getAs<ComplexType>()->getElementType();
+    QualType From
+      = E->getSubExpr()->getType()->getAs<ComplexType>()->getElementType();
+
+    Result.FloatReal
+      = HandleFloatToFloatCast(To, From, Result.FloatReal, Info.Ctx);
+    Result.FloatImag
+      = HandleFloatToFloatCast(To, From, Result.FloatImag, Info.Ctx);
+    return true;
+  }
+
+  case CK_FloatingComplexToIntegralComplex: {
+    if (!Visit(E->getSubExpr()))
+      return false;
+
+    QualType To = E->getType()->getAs<ComplexType>()->getElementType();
+    QualType From
+      = E->getSubExpr()->getType()->getAs<ComplexType>()->getElementType();
+    Result.makeComplexInt();
+    Result.IntReal = HandleFloatToIntCast(To, From, Result.FloatReal, Info.Ctx);
+    Result.IntImag = HandleFloatToIntCast(To, From, Result.FloatImag, Info.Ctx);
+    return true;
+  }
+
+  case CK_IntegralRealToComplex: {
     APSInt &Real = Result.IntReal;
-    if (!EvaluateInteger(SubExpr, Real, Info))
+    if (!EvaluateInteger(E->getSubExpr(), Real, Info))
       return false;
 
-    if (EltType->isRealFloatingType()) {
-      Result.makeComplexFloat();
-      Result.FloatReal
-        = HandleIntToFloatCast(EltType, SubType, Real, Info.Ctx);
-      Result.FloatImag = APFloat(Result.FloatReal.getSemantics());
-      return true;
-    } else {
-      Result.makeComplexInt();
-      Real = HandleIntToIntCast(EltType, SubType, Real, Info.Ctx);
-      Result.IntImag = APSInt(Real.getBitWidth(), !Real.isSigned());
-      return true;
-    }
-  } else if (const ComplexType *CT = SubType->getAs<ComplexType>()) {
-    if (!Visit(SubExpr))
+    Result.makeComplexInt();
+    Result.IntImag = APSInt(Real.getBitWidth(), !Real.isSigned());
+    return true;
+  }
+
+  case CK_IntegralComplexCast: {
+    if (!Visit(E->getSubExpr()))
       return false;
 
-    QualType SrcType = CT->getElementType();
+    QualType To = E->getType()->getAs<ComplexType>()->getElementType();
+    QualType From
+      = E->getSubExpr()->getType()->getAs<ComplexType>()->getElementType();
 
-    if (Result.isComplexFloat()) {
-      if (EltType->isRealFloatingType()) {
-        Result.makeComplexFloat();
-        Result.FloatReal = HandleFloatToFloatCast(EltType, SrcType,
-                                                  Result.FloatReal,
-                                                  Info.Ctx);
-        Result.FloatImag = HandleFloatToFloatCast(EltType, SrcType,
-                                                  Result.FloatImag,
-                                                  Info.Ctx);
-        return true;
-      } else {
-        Result.makeComplexInt();
-        Result.IntReal = HandleFloatToIntCast(EltType, SrcType,
-                                              Result.FloatReal,
-                                              Info.Ctx);
-        Result.IntImag = HandleFloatToIntCast(EltType, SrcType,
-                                              Result.FloatImag,
-                                              Info.Ctx);
-        return true;
-      }
-    } else {
-      assert(Result.isComplexInt() && "Invalid evaluate result.");
-      if (EltType->isRealFloatingType()) {
-        Result.makeComplexFloat();
-        Result.FloatReal = HandleIntToFloatCast(EltType, SrcType,
-                                                Result.IntReal,
-                                                Info.Ctx);
-        Result.FloatImag = HandleIntToFloatCast(EltType, SrcType,
-                                                Result.IntImag,
-                                                Info.Ctx);
-        return true;
-      } else {
-        Result.makeComplexInt();
-        Result.IntReal = HandleIntToIntCast(EltType, SrcType,
-                                            Result.IntReal,
-                                            Info.Ctx);
-        Result.IntImag = HandleIntToIntCast(EltType, SrcType,
-                                            Result.IntImag,
-                                            Info.Ctx);
-        return true;
-      }
-    }
+    Result.IntReal = HandleIntToIntCast(To, From, Result.IntReal, Info.Ctx);
+    Result.IntImag = HandleIntToIntCast(To, From, Result.IntImag, Info.Ctx);
+    return true;
+  }
+
+  case CK_IntegralComplexToFloatingComplex: {
+    if (!Visit(E->getSubExpr()))
+      return false;
+
+    QualType To = E->getType()->getAs<ComplexType>()->getElementType();
+    QualType From
+      = E->getSubExpr()->getType()->getAs<ComplexType>()->getElementType();
+    Result.makeComplexFloat();
+    Result.FloatReal = HandleIntToFloatCast(To, From, Result.IntReal, Info.Ctx);
+    Result.FloatImag = HandleIntToFloatCast(To, From, Result.IntImag, Info.Ctx);
+    return true;
+  }
   }
 
-  // FIXME: Handle more casts.
+  llvm_unreachable("unknown cast resulting in complex value");
   return false;
 }
 
 bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
+  if (E->getOpcode() == BO_Comma) {
+    if (!Visit(E->getRHS()))
+      return false;
+
+    // If we can't evaluate the LHS, it might have side effects;
+    // conservatively mark it.
+    if (!E->getLHS()->isEvaluatable(Info.Ctx))
+      Info.EvalResult.HasSideEffects = true;
+
+    return true;
+  }
   if (!Visit(E->getLHS()))
     return false;
 
@@ -2195,29 +2451,122 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
          LHS.getComplexIntImag() * RHS.getComplexIntReal());
     }
     break;
+  case BO_Div:
+    if (Result.isComplexFloat()) {
+      ComplexValue LHS = Result;
+      APFloat &LHS_r = LHS.getComplexFloatReal();
+      APFloat &LHS_i = LHS.getComplexFloatImag();
+      APFloat &RHS_r = RHS.getComplexFloatReal();
+      APFloat &RHS_i = RHS.getComplexFloatImag();
+      APFloat &Res_r = Result.getComplexFloatReal();
+      APFloat &Res_i = Result.getComplexFloatImag();
+
+      APFloat Den = RHS_r;
+      Den.multiply(RHS_r, APFloat::rmNearestTiesToEven);
+      APFloat Tmp = RHS_i;
+      Tmp.multiply(RHS_i, APFloat::rmNearestTiesToEven);
+      Den.add(Tmp, APFloat::rmNearestTiesToEven);
+
+      Res_r = LHS_r;
+      Res_r.multiply(RHS_r, APFloat::rmNearestTiesToEven);
+      Tmp = LHS_i;
+      Tmp.multiply(RHS_i, APFloat::rmNearestTiesToEven);
+      Res_r.add(Tmp, APFloat::rmNearestTiesToEven);
+      Res_r.divide(Den, APFloat::rmNearestTiesToEven);
+
+      Res_i = LHS_i;
+      Res_i.multiply(RHS_r, APFloat::rmNearestTiesToEven);
+      Tmp = LHS_r;
+      Tmp.multiply(RHS_i, APFloat::rmNearestTiesToEven);
+      Res_i.subtract(Tmp, APFloat::rmNearestTiesToEven);
+      Res_i.divide(Den, APFloat::rmNearestTiesToEven);
+    } else {
+      if (RHS.getComplexIntReal() == 0 && RHS.getComplexIntImag() == 0) {
+        // FIXME: what about diagnostics?
+        return false;
+      }
+      ComplexValue LHS = Result;
+      APSInt Den = RHS.getComplexIntReal() * RHS.getComplexIntReal() +
+        RHS.getComplexIntImag() * RHS.getComplexIntImag();
+      Result.getComplexIntReal() =
+        (LHS.getComplexIntReal() * RHS.getComplexIntReal() +
+         LHS.getComplexIntImag() * RHS.getComplexIntImag()) / Den;
+      Result.getComplexIntImag() =
+        (LHS.getComplexIntImag() * RHS.getComplexIntReal() -
+         LHS.getComplexIntReal() * RHS.getComplexIntImag()) / Den;
+    }
+    break;
   }
 
   return true;
 }
 
+bool ComplexExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
+  // Get the operand value into 'Result'.
+  if (!Visit(E->getSubExpr()))
+    return false;
+
+  switch (E->getOpcode()) {
+  default:
+    // FIXME: what about diagnostics?
+    return false;
+  case UO_Extension:
+    return true;
+  case UO_Plus:
+    // The result is always just the subexpr.
+    return true;
+  case UO_Minus:
+    if (Result.isComplexFloat()) {
+      Result.getComplexFloatReal().changeSign();
+      Result.getComplexFloatImag().changeSign();
+    }
+    else {
+      Result.getComplexIntReal() = -Result.getComplexIntReal();
+      Result.getComplexIntImag() = -Result.getComplexIntImag();
+    }
+    return true;
+  case UO_Not:
+    if (Result.isComplexFloat())
+      Result.getComplexFloatImag().changeSign();
+    else
+      Result.getComplexIntImag() = -Result.getComplexIntImag();
+    return true;
+  }
+}
+
+bool ComplexExprEvaluator::
+VisitBinaryConditionalOperator(const BinaryConditionalOperator *e) {
+  OpaqueValueEvaluation opaque(Info, e->getOpaqueValue(), e->getCommon());
+  if (opaque.hasError()) return false;
+
+  bool cond;
+  if (!HandleConversionToBool(e->getCond(), cond, Info))
+    return false;
+
+  return Visit(cond ? e->getTrueExpr() : e->getFalseExpr());
+}
+
+bool ComplexExprEvaluator::VisitConditionalOperator(const ConditionalOperator *E) {
+  bool Cond;
+  if (!HandleConversionToBool(E->getCond(), Cond, Info))
+    return false;
+
+  return Visit(Cond ? E->getTrueExpr() : E->getFalseExpr());
+}
+
 //===----------------------------------------------------------------------===//
 // Top level Expr::Evaluate method.
 //===----------------------------------------------------------------------===//
 
-/// Evaluate - Return true if this is a constant which we can fold using
-/// any crazy technique (that has nothing to do with language standards) that
-/// we want to.  If this function returns true, it returns the folded constant
-/// in Result.
-bool Expr::Evaluate(EvalResult &Result, ASTContext &Ctx) const {
-  const Expr *E = this;
-  EvalInfo Info(Ctx, Result);
+static bool Evaluate(EvalInfo &Info, const Expr *E) {
   if (E->getType()->isVectorType()) {
     if (!EvaluateVector(E, Info.EvalResult.Val, Info))
       return false;
   } else if (E->getType()->isIntegerType()) {
     if (!IntExprEvaluator(Info, Info.EvalResult.Val).Visit(const_cast<Expr*>(E)))
       return false;
-    if (Result.Val.isLValue() && !IsGlobalLValue(Result.Val.getLValueBase()))
+    if (Info.EvalResult.Val.isLValue() &&
+        !IsGlobalLValue(Info.EvalResult.Val.getLValueBase()))
       return false;
   } else if (E->getType()->hasPointerRepresentation()) {
     LValue LV;
@@ -2243,14 +2592,24 @@ bool Expr::Evaluate(EvalResult &Result, ASTContext &Ctx) const {
   return true;
 }
 
-bool Expr::EvaluateAsBooleanCondition(bool &Result, ASTContext &Ctx) const {
+/// Evaluate - Return true if this is a constant which we can fold using
+/// any crazy technique (that has nothing to do with language standards) that
+/// we want to.  If this function returns true, it returns the folded constant
+/// in Result.
+bool Expr::Evaluate(EvalResult &Result, const ASTContext &Ctx) const {
+  EvalInfo Info(Ctx, Result);
+  return ::Evaluate(Info, this);
+}
+
+bool Expr::EvaluateAsBooleanCondition(bool &Result,
+                                      const ASTContext &Ctx) const {
   EvalResult Scratch;
   EvalInfo Info(Ctx, Scratch);
 
   return HandleConversionToBool(this, Result, Info);
 }
 
-bool Expr::EvaluateAsLValue(EvalResult &Result, ASTContext &Ctx) const {
+bool Expr::EvaluateAsLValue(EvalResult &Result, const ASTContext &Ctx) const {
   EvalInfo Info(Ctx, Result);
 
   LValue LV;
@@ -2263,7 +2622,8 @@ bool Expr::EvaluateAsLValue(EvalResult &Result, ASTContext &Ctx) const {
   return false;
 }
 
-bool Expr::EvaluateAsAnyLValue(EvalResult &Result, ASTContext &Ctx) const {
+bool Expr::EvaluateAsAnyLValue(EvalResult &Result,
+                               const ASTContext &Ctx) const {
   EvalInfo Info(Ctx, Result);
 
   LValue LV;
@@ -2276,21 +2636,21 @@ bool Expr::EvaluateAsAnyLValue(EvalResult &Result, ASTContext &Ctx) const {
 
 /// isEvaluatable - Call Evaluate to see if this expression can be constant
 /// folded, but discard the result.
-bool Expr::isEvaluatable(ASTContext &Ctx) const {
+bool Expr::isEvaluatable(const ASTContext &Ctx) const {
   EvalResult Result;
   return Evaluate(Result, Ctx) && !Result.HasSideEffects;
 }
 
-bool Expr::HasSideEffects(ASTContext &Ctx) const {
+bool Expr::HasSideEffects(const ASTContext &Ctx) const {
   Expr::EvalResult Result;
   EvalInfo Info(Ctx, Result);
   return HasSideEffect(Info).Visit(const_cast<Expr*>(this));
 }
 
-APSInt Expr::EvaluateAsInt(ASTContext &Ctx) const {
+APSInt Expr::EvaluateAsInt(const ASTContext &Ctx) const {
   EvalResult EvalResult;
   bool Result = Evaluate(EvalResult, Ctx);
-  Result = Result;
+  (void)Result;
   assert(Result && "Could not evaluate expression");
   assert(EvalResult.Val.isInt() && "Expression did not evaluate to integer");
 
@@ -2358,6 +2718,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   }
 
   switch (E->getStmtClass()) {
+#define ABSTRACT_STMT(Node)
 #define STMT(Node, Base) case Expr::Node##Class:
 #define EXPR(Node, Base)
 #include "clang/AST/StmtNodes.inc"
@@ -2378,8 +2739,10 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::AddrLabelExprClass:
   case Expr::StmtExprClass:
   case Expr::CXXMemberCallExprClass:
+  case Expr::CUDAKernelCallExprClass:
   case Expr::CXXDynamicCastExprClass:
   case Expr::CXXTypeidExprClass:
+  case Expr::CXXUuidofExprClass:
   case Expr::CXXNullPtrLiteralExprClass:
   case Expr::CXXThisExprClass:
   case Expr::CXXThrowExprClass:
@@ -2390,7 +2753,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::DependentScopeDeclRefExprClass:
   case Expr::CXXConstructExprClass:
   case Expr::CXXBindTemporaryExprClass:
-  case Expr::CXXExprWithTemporariesClass:
+  case Expr::ExprWithCleanupsClass:
   case Expr::CXXTemporaryObjectExprClass:
   case Expr::CXXUnresolvedConstructExprClass:
   case Expr::CXXDependentScopeMemberExprClass:
@@ -2402,15 +2765,17 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::ObjCProtocolExprClass:
   case Expr::ObjCIvarRefExprClass:
   case Expr::ObjCPropertyRefExprClass:
-  case Expr::ObjCImplicitSetterGetterRefExprClass:
-  case Expr::ObjCSuperExprClass:
   case Expr::ObjCIsaExprClass:
   case Expr::ShuffleVectorExprClass:
   case Expr::BlockExprClass:
   case Expr::BlockDeclRefExprClass:
   case Expr::NoStmtClass:
+  case Expr::OpaqueValueExprClass:
+  case Expr::PackExpansionExprClass:
+  case Expr::SubstNonTypeTemplateParmPackExprClass:
     return ICEDiag(2, E->getLocStart());
 
+  case Expr::SizeOfPackExprClass:
   case Expr::GNUNullExprClass:
     // GCC considers the GNU __null value to be an integral constant expression.
     return NoDiag();
@@ -2421,8 +2786,9 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::CharacterLiteralClass:
   case Expr::CXXBoolLiteralExprClass:
   case Expr::CXXScalarValueInitExprClass:
-  case Expr::TypesCompatibleExprClass:
   case Expr::UnaryTypeTraitExprClass:
+  case Expr::BinaryTypeTraitExprClass:
+  case Expr::CXXNoexceptExprClass:
     return NoDiag();
   case Expr::CallExprClass:
   case Expr::CXXOperatorCallExprClass: {
@@ -2619,6 +2985,17 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
       return NoDiag();
     return ICEDiag(2, E->getLocStart());
   }
+  case Expr::BinaryConditionalOperatorClass: {
+    const BinaryConditionalOperator *Exp = cast<BinaryConditionalOperator>(E);
+    ICEDiag CommonResult = CheckICE(Exp->getCommon(), Ctx);
+    if (CommonResult.Val == 2) return CommonResult;
+    ICEDiag FalseResult = CheckICE(Exp->getFalseExpr(), Ctx);
+    if (FalseResult.Val == 2) return FalseResult;
+    if (CommonResult.Val == 1) return CommonResult;
+    if (FalseResult.Val == 1 &&
+        Exp->getCommon()->EvaluateAsInt(Ctx) == 0) return NoDiag();
+    return FalseResult;
+  }
   case Expr::ConditionalOperatorClass: {
     const ConditionalOperator *Exp = cast<ConditionalOperator>(E);
     // If the condition (ignoring parens) is a __builtin_constant_p call,
diff --git a/lib/AST/FullExpr.cpp b/lib/AST/FullExpr.cpp
deleted file mode 100644
index 93ee8d136000..000000000000
--- a/lib/AST/FullExpr.cpp
+++ /dev/null
@@ -1,45 +0,0 @@
-//===--- FullExpr.cpp - C++ full expression class ---------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the FullExpr interface, to be used for type safe handling
-//  of full expressions.
-//
-//  Full expressions are described in C++ [intro.execution]p12.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/FullExpr.h"
-#include "clang/AST/Expr.h"
-#include "clang/AST/ExprCXX.h"
-#include "llvm/Support/AlignOf.h"
-using namespace clang;
-
-FullExpr FullExpr::Create(ASTContext &Context, Expr *SubExpr,
-                          CXXTemporary **Temporaries, unsigned NumTemporaries) {
-  FullExpr E;
-
-  if (!NumTemporaries) {
-      E.SubExpr = SubExpr;
-      return E;
-  }
-
-  unsigned Size = sizeof(FullExpr) 
-      + sizeof(CXXTemporary *) * NumTemporaries;
-
-  unsigned Align = llvm::AlignOf<ExprAndTemporaries>::Alignment;
-  ExprAndTemporaries *ET = 
-      static_cast<ExprAndTemporaries *>(Context.Allocate(Size, Align));
-
-  ET->SubExpr = SubExpr;
-  std::copy(Temporaries, Temporaries + NumTemporaries, ET->temps_begin());
-    
-  return E;
-}
-
diff --git a/lib/AST/InheritViz.cpp b/lib/AST/InheritViz.cpp
index c47a9dadbadd..533a2329c583 100644
--- a/lib/AST/InheritViz.cpp
+++ b/lib/AST/InheritViz.cpp
@@ -17,6 +17,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/TypeOrdering.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/GraphWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include <map>
@@ -135,34 +136,28 @@ InheritanceHierarchyWriter::WriteNodeReference(QualType Type,
 /// class using GraphViz.
 void CXXRecordDecl::viewInheritance(ASTContext& Context) const {
   QualType Self = Context.getTypeDeclType(const_cast<CXXRecordDecl *>(this));
-  std::string ErrMsg;
-  sys::Path Filename = sys::Path::GetTemporaryDirectory(&ErrMsg);
-  if (Filename.isEmpty()) {
-    llvm::errs() << "Error: " << ErrMsg << "\n";
-    return;
-  }
-  Filename.appendComponent(Self.getAsString() + ".dot");
-  if (Filename.makeUnique(true,&ErrMsg)) {
-    llvm::errs() << "Error: " << ErrMsg << "\n";
+  // Create temp directory
+  SmallString<128> Filename;
+  int FileFD = 0;
+  if (error_code ec = sys::fs::unique_file(
+    "clang-class-inheritance-hierarchy-%%-%%-%%-%%-" +
+    Self.getAsString() + ".dot",
+    FileFD, Filename)) {
+    errs() << "Error creating temporary output file: " << ec.message() << '\n';
     return;
   }
 
-  llvm::errs() << "Writing '" << Filename.c_str() << "'... ";
+  llvm::errs() << "Writing '" << Filename << "'... ";
 
-  llvm::raw_fd_ostream O(Filename.c_str(), ErrMsg);
+  llvm::raw_fd_ostream O(FileFD, true);
+  InheritanceHierarchyWriter Writer(Context, O);
+  Writer.WriteGraph(Self);
 
-  if (ErrMsg.empty()) {
-    InheritanceHierarchyWriter Writer(Context, O);
-    Writer.WriteGraph(Self);
-    llvm::errs() << " done. \n";
+  llvm::errs() << " done. \n";
+  O.close();
 
-    O.close();
-
-    // Display the graph
-    DisplayGraph(Filename);
-  } else {
-    llvm::errs() << "error opening file for writing!\n";
-  }
+  // Display the graph
+  DisplayGraph(sys::Path(Filename));
 }
 
 }
diff --git a/lib/AST/ItaniumCXXABI.cpp b/lib/AST/ItaniumCXXABI.cpp
index c3fa4666537b..bed02b4c0010 100644
--- a/lib/AST/ItaniumCXXABI.cpp
+++ b/lib/AST/ItaniumCXXABI.cpp
@@ -19,7 +19,10 @@
 
 #include "CXXABI.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/Type.h"
+#include "clang/Basic/TargetInfo.h"
 
 using namespace clang;
 
@@ -35,6 +38,24 @@ public:
     if (Pointee->isFunctionType()) return 2;
     return 1;
   }
+
+  CallingConv getDefaultMethodCallConv() const {
+    return CC_C;
+  }
+
+  // We cheat and just check that the class has a vtable pointer, and that it's
+  // only big enough to have a vtable pointer and nothing more (or less).
+  bool isNearlyEmpty(const CXXRecordDecl *RD) const {
+
+    // Check that the class has a vtable pointer.
+    if (!RD->isDynamicClass())
+      return false;
+
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    CharUnits PointerSize = 
+      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+    return Layout.getNonVirtualSize() == PointerSize;
+  }
 };
 
 class ARMCXXABI : public ItaniumCXXABI {
diff --git a/lib/CodeGen/Mangle.cpp b/lib/AST/ItaniumMangle.cpp
index e1988743b7f9..d66c374cbec0 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/AST/ItaniumMangle.cpp
@@ -1,4 +1,4 @@
-//===--- Mangle.cpp - Mangle C++ Names --------------------------*- C++ -*-===//
+//===--- ItaniumMangle.cpp - Itanium C++ Name Mangling ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -14,18 +14,19 @@
 //   http://www.codesourcery.com/public/cxx-abi/abi.html
 //
 //===----------------------------------------------------------------------===//
-#include "Mangle.h"
+#include "clang/AST/Mangle.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/Basic/ABI.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "CGVTables.h"
 
 #define MANGLE_CHECKER 0
 
@@ -34,79 +35,17 @@
 #endif
 
 using namespace clang;
-using namespace CodeGen;
-
-MiscNameMangler::MiscNameMangler(MangleContext &C,
-                                 llvm::SmallVectorImpl<char> &Res)
-  : Context(C), Out(Res) { }
-
-void MiscNameMangler::mangleBlock(GlobalDecl GD, const BlockDecl *BD) {
-  // Mangle the context of the block.
-  // FIXME: We currently mimic GCC's mangling scheme, which leaves much to be
-  // desired. Come up with a better mangling scheme.
-  const DeclContext *DC = BD->getDeclContext();
-  while (isa<BlockDecl>(DC) || isa<EnumDecl>(DC))
-    DC = DC->getParent();
-  if (DC->isFunctionOrMethod()) {
-    Out << "__";
-    if (const ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(DC))
-      mangleObjCMethodName(Method);
-    else {
-      const NamedDecl *ND = cast<NamedDecl>(DC);
-      if (IdentifierInfo *II = ND->getIdentifier())
-        Out << II->getName();
-      else if (const CXXDestructorDecl *D = dyn_cast<CXXDestructorDecl>(ND)) {
-        llvm::SmallString<64> Buffer;
-        Context.mangleCXXDtor(D, GD.getDtorType(), Buffer);
-        Out << Buffer;
-      }
-      else if (const CXXConstructorDecl *D = dyn_cast<CXXConstructorDecl>(ND)) {
-        llvm::SmallString<64> Buffer;
-        Context.mangleCXXCtor(D, GD.getCtorType(), Buffer);
-        Out << Buffer;
-      }
-      else {
-        // FIXME: We were doing a mangleUnqualifiedName() before, but that's
-        // a private member of a class that will soon itself be private to the
-        // Itanium C++ ABI object. What should we do now? Right now, I'm just
-        // calling the mangleName() method on the MangleContext; is there a
-        // better way?
-        llvm::SmallString<64> Buffer;
-        Context.mangleName(ND, Buffer);
-        Out << Buffer;
-      }
-    }
-    Out << "_block_invoke_" << Context.getBlockId(BD, true);
-  } else {
-    Out << "__block_global_" << Context.getBlockId(BD, false);
-  }
-}
-
-void MiscNameMangler::mangleObjCMethodName(const ObjCMethodDecl *MD) {
-  llvm::SmallString<64> Name;
-  llvm::raw_svector_ostream OS(Name);
-  
-  const ObjCContainerDecl *CD =
-  dyn_cast<ObjCContainerDecl>(MD->getDeclContext());
-  assert (CD && "Missing container decl in GetNameForMethod");
-  OS << (MD->isInstanceMethod() ? '-' : '+') << '[' << CD->getName();
-  if (const ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(CD))
-    OS << '(' << CID << ')';
-  OS << ' ' << MD->getSelector().getAsString() << ']';
-  
-  Out << OS.str().size() << OS.str();
-}
 
 namespace {
 
-static const DeclContext *GetLocalClassFunctionDeclContext(
-                                                      const DeclContext *DC) {
-  if (isa<CXXRecordDecl>(DC)) {
-    while (!DC->isNamespace() && !DC->isTranslationUnit() &&
-           !isa<FunctionDecl>(DC))
-      DC = DC->getParent();
-    if (isa<FunctionDecl>(DC))
-      return DC;
+static const CXXRecordDecl *GetLocalClassDecl(const NamedDecl *ND) {
+  const DeclContext *DC = dyn_cast<DeclContext>(ND);
+  if (!DC)
+    DC = ND->getDeclContext();
+  while (!DC->isNamespace() && !DC->isTranslationUnit()) {
+    if (isa<FunctionDecl>(DC->getParent()))
+      return dyn_cast<CXXRecordDecl>(DC);
+    DC = DC->getParent();
   }
   return 0;
 }
@@ -127,10 +66,77 @@ static const CXXMethodDecl *getStructor(const CXXMethodDecl *MD) {
 
 static const unsigned UnknownArity = ~0U;
 
+class ItaniumMangleContext : public MangleContext {
+  llvm::DenseMap<const TagDecl *, uint64_t> AnonStructIds;
+  unsigned Discriminator;
+  llvm::DenseMap<const NamedDecl*, unsigned> Uniquifier;
+  
+public:
+  explicit ItaniumMangleContext(ASTContext &Context,
+                                Diagnostic &Diags)
+    : MangleContext(Context, Diags) { }
+
+  uint64_t getAnonymousStructId(const TagDecl *TD) {
+    std::pair<llvm::DenseMap<const TagDecl *,
+      uint64_t>::iterator, bool> Result =
+      AnonStructIds.insert(std::make_pair(TD, AnonStructIds.size()));
+    return Result.first->second;
+  }
+
+  void startNewFunction() {
+    MangleContext::startNewFunction();
+    mangleInitDiscriminator();
+  }
+
+  /// @name Mangler Entry Points
+  /// @{
+
+  bool shouldMangleDeclName(const NamedDecl *D);
+  void mangleName(const NamedDecl *D, llvm::raw_ostream &);
+  void mangleThunk(const CXXMethodDecl *MD,
+                   const ThunkInfo &Thunk,
+                   llvm::raw_ostream &);
+  void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
+                          const ThisAdjustment &ThisAdjustment,
+                          llvm::raw_ostream &);
+  void mangleReferenceTemporary(const VarDecl *D,
+                                llvm::raw_ostream &);
+  void mangleCXXVTable(const CXXRecordDecl *RD,
+                       llvm::raw_ostream &);
+  void mangleCXXVTT(const CXXRecordDecl *RD,
+                    llvm::raw_ostream &);
+  void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
+                           const CXXRecordDecl *Type,
+                           llvm::raw_ostream &);
+  void mangleCXXRTTI(QualType T, llvm::raw_ostream &);
+  void mangleCXXRTTIName(QualType T, llvm::raw_ostream &);
+  void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
+                     llvm::raw_ostream &);
+  void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
+                     llvm::raw_ostream &);
+
+  void mangleItaniumGuardVariable(const VarDecl *D, llvm::raw_ostream &);
+
+  void mangleInitDiscriminator() {
+    Discriminator = 0;
+  }
+
+  bool getNextDiscriminator(const NamedDecl *ND, unsigned &disc) {
+    unsigned &discriminator = Uniquifier[ND];
+    if (!discriminator)
+      discriminator = ++Discriminator;
+    if (discriminator == 1)
+      return false;
+    disc = discriminator-2;
+    return true;
+  }
+  /// @}
+};
+
 /// CXXNameMangler - Manage the mangling of a single name.
 class CXXNameMangler {
-  MangleContext &Context;
-  llvm::raw_svector_ostream Out;
+  ItaniumMangleContext &Context;
+  llvm::raw_ostream &Out;
 
   const CXXMethodDecl *Structor;
   unsigned StructorType;
@@ -143,15 +149,15 @@ class CXXNameMangler {
   ASTContext &getASTContext() const { return Context.getASTContext(); }
 
 public:
-  CXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res)
-    : Context(C), Out(Res), Structor(0), StructorType(0), SeqID(0) { }
-  CXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res,
+  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_)
+    : Context(C), Out(Out_), Structor(0), StructorType(0), SeqID(0) { }
+  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_,
                  const CXXConstructorDecl *D, CXXCtorType Type)
-    : Context(C), Out(Res), Structor(getStructor(D)), StructorType(Type),
+    : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
     SeqID(0) { }
-  CXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res,
+  CXXNameMangler(ItaniumMangleContext &C, llvm::raw_ostream &Out_,
                  const CXXDestructorDecl *D, CXXDtorType Type)
-    : Context(C), Out(Res), Structor(getStructor(D)), StructorType(Type),
+    : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
     SeqID(0) { }
 
 #if MANGLE_CHECKER
@@ -165,7 +171,7 @@ public:
     free(result);
   }
 #endif
-  llvm::raw_svector_ostream &getStream() { return Out; }
+  llvm::raw_ostream &getStream() { return Out; }
 
   void mangle(const NamedDecl *D, llvm::StringRef Prefix = "_Z");
   void mangleCallOffset(int64_t NonVirtual, int64_t Virtual);
@@ -222,6 +228,7 @@ private:
   void mangleTemplatePrefix(TemplateName Template);
   void mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity);
   void mangleQualifiers(Qualifiers Quals);
+  void mangleRefQualifier(RefQualifierKind RefQualifier);
 
   void mangleObjCMethodName(const ObjCMethodDecl *MD);
 
@@ -235,6 +242,7 @@ private:
   void mangleType(TemplateName);
   void mangleBareFunctionType(const FunctionType *T,
                               bool MangleReturnType);
+  void mangleNeonVectorType(const VectorType *T);
 
   void mangleIntegerLiteral(QualType T, const llvm::APSInt &Value);
   void mangleMemberExpr(const Expr *Base, bool IsArrow,
@@ -258,6 +266,7 @@ private:
 
   void mangleTemplateParameter(unsigned Index);
 };
+
 }
 
 static bool isInCLinkageSpecification(const Decl *D) {
@@ -271,7 +280,7 @@ static bool isInCLinkageSpecification(const Decl *D) {
   return false;
 }
 
-bool MangleContext::shouldMangleDeclName(const NamedDecl *D) {
+bool ItaniumMangleContext::shouldMangleDeclName(const NamedDecl *D) {
   // In C, functions with no attributes never need to be mangled. Fastpath them.
   if (!getASTContext().getLangOptions().CPlusPlus && !D->hasAttrs())
     return false;
@@ -320,7 +329,17 @@ void CXXNameMangler::mangle(const NamedDecl *D, llvm::StringRef Prefix) {
   // over all other naming in the .o file.
   if (const AsmLabelAttr *ALA = D->getAttr<AsmLabelAttr>()) {
     // If we have an asm name, then we use it as the mangling.
-    Out << '\01';  // LLVM IR Marker for __asm("foo")
+
+    // Adding the prefix can cause problems when one file has a "foo" and
+    // another has a "\01foo". That is known to happen on ELF with the
+    // tricks normally used for producing aliases (PR9177). Fortunately the
+    // llvm mangler on ELF is a nop, so we can just avoid adding the \01
+    // marker.
+    llvm::StringRef UserLabelPrefix =
+      getASTContext().Target.getUserLabelPrefix();
+    if (!UserLabelPrefix.empty())
+      Out << '\01';  // LLVM IR Marker for __asm("foo")
+
     Out << ALA->getLabel();
     return;
   }
@@ -373,7 +392,8 @@ void CXXNameMangler::mangleFunctionEncoding(const FunctionDecl *FD) {
 
   // Do the canonicalization out here because parameter types can
   // undergo additional canonicalization (e.g. array decay).
-  FunctionType *FT = cast<FunctionType>(Context.getASTContext()
+  const FunctionType *FT
+    = cast<FunctionType>(Context.getASTContext()
                                           .getCanonicalType(FD->getType()));
 
   mangleBareFunctionType(FT, MangleReturnType);
@@ -433,16 +453,15 @@ void CXXNameMangler::mangleName(const NamedDecl *ND) {
   //
   const DeclContext *DC = ND->getDeclContext();
 
-  if (GetLocalClassFunctionDeclContext(DC)) {
-    mangleLocalName(ND);
-    return;
-  }
-
   // If this is an extern variable declared locally, the relevant DeclContext
   // is that of the containing namespace, or the translation unit.
   if (isa<FunctionDecl>(DC) && ND->hasLinkage())
     while (!DC->isNamespace() && !DC->isTranslationUnit())
       DC = DC->getParent();
+  else if (GetLocalClassDecl(ND)) {
+    mangleLocalName(ND);
+    return;
+  }
 
   while (isa<LinkageSpecDecl>(DC))
     DC = DC->getParent();
@@ -525,7 +544,7 @@ void CXXNameMangler::mangleUnscopedTemplateName(TemplateName Template) {
     Diagnostic &Diags = Context.getDiags();
     unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
                                       "cannot mangle dependent operator name");
-    Diags.Report(FullSourceLoc(), DiagID);
+    Diags.Report(DiagID);
     return;
   }
   
@@ -809,13 +828,17 @@ void CXXNameMangler::mangleSourceName(const IdentifierInfo *II) {
 void CXXNameMangler::mangleNestedName(const NamedDecl *ND,
                                       const DeclContext *DC,
                                       bool NoFunction) {
-  // <nested-name> ::= N [<CV-qualifiers>] <prefix> <unqualified-name> E
-  //               ::= N [<CV-qualifiers>] <template-prefix> <template-args> E
+  // <nested-name> 
+  //   ::= N [<CV-qualifiers>] [<ref-qualifier>] <prefix> <unqualified-name> E
+  //   ::= N [<CV-qualifiers>] [<ref-qualifier>] <template-prefix> 
+  //       <template-args> E
 
   Out << 'N';
-  if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(ND))
+  if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(ND)) {
     mangleQualifiers(Qualifiers::fromCVRMask(Method->getTypeQualifiers()));
-
+    mangleRefQualifier(Method->getRefQualifier());
+  }
+  
   // Check if we have a template.
   const TemplateArgumentList *TemplateArgs = 0;
   if (const TemplateDecl *TD = isTemplate(ND, TemplateArgs)) {
@@ -853,15 +876,18 @@ void CXXNameMangler::mangleLocalName(const NamedDecl *ND) {
 
   if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(DC)) {
    mangleObjCMethodName(MD);
-  }
-  else if (const DeclContext *CDC = GetLocalClassFunctionDeclContext(DC)) {
-    mangleFunctionEncoding(cast<FunctionDecl>(CDC));
+  } else if (const CXXRecordDecl *RD = GetLocalClassDecl(ND)) {
+    mangleFunctionEncoding(cast<FunctionDecl>(RD->getDeclContext()));
     Out << 'E';
-    mangleNestedName(ND, DC, true /*NoFunction*/);
 
-    // FIXME. This still does not cover all cases.
+    // Mangle the name relative to the closest enclosing function.
+    if (ND == RD) // equality ok because RD derived from ND above
+      mangleUnqualifiedName(ND);
+    else
+      mangleNestedName(ND, DC, true /*NoFunction*/);
+
     unsigned disc;
-    if (Context.getNextDiscriminator(ND, disc)) {
+    if (Context.getNextDiscriminator(RD, disc)) {
       if (disc < 10)
         Out << '_' << disc;
       else
@@ -893,7 +919,9 @@ void CXXNameMangler::manglePrefix(const DeclContext *DC, bool NoFunction) {
   if (const BlockDecl *Block = dyn_cast<BlockDecl>(DC)) {
     manglePrefix(DC->getParent(), NoFunction);    
     llvm::SmallString<64> Name;
-    Context.mangleBlock(GlobalDecl(), Block, Name);
+    llvm::raw_svector_ostream NameStream(Name);
+    Context.mangleBlock(Block, NameStream);
+    NameStream.flush();
     Out << Name.size() << Name;
     return;
   }
@@ -1007,6 +1035,12 @@ void CXXNameMangler::mangleType(TemplateName TN) {
     break;
   }
 
+  case TemplateName::SubstTemplateTemplateParmPack: {
+    SubstTemplateTemplateParmPackStorage *SubstPack
+      = TN.getAsSubstTemplateTemplateParmPack();
+    mangleTemplateParameter(SubstPack->getParameterPack()->getIndex());
+    break;
+  }
   }
 
   addSubstitution(TN);
@@ -1145,27 +1179,57 @@ void CXXNameMangler::mangleQualifiers(Qualifiers Quals) {
   // FIXME: For now, just drop all extension qualifiers on the floor.
 }
 
+void CXXNameMangler::mangleRefQualifier(RefQualifierKind RefQualifier) {
+  // <ref-qualifier> ::= R                # lvalue reference
+  //                 ::= O                # rvalue-reference
+  // Proposal to Itanium C++ ABI list on 1/26/11
+  switch (RefQualifier) {
+  case RQ_None:
+    break;
+      
+  case RQ_LValue:
+    Out << 'R';
+    break;
+      
+  case RQ_RValue:
+    Out << 'O';
+    break;
+  }
+}
+
 void CXXNameMangler::mangleObjCMethodName(const ObjCMethodDecl *MD) {
-  llvm::SmallString<64> Buffer;
-  MiscNameMangler(Context, Buffer).mangleObjCMethodName(MD);
-  Out << Buffer;
+  Context.mangleObjCMethodName(MD, Out);
 }
 
-void CXXNameMangler::mangleType(QualType T) {
+void CXXNameMangler::mangleType(QualType nonCanon) {
   // Only operate on the canonical type!
-  T = Context.getASTContext().getCanonicalType(T);
+  QualType canon = nonCanon.getCanonicalType();
+
+  SplitQualType split = canon.split();
+  Qualifiers quals = split.second;
+  const Type *ty = split.first;
 
-  bool IsSubstitutable = T.hasLocalQualifiers() || !isa<BuiltinType>(T);
-  if (IsSubstitutable && mangleSubstitution(T))
+  bool isSubstitutable = quals || !isa<BuiltinType>(ty);
+  if (isSubstitutable && mangleSubstitution(canon))
     return;
 
-  if (Qualifiers Quals = T.getLocalQualifiers()) {
-    mangleQualifiers(Quals);
+  // If we're mangling a qualified array type, push the qualifiers to
+  // the element type.
+  if (quals && isa<ArrayType>(ty)) {
+    ty = Context.getASTContext().getAsArrayType(canon);
+    quals = Qualifiers();
+
+    // Note that we don't update canon: we want to add the
+    // substitution at the canonical type.
+  }
+
+  if (quals) {
+    mangleQualifiers(quals);
     // Recurse:  even if the qualified type isn't yet substitutable,
     // the unqualified type might be.
-    mangleType(T.getLocalUnqualifiedType());
+    mangleType(QualType(ty, 0));
   } else {
-    switch (T->getTypeClass()) {
+    switch (ty->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define NON_CANONICAL_TYPE(CLASS, PARENT) \
     case Type::CLASS: \
@@ -1173,15 +1237,15 @@ void CXXNameMangler::mangleType(QualType T) {
       return;
 #define TYPE(CLASS, PARENT) \
     case Type::CLASS: \
-      mangleType(static_cast<const CLASS##Type*>(T.getTypePtr())); \
+      mangleType(static_cast<const CLASS##Type*>(ty)); \
       break;
 #include "clang/AST/TypeNodes.def"
     }
   }
 
   // Add the substitution.
-  if (IsSubstitutable)
-    addSubstitution(T);
+  if (isSubstitutable)
+    addSubstitution(canon);
 }
 
 void CXXNameMangler::mangleNameOrStandardSubstitution(const NamedDecl *ND) {
@@ -1217,9 +1281,8 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
   // UNSUPPORTED:    ::= Dh # IEEE 754r half-precision floating point (16 bits)
   //                 ::= Di # char32_t
   //                 ::= Ds # char16_t
+  //                 ::= Dn # std::nullptr_t (i.e., decltype(nullptr))
   //                 ::= u <source-name>    # vendor extended type
-  // From our point of view, std::nullptr_t is a builtin, but as far as mangling
-  // is concerned, it's a type called std::nullptr_t.
   switch (T->getKind()) {
   case BuiltinType::Void: Out << 'v'; break;
   case BuiltinType::Bool: Out << 'b'; break;
@@ -1231,7 +1294,8 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
   case BuiltinType::ULongLong: Out << 'y'; break;
   case BuiltinType::UInt128: Out << 'o'; break;
   case BuiltinType::SChar: Out << 'a'; break;
-  case BuiltinType::WChar: Out << 'w'; break;
+  case BuiltinType::WChar_S:
+  case BuiltinType::WChar_U: Out << 'w'; break;
   case BuiltinType::Char16: Out << "Ds"; break;
   case BuiltinType::Char32: Out << "Di"; break;
   case BuiltinType::Short: Out << 's'; break;
@@ -1242,16 +1306,13 @@ void CXXNameMangler::mangleType(const BuiltinType *T) {
   case BuiltinType::Float: Out << 'f'; break;
   case BuiltinType::Double: Out << 'd'; break;
   case BuiltinType::LongDouble: Out << 'e'; break;
-  case BuiltinType::NullPtr: Out << "St9nullptr_t"; break;
+  case BuiltinType::NullPtr: Out << "Dn"; break;
 
   case BuiltinType::Overload:
   case BuiltinType::Dependent:
     assert(false &&
            "Overloaded and dependent types shouldn't get to name mangling");
     break;
-  case BuiltinType::UndeducedAuto:
-    assert(0 && "Should not see undeduced auto here");
-    break;
   case BuiltinType::ObjCId: Out << "11objc_object"; break;
   case BuiltinType::ObjCClass: Out << "10objc_class"; break;
   case BuiltinType::ObjCSel: Out << "13objc_selector"; break;
@@ -1322,7 +1383,9 @@ void CXXNameMangler::mangleType(const ConstantArrayType *T) {
 }
 void CXXNameMangler::mangleType(const VariableArrayType *T) {
   Out << 'A';
-  mangleExpression(T->getSizeExpr());
+  // decayed vla types (size 0) will just be skipped.
+  if (T->getSizeExpr())
+    mangleExpression(T->getSizeExpr());
   Out << '_';
   mangleType(T->getElementType());
 }
@@ -1333,7 +1396,7 @@ void CXXNameMangler::mangleType(const DependentSizedArrayType *T) {
   mangleType(T->getElementType());
 }
 void CXXNameMangler::mangleType(const IncompleteArrayType *T) {
-  Out << 'A' << '_';
+  Out << "A_";
   mangleType(T->getElementType());
 }
 
@@ -1345,6 +1408,7 @@ void CXXNameMangler::mangleType(const MemberPointerType *T) {
   QualType PointeeType = T->getPointeeType();
   if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(PointeeType)) {
     mangleQualifiers(Qualifiers::fromCVRMask(FPT->getTypeQuals()));
+    mangleRefQualifier(FPT->getRefQualifier());
     mangleType(FPT);
     
     // Itanium C++ ABI 5.1.8:
@@ -1371,6 +1435,11 @@ void CXXNameMangler::mangleType(const TemplateTypeParmType *T) {
   mangleTemplateParameter(T->getIndex());
 }
 
+// <type>           ::= <template-param>
+void CXXNameMangler::mangleType(const SubstTemplateTypeParmPackType *T) {
+  mangleTemplateParameter(T->getReplacedParameter()->getIndex());
+}
+
 // <type> ::= P <type>   # pointer-to
 void CXXNameMangler::mangleType(const PointerType *T) {
   Out << 'P';
@@ -1399,6 +1468,46 @@ void CXXNameMangler::mangleType(const ComplexType *T) {
   mangleType(T->getElementType());
 }
 
+// ARM's ABI for Neon vector types specifies that they should be mangled as
+// if they are structs (to match ARM's initial implementation).  The
+// vector type must be one of the special types predefined by ARM.
+void CXXNameMangler::mangleNeonVectorType(const VectorType *T) {
+  QualType EltType = T->getElementType();
+  assert(EltType->isBuiltinType() && "Neon vector element not a BuiltinType");
+  const char *EltName = 0;
+  if (T->getVectorKind() == VectorType::NeonPolyVector) {
+    switch (cast<BuiltinType>(EltType)->getKind()) {
+    case BuiltinType::SChar:     EltName = "poly8_t"; break;
+    case BuiltinType::Short:     EltName = "poly16_t"; break;
+    default: llvm_unreachable("unexpected Neon polynomial vector element type");
+    }
+  } else {
+    switch (cast<BuiltinType>(EltType)->getKind()) {
+    case BuiltinType::SChar:     EltName = "int8_t"; break;
+    case BuiltinType::UChar:     EltName = "uint8_t"; break;
+    case BuiltinType::Short:     EltName = "int16_t"; break;
+    case BuiltinType::UShort:    EltName = "uint16_t"; break;
+    case BuiltinType::Int:       EltName = "int32_t"; break;
+    case BuiltinType::UInt:      EltName = "uint32_t"; break;
+    case BuiltinType::LongLong:  EltName = "int64_t"; break;
+    case BuiltinType::ULongLong: EltName = "uint64_t"; break;
+    case BuiltinType::Float:     EltName = "float32_t"; break;
+    default: llvm_unreachable("unexpected Neon vector element type");
+    }
+  }
+  const char *BaseName = 0;
+  unsigned BitSize = (T->getNumElements() *
+                      getASTContext().getTypeSize(EltType));
+  if (BitSize == 64)
+    BaseName = "__simd64_";
+  else {
+    assert(BitSize == 128 && "Neon vector type not 64 or 128 bits");
+    BaseName = "__simd128_";
+  }
+  Out << strlen(BaseName) + strlen(EltName);
+  Out << BaseName << EltName;
+}
+
 // GNU extension: vector types
 // <type>                  ::= <vector-type>
 // <vector-type>           ::= Dv <positive dimension number> _
@@ -1407,10 +1516,15 @@ void CXXNameMangler::mangleType(const ComplexType *T) {
 // <extended element type> ::= <element type>
 //                         ::= p # AltiVec vector pixel
 void CXXNameMangler::mangleType(const VectorType *T) {
+  if ((T->getVectorKind() == VectorType::NeonVector ||
+       T->getVectorKind() == VectorType::NeonPolyVector)) {
+    mangleNeonVectorType(T);
+    return;
+  }
   Out << "Dv" << T->getNumElements() << '_';
-  if (T->getAltiVecSpecific() == VectorType::Pixel)
+  if (T->getVectorKind() == VectorType::AltiVecPixel)
     Out << 'p';
-  else if (T->getAltiVecSpecific() == VectorType::Bool)
+  else if (T->getVectorKind() == VectorType::AltiVecBool)
     Out << 'b';
   else
     mangleType(T->getElementType());
@@ -1425,6 +1539,12 @@ void CXXNameMangler::mangleType(const DependentSizedExtVectorType *T) {
   mangleType(T->getElementType());
 }
 
+void CXXNameMangler::mangleType(const PackExpansionType *T) {
+  // <type>  ::= Dp <type>          # pack expansion (C++0x)
+  Out << "Dp";
+  mangleType(T->getPattern());
+}
+
 void CXXNameMangler::mangleType(const ObjCInterfaceType *T) {
   mangleSourceName(T->getDecl()->getIdentifier());
 }
@@ -1525,6 +1645,12 @@ void CXXNameMangler::mangleType(const DecltypeType *T) {
   Out << 'E';
 }
 
+void CXXNameMangler::mangleType(const AutoType *T) {
+  QualType D = T->getDeducedType();
+  assert(!D.isNull() && "can't mangle undeduced auto type");
+  mangleType(D);
+}
+
 void CXXNameMangler::mangleIntegerLiteral(QualType T,
                                           const llvm::APSInt &Value) {
   //  <expr-primary> ::= L <type> <value number> E # integer literal
@@ -1570,12 +1696,14 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   //              ::= sr <type> <unqualified-name>                   # dependent name
   //              ::= sr <type> <unqualified-name> <template-args>   # dependent template-id
   //              ::= sZ <template-param>                            # size of a parameter pack
+  //              ::= sZ <function-param>    # size of a function parameter pack
   //              ::= <expr-primary>
   // <expr-primary> ::= L <type> <value number> E    # integer literal
   //                ::= L <type <value float> E      # floating literal
   //                ::= L <mangled-name> E           # external name
   switch (E->getStmtClass()) {
   case Expr::NoStmtClass:
+#define ABSTRACT_STMT(Type)
 #define EXPR(Type, Base)
 #define STMT(Type, Base) \
   case Expr::Type##Class:
@@ -1602,7 +1730,6 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::CompoundLiteralExprClass:
   case Expr::ExtVectorElementExprClass:
   case Expr::ObjCEncodeExprClass:
-  case Expr::ObjCImplicitSetterGetterRefExprClass:
   case Expr::ObjCIsaExprClass:
   case Expr::ObjCIvarRefExprClass:
   case Expr::ObjCMessageExprClass:
@@ -1610,25 +1737,40 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::ObjCProtocolExprClass:
   case Expr::ObjCSelectorExprClass:
   case Expr::ObjCStringLiteralClass:
-  case Expr::ObjCSuperExprClass:
   case Expr::OffsetOfExprClass:
   case Expr::PredefinedExprClass:
   case Expr::ShuffleVectorExprClass:
   case Expr::StmtExprClass:
-  case Expr::TypesCompatibleExprClass:
   case Expr::UnaryTypeTraitExprClass:
-  case Expr::VAArgExprClass: {
+  case Expr::BinaryTypeTraitExprClass:
+  case Expr::VAArgExprClass:
+  case Expr::CXXUuidofExprClass:
+  case Expr::CXXNoexceptExprClass:
+  case Expr::CUDAKernelCallExprClass: {
     // As bad as this diagnostic is, it's better than crashing.
     Diagnostic &Diags = Context.getDiags();
     unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
                                      "cannot yet mangle expression type %0");
-    Diags.Report(FullSourceLoc(E->getExprLoc(),
-                               getASTContext().getSourceManager()),
-                 DiagID)
+    Diags.Report(E->getExprLoc(), DiagID)
       << E->getStmtClassName() << E->getSourceRange();
     break;
   }
 
+  // Even gcc-4.5 doesn't mangle this.
+  case Expr::BinaryConditionalOperatorClass: {
+    Diagnostic &Diags = Context.getDiags();
+    unsigned DiagID =
+      Diags.getCustomDiagID(Diagnostic::Error,
+                "?: operator with omitted middle operand cannot be mangled");
+    Diags.Report(E->getExprLoc(), DiagID)
+      << E->getStmtClassName() << E->getSourceRange();
+    break;
+  }
+
+  // These are used for internal purposes and cannot be meaningfully mangled.
+  case Expr::OpaqueValueExprClass:
+    llvm_unreachable("cannot mangle opaque value; mangling wrong thing?");
+
   case Expr::CXXDefaultArgExprClass:
     mangleExpression(cast<CXXDefaultArgExpr>(E)->getExpr(), Arity);
     break;
@@ -1878,6 +2020,11 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
     break;
   }
 
+  case Expr::SubstNonTypeTemplateParmPackExprClass:
+    mangleTemplateParameter(
+     cast<SubstNonTypeTemplateParmPackExpr>(E)->getParameterPack()->getIndex());
+    break;
+      
   case Expr::DependentScopeDeclRefExprClass: {
     const DependentScopeDeclRefExpr *DRE = cast<DependentScopeDeclRefExpr>(E);
     NestedNameSpecifier *NNS = DRE->getQualifier();
@@ -1910,8 +2057,8 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
     mangleExpression(cast<CXXBindTemporaryExpr>(E)->getSubExpr());
     break;
 
-  case Expr::CXXExprWithTemporariesClass:
-    mangleExpression(cast<CXXExprWithTemporaries>(E)->getSubExpr(), Arity);
+  case Expr::ExprWithCleanupsClass:
+    mangleExpression(cast<ExprWithCleanups>(E)->getSubExpr(), Arity);
     break;
 
   case Expr::FloatingLiteralClass: {
@@ -1947,7 +2094,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   case Expr::ImaginaryLiteralClass: {
     const ImaginaryLiteral *IE = cast<ImaginaryLiteral>(E);
     // Mangle as if a complex literal.
-    // Proposal from David Vandervoorde, 2010.06.30.
+    // Proposal from David Vandevoorde, 2010.06.30.
     Out << 'L';
     mangleType(E->getType());
     if (const FloatingLiteral *Imag =
@@ -1957,7 +2104,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
       Out << '_';
       mangleFloat(Imag->getValue());
     } else {
-      Out << '0' << '_';
+      Out << "0_";
       llvm::APSInt Value(cast<IntegerLiteral>(IE->getSubExpr())->getValue());
       if (IE->getSubExpr()->getType()->isSignedIntegerType())
         Value.setIsSigned(true);
@@ -1986,7 +2133,33 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
     Out << "LDnE";
     break;
   }
-
+      
+  case Expr::PackExpansionExprClass:
+    Out << "sp";
+    mangleExpression(cast<PackExpansionExpr>(E)->getPattern());
+    break;
+      
+  case Expr::SizeOfPackExprClass: {
+    Out << "sZ";
+    const NamedDecl *Pack = cast<SizeOfPackExpr>(E)->getPack();
+    if (const TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Pack))
+      mangleTemplateParameter(TTP->getIndex());
+    else if (const NonTypeTemplateParmDecl *NTTP
+                = dyn_cast<NonTypeTemplateParmDecl>(Pack))
+      mangleTemplateParameter(NTTP->getIndex());
+    else if (const TemplateTemplateParmDecl *TempTP
+                                    = dyn_cast<TemplateTemplateParmDecl>(Pack))
+      mangleTemplateParameter(TempTP->getIndex());
+    else {
+      // Note: proposed by Mike Herrick on 11/30/10
+      // <expression> ::= sZ <function-param>  # size of function parameter pack
+      Diagnostic &Diags = Context.getDiags();
+      unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+                            "cannot mangle sizeof...(function parameter pack)");
+      Diags.Report(DiagID);
+      return;
+    }
+  }
   }
 }
 
@@ -2073,11 +2246,12 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
   // <template-arg> ::= <type>              # type or template
   //                ::= X <expression> E    # expression
   //                ::= <expr-primary>      # simple expressions
-  //                ::= I <template-arg>* E # argument pack
+  //                ::= J <template-arg>* E # argument pack
   //                ::= sp <expression>     # pack expansion of (C++0x)
   switch (A.getKind()) {
-  default:
-    assert(0 && "Unknown template argument kind!");
+  case TemplateArgument::Null:
+    llvm_unreachable("Cannot mangle NULL template argument");
+      
   case TemplateArgument::Type:
     mangleType(A.getAsType());
     break;
@@ -2085,6 +2259,11 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
     // This is mangled as <type>.
     mangleType(A.getAsTemplate());
     break;
+  case TemplateArgument::TemplateExpansion:
+    // <type>  ::= Dp <type>          # pack expansion (C++0x)
+    Out << "Dp";
+    mangleType(A.getAsTemplateOrTemplatePattern());
+    break;
   case TemplateArgument::Expression:
     Out << 'X';
     mangleExpression(A.getAsExpr());
@@ -2126,6 +2305,16 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
 
     break;
   }
+      
+  case TemplateArgument::Pack: {
+    // Note: proposal by Mike Herrick on 12/20/10
+    Out << 'J';
+    for (TemplateArgument::pack_iterator PA = A.pack_begin(), 
+                                      PAEnd = A.pack_end();
+         PA != PAEnd; ++PA)
+      mangleTemplateArg(P, *PA);
+    Out << 'E';
+  }
   }
 }
 
@@ -2240,8 +2429,8 @@ static bool isCharSpecialization(QualType T, const char *Name) {
 }
 
 template <std::size_t StrLen>
-bool isStreamCharSpecialization(const ClassTemplateSpecializationDecl *SD,
-                                const char (&Str)[StrLen]) {
+static bool isStreamCharSpecialization(const ClassTemplateSpecializationDecl*SD,
+                                       const char (&Str)[StrLen]) {
   if (!SD->getIdentifier()->isStr(Str))
     return false;
 
@@ -2370,8 +2559,8 @@ void CXXNameMangler::addSubstitution(uintptr_t Ptr) {
 /// and this routine will return false. In this case, the caller should just
 /// emit the identifier of the declaration (\c D->getIdentifier()) as its
 /// name.
-void MangleContext::mangleName(const NamedDecl *D,
-                               llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleName(const NamedDecl *D,
+                                      llvm::raw_ostream &Out) {
   assert((isa<FunctionDecl>(D) || isa<VarDecl>(D)) &&
           "Invalid mangleName() call, argument is not a variable or function!");
   assert(!isa<CXXConstructorDecl>(D) && !isa<CXXDestructorDecl>(D) &&
@@ -2381,31 +2570,27 @@ void MangleContext::mangleName(const NamedDecl *D,
                                  getASTContext().getSourceManager(),
                                  "Mangling declaration");
 
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   return Mangler.mangle(D);
 }
 
-void MangleContext::mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                                  llvm::SmallVectorImpl<char> &Res) {
-  CXXNameMangler Mangler(*this, Res, D, Type);
+void ItaniumMangleContext::mangleCXXCtor(const CXXConstructorDecl *D,
+                                         CXXCtorType Type,
+                                         llvm::raw_ostream &Out) {
+  CXXNameMangler Mangler(*this, Out, D, Type);
   Mangler.mangle(D);
 }
 
-void MangleContext::mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                                  llvm::SmallVectorImpl<char> &Res) {
-  CXXNameMangler Mangler(*this, Res, D, Type);
+void ItaniumMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
+                                         CXXDtorType Type,
+                                         llvm::raw_ostream &Out) {
+  CXXNameMangler Mangler(*this, Out, D, Type);
   Mangler.mangle(D);
 }
 
-void MangleContext::mangleBlock(GlobalDecl GD, const BlockDecl *BD,
-                                llvm::SmallVectorImpl<char> &Res) {
-  MiscNameMangler Mangler(*this, Res);
-  Mangler.mangleBlock(GD, BD);
-}
-
-void MangleContext::mangleThunk(const CXXMethodDecl *MD,
-                                const ThunkInfo &Thunk,
-                                llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleThunk(const CXXMethodDecl *MD,
+                                       const ThunkInfo &Thunk,
+                                       llvm::raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   //  <special-name> ::= Tc <call-offset> <call-offset> <base encoding>
@@ -2415,8 +2600,7 @@ void MangleContext::mangleThunk(const CXXMethodDecl *MD,
   
   assert(!isa<CXXDestructorDecl>(MD) &&
          "Use mangleCXXDtor for destructor decls!");
-  
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZT";
   if (!Thunk.Return.isEmpty())
     Mangler.getStream() << 'c';
@@ -2433,13 +2617,13 @@ void MangleContext::mangleThunk(const CXXMethodDecl *MD,
 }
 
 void 
-MangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
-                                  const ThisAdjustment &ThisAdjustment,
-                                  llvm::SmallVectorImpl<char> &Res) {
+ItaniumMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
+                                         CXXDtorType Type,
+                                         const ThisAdjustment &ThisAdjustment,
+                                         llvm::raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
-  
-  CXXNameMangler Mangler(*this, Res, DD, Type);
+  CXXNameMangler Mangler(*this, Out, DD, Type);
   Mangler.getStream() << "_ZT";
 
   // Mangle the 'this' pointer adjustment.
@@ -2451,45 +2635,46 @@ MangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
 
 /// mangleGuardVariable - Returns the mangled name for a guard variable
 /// for the passed in VarDecl.
-void MangleContext::mangleGuardVariable(const VarDecl *D,
-                                        llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleItaniumGuardVariable(const VarDecl *D,
+                                                      llvm::raw_ostream &Out) {
   //  <special-name> ::= GV <object name>       # Guard variable for one-time
   //                                            # initialization
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZGV";
   Mangler.mangleName(D);
 }
 
-void MangleContext::mangleReferenceTemporary(const VarDecl *D,
-                                             llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleReferenceTemporary(const VarDecl *D,
+                                                    llvm::raw_ostream &Out) {
   // We match the GCC mangling here.
   //  <special-name> ::= GR <object name>
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZGR";
   Mangler.mangleName(D);
 }
 
-void MangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
-                                    llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
+                                           llvm::raw_ostream &Out) {
   // <special-name> ::= TV <type>  # virtual table
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTV";
   Mangler.mangleNameOrStandardSubstitution(RD);
 }
 
-void MangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
-                                 llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
+                                        llvm::raw_ostream &Out) {
   // <special-name> ::= TT <type>  # VTT structure
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTT";
   Mangler.mangleNameOrStandardSubstitution(RD);
 }
 
-void MangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
-                                        const CXXRecordDecl *Type,
-                                        llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
+                                               int64_t Offset,
+                                               const CXXRecordDecl *Type,
+                                               llvm::raw_ostream &Out) {
   // <special-name> ::= TC <type> <offset number> _ <base type>
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTC";
   Mangler.mangleNameOrStandardSubstitution(RD);
   Mangler.getStream() << Offset;
@@ -2497,19 +2682,24 @@ void MangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
   Mangler.mangleNameOrStandardSubstitution(Type);
 }
 
-void MangleContext::mangleCXXRTTI(QualType Ty,
-                                  llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleCXXRTTI(QualType Ty,
+                                         llvm::raw_ostream &Out) {
   // <special-name> ::= TI <type>  # typeinfo structure
   assert(!Ty.hasQualifiers() && "RTTI info cannot have top-level qualifiers");
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTI";
   Mangler.mangleType(Ty);
 }
 
-void MangleContext::mangleCXXRTTIName(QualType Ty,
-                                      llvm::SmallVectorImpl<char> &Res) {
+void ItaniumMangleContext::mangleCXXRTTIName(QualType Ty,
+                                             llvm::raw_ostream &Out) {
   // <special-name> ::= TS <type>  # typeinfo name (null terminated byte string)
-  CXXNameMangler Mangler(*this, Res);
+  CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTS";
   Mangler.mangleType(Ty);
 }
+
+MangleContext *clang::createItaniumMangleContext(ASTContext &Context,
+                                                 Diagnostic &Diags) {
+  return new ItaniumMangleContext(Context, Diags);
+}
diff --git a/lib/AST/Mangle.cpp b/lib/AST/Mangle.cpp
new file mode 100644
index 000000000000..3a0b9094307f
--- /dev/null
+++ b/lib/AST/Mangle.cpp
@@ -0,0 +1,135 @@
+//===--- Mangle.cpp - Mangle C++ Names --------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements generic name mangling support for blocks and Objective-C.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/AST/Mangle.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/Basic/ABI.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#define MANGLE_CHECKER 0
+
+#if MANGLE_CHECKER
+#include <cxxabi.h>
+#endif
+
+using namespace clang;
+
+// FIXME: For blocks we currently mimic GCC's mangling scheme, which leaves
+// much to be desired. Come up with a better mangling scheme.
+
+namespace {
+
+static void mangleFunctionBlock(MangleContext &Context,
+                                llvm::StringRef Outer,
+                                const BlockDecl *BD,
+                                llvm::raw_ostream &Out) {
+  Out << "__" << Outer << "_block_invoke_" << Context.getBlockId(BD, true);
+}
+
+static void checkMangleDC(const DeclContext *DC, const BlockDecl *BD) {
+#ifndef NDEBUG
+  const DeclContext *ExpectedDC = BD->getDeclContext();
+  while (isa<BlockDecl>(ExpectedDC) || isa<EnumDecl>(ExpectedDC))
+    ExpectedDC = ExpectedDC->getParent();
+  assert(DC == ExpectedDC && "Given decl context did not match expected!");
+#endif
+}
+
+}
+
+void MangleContext::mangleGlobalBlock(const BlockDecl *BD,
+                                      llvm::raw_ostream &Out) {
+  Out << "__block_global_" << getBlockId(BD, false);
+}
+
+void MangleContext::mangleCtorBlock(const CXXConstructorDecl *CD,
+                                    CXXCtorType CT, const BlockDecl *BD,
+                                    llvm::raw_ostream &ResStream) {
+  checkMangleDC(CD, BD);
+  llvm::SmallString<64> Buffer;
+  llvm::raw_svector_ostream Out(Buffer);
+  mangleCXXCtor(CD, CT, Out);
+  Out.flush();
+  mangleFunctionBlock(*this, Buffer, BD, ResStream);
+}
+
+void MangleContext::mangleDtorBlock(const CXXDestructorDecl *DD,
+                                    CXXDtorType DT, const BlockDecl *BD,
+                                    llvm::raw_ostream &ResStream) {
+  checkMangleDC(DD, BD);
+  llvm::SmallString<64> Buffer;
+  llvm::raw_svector_ostream Out(Buffer);
+  mangleCXXDtor(DD, DT, Out);
+  Out.flush();
+  mangleFunctionBlock(*this, Buffer, BD, ResStream);
+}
+
+void MangleContext::mangleBlock(const DeclContext *DC, const BlockDecl *BD,
+                                llvm::raw_ostream &Out) {
+  assert(!isa<CXXConstructorDecl>(DC) && !isa<CXXDestructorDecl>(DC));
+  checkMangleDC(DC, BD);
+
+  llvm::SmallString<64> Buffer;
+  llvm::raw_svector_ostream Stream(Buffer);
+  if (const ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(DC)) {
+    mangleObjCMethodName(Method, Stream);
+  } else {
+    const NamedDecl *ND = cast<NamedDecl>(DC);
+    if (IdentifierInfo *II = ND->getIdentifier())
+      Stream << II->getName();
+    else {
+      // FIXME: We were doing a mangleUnqualifiedName() before, but that's
+      // a private member of a class that will soon itself be private to the
+      // Itanium C++ ABI object. What should we do now? Right now, I'm just
+      // calling the mangleName() method on the MangleContext; is there a
+      // better way?
+      mangleName(ND, Stream);
+    }
+  }
+  Stream.flush();
+  mangleFunctionBlock(*this, Buffer, BD, Out);
+}
+
+void MangleContext::mangleObjCMethodName(const ObjCMethodDecl *MD,
+                                         llvm::raw_ostream &Out) {
+  llvm::SmallString<64> Name;
+  llvm::raw_svector_ostream OS(Name);
+  
+  const ObjCContainerDecl *CD =
+  dyn_cast<ObjCContainerDecl>(MD->getDeclContext());
+  assert (CD && "Missing container decl in GetNameForMethod");
+  OS << (MD->isInstanceMethod() ? '-' : '+') << '[' << CD->getName();
+  if (const ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(CD))
+    OS << '(' << CID << ')';
+  OS << ' ' << MD->getSelector().getAsString() << ']';
+  
+  Out << OS.str().size() << OS.str();
+}
+
+void MangleContext::mangleBlock(const BlockDecl *BD,
+                                llvm::raw_ostream &Out) {
+  const DeclContext *DC = BD->getDeclContext();
+  while (isa<BlockDecl>(DC) || isa<EnumDecl>(DC))
+    DC = DC->getParent();
+  if (DC->isFunctionOrMethod())
+    mangleBlock(DC, BD, Out);
+  else
+    mangleGlobalBlock(BD, Out);
+}
diff --git a/lib/AST/MicrosoftCXXABI.cpp b/lib/AST/MicrosoftCXXABI.cpp
index 87b77673925f..4de93bb4beab 100644
--- a/lib/AST/MicrosoftCXXABI.cpp
+++ b/lib/AST/MicrosoftCXXABI.cpp
@@ -14,8 +14,10 @@
 
 #include "CXXABI.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/Type.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/TargetInfo.h"
 
 using namespace clang;
 
@@ -26,6 +28,27 @@ public:
   MicrosoftCXXABI(ASTContext &Ctx) : Context(Ctx) { }
 
   unsigned getMemberPointerSize(const MemberPointerType *MPT) const;
+
+  CallingConv getDefaultMethodCallConv() const {
+    if (Context.Target.getTriple().getArch() == llvm::Triple::x86)
+      return CC_X86ThisCall;
+    else
+      return CC_C;
+  }
+
+  bool isNearlyEmpty(const CXXRecordDecl *RD) const {
+    // FIXME: Audit the corners
+    if (!RD->isDynamicClass())
+      return false;
+
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    
+    // In the Microsoft ABI, classes can have one or two vtable pointers.
+    CharUnits PointerSize = 
+      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(0));
+    return Layout.getNonVirtualSize() == PointerSize ||
+      Layout.getNonVirtualSize() == PointerSize * 2;
+  }    
 };
 }
 
diff --git a/lib/AST/MicrosoftMangle.cpp b/lib/AST/MicrosoftMangle.cpp
new file mode 100644
index 000000000000..4bf7f23a0a9d
--- /dev/null
+++ b/lib/AST/MicrosoftMangle.cpp
@@ -0,0 +1,1188 @@
+//===--- MicrosoftMangle.cpp - Microsoft Visual C++ Name Mangling ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides C++ name mangling targetting the Microsoft Visual C++ ABI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/Mangle.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/CharUnits.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/Basic/ABI.h"
+
+using namespace clang;
+
+namespace {
+
+/// MicrosoftCXXNameMangler - Manage the mangling of a single name for the
+/// Microsoft Visual C++ ABI.
+class MicrosoftCXXNameMangler {
+  MangleContext &Context;
+  llvm::raw_ostream &Out;
+
+  ASTContext &getASTContext() const { return Context.getASTContext(); }
+
+public:
+  MicrosoftCXXNameMangler(MangleContext &C, llvm::raw_ostream &Out_)
+  : Context(C), Out(Out_) { }
+
+  void mangle(const NamedDecl *D, llvm::StringRef Prefix = "?");
+  void mangleName(const NamedDecl *ND);
+  void mangleFunctionEncoding(const FunctionDecl *FD);
+  void mangleVariableEncoding(const VarDecl *VD);
+  void mangleNumber(int64_t Number);
+  void mangleType(QualType T);
+
+private:
+  void mangleUnqualifiedName(const NamedDecl *ND) {
+    mangleUnqualifiedName(ND, ND->getDeclName());
+  }
+  void mangleUnqualifiedName(const NamedDecl *ND, DeclarationName Name);
+  void mangleSourceName(const IdentifierInfo *II);
+  void manglePostfix(const DeclContext *DC, bool NoFunction=false);
+  void mangleOperatorName(OverloadedOperatorKind OO);
+  void mangleQualifiers(Qualifiers Quals, bool IsMember);
+
+  void mangleObjCMethodName(const ObjCMethodDecl *MD);
+
+  // Declare manglers for every type class.
+#define ABSTRACT_TYPE(CLASS, PARENT)
+#define NON_CANONICAL_TYPE(CLASS, PARENT)
+#define TYPE(CLASS, PARENT) void mangleType(const CLASS##Type *T);
+#include "clang/AST/TypeNodes.def"
+  
+  void mangleType(const TagType*);
+  void mangleType(const FunctionType *T, const FunctionDecl *D,
+                  bool IsStructor, bool IsInstMethod);
+  void mangleType(const ArrayType *T, bool IsGlobal);
+  void mangleExtraDimensions(QualType T);
+  void mangleFunctionClass(const FunctionDecl *FD);
+  void mangleCallingConvention(const FunctionType *T, bool IsInstMethod = false);
+  void mangleThrowSpecification(const FunctionProtoType *T);
+
+};
+
+/// MicrosoftMangleContext - Overrides the default MangleContext for the
+/// Microsoft Visual C++ ABI.
+class MicrosoftMangleContext : public MangleContext {
+public:
+  MicrosoftMangleContext(ASTContext &Context,
+                         Diagnostic &Diags) : MangleContext(Context, Diags) { }
+  virtual bool shouldMangleDeclName(const NamedDecl *D);
+  virtual void mangleName(const NamedDecl *D, llvm::raw_ostream &Out);
+  virtual void mangleThunk(const CXXMethodDecl *MD,
+                           const ThunkInfo &Thunk,
+                           llvm::raw_ostream &);
+  virtual void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
+                                  const ThisAdjustment &ThisAdjustment,
+                                  llvm::raw_ostream &);
+  virtual void mangleCXXVTable(const CXXRecordDecl *RD,
+                               llvm::raw_ostream &);
+  virtual void mangleCXXVTT(const CXXRecordDecl *RD,
+                            llvm::raw_ostream &);
+  virtual void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
+                                   const CXXRecordDecl *Type,
+                                   llvm::raw_ostream &);
+  virtual void mangleCXXRTTI(QualType T, llvm::raw_ostream &);
+  virtual void mangleCXXRTTIName(QualType T, llvm::raw_ostream &);
+  virtual void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
+                             llvm::raw_ostream &);
+  virtual void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
+                             llvm::raw_ostream &);
+  virtual void mangleReferenceTemporary(const clang::VarDecl *,
+                                        llvm::raw_ostream &);
+};
+
+}
+
+static bool isInCLinkageSpecification(const Decl *D) {
+  D = D->getCanonicalDecl();
+  for (const DeclContext *DC = D->getDeclContext();
+       !DC->isTranslationUnit(); DC = DC->getParent()) {
+    if (const LinkageSpecDecl *Linkage = dyn_cast<LinkageSpecDecl>(DC))
+      return Linkage->getLanguage() == LinkageSpecDecl::lang_c;
+  }
+
+  return false;
+}
+
+bool MicrosoftMangleContext::shouldMangleDeclName(const NamedDecl *D) {
+  // In C, functions with no attributes never need to be mangled. Fastpath them.
+  if (!getASTContext().getLangOptions().CPlusPlus && !D->hasAttrs())
+    return false;
+
+  // Any decl can be declared with __asm("foo") on it, and this takes precedence
+  // over all other naming in the .o file.
+  if (D->hasAttr<AsmLabelAttr>())
+    return true;
+
+  // Clang's "overloadable" attribute extension to C/C++ implies name mangling
+  // (always) as does passing a C++ member function and a function
+  // whose name is not a simple identifier.
+  const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+  if (FD && (FD->hasAttr<OverloadableAttr>() || isa<CXXMethodDecl>(FD) ||
+             !FD->getDeclName().isIdentifier()))
+    return true;
+
+  // Otherwise, no mangling is done outside C++ mode.
+  if (!getASTContext().getLangOptions().CPlusPlus)
+    return false;
+
+  // Variables at global scope with internal linkage are not mangled.
+  if (!FD) {
+    const DeclContext *DC = D->getDeclContext();
+    if (DC->isTranslationUnit() && D->getLinkage() == InternalLinkage)
+      return false;
+  }
+
+  // C functions and "main" are not mangled.
+  if ((FD && FD->isMain()) || isInCLinkageSpecification(D))
+    return false;
+
+  return true;
+}
+
+void MicrosoftCXXNameMangler::mangle(const NamedDecl *D,
+                                     llvm::StringRef Prefix) {
+  // MSVC doesn't mangle C++ names the same way it mangles extern "C" names.
+  // Therefore it's really important that we don't decorate the
+  // name with leading underscores or leading/trailing at signs. So, emit a
+  // asm marker at the start so we get the name right.
+  Out << '\01';  // LLVM IR Marker for __asm("foo")
+
+  // Any decl can be declared with __asm("foo") on it, and this takes precedence
+  // over all other naming in the .o file.
+  if (const AsmLabelAttr *ALA = D->getAttr<AsmLabelAttr>()) {
+    // If we have an asm name, then we use it as the mangling.
+    Out << ALA->getLabel();
+    return;
+  }
+
+  // <mangled-name> ::= ? <name> <type-encoding>
+  Out << Prefix;
+  mangleName(D);
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
+    mangleFunctionEncoding(FD);
+  else if (const VarDecl *VD = dyn_cast<VarDecl>(D))
+    mangleVariableEncoding(VD);
+  // TODO: Fields? Can MSVC even mangle them?
+}
+
+void MicrosoftCXXNameMangler::mangleFunctionEncoding(const FunctionDecl *FD) {
+  // <type-encoding> ::= <function-class> <function-type>
+
+  // Don't mangle in the type if this isn't a decl we should typically mangle.
+  if (!Context.shouldMangleDeclName(FD))
+    return;
+  
+  // We should never ever see a FunctionNoProtoType at this point.
+  // We don't even know how to mangle their types anyway :).
+  const FunctionProtoType *FT = cast<FunctionProtoType>(FD->getType());
+
+  bool InStructor = false, InInstMethod = false;
+  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
+  if (MD) {
+    if (MD->isInstance())
+      InInstMethod = true;
+    if (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD))
+      InStructor = true;
+  }
+
+  // First, the function class.
+  mangleFunctionClass(FD);
+
+  mangleType(FT, FD, InStructor, InInstMethod);
+}
+
+void MicrosoftCXXNameMangler::mangleVariableEncoding(const VarDecl *VD) {
+  // <type-encoding> ::= <storage-class> <variable-type>
+  // <storage-class> ::= 0  # private static member
+  //                 ::= 1  # protected static member
+  //                 ::= 2  # public static member
+  //                 ::= 3  # global
+  //                 ::= 4  # static local
+  
+  // The first character in the encoding (after the name) is the storage class.
+  if (VD->isStaticDataMember()) {
+    // If it's a static member, it also encodes the access level.
+    switch (VD->getAccess()) {
+      default:
+      case AS_private: Out << '0'; break;
+      case AS_protected: Out << '1'; break;
+      case AS_public: Out << '2'; break;
+    }
+  }
+  else if (!VD->isStaticLocal())
+    Out << '3';
+  else
+    Out << '4';
+  // Now mangle the type.
+  // <variable-type> ::= <type> <cvr-qualifiers>
+  //                 ::= <type> A # pointers, references, arrays
+  // Pointers and references are odd. The type of 'int * const foo;' gets
+  // mangled as 'QAHA' instead of 'PAHB', for example.
+  QualType Ty = VD->getType();
+  if (Ty->isPointerType() || Ty->isReferenceType()) {
+    mangleType(Ty);
+    Out << 'A';
+  } else if (Ty->isArrayType()) {
+    // Global arrays are funny, too.
+    mangleType(cast<ArrayType>(Ty.getTypePtr()), true);
+    Out << 'A';
+  } else {
+    mangleType(Ty.getLocalUnqualifiedType());
+    mangleQualifiers(Ty.getLocalQualifiers(), false);
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleName(const NamedDecl *ND) {
+  // <name> ::= <unscoped-name> {[<named-scope>]+ | [<nested-name>]}? @
+  const DeclContext *DC = ND->getDeclContext();
+
+  // Always start with the unqualified name.
+  mangleUnqualifiedName(ND);    
+
+  // If this is an extern variable declared locally, the relevant DeclContext
+  // is that of the containing namespace, or the translation unit.
+  if (isa<FunctionDecl>(DC) && ND->hasLinkage())
+    while (!DC->isNamespace() && !DC->isTranslationUnit())
+      DC = DC->getParent();
+
+  manglePostfix(DC);
+
+  // Terminate the whole name with an '@'.
+  Out << '@';
+}
+
+void MicrosoftCXXNameMangler::mangleNumber(int64_t Number) {
+  // <number> ::= [?] <decimal digit> # <= 9
+  //          ::= [?] <hex digit>+ @ # > 9; A = 0, B = 1, etc...
+  if (Number < 0) {
+    Out << '?';
+    Number = -Number;
+  }
+  if (Number >= 1 && Number <= 10) {
+    Out << Number-1;
+  } else {
+    // We have to build up the encoding in reverse order, so it will come
+    // out right when we write it out.
+    char Encoding[16];
+    char *EndPtr = Encoding+sizeof(Encoding);
+    char *CurPtr = EndPtr;
+    while (Number) {
+      *--CurPtr = 'A' + (Number % 16);
+      Number /= 16;
+    }
+    Out.write(CurPtr, EndPtr-CurPtr);
+    Out << '@';
+  }
+}
+
+void
+MicrosoftCXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
+                                               DeclarationName Name) {
+  //  <unqualified-name> ::= <operator-name>
+  //                     ::= <ctor-dtor-name>
+  //                     ::= <source-name>
+  switch (Name.getNameKind()) {
+    case DeclarationName::Identifier: {
+      if (const IdentifierInfo *II = Name.getAsIdentifierInfo()) {
+        mangleSourceName(II);
+        break;
+      }
+      
+      // Otherwise, an anonymous entity.  We must have a declaration.
+      assert(ND && "mangling empty name without declaration");
+      
+      if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
+        if (NS->isAnonymousNamespace()) {
+          Out << "?A";
+          break;
+        }
+      }
+      
+      // We must have an anonymous struct.
+      const TagDecl *TD = cast<TagDecl>(ND);
+      if (const TypedefDecl *D = TD->getTypedefForAnonDecl()) {
+        assert(TD->getDeclContext() == D->getDeclContext() &&
+               "Typedef should not be in another decl context!");
+        assert(D->getDeclName().getAsIdentifierInfo() &&
+               "Typedef was not named!");
+        mangleSourceName(D->getDeclName().getAsIdentifierInfo());
+        break;
+      }
+
+      // When VC encounters an anonymous type with no tag and no typedef,
+      // it literally emits '<unnamed-tag>'.
+      Out << "<unnamed-tag>";
+      break;
+    }
+      
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      assert(false && "Can't mangle Objective-C selector names here!");
+      break;
+      
+    case DeclarationName::CXXConstructorName:
+      assert(false && "Can't mangle constructors yet!");
+      break;
+      
+    case DeclarationName::CXXDestructorName:
+      assert(false && "Can't mangle destructors yet!");
+      break;
+      
+    case DeclarationName::CXXConversionFunctionName:
+      // <operator-name> ::= ?B # (cast)
+      // The target type is encoded as the return type.
+      Out << "?B";
+      break;
+      
+    case DeclarationName::CXXOperatorName:
+      mangleOperatorName(Name.getCXXOverloadedOperator());
+      break;
+      
+    case DeclarationName::CXXLiteralOperatorName:
+      // FIXME: Was this added in VS2010? Does MS even know how to mangle this?
+      assert(false && "Don't know how to mangle literal operators yet!");
+      break;
+      
+    case DeclarationName::CXXUsingDirective:
+      assert(false && "Can't mangle a using directive name!");
+      break;
+  }
+}
+
+void MicrosoftCXXNameMangler::manglePostfix(const DeclContext *DC,
+                                            bool NoFunction) {
+  // <postfix> ::= <unqualified-name> [<postfix>]
+  //           ::= <template-postfix> <template-args> [<postfix>]
+  //           ::= <template-param>
+  //           ::= <substitution> [<postfix>]
+
+  if (!DC) return;
+
+  while (isa<LinkageSpecDecl>(DC))
+    DC = DC->getParent();
+
+  if (DC->isTranslationUnit())
+    return;
+
+  if (const BlockDecl *BD = dyn_cast<BlockDecl>(DC)) {
+    Context.mangleBlock(BD, Out);
+    Out << '@';
+    return manglePostfix(DC->getParent(), NoFunction);
+  }
+
+  if (NoFunction && (isa<FunctionDecl>(DC) || isa<ObjCMethodDecl>(DC)))
+    return;
+  else if (const ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(DC))
+    mangleObjCMethodName(Method);
+  else {
+    mangleUnqualifiedName(cast<NamedDecl>(DC));
+    manglePostfix(DC->getParent(), NoFunction);
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO) {
+  switch (OO) {
+  //                     ?0 # constructor
+  //                     ?1 # destructor
+  // <operator-name> ::= ?2 # new
+  case OO_New: Out << "?2"; break;
+  // <operator-name> ::= ?3 # delete
+  case OO_Delete: Out << "?3"; break;
+  // <operator-name> ::= ?4 # =
+  case OO_Equal: Out << "?4"; break;
+  // <operator-name> ::= ?5 # >>
+  case OO_GreaterGreater: Out << "?5"; break;
+  // <operator-name> ::= ?6 # <<
+  case OO_LessLess: Out << "?6"; break;
+  // <operator-name> ::= ?7 # !
+  case OO_Exclaim: Out << "?7"; break;
+  // <operator-name> ::= ?8 # ==
+  case OO_EqualEqual: Out << "?8"; break;
+  // <operator-name> ::= ?9 # !=
+  case OO_ExclaimEqual: Out << "?9"; break;
+  // <operator-name> ::= ?A # []
+  case OO_Subscript: Out << "?A"; break;
+  //                     ?B # conversion
+  // <operator-name> ::= ?C # ->
+  case OO_Arrow: Out << "?C"; break;
+  // <operator-name> ::= ?D # *
+  case OO_Star: Out << "?D"; break;
+  // <operator-name> ::= ?E # ++
+  case OO_PlusPlus: Out << "?E"; break;
+  // <operator-name> ::= ?F # --
+  case OO_MinusMinus: Out << "?F"; break;
+  // <operator-name> ::= ?G # -
+  case OO_Minus: Out << "?G"; break;
+  // <operator-name> ::= ?H # +
+  case OO_Plus: Out << "?H"; break;
+  // <operator-name> ::= ?I # &
+  case OO_Amp: Out << "?I"; break;
+  // <operator-name> ::= ?J # ->*
+  case OO_ArrowStar: Out << "?J"; break;
+  // <operator-name> ::= ?K # /
+  case OO_Slash: Out << "?K"; break;
+  // <operator-name> ::= ?L # %
+  case OO_Percent: Out << "?L"; break;
+  // <operator-name> ::= ?M # <
+  case OO_Less: Out << "?M"; break;
+  // <operator-name> ::= ?N # <=
+  case OO_LessEqual: Out << "?N"; break;
+  // <operator-name> ::= ?O # >
+  case OO_Greater: Out << "?O"; break;
+  // <operator-name> ::= ?P # >=
+  case OO_GreaterEqual: Out << "?P"; break;
+  // <operator-name> ::= ?Q # ,
+  case OO_Comma: Out << "?Q"; break;
+  // <operator-name> ::= ?R # ()
+  case OO_Call: Out << "?R"; break;
+  // <operator-name> ::= ?S # ~
+  case OO_Tilde: Out << "?S"; break;
+  // <operator-name> ::= ?T # ^
+  case OO_Caret: Out << "?T"; break;
+  // <operator-name> ::= ?U # |
+  case OO_Pipe: Out << "?U"; break;
+  // <operator-name> ::= ?V # &&
+  case OO_AmpAmp: Out << "?V"; break;
+  // <operator-name> ::= ?W # ||
+  case OO_PipePipe: Out << "?W"; break;
+  // <operator-name> ::= ?X # *=
+  case OO_StarEqual: Out << "?X"; break;
+  // <operator-name> ::= ?Y # +=
+  case OO_PlusEqual: Out << "?Y"; break;
+  // <operator-name> ::= ?Z # -=
+  case OO_MinusEqual: Out << "?Z"; break;
+  // <operator-name> ::= ?_0 # /=
+  case OO_SlashEqual: Out << "?_0"; break;
+  // <operator-name> ::= ?_1 # %=
+  case OO_PercentEqual: Out << "?_1"; break;
+  // <operator-name> ::= ?_2 # >>=
+  case OO_GreaterGreaterEqual: Out << "?_2"; break;
+  // <operator-name> ::= ?_3 # <<=
+  case OO_LessLessEqual: Out << "?_3"; break;
+  // <operator-name> ::= ?_4 # &=
+  case OO_AmpEqual: Out << "?_4"; break;
+  // <operator-name> ::= ?_5 # |=
+  case OO_PipeEqual: Out << "?_5"; break;
+  // <operator-name> ::= ?_6 # ^=
+  case OO_CaretEqual: Out << "?_6"; break;
+  //                     ?_7 # vftable
+  //                     ?_8 # vbtable
+  //                     ?_9 # vcall
+  //                     ?_A # typeof
+  //                     ?_B # local static guard
+  //                     ?_C # string
+  //                     ?_D # vbase destructor
+  //                     ?_E # vector deleting destructor
+  //                     ?_F # default constructor closure
+  //                     ?_G # scalar deleting destructor
+  //                     ?_H # vector constructor iterator
+  //                     ?_I # vector destructor iterator
+  //                     ?_J # vector vbase constructor iterator
+  //                     ?_K # virtual displacement map
+  //                     ?_L # eh vector constructor iterator
+  //                     ?_M # eh vector destructor iterator
+  //                     ?_N # eh vector vbase constructor iterator
+  //                     ?_O # copy constructor closure
+  //                     ?_P<name> # udt returning <name>
+  //                     ?_Q # <unknown>
+  //                     ?_R0 # RTTI Type Descriptor
+  //                     ?_R1 # RTTI Base Class Descriptor at (a,b,c,d)
+  //                     ?_R2 # RTTI Base Class Array
+  //                     ?_R3 # RTTI Class Hierarchy Descriptor
+  //                     ?_R4 # RTTI Complete Object Locator
+  //                     ?_S # local vftable
+  //                     ?_T # local vftable constructor closure
+  // <operator-name> ::= ?_U # new[]
+  case OO_Array_New: Out << "?_U"; break;
+  // <operator-name> ::= ?_V # delete[]
+  case OO_Array_Delete: Out << "?_V"; break;
+    
+  case OO_Conditional:
+    assert(false && "Don't know how to mangle ?:");
+    break;
+    
+  case OO_None:
+  case NUM_OVERLOADED_OPERATORS:
+    assert(false && "Not an overloaded operator");
+    break;
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleSourceName(const IdentifierInfo *II) {
+  // <source name> ::= <identifier> @
+  Out << II->getName() << '@';
+}
+
+void MicrosoftCXXNameMangler::mangleObjCMethodName(const ObjCMethodDecl *MD) {
+  Context.mangleObjCMethodName(MD, Out);
+}
+
+void MicrosoftCXXNameMangler::mangleQualifiers(Qualifiers Quals,
+                                               bool IsMember) {
+  // <cvr-qualifiers> ::= [E] [F] [I] <base-cvr-qualifiers>
+  // 'E' means __ptr64 (32-bit only); 'F' means __unaligned (32/64-bit only);
+  // 'I' means __restrict (32/64-bit).
+  // Note that the MSVC __restrict keyword isn't the same as the C99 restrict
+  // keyword!
+  // <base-cvr-qualifiers> ::= A  # near
+  //                       ::= B  # near const
+  //                       ::= C  # near volatile
+  //                       ::= D  # near const volatile
+  //                       ::= E  # far (16-bit)
+  //                       ::= F  # far const (16-bit)
+  //                       ::= G  # far volatile (16-bit)
+  //                       ::= H  # far const volatile (16-bit)
+  //                       ::= I  # huge (16-bit)
+  //                       ::= J  # huge const (16-bit)
+  //                       ::= K  # huge volatile (16-bit)
+  //                       ::= L  # huge const volatile (16-bit)
+  //                       ::= M <basis> # based
+  //                       ::= N <basis> # based const
+  //                       ::= O <basis> # based volatile
+  //                       ::= P <basis> # based const volatile
+  //                       ::= Q  # near member
+  //                       ::= R  # near const member
+  //                       ::= S  # near volatile member
+  //                       ::= T  # near const volatile member
+  //                       ::= U  # far member (16-bit)
+  //                       ::= V  # far const member (16-bit)
+  //                       ::= W  # far volatile member (16-bit)
+  //                       ::= X  # far const volatile member (16-bit)
+  //                       ::= Y  # huge member (16-bit)
+  //                       ::= Z  # huge const member (16-bit)
+  //                       ::= 0  # huge volatile member (16-bit)
+  //                       ::= 1  # huge const volatile member (16-bit)
+  //                       ::= 2 <basis> # based member
+  //                       ::= 3 <basis> # based const member
+  //                       ::= 4 <basis> # based volatile member
+  //                       ::= 5 <basis> # based const volatile member
+  //                       ::= 6  # near function (pointers only)
+  //                       ::= 7  # far function (pointers only)
+  //                       ::= 8  # near method (pointers only)
+  //                       ::= 9  # far method (pointers only)
+  //                       ::= _A <basis> # based function (pointers only)
+  //                       ::= _B <basis> # based function (far?) (pointers only)
+  //                       ::= _C <basis> # based method (pointers only)
+  //                       ::= _D <basis> # based method (far?) (pointers only)
+  //                       ::= _E # block (Clang)
+  // <basis> ::= 0 # __based(void)
+  //         ::= 1 # __based(segment)?
+  //         ::= 2 <name> # __based(name)
+  //         ::= 3 # ?
+  //         ::= 4 # ?
+  //         ::= 5 # not really based
+  if (!IsMember) {
+    if (!Quals.hasVolatile()) {
+      if (!Quals.hasConst())
+        Out << 'A';
+      else
+        Out << 'B';
+    } else {
+      if (!Quals.hasConst())
+        Out << 'C';
+      else
+        Out << 'D';
+    }
+  } else {
+    if (!Quals.hasVolatile()) {
+      if (!Quals.hasConst())
+        Out << 'Q';
+      else
+        Out << 'R';
+    } else {
+      if (!Quals.hasConst())
+        Out << 'S';
+      else
+        Out << 'T';
+    }
+  }
+
+  // FIXME: For now, just drop all extension qualifiers on the floor.
+}
+
+void MicrosoftCXXNameMangler::mangleType(QualType T) {
+  // Only operate on the canonical type!
+  T = getASTContext().getCanonicalType(T);
+  
+  Qualifiers Quals = T.getLocalQualifiers();
+  if (Quals) {
+    // We have to mangle these now, while we still have enough information.
+    // <pointer-cvr-qualifiers> ::= P  # pointer
+    //                          ::= Q  # const pointer
+    //                          ::= R  # volatile pointer
+    //                          ::= S  # const volatile pointer
+    if (T->isAnyPointerType() || T->isMemberPointerType() ||
+        T->isBlockPointerType()) {
+      if (!Quals.hasVolatile())
+        Out << 'Q';
+      else {
+        if (!Quals.hasConst())
+          Out << 'R';
+        else
+          Out << 'S';
+      }
+    } else
+      // Just emit qualifiers like normal.
+      // NB: When we mangle a pointer/reference type, and the pointee
+      // type has no qualifiers, the lack of qualifier gets mangled
+      // in there.
+      mangleQualifiers(Quals, false);
+  } else if (T->isAnyPointerType() || T->isMemberPointerType() ||
+             T->isBlockPointerType()) {
+    Out << 'P';
+  }
+  switch (T->getTypeClass()) {
+#define ABSTRACT_TYPE(CLASS, PARENT)
+#define NON_CANONICAL_TYPE(CLASS, PARENT) \
+case Type::CLASS: \
+llvm_unreachable("can't mangle non-canonical type " #CLASS "Type"); \
+return;
+#define TYPE(CLASS, PARENT) \
+case Type::CLASS: \
+mangleType(static_cast<const CLASS##Type*>(T.getTypePtr())); \
+break;
+#include "clang/AST/TypeNodes.def"
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T) {
+  //  <type>         ::= <builtin-type>
+  //  <builtin-type> ::= X  # void
+  //                 ::= C  # signed char
+  //                 ::= D  # char
+  //                 ::= E  # unsigned char
+  //                 ::= F  # short
+  //                 ::= G  # unsigned short (or wchar_t if it's not a builtin)
+  //                 ::= H  # int
+  //                 ::= I  # unsigned int
+  //                 ::= J  # long
+  //                 ::= K  # unsigned long
+  //                     L  # <none>
+  //                 ::= M  # float
+  //                 ::= N  # double
+  //                 ::= O  # long double (__float80 is mangled differently)
+  //                 ::= _D # __int8 (yup, it's a distinct type in MSVC)
+  //                 ::= _E # unsigned __int8
+  //                 ::= _F # __int16
+  //                 ::= _G # unsigned __int16
+  //                 ::= _H # __int32
+  //                 ::= _I # unsigned __int32
+  //                 ::= _J # long long, __int64
+  //                 ::= _K # unsigned long long, __int64
+  //                 ::= _L # __int128
+  //                 ::= _M # unsigned __int128
+  //                 ::= _N # bool
+  //                     _O # <array in parameter>
+  //                 ::= _T # __float80 (Intel)
+  //                 ::= _W # wchar_t
+  //                 ::= _Z # __float80 (Digital Mars)
+  switch (T->getKind()) {
+  case BuiltinType::Void: Out << 'X'; break;
+  case BuiltinType::SChar: Out << 'C'; break;
+  case BuiltinType::Char_U: case BuiltinType::Char_S: Out << 'D'; break;
+  case BuiltinType::UChar: Out << 'E'; break;
+  case BuiltinType::Short: Out << 'F'; break;
+  case BuiltinType::UShort: Out << 'G'; break;
+  case BuiltinType::Int: Out << 'H'; break;
+  case BuiltinType::UInt: Out << 'I'; break;
+  case BuiltinType::Long: Out << 'J'; break;
+  case BuiltinType::ULong: Out << 'K'; break;
+  case BuiltinType::Float: Out << 'M'; break;
+  case BuiltinType::Double: Out << 'N'; break;
+  // TODO: Determine size and mangle accordingly
+  case BuiltinType::LongDouble: Out << 'O'; break;
+  // TODO: __int8 and friends
+  case BuiltinType::LongLong: Out << "_J"; break;
+  case BuiltinType::ULongLong: Out << "_K"; break;
+  case BuiltinType::Int128: Out << "_L"; break;
+  case BuiltinType::UInt128: Out << "_M"; break;
+  case BuiltinType::Bool: Out << "_N"; break;
+  case BuiltinType::WChar_S:
+  case BuiltinType::WChar_U: Out << "_W"; break;
+
+  case BuiltinType::Overload:
+  case BuiltinType::Dependent:
+    assert(false &&
+           "Overloaded and dependent types shouldn't get to name mangling");
+    break;
+  case BuiltinType::ObjCId: Out << "PAUobjc_object@@"; break;
+  case BuiltinType::ObjCClass: Out << "PAUobjc_class@@"; break;
+  case BuiltinType::ObjCSel: Out << "PAUobjc_selector@@"; break;
+
+  case BuiltinType::Char16:
+  case BuiltinType::Char32:
+  case BuiltinType::NullPtr:
+    assert(false && "Don't know how to mangle this type");
+    break;
+  }
+}
+
+// <type>          ::= <function-type>
+void MicrosoftCXXNameMangler::mangleType(const FunctionProtoType *T) {
+  // Structors only appear in decls, so at this point we know it's not a
+  // structor type.
+  // I'll probably have mangleType(MemberPointerType) call the mangleType()
+  // method directly.
+  mangleType(T, NULL, false, false);
+}
+void MicrosoftCXXNameMangler::mangleType(const FunctionNoProtoType *T) {
+  llvm_unreachable("Can't mangle K&R function prototypes");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const FunctionType *T,
+                                         const FunctionDecl *D,
+                                         bool IsStructor,
+                                         bool IsInstMethod) {
+  // <function-type> ::= <this-cvr-qualifiers> <calling-convention>
+  //                     <return-type> <argument-list> <throw-spec>
+  const FunctionProtoType *Proto = cast<FunctionProtoType>(T);
+
+  // If this is a C++ instance method, mangle the CVR qualifiers for the
+  // this pointer.
+  if (IsInstMethod)
+    mangleQualifiers(Qualifiers::fromCVRMask(Proto->getTypeQuals()), false);
+
+  mangleCallingConvention(T, IsInstMethod);
+
+  // <return-type> ::= <type>
+  //               ::= @ # structors (they have no declared return type)
+  if (IsStructor)
+    Out << '@';
+  else
+    mangleType(Proto->getResultType());
+
+  // <argument-list> ::= X # void
+  //                 ::= <type>+ @
+  //                 ::= <type>* Z # varargs
+  if (Proto->getNumArgs() == 0 && !Proto->isVariadic()) {
+    Out << 'X';
+  } else {
+    if (D) {
+      // If we got a decl, use the "types-as-written" to make sure arrays
+      // get mangled right.
+      for (FunctionDecl::param_const_iterator Parm = D->param_begin(),
+           ParmEnd = D->param_end();
+           Parm != ParmEnd; ++Parm)
+        mangleType((*Parm)->getTypeSourceInfo()->getType());
+    } else {
+      for (FunctionProtoType::arg_type_iterator Arg = Proto->arg_type_begin(),
+           ArgEnd = Proto->arg_type_end();
+           Arg != ArgEnd; ++Arg)
+        mangleType(*Arg);
+    }
+    // <builtin-type>      ::= Z  # ellipsis
+    if (Proto->isVariadic())
+      Out << 'Z';
+    else
+      Out << '@';
+  }
+
+  mangleThrowSpecification(Proto);
+}
+
+void MicrosoftCXXNameMangler::mangleFunctionClass(const FunctionDecl *FD) {
+  // <function-class> ::= A # private: near
+  //                  ::= B # private: far
+  //                  ::= C # private: static near
+  //                  ::= D # private: static far
+  //                  ::= E # private: virtual near
+  //                  ::= F # private: virtual far
+  //                  ::= G # private: thunk near
+  //                  ::= H # private: thunk far
+  //                  ::= I # protected: near
+  //                  ::= J # protected: far
+  //                  ::= K # protected: static near
+  //                  ::= L # protected: static far
+  //                  ::= M # protected: virtual near
+  //                  ::= N # protected: virtual far
+  //                  ::= O # protected: thunk near
+  //                  ::= P # protected: thunk far
+  //                  ::= Q # public: near
+  //                  ::= R # public: far
+  //                  ::= S # public: static near
+  //                  ::= T # public: static far
+  //                  ::= U # public: virtual near
+  //                  ::= V # public: virtual far
+  //                  ::= W # public: thunk near
+  //                  ::= X # public: thunk far
+  //                  ::= Y # global near
+  //                  ::= Z # global far
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
+    switch (MD->getAccess()) {
+      default:
+      case AS_private:
+        if (MD->isStatic())
+          Out << 'C';
+        else if (MD->isVirtual())
+          Out << 'E';
+        else
+          Out << 'A';
+        break;
+      case AS_protected:
+        if (MD->isStatic())
+          Out << 'K';
+        else if (MD->isVirtual())
+          Out << 'M';
+        else
+          Out << 'I';
+        break;
+      case AS_public:
+        if (MD->isStatic())
+          Out << 'S';
+        else if (MD->isVirtual())
+          Out << 'U';
+        else
+          Out << 'Q';
+    }
+  } else
+    Out << 'Y';
+}
+void MicrosoftCXXNameMangler::mangleCallingConvention(const FunctionType *T,
+                                                      bool IsInstMethod) {
+  // <calling-convention> ::= A # __cdecl
+  //                      ::= B # __export __cdecl
+  //                      ::= C # __pascal
+  //                      ::= D # __export __pascal
+  //                      ::= E # __thiscall
+  //                      ::= F # __export __thiscall
+  //                      ::= G # __stdcall
+  //                      ::= H # __export __stdcall
+  //                      ::= I # __fastcall
+  //                      ::= J # __export __fastcall
+  // The 'export' calling conventions are from a bygone era
+  // (*cough*Win16*cough*) when functions were declared for export with
+  // that keyword. (It didn't actually export them, it just made them so
+  // that they could be in a DLL and somebody from another module could call
+  // them.)
+  CallingConv CC = T->getCallConv();
+  if (CC == CC_Default)
+    CC = IsInstMethod ? getASTContext().getDefaultMethodCallConv() : CC_C;
+  switch (CC) {
+    case CC_Default:
+    case CC_C: Out << 'A'; break;
+    case CC_X86Pascal: Out << 'C'; break;
+    case CC_X86ThisCall: Out << 'E'; break;
+    case CC_X86StdCall: Out << 'G'; break;
+    case CC_X86FastCall: Out << 'I'; break;
+  }
+}
+void MicrosoftCXXNameMangler::mangleThrowSpecification(
+                                                const FunctionProtoType *FT) {
+  // <throw-spec> ::= Z # throw(...) (default)
+  //              ::= @ # throw() or __declspec/__attribute__((nothrow))
+  //              ::= <type>+
+  // NOTE: Since the Microsoft compiler ignores throw specifications, they are
+  // all actually mangled as 'Z'. (They're ignored because their associated
+  // functionality isn't implemented, and probably never will be.)
+  Out << 'Z';
+}
+
+void MicrosoftCXXNameMangler::mangleType(const UnresolvedUsingType *T) {
+  assert(false && "Don't know how to mangle UnresolvedUsingTypes yet!");
+}
+
+// <type>        ::= <union-type> | <struct-type> | <class-type> | <enum-type>
+// <union-type>  ::= T <name>
+// <struct-type> ::= U <name>
+// <class-type>  ::= V <name>
+// <enum-type>   ::= W <size> <name>
+void MicrosoftCXXNameMangler::mangleType(const EnumType *T) {
+  mangleType(static_cast<const TagType*>(T));
+}
+void MicrosoftCXXNameMangler::mangleType(const RecordType *T) {
+  mangleType(static_cast<const TagType*>(T));
+}
+void MicrosoftCXXNameMangler::mangleType(const TagType *T) {
+  switch (T->getDecl()->getTagKind()) {
+    case TTK_Union:
+      Out << 'T';
+      break;
+    case TTK_Struct:
+      Out << 'U';
+      break;
+    case TTK_Class:
+      Out << 'V';
+      break;
+    case TTK_Enum:
+      Out << 'W';
+      Out << getASTContext().getTypeSizeInChars(
+                cast<EnumDecl>(T->getDecl())->getIntegerType()).getQuantity();
+      break;
+  }
+  mangleName(T->getDecl());
+}
+
+// <type>       ::= <array-type>
+// <array-type> ::= P <cvr-qualifiers> [Y <dimension-count> <dimension>+]
+//                                                  <element-type> # as global
+//              ::= Q <cvr-qualifiers> [Y <dimension-count> <dimension>+]
+//                                                  <element-type> # as param
+// It's supposed to be the other way around, but for some strange reason, it
+// isn't. Today this behavior is retained for the sole purpose of backwards
+// compatibility.
+void MicrosoftCXXNameMangler::mangleType(const ArrayType *T, bool IsGlobal) {
+  // This isn't a recursive mangling, so now we have to do it all in this
+  // one call.
+  if (IsGlobal)
+    Out << 'P';
+  else
+    Out << 'Q';
+  mangleExtraDimensions(T->getElementType());
+}
+void MicrosoftCXXNameMangler::mangleType(const ConstantArrayType *T) {
+  mangleType(static_cast<const ArrayType *>(T), false);
+}
+void MicrosoftCXXNameMangler::mangleType(const VariableArrayType *T) {
+  mangleType(static_cast<const ArrayType *>(T), false);
+}
+void MicrosoftCXXNameMangler::mangleType(const DependentSizedArrayType *T) {
+  mangleType(static_cast<const ArrayType *>(T), false);
+}
+void MicrosoftCXXNameMangler::mangleType(const IncompleteArrayType *T) {
+  mangleType(static_cast<const ArrayType *>(T), false);
+}
+void MicrosoftCXXNameMangler::mangleExtraDimensions(QualType ElementTy) {
+  llvm::SmallVector<llvm::APInt, 3> Dimensions;
+  for (;;) {
+    if (ElementTy->isConstantArrayType()) {
+      const ConstantArrayType *CAT =
+      static_cast<const ConstantArrayType *>(ElementTy.getTypePtr());
+      Dimensions.push_back(CAT->getSize());
+      ElementTy = CAT->getElementType();
+    } else if (ElementTy->isVariableArrayType()) {
+      assert(false && "Don't know how to mangle VLAs!");
+    } else if (ElementTy->isDependentSizedArrayType()) {
+      // The dependent expression has to be folded into a constant (TODO).
+      assert(false && "Don't know how to mangle dependent-sized arrays!");
+    } else if (ElementTy->isIncompleteArrayType()) continue;
+    else break;
+  }
+  mangleQualifiers(ElementTy.getQualifiers(), false);
+  // If there are any additional dimensions, mangle them now.
+  if (Dimensions.size() > 0) {
+    Out << 'Y';
+    // <dimension-count> ::= <number> # number of extra dimensions
+    mangleNumber(Dimensions.size());
+    for (unsigned Dim = 0; Dim < Dimensions.size(); ++Dim) {
+      mangleNumber(Dimensions[Dim].getLimitedValue());
+    }
+  }
+  mangleType(ElementTy.getLocalUnqualifiedType());
+}
+
+// <type>                   ::= <pointer-to-member-type>
+// <pointer-to-member-type> ::= <pointer-cvr-qualifiers> <cvr-qualifiers>
+//                                                          <class name> <type>
+void MicrosoftCXXNameMangler::mangleType(const MemberPointerType *T) {
+  QualType PointeeType = T->getPointeeType();
+  if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(PointeeType)) {
+    Out << '8';
+    mangleName(cast<RecordType>(T->getClass())->getDecl());
+    mangleType(FPT, NULL, false, true);
+  } else {
+    mangleQualifiers(PointeeType.getQualifiers(), true);
+    mangleName(cast<RecordType>(T->getClass())->getDecl());
+    mangleType(PointeeType.getLocalUnqualifiedType());
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleType(const TemplateTypeParmType *T) {
+  assert(false && "Don't know how to mangle TemplateTypeParmTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(
+                                       const SubstTemplateTypeParmPackType *T) {
+  assert(false && 
+         "Don't know how to mangle SubstTemplateTypeParmPackTypes yet!");
+}
+
+// <type> ::= <pointer-type>
+// <pointer-type> ::= <pointer-cvr-qualifiers> <cvr-qualifiers> <type>
+void MicrosoftCXXNameMangler::mangleType(const PointerType *T) {
+  QualType PointeeTy = T->getPointeeType();
+  if (PointeeTy->isArrayType()) {
+    // Pointers to arrays are mangled like arrays.
+    mangleExtraDimensions(T->getPointeeType());
+  } else if (PointeeTy->isFunctionType()) {
+    // Function pointers are special.
+    Out << '6';
+    mangleType(static_cast<const FunctionType *>(PointeeTy.getTypePtr()),
+               NULL, false, false);
+  } else {
+    if (!PointeeTy.hasQualifiers())
+      // Lack of qualifiers is mangled as 'A'.
+      Out << 'A';
+    mangleType(PointeeTy);
+  }
+}
+void MicrosoftCXXNameMangler::mangleType(const ObjCObjectPointerType *T) {
+  // Object pointers never have qualifiers.
+  Out << 'A';
+  mangleType(T->getPointeeType());
+}
+
+// <type> ::= <reference-type>
+// <reference-type> ::= A <cvr-qualifiers> <type>
+void MicrosoftCXXNameMangler::mangleType(const LValueReferenceType *T) {
+  Out << 'A';
+  QualType PointeeTy = T->getPointeeType();
+  if (!PointeeTy.hasQualifiers())
+    // Lack of qualifiers is mangled as 'A'.
+    Out << 'A';
+  mangleType(PointeeTy);
+}
+
+void MicrosoftCXXNameMangler::mangleType(const RValueReferenceType *T) {
+  assert(false && "Don't know how to mangle RValueReferenceTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const ComplexType *T) {
+  assert(false && "Don't know how to mangle ComplexTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const VectorType *T) {
+  assert(false && "Don't know how to mangle VectorTypes yet!");
+}
+void MicrosoftCXXNameMangler::mangleType(const ExtVectorType *T) {
+  assert(false && "Don't know how to mangle ExtVectorTypes yet!");
+}
+void MicrosoftCXXNameMangler::mangleType(const DependentSizedExtVectorType *T) {
+  assert(false && "Don't know how to mangle DependentSizedExtVectorTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const ObjCInterfaceType *T) {
+  // ObjC interfaces have structs underlying them.
+  Out << 'U';
+  mangleName(T->getDecl());
+}
+
+void MicrosoftCXXNameMangler::mangleType(const ObjCObjectType *T) {
+  // We don't allow overloading by different protocol qualification,
+  // so mangling them isn't necessary.
+  mangleType(T->getBaseType());
+}
+
+void MicrosoftCXXNameMangler::mangleType(const BlockPointerType *T) {
+  Out << "_E";
+  mangleType(T->getPointeeType());
+}
+
+void MicrosoftCXXNameMangler::mangleType(const InjectedClassNameType *T) {
+  assert(false && "Don't know how to mangle InjectedClassNameTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const TemplateSpecializationType *T) {
+  assert(false && "Don't know how to mangle TemplateSpecializationTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const DependentNameType *T) {
+  assert(false && "Don't know how to mangle DependentNameTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(
+                                 const DependentTemplateSpecializationType *T) {
+  assert(false &&
+         "Don't know how to mangle DependentTemplateSpecializationTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const PackExpansionType *T) {
+  assert(false && "Don't know how to mangle PackExpansionTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const TypeOfType *T) {
+  assert(false && "Don't know how to mangle TypeOfTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const TypeOfExprType *T) {
+  assert(false && "Don't know how to mangle TypeOfExprTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const DecltypeType *T) {
+  assert(false && "Don't know how to mangle DecltypeTypes yet!");
+}
+
+void MicrosoftCXXNameMangler::mangleType(const AutoType *T) {
+  assert(false && "Don't know how to mangle AutoTypes yet!");
+}
+
+void MicrosoftMangleContext::mangleName(const NamedDecl *D,
+                                        llvm::raw_ostream &Out) {
+  assert((isa<FunctionDecl>(D) || isa<VarDecl>(D)) &&
+         "Invalid mangleName() call, argument is not a variable or function!");
+  assert(!isa<CXXConstructorDecl>(D) && !isa<CXXDestructorDecl>(D) &&
+         "Invalid mangleName() call on 'structor decl!");
+
+  PrettyStackTraceDecl CrashInfo(D, SourceLocation(),
+                                 getASTContext().getSourceManager(),
+                                 "Mangling declaration");
+
+  MicrosoftCXXNameMangler Mangler(*this, Out);
+  return Mangler.mangle(D);
+}
+void MicrosoftMangleContext::mangleThunk(const CXXMethodDecl *MD,
+                                         const ThunkInfo &Thunk,
+                                         llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle thunks!");
+}
+void MicrosoftMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
+                                                CXXDtorType Type,
+                                                const ThisAdjustment &,
+                                                llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle destructor thunks!");
+}
+void MicrosoftMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
+                                             llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle virtual tables!");
+}
+void MicrosoftMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
+                                          llvm::raw_ostream &) {
+  llvm_unreachable("The MS C++ ABI does not have virtual table tables!");
+}
+void MicrosoftMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
+                                                 int64_t Offset,
+                                                 const CXXRecordDecl *Type,
+                                                 llvm::raw_ostream &) {
+  llvm_unreachable("The MS C++ ABI does not have constructor vtables!");
+}
+void MicrosoftMangleContext::mangleCXXRTTI(QualType T,
+                                           llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle RTTI!");
+}
+void MicrosoftMangleContext::mangleCXXRTTIName(QualType T,
+                                               llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle RTTI names!");
+}
+void MicrosoftMangleContext::mangleCXXCtor(const CXXConstructorDecl *D,
+                                           CXXCtorType Type,
+                                           llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle constructors!");
+}
+void MicrosoftMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
+                                           CXXDtorType Type,
+                                           llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle destructors!");
+}
+void MicrosoftMangleContext::mangleReferenceTemporary(const clang::VarDecl *,
+                                                      llvm::raw_ostream &) {
+  assert(false && "Can't yet mangle reference temporaries!");
+}
+
+MangleContext *clang::createMicrosoftMangleContext(ASTContext &Context,
+                                                   Diagnostic &Diags) {
+  return new MicrosoftMangleContext(Context, Diags);
+}
diff --git a/lib/AST/NestedNameSpecifier.cpp b/lib/AST/NestedNameSpecifier.cpp
index 212def8565ea..650321d76ee5 100644
--- a/lib/AST/NestedNameSpecifier.cpp
+++ b/lib/AST/NestedNameSpecifier.cpp
@@ -22,7 +22,7 @@
 using namespace clang;
 
 NestedNameSpecifier *
-NestedNameSpecifier::FindOrInsert(ASTContext &Context,
+NestedNameSpecifier::FindOrInsert(const ASTContext &Context,
                                   const NestedNameSpecifier &Mockup) {
   llvm::FoldingSetNodeID ID;
   Mockup.Profile(ID);
@@ -39,8 +39,8 @@ NestedNameSpecifier::FindOrInsert(ASTContext &Context,
 }
 
 NestedNameSpecifier *
-NestedNameSpecifier::Create(ASTContext &Context, NestedNameSpecifier *Prefix,
-                            IdentifierInfo *II) {
+NestedNameSpecifier::Create(const ASTContext &Context,
+                            NestedNameSpecifier *Prefix, IdentifierInfo *II) {
   assert(II && "Identifier cannot be NULL");
   assert((!Prefix || Prefix->isDependent()) && "Prefix must be dependent");
 
@@ -52,8 +52,8 @@ NestedNameSpecifier::Create(ASTContext &Context, NestedNameSpecifier *Prefix,
 }
 
 NestedNameSpecifier *
-NestedNameSpecifier::Create(ASTContext &Context, NestedNameSpecifier *Prefix,
-                            NamespaceDecl *NS) {
+NestedNameSpecifier::Create(const ASTContext &Context,
+                            NestedNameSpecifier *Prefix, NamespaceDecl *NS) {
   assert(NS && "Namespace cannot be NULL");
   assert((!Prefix ||
           (Prefix->getAsType() == 0 && Prefix->getAsIdentifier() == 0)) &&
@@ -66,18 +66,19 @@ NestedNameSpecifier::Create(ASTContext &Context, NestedNameSpecifier *Prefix,
 }
 
 NestedNameSpecifier *
-NestedNameSpecifier::Create(ASTContext &Context, NestedNameSpecifier *Prefix,
-                            bool Template, Type *T) {
+NestedNameSpecifier::Create(const ASTContext &Context,
+                            NestedNameSpecifier *Prefix,
+                            bool Template, const Type *T) {
   assert(T && "Type cannot be NULL");
   NestedNameSpecifier Mockup;
   Mockup.Prefix.setPointer(Prefix);
   Mockup.Prefix.setInt(Template? TypeSpecWithTemplate : TypeSpec);
-  Mockup.Specifier = T;
+  Mockup.Specifier = const_cast<Type*>(T);
   return FindOrInsert(Context, Mockup);
 }
 
 NestedNameSpecifier *
-NestedNameSpecifier::Create(ASTContext &Context, IdentifierInfo *II) {
+NestedNameSpecifier::Create(const ASTContext &Context, IdentifierInfo *II) {
   assert(II && "Identifier cannot be NULL");
   NestedNameSpecifier Mockup;
   Mockup.Prefix.setPointer(0);
@@ -86,7 +87,8 @@ NestedNameSpecifier::Create(ASTContext &Context, IdentifierInfo *II) {
   return FindOrInsert(Context, Mockup);
 }
 
-NestedNameSpecifier *NestedNameSpecifier::GlobalSpecifier(ASTContext &Context) {
+NestedNameSpecifier *
+NestedNameSpecifier::GlobalSpecifier(const ASTContext &Context) {
   if (!Context.GlobalNestedNameSpecifier)
     Context.GlobalNestedNameSpecifier = new (Context, 4) NestedNameSpecifier();
   return Context.GlobalNestedNameSpecifier;
@@ -113,6 +115,24 @@ bool NestedNameSpecifier::isDependent() const {
   return false;
 }
 
+bool NestedNameSpecifier::containsUnexpandedParameterPack() const {
+  switch (getKind()) {
+  case Identifier:
+    return getPrefix() && getPrefix()->containsUnexpandedParameterPack();
+
+  case Namespace:
+  case Global:
+    return false;
+
+  case TypeSpec:
+  case TypeSpecWithTemplate:
+    return getAsType()->containsUnexpandedParameterPack();
+  }
+
+  // Necessary to suppress a GCC warning.
+  return false;  
+}
+
 /// \brief Print this nested name specifier to the given output
 /// stream.
 void
@@ -139,7 +159,7 @@ NestedNameSpecifier::print(llvm::raw_ostream &OS,
 
   case TypeSpec: {
     std::string TypeStr;
-    Type *T = getAsType();
+    const Type *T = getAsType();
 
     PrintingPolicy InnerPolicy(Policy);
     InnerPolicy.SuppressScope = true;
diff --git a/lib/AST/ParentMap.cpp b/lib/AST/ParentMap.cpp
index 5fe873acf7ac..eca351aec8dd 100644
--- a/lib/AST/ParentMap.cpp
+++ b/lib/AST/ParentMap.cpp
@@ -21,7 +21,7 @@ using namespace clang;
 typedef llvm::DenseMap<Stmt*, Stmt*> MapTy;
 
 static void BuildParentMap(MapTy& M, Stmt* S) {
-  for (Stmt::child_iterator I=S->child_begin(), E=S->child_end(); I!=E; ++I)
+  for (Stmt::child_range I = S->children(); I; ++I)
     if (*I) {
       M[*I] = S;
       BuildParentMap(M, *I);
@@ -40,6 +40,12 @@ ParentMap::~ParentMap() {
   delete (MapTy*) Impl;
 }
 
+void ParentMap::addStmt(Stmt* S) {
+  if (S) {
+    BuildParentMap(*(MapTy*) Impl, S);
+  }
+}
+
 Stmt* ParentMap::getParent(Stmt* S) const {
   MapTy* M = (MapTy*) Impl;
   MapTy::iterator I = M->find(S);
@@ -51,6 +57,15 @@ Stmt *ParentMap::getParentIgnoreParens(Stmt *S) const {
   return S;
 }
 
+Stmt *ParentMap::getParentIgnoreParenCasts(Stmt *S) const {
+  do {
+    S = getParent(S);
+  }
+  while (S && (isa<ParenExpr>(S) || isa<CastExpr>(S)));
+
+  return S;  
+}
+
 bool ParentMap::isConsumedExpr(Expr* E) const {
   Stmt *P = getParent(E);
   Stmt *DirectChild = E;
diff --git a/lib/AST/RecordLayout.cpp b/lib/AST/RecordLayout.cpp
index 4d9c51633626..035c48fd0827 100644
--- a/lib/AST/RecordLayout.cpp
+++ b/lib/AST/RecordLayout.cpp
@@ -27,9 +27,10 @@ void ASTRecordLayout::Destroy(ASTContext &Ctx) {
   Ctx.Deallocate(this);
 }
 
-ASTRecordLayout::ASTRecordLayout(ASTContext &Ctx, uint64_t size, unsigned alignment,
-                unsigned datasize, const uint64_t *fieldoffsets,
-                unsigned fieldcount)
+ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx, CharUnits size,
+                                 CharUnits alignment, CharUnits datasize,
+                                 const uint64_t *fieldoffsets,
+                                 unsigned fieldcount)
   : Size(size), DataSize(datasize), FieldOffsets(0), Alignment(alignment),
     FieldCount(fieldcount), CXXInfo(0) {
   if (FieldCount > 0)  {
@@ -39,16 +40,16 @@ ASTRecordLayout::ASTRecordLayout(ASTContext &Ctx, uint64_t size, unsigned alignm
 }
 
 // Constructor for C++ records.
-ASTRecordLayout::ASTRecordLayout(ASTContext &Ctx,
-                                 uint64_t size, unsigned alignment,
-                                 uint64_t datasize,
+ASTRecordLayout::ASTRecordLayout(const ASTContext &Ctx,
+                                 CharUnits size, CharUnits alignment,
+                                 CharUnits datasize,
                                  const uint64_t *fieldoffsets,
                                  unsigned fieldcount,
-                                 uint64_t nonvirtualsize,
-                                 unsigned nonvirtualalign,
-                                 uint64_t SizeOfLargestEmptySubobject,
+                                 CharUnits nonvirtualsize,
+                                 CharUnits nonvirtualalign,
+                                 CharUnits SizeOfLargestEmptySubobject,
                                  const CXXRecordDecl *PrimaryBase,
-                                 bool PrimaryBaseIsVirtual,
+                                 bool IsPrimaryBaseVirtual,
                                  const BaseOffsetsMapTy& BaseOffsets,
                                  const BaseOffsetsMapTy& VBaseOffsets)
   : Size(size), DataSize(datasize), FieldOffsets(0), Alignment(alignment),
@@ -59,7 +60,8 @@ ASTRecordLayout::ASTRecordLayout(ASTContext &Ctx,
     memcpy(FieldOffsets, fieldoffsets, FieldCount * sizeof(*FieldOffsets));
   }
 
-  CXXInfo->PrimaryBase = PrimaryBaseInfo(PrimaryBase, PrimaryBaseIsVirtual);
+  CXXInfo->PrimaryBase.setPointer(PrimaryBase);
+  CXXInfo->PrimaryBase.setInt(IsPrimaryBaseVirtual);
   CXXInfo->NonVirtualSize = nonvirtualsize;
   CXXInfo->NonVirtualAlign = nonvirtualalign;
   CXXInfo->SizeOfLargestEmptySubobject = SizeOfLargestEmptySubobject;
@@ -68,11 +70,11 @@ ASTRecordLayout::ASTRecordLayout(ASTContext &Ctx,
 
 #ifndef NDEBUG
     if (const CXXRecordDecl *PrimaryBase = getPrimaryBase()) {
-      if (getPrimaryBaseWasVirtual())
-        assert(getVBaseClassOffset(PrimaryBase) == 0 &&
+      if (isPrimaryBaseVirtual())
+        assert(getVBaseClassOffset(PrimaryBase).isZero() &&
                "Primary virtual base must be at offset 0!");
       else
-        assert(getBaseClassOffset(PrimaryBase) == 0 &&
+        assert(getBaseClassOffsetInBits(PrimaryBase) == 0 &&
                "Primary base must be at offset 0!");
     }
 #endif        
diff --git a/lib/AST/RecordLayoutBuilder.cpp b/lib/AST/RecordLayoutBuilder.cpp
index 13fae299d877..cf14eba1405d 100644
--- a/lib/AST/RecordLayoutBuilder.cpp
+++ b/lib/AST/RecordLayoutBuilder.cpp
@@ -8,12 +8,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/Attr.h"
+#include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/SemaDiagnostic.h"
 #include "llvm/Support/Format.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/MathExtras.h"
@@ -55,62 +57,71 @@ struct BaseSubobjectInfo {
 /// EmptySubobjectMap - Keeps track of which empty subobjects exist at different
 /// offsets while laying out a C++ class.
 class EmptySubobjectMap {
-  ASTContext &Context;
-
+  const ASTContext &Context;
+  uint64_t CharWidth;
+  
   /// Class - The class whose empty entries we're keeping track of.
   const CXXRecordDecl *Class;
 
   /// EmptyClassOffsets - A map from offsets to empty record decls.
   typedef llvm::SmallVector<const CXXRecordDecl *, 1> ClassVectorTy;
-  typedef llvm::DenseMap<uint64_t, ClassVectorTy> EmptyClassOffsetsMapTy;
+  typedef llvm::DenseMap<CharUnits, ClassVectorTy> EmptyClassOffsetsMapTy;
   EmptyClassOffsetsMapTy EmptyClassOffsets;
   
   /// MaxEmptyClassOffset - The highest offset known to contain an empty
   /// base subobject.
-  uint64_t MaxEmptyClassOffset;
+  CharUnits MaxEmptyClassOffset;
   
   /// ComputeEmptySubobjectSizes - Compute the size of the largest base or
   /// member subobject that is empty.
   void ComputeEmptySubobjectSizes();
   
-  void AddSubobjectAtOffset(const CXXRecordDecl *RD, uint64_t Offset);
+  void AddSubobjectAtOffset(const CXXRecordDecl *RD, CharUnits Offset);
   
   void UpdateEmptyBaseSubobjects(const BaseSubobjectInfo *Info,
-                                 uint64_t Offset, bool PlacingEmptyBase);
+                                 CharUnits Offset, bool PlacingEmptyBase);
   
   void UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD, 
                                   const CXXRecordDecl *Class,
-                                  uint64_t Offset);
-  void UpdateEmptyFieldSubobjects(const FieldDecl *FD, uint64_t Offset);
+                                  CharUnits Offset);
+  void UpdateEmptyFieldSubobjects(const FieldDecl *FD, CharUnits Offset);
   
   /// AnyEmptySubobjectsBeyondOffset - Returns whether there are any empty
   /// subobjects beyond the given offset.
-  bool AnyEmptySubobjectsBeyondOffset(uint64_t Offset) const {
+  bool AnyEmptySubobjectsBeyondOffset(CharUnits Offset) const {
     return Offset <= MaxEmptyClassOffset;
   }
 
+  CharUnits 
+  getFieldOffset(const ASTRecordLayout &Layout, unsigned FieldNo) const {
+    uint64_t FieldOffset = Layout.getFieldOffset(FieldNo);
+    assert(FieldOffset % CharWidth == 0 && 
+           "Field offset not at char boundary!");
+
+    return Context.toCharUnitsFromBits(FieldOffset);
+  }
+
 protected:
-  bool CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD, 
-                                 uint64_t Offset) const;
+  bool CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD,
+                                 CharUnits Offset) const;
 
   bool CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
-                                     uint64_t Offset);
+                                     CharUnits Offset);
 
   bool CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD, 
                                       const CXXRecordDecl *Class,
-                                      uint64_t Offset) const;
+                                      CharUnits Offset) const;
   bool CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
-                                      uint64_t Offset) const;
+                                      CharUnits Offset) const;
 
 public:
   /// This holds the size of the largest empty subobject (either a base
   /// or a member). Will be zero if the record being built doesn't contain
   /// any empty classes.
-  uint64_t SizeOfLargestEmptySubobject;
+  CharUnits SizeOfLargestEmptySubobject;
 
-  EmptySubobjectMap(ASTContext &Context, const CXXRecordDecl *Class)
-    : Context(Context), Class(Class), MaxEmptyClassOffset(0),
-    SizeOfLargestEmptySubobject(0) {
+  EmptySubobjectMap(const ASTContext &Context, const CXXRecordDecl *Class)
+  : Context(Context), CharWidth(Context.getCharWidth()), Class(Class) {
       ComputeEmptySubobjectSizes();
   }
 
@@ -119,11 +130,11 @@ public:
   /// Returns false if placing the record will result in two components
   /// (direct or indirect) of the same type having the same offset.
   bool CanPlaceBaseAtOffset(const BaseSubobjectInfo *Info,
-                            uint64_t Offset);
+                            CharUnits Offset);
 
   /// CanPlaceFieldAtOffset - Return whether a field can be placed at the given
   /// offset.
-  bool CanPlaceFieldAtOffset(const FieldDecl *FD, uint64_t Offset);
+  bool CanPlaceFieldAtOffset(const FieldDecl *FD, CharUnits Offset);
 };
 
 void EmptySubobjectMap::ComputeEmptySubobjectSizes() {
@@ -133,7 +144,7 @@ void EmptySubobjectMap::ComputeEmptySubobjectSizes() {
     const CXXRecordDecl *BaseDecl =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
-    uint64_t EmptySize = 0;
+    CharUnits EmptySize;
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(BaseDecl);
     if (BaseDecl->isEmpty()) {
       // If the class decl is empty, get its size.
@@ -143,8 +154,8 @@ void EmptySubobjectMap::ComputeEmptySubobjectSizes() {
       EmptySize = Layout.getSizeOfLargestEmptySubobject();
     }
 
-    SizeOfLargestEmptySubobject = std::max(SizeOfLargestEmptySubobject,
-                                           EmptySize);
+    if (EmptySize > SizeOfLargestEmptySubobject)
+      SizeOfLargestEmptySubobject = EmptySize;
   }
 
   // Check the fields.
@@ -159,7 +170,7 @@ void EmptySubobjectMap::ComputeEmptySubobjectSizes() {
     if (!RT)
       continue;
 
-    uint64_t EmptySize = 0;
+    CharUnits EmptySize;
     const CXXRecordDecl *MemberDecl = cast<CXXRecordDecl>(RT->getDecl());
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(MemberDecl);
     if (MemberDecl->isEmpty()) {
@@ -170,14 +181,14 @@ void EmptySubobjectMap::ComputeEmptySubobjectSizes() {
       EmptySize = Layout.getSizeOfLargestEmptySubobject();
     }
 
-   SizeOfLargestEmptySubobject = std::max(SizeOfLargestEmptySubobject,
-                                          EmptySize);
+    if (EmptySize > SizeOfLargestEmptySubobject)
+      SizeOfLargestEmptySubobject = EmptySize;
   }
 }
 
 bool
 EmptySubobjectMap::CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD, 
-                                             uint64_t Offset) const {
+                                             CharUnits Offset) const {
   // We only need to check empty bases.
   if (!RD->isEmpty())
     return true;
@@ -195,24 +206,27 @@ EmptySubobjectMap::CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD,
 }
   
 void EmptySubobjectMap::AddSubobjectAtOffset(const CXXRecordDecl *RD, 
-                                             uint64_t Offset) {
+                                             CharUnits Offset) {
   // We only care about empty bases.
   if (!RD->isEmpty())
     return;
 
+  // If we have empty structures inside an union, we can assign both
+  // the same offset. Just avoid pushing them twice in the list.
   ClassVectorTy& Classes = EmptyClassOffsets[Offset];
-  assert(std::find(Classes.begin(), Classes.end(), RD) == Classes.end() &&
-         "Duplicate empty class detected!");
-
+  if (std::find(Classes.begin(), Classes.end(), RD) != Classes.end())
+    return;
+  
   Classes.push_back(RD);
   
   // Update the empty class offset.
-  MaxEmptyClassOffset = std::max(MaxEmptyClassOffset, Offset);
+  if (Offset > MaxEmptyClassOffset)
+    MaxEmptyClassOffset = Offset;
 }
 
 bool
-EmptySubobjectMap::CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info, 
-                                                 uint64_t Offset) {
+EmptySubobjectMap::CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
+                                                 CharUnits Offset) {
   // We don't have to keep looking past the maximum offset that's known to
   // contain an empty class.
   if (!AnyEmptySubobjectsBeyondOffset(Offset))
@@ -228,7 +242,7 @@ EmptySubobjectMap::CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
     if (Base->IsVirtual)
       continue;
 
-    uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
 
     if (!CanPlaceBaseSubobjectAtOffset(Base, BaseOffset))
       return false;
@@ -248,8 +262,10 @@ EmptySubobjectMap::CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
   for (CXXRecordDecl::field_iterator I = Info->Class->field_begin(), 
        E = Info->Class->field_end(); I != E; ++I, ++FieldNo) {
     const FieldDecl *FD = *I;
-
-    uint64_t FieldOffset = Offset + Layout.getFieldOffset(FieldNo);
+    if (FD->isBitField())
+      continue;
+  
+    CharUnits FieldOffset = Offset + getFieldOffset(Layout, FieldNo);
     if (!CanPlaceFieldSubobjectAtOffset(FD, FieldOffset))
       return false;
   }
@@ -258,7 +274,7 @@ EmptySubobjectMap::CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
 }
 
 void EmptySubobjectMap::UpdateEmptyBaseSubobjects(const BaseSubobjectInfo *Info, 
-                                                  uint64_t Offset,
+                                                  CharUnits Offset,
                                                   bool PlacingEmptyBase) {
   if (!PlacingEmptyBase && Offset >= SizeOfLargestEmptySubobject) {
     // We know that the only empty subobjects that can conflict with empty
@@ -278,7 +294,7 @@ void EmptySubobjectMap::UpdateEmptyBaseSubobjects(const BaseSubobjectInfo *Info,
     if (Base->IsVirtual)
       continue;
 
-    uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
     UpdateEmptyBaseSubobjects(Base, BaseOffset, PlacingEmptyBase);
   }
 
@@ -295,17 +311,19 @@ void EmptySubobjectMap::UpdateEmptyBaseSubobjects(const BaseSubobjectInfo *Info,
   for (CXXRecordDecl::field_iterator I = Info->Class->field_begin(), 
        E = Info->Class->field_end(); I != E; ++I, ++FieldNo) {
     const FieldDecl *FD = *I;
+    if (FD->isBitField())
+      continue;
 
-    uint64_t FieldOffset = Offset + Layout.getFieldOffset(FieldNo);
+    CharUnits FieldOffset = Offset + getFieldOffset(Layout, FieldNo);
     UpdateEmptyFieldSubobjects(FD, FieldOffset);
   }
 }
 
 bool EmptySubobjectMap::CanPlaceBaseAtOffset(const BaseSubobjectInfo *Info,
-                                             uint64_t Offset) {
+                                             CharUnits Offset) {
   // If we know this class doesn't have any empty subobjects we don't need to
   // bother checking.
-  if (!SizeOfLargestEmptySubobject)
+  if (SizeOfLargestEmptySubobject.isZero())
     return true;
 
   if (!CanPlaceBaseSubobjectAtOffset(Info, Offset))
@@ -320,7 +338,7 @@ bool EmptySubobjectMap::CanPlaceBaseAtOffset(const BaseSubobjectInfo *Info,
 bool
 EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD, 
                                                   const CXXRecordDecl *Class,
-                                                  uint64_t Offset) const {
+                                                  CharUnits Offset) const {
   // We don't have to keep looking past the maximum offset that's known to
   // contain an empty class.
   if (!AnyEmptySubobjectsBeyondOffset(Offset))
@@ -340,7 +358,7 @@ EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD,
     const CXXRecordDecl *BaseDecl =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
-    uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(BaseDecl);
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(BaseDecl);
     if (!CanPlaceFieldSubobjectAtOffset(BaseDecl, Class, BaseOffset))
       return false;
   }
@@ -352,7 +370,7 @@ EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD,
       const CXXRecordDecl *VBaseDecl =
         cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
       
-      uint64_t VBaseOffset = Offset + Layout.getVBaseClassOffset(VBaseDecl);
+      CharUnits VBaseOffset = Offset + Layout.getVBaseClassOffset(VBaseDecl);
       if (!CanPlaceFieldSubobjectAtOffset(VBaseDecl, Class, VBaseOffset))
         return false;
     }
@@ -363,8 +381,10 @@ EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD,
   for (CXXRecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end();
        I != E; ++I, ++FieldNo) {
     const FieldDecl *FD = *I;
-    
-    uint64_t FieldOffset = Offset + Layout.getFieldOffset(FieldNo);
+    if (FD->isBitField())
+      continue;
+
+    CharUnits FieldOffset = Offset + getFieldOffset(Layout, FieldNo);
     
     if (!CanPlaceFieldSubobjectAtOffset(FD, FieldOffset))
       return false;
@@ -373,8 +393,9 @@ EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD,
   return true;
 }
 
-bool EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
-                                                       uint64_t Offset) const {
+bool
+EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
+                                                  CharUnits Offset) const {
   // We don't have to keep looking past the maximum offset that's known to
   // contain an empty class.
   if (!AnyEmptySubobjectsBeyondOffset(Offset))
@@ -397,7 +418,7 @@ bool EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
 
     uint64_t NumElements = Context.getConstantArrayElementCount(AT);
-    uint64_t ElementOffset = Offset;
+    CharUnits ElementOffset = Offset;
     for (uint64_t I = 0; I != NumElements; ++I) {
       // We don't have to keep looking past the maximum offset that's known to
       // contain an empty class.
@@ -415,7 +436,8 @@ bool EmptySubobjectMap::CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
 }
 
 bool
-EmptySubobjectMap::CanPlaceFieldAtOffset(const FieldDecl *FD, uint64_t Offset) {
+EmptySubobjectMap::CanPlaceFieldAtOffset(const FieldDecl *FD, 
+                                         CharUnits Offset) {
   if (!CanPlaceFieldSubobjectAtOffset(FD, Offset))
     return false;
   
@@ -427,7 +449,7 @@ EmptySubobjectMap::CanPlaceFieldAtOffset(const FieldDecl *FD, uint64_t Offset) {
 
 void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD, 
                                                    const CXXRecordDecl *Class,
-                                                   uint64_t Offset) {
+                                                   CharUnits Offset) {
   // We know that the only empty subobjects that can conflict with empty
   // field subobjects are subobjects of empty bases that can be placed at offset
   // zero. Because of this, we only need to keep track of empty field 
@@ -449,7 +471,7 @@ void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD,
     const CXXRecordDecl *BaseDecl =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
-    uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(BaseDecl);
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(BaseDecl);
     UpdateEmptyFieldSubobjects(BaseDecl, Class, BaseOffset);
   }
 
@@ -460,7 +482,7 @@ void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD,
       const CXXRecordDecl *VBaseDecl =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
       
-      uint64_t VBaseOffset = Offset + Layout.getVBaseClassOffset(VBaseDecl);
+      CharUnits VBaseOffset = Offset + Layout.getVBaseClassOffset(VBaseDecl);
       UpdateEmptyFieldSubobjects(VBaseDecl, Class, VBaseOffset);
     }
   }
@@ -470,15 +492,17 @@ void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD,
   for (CXXRecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end();
        I != E; ++I, ++FieldNo) {
     const FieldDecl *FD = *I;
-    
-    uint64_t FieldOffset = Offset + Layout.getFieldOffset(FieldNo);
+    if (FD->isBitField())
+      continue;
+
+    CharUnits FieldOffset = Offset + getFieldOffset(Layout, FieldNo);
 
     UpdateEmptyFieldSubobjects(FD, FieldOffset);
   }
 }
   
 void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const FieldDecl *FD,
-                                                   uint64_t Offset) {
+                                                   CharUnits Offset) {
   QualType T = FD->getType();
   if (const RecordType *RT = T->getAs<RecordType>()) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
@@ -497,7 +521,7 @@ void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const FieldDecl *FD,
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
     
     uint64_t NumElements = Context.getConstantArrayElementCount(AT);
-    uint64_t ElementOffset = Offset;
+    CharUnits ElementOffset = Offset;
     
     for (uint64_t I = 0; I != NumElements; ++I) {
       // We know that the only empty subobjects that can conflict with empty
@@ -519,7 +543,7 @@ protected:
   // FIXME: Remove this and make the appropriate fields public.
   friend class clang::ASTContext;
 
-  ASTContext &Context;
+  const ASTContext &Context;
 
   EmptySubobjectMap *EmptySubobjects;
 
@@ -527,7 +551,10 @@ protected:
   uint64_t Size;
 
   /// Alignment - The current alignment of the record layout.
-  unsigned Alignment;
+  CharUnits Alignment;
+
+  /// \brief The alignment if attribute packed is not used.
+  CharUnits UnpackedAlignment;
 
   llvm::SmallVector<uint64_t, 16> FieldOffsets;
 
@@ -545,13 +572,13 @@ protected:
 
   /// MaxFieldAlignment - The maximum allowed field alignment. This is set by
   /// #pragma pack.
-  unsigned MaxFieldAlignment;
+  CharUnits MaxFieldAlignment;
 
   /// DataSize - The data size of the record being laid out.
   uint64_t DataSize;
 
-  uint64_t NonVirtualSize;
-  unsigned NonVirtualAlignment;
+  CharUnits NonVirtualSize;
+  CharUnits NonVirtualAlignment;
 
   /// PrimaryBase - the primary base class (if one exists) of the class
   /// we're laying out.
@@ -561,7 +588,7 @@ protected:
   /// out is virtual.
   bool PrimaryBaseIsVirtual;
 
-  typedef llvm::DenseMap<const CXXRecordDecl *, uint64_t> BaseOffsetsMapTy;
+  typedef llvm::DenseMap<const CXXRecordDecl *, CharUnits> BaseOffsetsMapTy;
 
   /// Bases - base classes and their offsets in the record.
   BaseOffsetsMapTy Bases;
@@ -571,7 +598,7 @@ protected:
 
   /// IndirectPrimaryBases - Virtual base classes, direct or indirect, that are
   /// primary base classes for some other direct or indirect base class.
-  llvm::SmallSet<const CXXRecordDecl*, 32> IndirectPrimaryBases;
+  CXXIndirectPrimaryBaseSet IndirectPrimaryBases;
 
   /// FirstNearlyEmptyVBase - The first nearly empty virtual base class in
   /// inheritance graph order. Used for determining the primary base class.
@@ -581,11 +608,14 @@ protected:
   /// avoid visiting virtual bases more than once.
   llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBases;
 
-  RecordLayoutBuilder(ASTContext &Context, EmptySubobjectMap *EmptySubobjects)
-    : Context(Context), EmptySubobjects(EmptySubobjects), Size(0), Alignment(8),
-      Packed(false), IsUnion(false), IsMac68kAlign(false),
-      UnfilledBitsInLastByte(0), MaxFieldAlignment(0), DataSize(0),
-      NonVirtualSize(0), NonVirtualAlignment(8), PrimaryBase(0),
+  RecordLayoutBuilder(const ASTContext &Context, EmptySubobjectMap
+                      *EmptySubobjects)
+    : Context(Context), EmptySubobjects(EmptySubobjects), Size(0), 
+      Alignment(CharUnits::One()), UnpackedAlignment(Alignment),
+      Packed(false), IsUnion(false), IsMac68kAlign(false), 
+      UnfilledBitsInLastByte(0), MaxFieldAlignment(CharUnits::Zero()), 
+      DataSize(0), NonVirtualSize(CharUnits::Zero()), 
+      NonVirtualAlignment(CharUnits::One()), PrimaryBase(0), 
       PrimaryBaseIsVirtual(false), FirstNearlyEmptyVBase(0) { }
 
   void Layout(const RecordDecl *D);
@@ -594,7 +624,8 @@ protected:
 
   void LayoutFields(const RecordDecl *D);
   void LayoutField(const FieldDecl *D);
-  void LayoutWideBitField(uint64_t FieldSize, uint64_t TypeSize);
+  void LayoutWideBitField(uint64_t FieldSize, uint64_t TypeSize,
+                          bool FieldPacked, const FieldDecl *D);
   void LayoutBitField(const FieldDecl *D);
 
   /// BaseSubobjectInfoAllocator - Allocator for BaseSubobjectInfo objects.
@@ -628,13 +659,6 @@ protected:
 
   virtual uint64_t GetVirtualPointersSize(const CXXRecordDecl *RD) const;
 
-  /// IdentifyPrimaryBases - Identify all virtual base classes, direct or
-  /// indirect, that are primary base classes for some other direct or indirect
-  /// base class.
-  void IdentifyPrimaryBases(const CXXRecordDecl *RD);
-
-  virtual bool IsNearlyEmpty(const CXXRecordDecl *RD) const;
-
   /// LayoutNonVirtualBases - Determines the primary base class (if any) and
   /// lays it out. Will then proceed to lay out all non-virtual base clasess.
   void LayoutNonVirtualBases(const CXXRecordDecl *RD);
@@ -642,8 +666,8 @@ protected:
   /// LayoutNonVirtualBase - Lays out a single non-virtual base.
   void LayoutNonVirtualBase(const BaseSubobjectInfo *Base);
 
-  void AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info, 
-                                            uint64_t Offset);
+  void AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info,
+                                    CharUnits Offset);
 
   /// LayoutVirtualBases - Lays out all the virtual bases.
   void LayoutVirtualBases(const CXXRecordDecl *RD,
@@ -653,17 +677,26 @@ protected:
   void LayoutVirtualBase(const BaseSubobjectInfo *Base);
 
   /// LayoutBase - Will lay out a base and return the offset where it was
-  /// placed, in bits.
-  uint64_t LayoutBase(const BaseSubobjectInfo *Base);
+  /// placed, in chars.
+  CharUnits LayoutBase(const BaseSubobjectInfo *Base);
 
   /// InitializeLayout - Initialize record layout for the given record decl.
   void InitializeLayout(const Decl *D);
 
   /// FinishLayout - Finalize record layout. Adjust record size based on the
   /// alignment.
-  void FinishLayout();
+  void FinishLayout(const NamedDecl *D);
+
+  void UpdateAlignment(CharUnits NewAlignment, CharUnits UnpackedNewAlignment);
+  void UpdateAlignment(CharUnits NewAlignment) {
+    UpdateAlignment(NewAlignment, NewAlignment);
+  }
+
+  void CheckFieldPadding(uint64_t Offset, uint64_t UnpaddedOffset,
+                          uint64_t UnpackedOffset, unsigned UnpackedAlign,
+                          bool isPacked, const FieldDecl *D);
 
-  void UpdateAlignment(unsigned NewAlignment);
+  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID);
 
   RecordLayoutBuilder(const RecordLayoutBuilder&);   // DO NOT IMPLEMENT
   void operator=(const RecordLayoutBuilder&); // DO NOT IMPLEMENT
@@ -674,42 +707,6 @@ public:
 };
 } // end anonymous namespace
 
-/// IsNearlyEmpty - Indicates when a class has a vtable pointer, but
-/// no other data.
-bool RecordLayoutBuilder::IsNearlyEmpty(const CXXRecordDecl *RD) const {
-  // FIXME: Audit the corners
-  if (!RD->isDynamicClass())
-    return false;
-  const ASTRecordLayout &BaseInfo = Context.getASTRecordLayout(RD);
-  if (BaseInfo.getNonVirtualSize() == Context.Target.getPointerWidth(0))
-    return true;
-  return false;
-}
-
-void RecordLayoutBuilder::IdentifyPrimaryBases(const CXXRecordDecl *RD) {
-  const ASTRecordLayout::PrimaryBaseInfo &BaseInfo =
-    Context.getASTRecordLayout(RD).getPrimaryBaseInfo();
-
-  // If the record has a primary base class that is virtual, add it to the set
-  // of primary bases.
-  if (BaseInfo.isVirtual())
-    IndirectPrimaryBases.insert(BaseInfo.getBase());
-
-  // Now traverse all bases and find primary bases for them.
-  for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
-         e = RD->bases_end(); i != e; ++i) {
-    assert(!i->getType()->isDependentType() &&
-           "Cannot layout class with dependent bases.");
-    const CXXRecordDecl *Base =
-      cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
-
-    // Only bases with virtual bases participate in computing the
-    // indirect primary virtual base classes.
-    if (Base->getNumVBases())
-      IdentifyPrimaryBases(Base);
-  }
-}
-
 void
 RecordLayoutBuilder::SelectPrimaryVBase(const CXXRecordDecl *RD) {
   for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
@@ -721,7 +718,7 @@ RecordLayoutBuilder::SelectPrimaryVBase(const CXXRecordDecl *RD) {
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
     // Check if this is a nearly empty virtual base.
-    if (I->isVirtual() && IsNearlyEmpty(Base)) {
+    if (I->isVirtual() && Context.isNearlyEmpty(Base)) {
       // If it's not an indirect primary base, then we've found our primary
       // base.
       if (!IndirectPrimaryBases.count(Base)) {
@@ -754,14 +751,7 @@ void RecordLayoutBuilder::DeterminePrimaryBase(const CXXRecordDecl *RD) {
 
   // Compute all the primary virtual bases for all of our direct and
   // indirect bases, and record all their primary virtual base classes.
-  for (CXXRecordDecl::base_class_const_iterator i = RD->bases_begin(),
-         e = RD->bases_end(); i != e; ++i) {
-    assert(!i->getType()->isDependentType() &&
-           "Cannot lay out class with dependent bases.");
-    const CXXRecordDecl *Base =
-      cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
-    IdentifyPrimaryBases(Base);
-  }
+  RD->getIndirectPrimaryBases(IndirectPrimaryBases);
 
   // If the record has a dynamic base class, attempt to choose a primary base
   // class. It is the first (in direct base class order) non-virtual dynamic
@@ -809,8 +799,18 @@ void RecordLayoutBuilder::DeterminePrimaryBase(const CXXRecordDecl *RD) {
   Size += GetVirtualPointersSize(RD);
   DataSize = Size;
 
+  CharUnits UnpackedBaseAlign = 
+    Context.toCharUnitsFromBits(Context.Target.getPointerAlign(0));
+  CharUnits BaseAlign = (Packed) ? CharUnits::One() : UnpackedBaseAlign;
+
+  // The maximum field alignment overrides base align.
+  if (!MaxFieldAlignment.isZero()) {
+    BaseAlign = std::min(BaseAlign, MaxFieldAlignment);
+    UnpackedBaseAlign = std::min(UnpackedBaseAlign, MaxFieldAlignment);
+  }
+
   // Update the alignment.
-  UpdateAlignment(Context.Target.getPointerAlign(0));
+  UpdateAlignment(BaseAlign, UnpackedBaseAlign);
 }
 
 BaseSubobjectInfo *
@@ -845,7 +845,7 @@ RecordLayoutBuilder::ComputeBaseSubobjectInfo(const CXXRecordDecl *RD,
   // Check if this base has a primary virtual base.
   if (RD->getNumVBases()) {
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-    if (Layout.getPrimaryBaseWasVirtual()) {
+    if (Layout.isPrimaryBaseVirtual()) {
       // This base does have a primary virtual base.
       PrimaryVirtualBase = Layout.getPrimaryBase();
       assert(PrimaryVirtualBase && "Didn't have a primary virtual base!");
@@ -977,7 +977,7 @@ RecordLayoutBuilder::LayoutNonVirtualBases(const CXXRecordDecl *RD) {
 
 void RecordLayoutBuilder::LayoutNonVirtualBase(const BaseSubobjectInfo *Base) {
   // Layout the base.
-  uint64_t Offset = LayoutBase(Base);
+  CharUnits Offset = LayoutBase(Base);
 
   // Add its base class offset.
   assert(!Bases.count(Base->Class) && "base offset already exists!");
@@ -988,7 +988,7 @@ void RecordLayoutBuilder::LayoutNonVirtualBase(const BaseSubobjectInfo *Base) {
 
 void
 RecordLayoutBuilder::AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info, 
-                                                  uint64_t Offset) {
+                                                  CharUnits Offset) {
   // This base isn't interesting, it has no virtual bases.
   if (!Info->Class->getNumVBases())
     return;
@@ -1016,7 +1016,7 @@ RecordLayoutBuilder::AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info,
     if (Base->IsVirtual)
       continue;
 
-    uint64_t BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(Base->Class);
     AddPrimaryVirtualBaseOffsets(Base, BaseOffset);
   }
 }
@@ -1033,7 +1033,7 @@ RecordLayoutBuilder::LayoutVirtualBases(const CXXRecordDecl *RD,
   } else {
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
     PrimaryBase = Layout.getPrimaryBase();
-    PrimaryBaseIsVirtual = Layout.getPrimaryBaseWasVirtual();
+    PrimaryBaseIsVirtual = Layout.isPrimaryBaseVirtual();
   }
 
   for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
@@ -1074,7 +1074,7 @@ void RecordLayoutBuilder::LayoutVirtualBase(const BaseSubobjectInfo *Base) {
   assert(!Base->Derived && "Trying to lay out a primary virtual base!");
   
   // Layout the base.
-  uint64_t Offset = LayoutBase(Base);
+  CharUnits Offset = LayoutBase(Base);
 
   // Add its base class offset.
   assert(!VBases.count(Base->Class) && "vbase offset already exists!");
@@ -1083,38 +1083,48 @@ void RecordLayoutBuilder::LayoutVirtualBase(const BaseSubobjectInfo *Base) {
   AddPrimaryVirtualBaseOffsets(Base, Offset);
 }
 
-uint64_t RecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
+CharUnits RecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
   const ASTRecordLayout &Layout = Context.getASTRecordLayout(Base->Class);
 
   // If we have an empty base class, try to place it at offset 0.
   if (Base->Class->isEmpty() &&
-      EmptySubobjects->CanPlaceBaseAtOffset(Base, 0)) {
-    Size = std::max(Size, Layout.getSize());
+      EmptySubobjects->CanPlaceBaseAtOffset(Base, CharUnits::Zero())) {
+    uint64_t RecordSizeInBits = Context.toBits(Layout.getSize());
+    Size = std::max(Size, RecordSizeInBits);
 
-    return 0;
+    return CharUnits::Zero();
   }
 
-  unsigned BaseAlign = Layout.getNonVirtualAlign();
+  CharUnits UnpackedBaseAlign = Layout.getNonVirtualAlign();
+  CharUnits BaseAlign = (Packed) ? CharUnits::One() : UnpackedBaseAlign;
+
+  // The maximum field alignment overrides base align.
+  if (!MaxFieldAlignment.isZero()) {
+    BaseAlign = std::min(BaseAlign, MaxFieldAlignment);
+    UnpackedBaseAlign = std::min(UnpackedBaseAlign, MaxFieldAlignment);
+  }
 
   // Round up the current record size to the base's alignment boundary.
-  uint64_t Offset = llvm::RoundUpToAlignment(DataSize, BaseAlign);
+  uint64_t Offset = 
+    llvm::RoundUpToAlignment(DataSize, Context.toBits(BaseAlign));
 
   // Try to place the base.
-  while (!EmptySubobjects->CanPlaceBaseAtOffset(Base, Offset))
-    Offset += BaseAlign;
+  while (!EmptySubobjects->CanPlaceBaseAtOffset(Base, 
+                                          Context.toCharUnitsFromBits(Offset)))
+    Offset += Context.toBits(BaseAlign);
 
   if (!Base->Class->isEmpty()) {
     // Update the data size.
-    DataSize = Offset + Layout.getNonVirtualSize();
+    DataSize = Offset + Context.toBits(Layout.getNonVirtualSize());
 
     Size = std::max(Size, DataSize);
   } else
-    Size = std::max(Size, Offset + Layout.getSize());
+    Size = std::max(Size, Offset + Context.toBits(Layout.getSize()));
 
   // Remember max struct/class alignment.
-  UpdateAlignment(BaseAlign);
+  UpdateAlignment(BaseAlign, UnpackedBaseAlign);
 
-  return Offset;
+  return Context.toCharUnitsFromBits(Offset);
 }
 
 void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
@@ -1129,14 +1139,14 @@ void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
   // to bit-fields, but gcc appears not to follow that.
   if (D->hasAttr<AlignMac68kAttr>()) {
     IsMac68kAlign = true;
-    MaxFieldAlignment = 2 * 8;
-    Alignment = 2 * 8;
+    MaxFieldAlignment = CharUnits::fromQuantity(2);
+    Alignment = CharUnits::fromQuantity(2);
   } else {
     if (const MaxFieldAlignmentAttr *MFAA = D->getAttr<MaxFieldAlignmentAttr>())
-      MaxFieldAlignment = MFAA->getAlignment();
+      MaxFieldAlignment = Context.toCharUnitsFromBits(MFAA->getAlignment());
 
     if (unsigned MaxAlign = D->getMaxAlignment())
-      UpdateAlignment(MaxAlign);
+      UpdateAlignment(Context.toCharUnitsFromBits(MaxAlign));
   }
 }
 
@@ -1146,7 +1156,7 @@ void RecordLayoutBuilder::Layout(const RecordDecl *D) {
 
   // Finally, round the size of the total struct up to the alignment of the
   // struct itself.
-  FinishLayout();
+  FinishLayout(D);
 }
 
 void RecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
@@ -1157,7 +1167,7 @@ void RecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
 
   LayoutFields(RD);
 
-  NonVirtualSize = Size;
+  NonVirtualSize = Context.toCharUnitsFromBits(Size);
   NonVirtualAlignment = Alignment;
 
   // Lay out the virtual bases and add the primary virtual base offsets.
@@ -1167,7 +1177,7 @@ void RecordLayoutBuilder::Layout(const CXXRecordDecl *RD) {
 
   // Finally, round the size of the total struct up to the alignment of the
   // struct itself.
-  FinishLayout();
+  FinishLayout(RD);
 
 #ifndef NDEBUG
   // Check that we have base offsets for all bases.
@@ -1201,7 +1211,7 @@ void RecordLayoutBuilder::Layout(const ObjCInterfaceDecl *D) {
 
     // We start laying out ivars not at the end of the superclass
     // structure, but at the next byte following the last field.
-    Size = llvm::RoundUpToAlignment(SL.getDataSize(), 8);
+    Size = Context.toBits(SL.getDataSize());
     DataSize = Size;
   }
 
@@ -1215,7 +1225,7 @@ void RecordLayoutBuilder::Layout(const ObjCInterfaceDecl *D) {
 
   // Finally, round the size of the total struct up to the alignment of the
   // struct itself.
-  FinishLayout();
+  FinishLayout(D);
 }
 
 void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
@@ -1227,7 +1237,9 @@ void RecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
 }
 
 void RecordLayoutBuilder::LayoutWideBitField(uint64_t FieldSize,
-                                                uint64_t TypeSize) {
+                                             uint64_t TypeSize,
+                                             bool FieldPacked,
+                                             const FieldDecl *D) {
   assert(Context.getLangOptions().CPlusPlus &&
          "Can only have wide bit-fields in C++!");
 
@@ -1258,6 +1270,7 @@ void RecordLayoutBuilder::LayoutWideBitField(uint64_t FieldSize,
   UnfilledBitsInLastByte = 0;
 
   uint64_t FieldOffset;
+  uint64_t UnpaddedFieldOffset = DataSize - UnfilledBitsInLastByte;
 
   if (IsUnion) {
     DataSize = std::max(DataSize, FieldSize);
@@ -1276,16 +1289,20 @@ void RecordLayoutBuilder::LayoutWideBitField(uint64_t FieldSize,
   // Place this field at the current location.
   FieldOffsets.push_back(FieldOffset);
 
+  CheckFieldPadding(FieldOffset, UnpaddedFieldOffset, FieldOffset,
+                    TypeAlign, FieldPacked, D);
+
   // Update the size.
   Size = std::max(Size, DataSize);
 
   // Remember max struct/class alignment.
-  UpdateAlignment(TypeAlign);
+  UpdateAlignment(Context.toCharUnitsFromBits(TypeAlign));
 }
 
 void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   bool FieldPacked = Packed || D->hasAttr<PackedAttr>();
-  uint64_t FieldOffset = IsUnion ? 0 : (DataSize - UnfilledBitsInLastByte);
+  uint64_t UnpaddedFieldOffset = DataSize - UnfilledBitsInLastByte;
+  uint64_t FieldOffset = IsUnion ? 0 : UnpaddedFieldOffset;
   uint64_t FieldSize = D->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
 
   std::pair<uint64_t, unsigned> FieldInfo = Context.getTypeInfo(D->getType());
@@ -1293,29 +1310,48 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   unsigned FieldAlign = FieldInfo.second;
 
   if (FieldSize > TypeSize) {
-    LayoutWideBitField(FieldSize, TypeSize);
+    LayoutWideBitField(FieldSize, TypeSize, FieldPacked, D);
     return;
   }
 
+  // The align if the field is not packed. This is to check if the attribute
+  // was unnecessary (-Wpacked).
+  unsigned UnpackedFieldAlign = FieldAlign;
+  uint64_t UnpackedFieldOffset = FieldOffset;
+  if (!Context.Target.useBitFieldTypeAlignment())
+    UnpackedFieldAlign = 1;
+
   if (FieldPacked || !Context.Target.useBitFieldTypeAlignment())
     FieldAlign = 1;
   FieldAlign = std::max(FieldAlign, D->getMaxAlignment());
+  UnpackedFieldAlign = std::max(UnpackedFieldAlign, D->getMaxAlignment());
 
   // The maximum field alignment overrides the aligned attribute.
-  if (MaxFieldAlignment)
-    FieldAlign = std::min(FieldAlign, MaxFieldAlignment);
+  if (!MaxFieldAlignment.isZero()) {
+    unsigned MaxFieldAlignmentInBits = Context.toBits(MaxFieldAlignment);
+    FieldAlign = std::min(FieldAlign, MaxFieldAlignmentInBits);
+    UnpackedFieldAlign = std::min(UnpackedFieldAlign, MaxFieldAlignmentInBits);
+  }
 
   // Check if we need to add padding to give the field the correct alignment.
   if (FieldSize == 0 || (FieldOffset & (FieldAlign-1)) + FieldSize > TypeSize)
     FieldOffset = llvm::RoundUpToAlignment(FieldOffset, FieldAlign);
 
+  if (FieldSize == 0 ||
+      (UnpackedFieldOffset & (UnpackedFieldAlign-1)) + FieldSize > TypeSize)
+    UnpackedFieldOffset = llvm::RoundUpToAlignment(UnpackedFieldOffset,
+                                                   UnpackedFieldAlign);
+
   // Padding members don't affect overall alignment.
   if (!D->getIdentifier())
-    FieldAlign = 1;
+    FieldAlign = UnpackedFieldAlign = 1;
 
   // Place this field at the current location.
   FieldOffsets.push_back(FieldOffset);
 
+  CheckFieldPadding(FieldOffset, UnpaddedFieldOffset, UnpackedFieldOffset,
+                    UnpackedFieldAlign, FieldPacked, D);
+
   // Update DataSize to include the last byte containing (part of) the bitfield.
   if (IsUnion) {
     // FIXME: I think FieldSize should be TypeSize here.
@@ -1331,7 +1367,8 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
   Size = std::max(Size, DataSize);
 
   // Remember max struct/class alignment.
-  UpdateAlignment(FieldAlign);
+  UpdateAlignment(Context.toCharUnitsFromBits(FieldAlign), 
+                  Context.toCharUnitsFromBits(UnpackedFieldAlign));
 }
 
 void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
@@ -1340,42 +1377,74 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
     return;
   }
 
+  uint64_t UnpaddedFieldOffset = DataSize - UnfilledBitsInLastByte;
+
   // Reset the unfilled bits.
   UnfilledBitsInLastByte = 0;
 
   bool FieldPacked = Packed || D->hasAttr<PackedAttr>();
-  uint64_t FieldOffset = IsUnion ? 0 : DataSize;
-  uint64_t FieldSize;
-  unsigned FieldAlign;
+  CharUnits FieldOffset = 
+    IsUnion ? CharUnits::Zero() : Context.toCharUnitsFromBits(DataSize);
+  CharUnits FieldSize;
+  CharUnits FieldAlign;
 
   if (D->getType()->isIncompleteArrayType()) {
     // This is a flexible array member; we can't directly
     // query getTypeInfo about these, so we figure it out here.
     // Flexible array members don't have any size, but they
     // have to be aligned appropriately for their element type.
-    FieldSize = 0;
+    FieldSize = CharUnits::Zero();
     const ArrayType* ATy = Context.getAsArrayType(D->getType());
-    FieldAlign = Context.getTypeAlign(ATy->getElementType());
+    FieldAlign = Context.getTypeAlignInChars(ATy->getElementType());
   } else if (const ReferenceType *RT = D->getType()->getAs<ReferenceType>()) {
     unsigned AS = RT->getPointeeType().getAddressSpace();
-    FieldSize = Context.Target.getPointerWidth(AS);
-    FieldAlign = Context.Target.getPointerAlign(AS);
+    FieldSize = 
+      Context.toCharUnitsFromBits(Context.Target.getPointerWidth(AS));
+    FieldAlign = 
+      Context.toCharUnitsFromBits(Context.Target.getPointerAlign(AS));
   } else {
-    std::pair<uint64_t, unsigned> FieldInfo = Context.getTypeInfo(D->getType());
+    std::pair<CharUnits, CharUnits> FieldInfo = 
+      Context.getTypeInfoInChars(D->getType());
     FieldSize = FieldInfo.first;
     FieldAlign = FieldInfo.second;
+
+    if (Context.getLangOptions().MSBitfields) {
+      // If MS bitfield layout is required, figure out what type is being
+      // laid out and align the field to the width of that type.
+      
+      // Resolve all typedefs down to their base type and round up the field
+      // alignment if necessary.
+      QualType T = Context.getBaseElementType(D->getType());
+      if (const BuiltinType *BTy = T->getAs<BuiltinType>()) {
+        CharUnits TypeSize = Context.getTypeSizeInChars(BTy);
+        if (TypeSize > FieldAlign)
+          FieldAlign = TypeSize;
+      }
+    }
   }
 
+  // The align if the field is not packed. This is to check if the attribute
+  // was unnecessary (-Wpacked).
+  CharUnits UnpackedFieldAlign = FieldAlign;
+  CharUnits UnpackedFieldOffset = FieldOffset;
+
   if (FieldPacked)
-    FieldAlign = 8;
-  FieldAlign = std::max(FieldAlign, D->getMaxAlignment());
+    FieldAlign = CharUnits::One();
+  CharUnits MaxAlignmentInChars = 
+    Context.toCharUnitsFromBits(D->getMaxAlignment());
+  FieldAlign = std::max(FieldAlign, MaxAlignmentInChars);
+  UnpackedFieldAlign = std::max(UnpackedFieldAlign, MaxAlignmentInChars);
 
   // The maximum field alignment overrides the aligned attribute.
-  if (MaxFieldAlignment)
+  if (!MaxFieldAlignment.isZero()) {
     FieldAlign = std::min(FieldAlign, MaxFieldAlignment);
+    UnpackedFieldAlign = std::min(UnpackedFieldAlign, MaxFieldAlignment);
+  }
 
   // Round up the current record size to the field's alignment boundary.
-  FieldOffset = llvm::RoundUpToAlignment(FieldOffset, FieldAlign);
+  FieldOffset = FieldOffset.RoundUpToAlignment(FieldAlign);
+  UnpackedFieldOffset = 
+    UnpackedFieldOffset.RoundUpToAlignment(UnpackedFieldAlign);
 
   if (!IsUnion && EmptySubobjects) {
     // Check if we can place the field at this offset.
@@ -1386,41 +1455,130 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
   }
 
   // Place this field at the current location.
-  FieldOffsets.push_back(FieldOffset);
+  FieldOffsets.push_back(Context.toBits(FieldOffset));
+
+  CheckFieldPadding(Context.toBits(FieldOffset), UnpaddedFieldOffset, 
+                    Context.toBits(UnpackedFieldOffset),
+                    Context.toBits(UnpackedFieldAlign), FieldPacked, D);
 
   // Reserve space for this field.
+  uint64_t FieldSizeInBits = Context.toBits(FieldSize);
   if (IsUnion)
-    Size = std::max(Size, FieldSize);
+    Size = std::max(Size, FieldSizeInBits);
   else
-    Size = FieldOffset + FieldSize;
+    Size = Context.toBits(FieldOffset) + FieldSizeInBits;
 
   // Update the data size.
   DataSize = Size;
 
   // Remember max struct/class alignment.
-  UpdateAlignment(FieldAlign);
+  UpdateAlignment(FieldAlign, UnpackedFieldAlign);
 }
 
-void RecordLayoutBuilder::FinishLayout() {
+void RecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
   // In C++, records cannot be of size 0.
-  if (Context.getLangOptions().CPlusPlus && Size == 0)
-    Size = 8;
+  if (Context.getLangOptions().CPlusPlus && Size == 0) {
+    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
+      // Compatibility with gcc requires a class (pod or non-pod)
+      // which is not empty but of size 0; such as having fields of
+      // array of zero-length, remains of Size 0
+      if (RD->isEmpty())
+        Size = 8;
+    }
+    else
+      Size = 8;
+  }
   // Finally, round the size of the record up to the alignment of the
   // record itself.
-  Size = llvm::RoundUpToAlignment(Size, Alignment);
+  uint64_t UnpaddedSize = Size - UnfilledBitsInLastByte;
+  uint64_t UnpackedSize = 
+    llvm::RoundUpToAlignment(Size, Context.toBits(UnpackedAlignment));
+  Size = llvm::RoundUpToAlignment(Size, Context.toBits(Alignment));
+
+  unsigned CharBitNum = Context.Target.getCharWidth();
+  if (const RecordDecl *RD = dyn_cast<RecordDecl>(D)) {
+    // Warn if padding was introduced to the struct/class/union.
+    if (Size > UnpaddedSize) {
+      unsigned PadSize = Size - UnpaddedSize;
+      bool InBits = true;
+      if (PadSize % CharBitNum == 0) {
+        PadSize = PadSize / CharBitNum;
+        InBits = false;
+      }
+      Diag(RD->getLocation(), diag::warn_padded_struct_size)
+          << Context.getTypeDeclType(RD)
+          << PadSize
+          << (InBits ? 1 : 0) /*(byte|bit)*/ << (PadSize > 1); // plural or not
+    }
+
+    // Warn if we packed it unnecessarily. If the alignment is 1 byte don't
+    // bother since there won't be alignment issues.
+    if (Packed && UnpackedAlignment > CharUnits::One() && Size == UnpackedSize)
+      Diag(D->getLocation(), diag::warn_unnecessary_packed)
+          << Context.getTypeDeclType(RD);
+  }
 }
 
-void RecordLayoutBuilder::UpdateAlignment(unsigned NewAlignment) {
+void RecordLayoutBuilder::UpdateAlignment(CharUnits NewAlignment,
+                                          CharUnits UnpackedNewAlignment) {
   // The alignment is not modified when using 'mac68k' alignment.
   if (IsMac68kAlign)
     return;
 
-  if (NewAlignment <= Alignment)
+  if (NewAlignment > Alignment) {
+    assert(llvm::isPowerOf2_32(NewAlignment.getQuantity() && 
+           "Alignment not a power of 2"));
+    Alignment = NewAlignment;
+  }
+
+  if (UnpackedNewAlignment > UnpackedAlignment) {
+    assert(llvm::isPowerOf2_32(UnpackedNewAlignment.getQuantity() &&
+           "Alignment not a power of 2"));
+    UnpackedAlignment = UnpackedNewAlignment;
+  }
+}
+
+void RecordLayoutBuilder::CheckFieldPadding(uint64_t Offset,
+                                            uint64_t UnpaddedOffset,
+                                            uint64_t UnpackedOffset,
+                                            unsigned UnpackedAlign,
+                                            bool isPacked,
+                                            const FieldDecl *D) {
+  // We let objc ivars without warning, objc interfaces generally are not used
+  // for padding tricks.
+  if (isa<ObjCIvarDecl>(D))
     return;
 
-  assert(llvm::isPowerOf2_32(NewAlignment && "Alignment not a power of 2"));
+  unsigned CharBitNum = Context.Target.getCharWidth();
 
-  Alignment = NewAlignment;
+  // Warn if padding was introduced to the struct/class.
+  if (!IsUnion && Offset > UnpaddedOffset) {
+    unsigned PadSize = Offset - UnpaddedOffset;
+    bool InBits = true;
+    if (PadSize % CharBitNum == 0) {
+      PadSize = PadSize / CharBitNum;
+      InBits = false;
+    }
+    if (D->getIdentifier())
+      Diag(D->getLocation(), diag::warn_padded_struct_field)
+          << (D->getParent()->isStruct() ? 0 : 1) // struct|class
+          << Context.getTypeDeclType(D->getParent())
+          << PadSize
+          << (InBits ? 1 : 0) /*(byte|bit)*/ << (PadSize > 1) // plural or not
+          << D->getIdentifier();
+    else
+      Diag(D->getLocation(), diag::warn_padded_struct_anon_field)
+          << (D->getParent()->isStruct() ? 0 : 1) // struct|class
+          << Context.getTypeDeclType(D->getParent())
+          << PadSize
+          << (InBits ? 1 : 0) /*(byte|bit)*/ << (PadSize > 1); // plural or not
+  }
+
+  // Warn if we packed it unnecessarily. If the alignment is 1 byte don't
+  // bother since there won't be alignment issues.
+  if (isPacked && UnpackedAlign > CharBitNum && Offset == UnpackedOffset)
+    Diag(D->getLocation(), diag::warn_unnecessary_packed)
+        << D->getIdentifier();
 }
 
 const CXXMethodDecl *
@@ -1435,6 +1593,13 @@ RecordLayoutBuilder::ComputeKeyFunction(const CXXRecordDecl *RD) {
   if (RD->isInAnonymousNamespace())
     return 0;
 
+  // Template instantiations don't have key functions,see Itanium C++ ABI 5.2.6.
+  // Same behavior as GCC.
+  TemplateSpecializationKind TSK = RD->getTemplateSpecializationKind();
+  if (TSK == TSK_ImplicitInstantiation ||
+      TSK == TSK_ExplicitInstantiationDefinition)
+    return 0;
+
   for (CXXRecordDecl::method_iterator I = RD->method_begin(),
          E = RD->method_end(); I != E; ++I) {
     const CXXMethodDecl *MD = *I;
@@ -1463,26 +1628,21 @@ RecordLayoutBuilder::ComputeKeyFunction(const CXXRecordDecl *RD) {
   return 0;
 }
 
-// This class implements layout specific to the Microsoft ABI.
-class MSRecordLayoutBuilder: public RecordLayoutBuilder {
-public:
-  MSRecordLayoutBuilder(ASTContext& Ctx, EmptySubobjectMap *EmptySubobjects):
-    RecordLayoutBuilder(Ctx, EmptySubobjects) {}
-
-  virtual bool IsNearlyEmpty(const CXXRecordDecl *RD) const;
-  virtual uint64_t GetVirtualPointersSize(const CXXRecordDecl *RD) const;
-};
+DiagnosticBuilder
+RecordLayoutBuilder::Diag(SourceLocation Loc, unsigned DiagID) {
+  return Context.getDiagnostics().Report(Loc, DiagID);
+}
 
-bool MSRecordLayoutBuilder::IsNearlyEmpty(const CXXRecordDecl *RD) const {
-  // FIXME: Audit the corners
-  if (!RD->isDynamicClass())
-    return false;
-  const ASTRecordLayout &BaseInfo = Context.getASTRecordLayout(RD);
-  // In the Microsoft ABI, classes can have one or two vtable pointers.
-  if (BaseInfo.getNonVirtualSize() == Context.Target.getPointerWidth(0) ||
-      BaseInfo.getNonVirtualSize() == Context.Target.getPointerWidth(0) * 2)
-    return true;
-  return false;
+namespace {
+  // This class implements layout specific to the Microsoft ABI.
+  class MSRecordLayoutBuilder : public RecordLayoutBuilder {
+  public:
+    MSRecordLayoutBuilder(const ASTContext& Ctx,
+                          EmptySubobjectMap *EmptySubobjects) :
+      RecordLayoutBuilder(Ctx, EmptySubobjects) {}
+
+    virtual uint64_t GetVirtualPointersSize(const CXXRecordDecl *RD) const;
+  };
 }
 
 uint64_t
@@ -1497,7 +1657,8 @@ MSRecordLayoutBuilder::GetVirtualPointersSize(const CXXRecordDecl *RD) const {
 /// getASTRecordLayout - Get or compute information about the layout of the
 /// specified record (struct/union/class), which indicates its size and field
 /// position information.
-const ASTRecordLayout &ASTContext::getASTRecordLayout(const RecordDecl *D) {
+const ASTRecordLayout &
+ASTContext::getASTRecordLayout(const RecordDecl *D) const {
   D = D->getDefinition();
   assert(D && "Cannot get layout of forward declarations!");
 
@@ -1531,12 +1692,16 @@ const ASTRecordLayout &ASTContext::getASTRecordLayout(const RecordDecl *D) {
     // FIXME: This should be done in FinalizeLayout.
     uint64_t DataSize =
       IsPODForThePurposeOfLayout ? Builder->Size : Builder->DataSize;
-    uint64_t NonVirtualSize =
-      IsPODForThePurposeOfLayout ? DataSize : Builder->NonVirtualSize;
+    CharUnits NonVirtualSize =
+      IsPODForThePurposeOfLayout ? 
+        toCharUnitsFromBits(DataSize) : Builder->NonVirtualSize;
 
+    CharUnits RecordSize = toCharUnitsFromBits(Builder->Size);
     NewEntry =
-      new (*this) ASTRecordLayout(*this, Builder->Size, Builder->Alignment,
-                                  DataSize, Builder->FieldOffsets.data(),
+      new (*this) ASTRecordLayout(*this, RecordSize, 
+                                  Builder->Alignment,
+                                  toCharUnitsFromBits(DataSize), 
+                                  Builder->FieldOffsets.data(),
                                   Builder->FieldOffsets.size(),
                                   NonVirtualSize,
                                   Builder->NonVirtualAlignment,
@@ -1548,9 +1713,12 @@ const ASTRecordLayout &ASTContext::getASTRecordLayout(const RecordDecl *D) {
     RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0);
     Builder.Layout(D);
 
+    CharUnits RecordSize = toCharUnitsFromBits(Builder.Size);
+
     NewEntry =
-      new (*this) ASTRecordLayout(*this, Builder.Size, Builder.Alignment,
-                                  Builder.Size,
+      new (*this) ASTRecordLayout(*this, RecordSize, 
+                                  Builder.Alignment,
+                                  toCharUnitsFromBits(Builder.Size),
                                   Builder.FieldOffsets.data(),
                                   Builder.FieldOffsets.size());
   }
@@ -1572,9 +1740,6 @@ const CXXMethodDecl *ASTContext::getKeyFunction(const CXXRecordDecl *RD) {
   const CXXMethodDecl *&Entry = KeyFunctions[RD];
   if (!Entry)
     Entry = RecordLayoutBuilder::ComputeKeyFunction(RD);
-  else
-    assert(Entry == RecordLayoutBuilder::ComputeKeyFunction(RD) &&
-           "Key function changed!");
 
   return Entry;
 }
@@ -1586,7 +1751,7 @@ const CXXMethodDecl *ASTContext::getKeyFunction(const CXXRecordDecl *RD) {
 /// implementation. This may differ by including synthesized ivars.
 const ASTRecordLayout &
 ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
-                          const ObjCImplementationDecl *Impl) {
+                          const ObjCImplementationDecl *Impl) const {
   assert(!D->isForwardDecl() && "Invalid interface decl!");
 
   // Look up this layout, if already laid out, return what we have.
@@ -1609,9 +1774,12 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
   RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0);
   Builder.Layout(D);
 
+  CharUnits RecordSize = toCharUnitsFromBits(Builder.Size);
+
   const ASTRecordLayout *NewEntry =
-    new (*this) ASTRecordLayout(*this, Builder.Size, Builder.Alignment,
-                                Builder.DataSize,
+    new (*this) ASTRecordLayout(*this, RecordSize, 
+                                Builder.Alignment,
+                                toCharUnitsFromBits(Builder.DataSize),
                                 Builder.FieldOffsets.data(),
                                 Builder.FieldOffsets.size());
 
@@ -1621,18 +1789,18 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
 }
 
 static void PrintOffset(llvm::raw_ostream &OS,
-                        uint64_t Offset, unsigned IndentLevel) {
-  OS << llvm::format("%4d | ", Offset);
+                        CharUnits Offset, unsigned IndentLevel) {
+  OS << llvm::format("%4d | ", Offset.getQuantity());
   OS.indent(IndentLevel * 2);
 }
 
 static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
-                                const CXXRecordDecl *RD, ASTContext &C,
-                                uint64_t Offset,
+                                const CXXRecordDecl *RD, const ASTContext &C,
+                                CharUnits Offset,
                                 unsigned IndentLevel,
                                 const char* Description,
                                 bool IncludeVirtualBases) {
-  const ASTRecordLayout &Info = C.getASTRecordLayout(RD);
+  const ASTRecordLayout &Layout = C.getASTRecordLayout(RD);
 
   PrintOffset(OS, Offset, IndentLevel);
   OS << C.getTypeDeclType(const_cast<CXXRecordDecl *>(RD)).getAsString();
@@ -1644,7 +1812,7 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
 
   IndentLevel++;
 
-  const CXXRecordDecl *PrimaryBase = Info.getPrimaryBase();
+  const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
 
   // Vtable pointer.
   if (RD->isDynamicClass() && !PrimaryBase) {
@@ -1662,7 +1830,7 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
     const CXXRecordDecl *Base =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
-    uint64_t BaseOffset = Offset + Info.getBaseClassOffset(Base) / 8;
+    CharUnits BaseOffset = Offset + Layout.getBaseClassOffset(Base);
 
     DumpCXXRecordLayout(OS, Base, C, BaseOffset, IndentLevel,
                         Base == PrimaryBase ? "(primary base)" : "(base)",
@@ -1674,7 +1842,8 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
   for (CXXRecordDecl::field_iterator I = RD->field_begin(),
          E = RD->field_end(); I != E; ++I, ++FieldNo) {
     const FieldDecl *Field = *I;
-    uint64_t FieldOffset = Offset + Info.getFieldOffset(FieldNo) / 8;
+    CharUnits FieldOffset = Offset + 
+      C.toCharUnitsFromBits(Layout.getFieldOffset(FieldNo));
 
     if (const RecordType *RT = Field->getType()->getAs<RecordType>()) {
       if (const CXXRecordDecl *D = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
@@ -1699,27 +1868,27 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
     const CXXRecordDecl *VBase =
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
-    uint64_t VBaseOffset = Offset + Info.getVBaseClassOffset(VBase) / 8;
+    CharUnits VBaseOffset = Offset + Layout.getVBaseClassOffset(VBase);
     DumpCXXRecordLayout(OS, VBase, C, VBaseOffset, IndentLevel,
                         VBase == PrimaryBase ?
                         "(primary virtual base)" : "(virtual base)",
                         /*IncludeVirtualBases=*/false);
   }
 
-  OS << "  sizeof=" << Info.getSize() / 8;
-  OS << ", dsize=" << Info.getDataSize() / 8;
-  OS << ", align=" << Info.getAlignment() / 8 << '\n';
-  OS << "  nvsize=" << Info.getNonVirtualSize() / 8;
-  OS << ", nvalign=" << Info.getNonVirtualAlign() / 8 << '\n';
+  OS << "  sizeof=" << Layout.getSize().getQuantity();
+  OS << ", dsize=" << Layout.getDataSize().getQuantity();
+  OS << ", align=" << Layout.getAlignment().getQuantity() << '\n';
+  OS << "  nvsize=" << Layout.getNonVirtualSize().getQuantity();
+  OS << ", nvalign=" << Layout.getNonVirtualAlign().getQuantity() << '\n';
   OS << '\n';
 }
 
 void ASTContext::DumpRecordLayout(const RecordDecl *RD,
-                                  llvm::raw_ostream &OS) {
+                                  llvm::raw_ostream &OS) const {
   const ASTRecordLayout &Info = getASTRecordLayout(RD);
 
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
-    return DumpCXXRecordLayout(OS, CXXRD, *this, 0, 0, 0,
+    return DumpCXXRecordLayout(OS, CXXRD, *this, CharUnits(), 0, 0,
                                /*IncludeVirtualBases=*/true);
 
   OS << "Type: " << getTypeDeclType(RD).getAsString() << "\n";
@@ -1727,9 +1896,9 @@ void ASTContext::DumpRecordLayout(const RecordDecl *RD,
   RD->dump();
   OS << "\nLayout: ";
   OS << "<ASTRecordLayout\n";
-  OS << "  Size:" << Info.getSize() << "\n";
-  OS << "  DataSize:" << Info.getDataSize() << "\n";
-  OS << "  Alignment:" << Info.getAlignment() << "\n";
+  OS << "  Size:" << toBits(Info.getSize()) << "\n";
+  OS << "  DataSize:" << toBits(Info.getDataSize()) << "\n";
+  OS << "  Alignment:" << toBits(Info.getAlignment()) << "\n";
   OS << "  FieldOffsets: [";
   for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i) {
     if (i) OS << ", ";
diff --git a/lib/AST/Stmt.cpp b/lib/AST/Stmt.cpp
index fc8898173f3e..7e73f02e67e7 100644
--- a/lib/AST/Stmt.cpp
+++ b/lib/AST/Stmt.cpp
@@ -46,7 +46,7 @@ static StmtClassNameTable &getStmtInfoTableEntry(Stmt::StmtClass E) {
 }
 
 const char *Stmt::getStmtClassName() const {
-  return getStmtInfoTableEntry((StmtClass)sClass).Name;
+  return getStmtInfoTableEntry((StmtClass) StmtBits.sClass).Name;
 }
 
 void Stmt::PrintStats() {
@@ -84,17 +84,84 @@ bool Stmt::CollectingStats(bool Enable) {
   return StatSwitch;
 }
 
+namespace {
+  struct good {};
+  struct bad {};
+
+  // These silly little functions have to be static inline to suppress
+  // unused warnings, and they have to be defined to suppress other
+  // warnings.
+  static inline good is_good(good) { return good(); }
+
+  typedef Stmt::child_range children_t();
+  template <class T> good implements_children(children_t T::*) {
+    return good();
+  }
+  static inline bad implements_children(children_t Stmt::*) {
+    return bad();
+  }
+
+  typedef SourceRange getSourceRange_t() const;
+  template <class T> good implements_getSourceRange(getSourceRange_t T::*) {
+    return good();
+  }
+  static inline bad implements_getSourceRange(getSourceRange_t Stmt::*) {
+    return bad();
+  }
+
+#define ASSERT_IMPLEMENTS_children(type) \
+  (void) sizeof(is_good(implements_children(&type::children)))
+#define ASSERT_IMPLEMENTS_getSourceRange(type) \
+  (void) sizeof(is_good(implements_getSourceRange(&type::getSourceRange)))
+}
+
+/// Check whether the various Stmt classes implement their member
+/// functions.
+static inline void check_implementations() {
+#define ABSTRACT_STMT(type)
+#define STMT(type, base) \
+  ASSERT_IMPLEMENTS_children(type); \
+  ASSERT_IMPLEMENTS_getSourceRange(type);
+#include "clang/AST/StmtNodes.inc"
+}
+
+Stmt::child_range Stmt::children() {
+  switch (getStmtClass()) {
+  case Stmt::NoStmtClass: llvm_unreachable("statement without class");
+#define ABSTRACT_STMT(type)
+#define STMT(type, base) \
+  case Stmt::type##Class: \
+    return static_cast<type*>(this)->children();
+#include "clang/AST/StmtNodes.inc"
+  }
+  llvm_unreachable("unknown statement kind!");
+  return child_range();
+}
+
+SourceRange Stmt::getSourceRange() const {
+  switch (getStmtClass()) {
+  case Stmt::NoStmtClass: llvm_unreachable("statement without class");
+#define ABSTRACT_STMT(type)
+#define STMT(type, base) \
+  case Stmt::type##Class: \
+    return static_cast<const type*>(this)->getSourceRange();
+#include "clang/AST/StmtNodes.inc"
+  }
+  llvm_unreachable("unknown statement kind!");
+  return SourceRange();
+}
+
 void CompoundStmt::setStmts(ASTContext &C, Stmt **Stmts, unsigned NumStmts) {
   if (this->Body)
     C.Deallocate(Body);
-  this->NumStmts = NumStmts;
+  this->CompoundStmtBits.NumStmts = NumStmts;
 
   Body = new (C) Stmt*[NumStmts];
   memcpy(Body, Stmts, sizeof(Stmt *) * NumStmts);
 }
 
 const char *LabelStmt::getName() const {
-  return getID()->getNameStart();
+  return getDecl()->getIdentifier()->getNameStart();
 }
 
 // This is defined here to avoid polluting Stmt.h with importing Expr.h
@@ -106,7 +173,7 @@ SourceRange ReturnStmt::getSourceRange() const {
 }
 
 bool Stmt::hasImplicitControlFlow() const {
-  switch (sClass) {
+  switch (StmtBits.sClass) {
     default:
       return false;
 
@@ -416,7 +483,7 @@ ObjCAtTryStmt *ObjCAtTryStmt::Create(ASTContext &Context,
                                      Stmt *atFinallyStmt) {
   unsigned Size = sizeof(ObjCAtTryStmt) + 
     (1 + NumCatchStmts + (atFinallyStmt != 0)) * sizeof(Stmt *);
-  void *Mem = Context.Allocate(Size, llvm::alignof<ObjCAtTryStmt>());
+  void *Mem = Context.Allocate(Size, llvm::alignOf<ObjCAtTryStmt>());
   return new (Mem) ObjCAtTryStmt(atTryLoc, atTryStmt, CatchStmts, NumCatchStmts,
                                  atFinallyStmt);
 }
@@ -426,7 +493,7 @@ ObjCAtTryStmt *ObjCAtTryStmt::CreateEmpty(ASTContext &Context,
                                                  bool HasFinally) {
   unsigned Size = sizeof(ObjCAtTryStmt) + 
     (1 + NumCatchStmts + HasFinally) * sizeof(Stmt *);
-  void *Mem = Context.Allocate(Size, llvm::alignof<ObjCAtTryStmt>());
+  void *Mem = Context.Allocate(Size, llvm::alignOf<ObjCAtTryStmt>());
   return new (Mem) ObjCAtTryStmt(EmptyShell(), NumCatchStmts, HasFinally);  
 }
 
@@ -448,7 +515,7 @@ CXXTryStmt *CXXTryStmt::Create(ASTContext &C, SourceLocation tryLoc,
   std::size_t Size = sizeof(CXXTryStmt);
   Size += ((numHandlers + 1) * sizeof(Stmt));
 
-  void *Mem = C.Allocate(Size, llvm::alignof<CXXTryStmt>());
+  void *Mem = C.Allocate(Size, llvm::alignOf<CXXTryStmt>());
   return new (Mem) CXXTryStmt(tryLoc, tryBlock, handlers, numHandlers);
 }
 
@@ -457,7 +524,7 @@ CXXTryStmt *CXXTryStmt::Create(ASTContext &C, EmptyShell Empty,
   std::size_t Size = sizeof(CXXTryStmt);
   Size += ((numHandlers + 1) * sizeof(Stmt));
 
-  void *Mem = C.Allocate(Size, llvm::alignof<CXXTryStmt>());
+  void *Mem = C.Allocate(Size, llvm::alignOf<CXXTryStmt>());
   return new (Mem) CXXTryStmt(Empty, numHandlers);
 }
 
@@ -530,7 +597,7 @@ void ForStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
 }
 
 SwitchStmt::SwitchStmt(ASTContext &C, VarDecl *Var, Expr *cond) 
-  : Stmt(SwitchStmtClass), FirstCase(0) 
+  : Stmt(SwitchStmtClass), FirstCase(0), AllEnumCasesCovered(0) 
 {
   setConditionVariable(C, Var);
   SubExprs[COND] = reinterpret_cast<Stmt*>(cond);
@@ -556,6 +623,11 @@ void SwitchStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                    V->getSourceRange().getEnd());
 }
 
+Stmt *SwitchCase::getSubStmt() {
+  if (isa<CaseStmt>(this)) return cast<CaseStmt>(this)->getSubStmt();
+  return cast<DefaultStmt>(this)->getSubStmt();
+}
+
 WhileStmt::WhileStmt(ASTContext &C, VarDecl *Var, Expr *cond, Stmt *body, 
                      SourceLocation WL)
 : Stmt(WhileStmtClass)
@@ -585,101 +657,13 @@ void WhileStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                    V->getSourceRange().getEnd());
 }
 
-//===----------------------------------------------------------------------===//
-//  Child Iterators for iterating over subexpressions/substatements
-//===----------------------------------------------------------------------===//
-
-// DeclStmt
-Stmt::child_iterator DeclStmt::child_begin() {
-  return StmtIterator(DG.begin(), DG.end());
-}
-
-Stmt::child_iterator DeclStmt::child_end() {
-  return StmtIterator(DG.end(), DG.end());
-}
-
-// NullStmt
-Stmt::child_iterator NullStmt::child_begin() { return child_iterator(); }
-Stmt::child_iterator NullStmt::child_end() { return child_iterator(); }
-
-// CompoundStmt
-Stmt::child_iterator CompoundStmt::child_begin() { return &Body[0]; }
-Stmt::child_iterator CompoundStmt::child_end() { return &Body[0]+NumStmts; }
-
-// CaseStmt
-Stmt::child_iterator CaseStmt::child_begin() { return &SubExprs[0]; }
-Stmt::child_iterator CaseStmt::child_end() { return &SubExprs[END_EXPR]; }
-
-// DefaultStmt
-Stmt::child_iterator DefaultStmt::child_begin() { return &SubStmt; }
-Stmt::child_iterator DefaultStmt::child_end() { return &SubStmt+1; }
-
-// LabelStmt
-Stmt::child_iterator LabelStmt::child_begin() { return &SubStmt; }
-Stmt::child_iterator LabelStmt::child_end() { return &SubStmt+1; }
-
-// IfStmt
-Stmt::child_iterator IfStmt::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator IfStmt::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// SwitchStmt
-Stmt::child_iterator SwitchStmt::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator SwitchStmt::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// WhileStmt
-Stmt::child_iterator WhileStmt::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator WhileStmt::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// DoStmt
-Stmt::child_iterator DoStmt::child_begin() { return &SubExprs[0]; }
-Stmt::child_iterator DoStmt::child_end() { return &SubExprs[0]+END_EXPR; }
-
-// ForStmt
-Stmt::child_iterator ForStmt::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator ForStmt::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// ObjCForCollectionStmt
-Stmt::child_iterator ObjCForCollectionStmt::child_begin() {
-  return &SubExprs[0];
-}
-Stmt::child_iterator ObjCForCollectionStmt::child_end() {
-  return &SubExprs[0]+END_EXPR;
-}
-
-// GotoStmt
-Stmt::child_iterator GotoStmt::child_begin() { return child_iterator(); }
-Stmt::child_iterator GotoStmt::child_end() { return child_iterator(); }
-
 // IndirectGotoStmt
-Expr* IndirectGotoStmt::getTarget() { return cast<Expr>(Target); }
-const Expr* IndirectGotoStmt::getTarget() const { return cast<Expr>(Target); }
-
-Stmt::child_iterator IndirectGotoStmt::child_begin() { return &Target; }
-Stmt::child_iterator IndirectGotoStmt::child_end() { return &Target+1; }
-
-// ContinueStmt
-Stmt::child_iterator ContinueStmt::child_begin() { return child_iterator(); }
-Stmt::child_iterator ContinueStmt::child_end() { return child_iterator(); }
-
-// BreakStmt
-Stmt::child_iterator BreakStmt::child_begin() { return child_iterator(); }
-Stmt::child_iterator BreakStmt::child_end() { return child_iterator(); }
+LabelDecl *IndirectGotoStmt::getConstantTarget() {
+  if (AddrLabelExpr *E =
+        dyn_cast<AddrLabelExpr>(getTarget()->IgnoreParenImpCasts()))
+    return E->getLabel();
+  return 0;
+}
 
 // ReturnStmt
 const Expr* ReturnStmt::getRetValue() const {
@@ -688,69 +672,3 @@ const Expr* ReturnStmt::getRetValue() const {
 Expr* ReturnStmt::getRetValue() {
   return cast_or_null<Expr>(RetExpr);
 }
-
-Stmt::child_iterator ReturnStmt::child_begin() {
-  return &RetExpr;
-}
-Stmt::child_iterator ReturnStmt::child_end() {
-  return RetExpr ? &RetExpr+1 : &RetExpr;
-}
-
-// AsmStmt
-Stmt::child_iterator AsmStmt::child_begin() {
-  return NumOutputs + NumInputs == 0 ? 0 : &Exprs[0];
-}
-Stmt::child_iterator AsmStmt::child_end() {
-  return NumOutputs + NumInputs == 0 ? 0 : &Exprs[0] + NumOutputs + NumInputs;
-}
-
-// ObjCAtCatchStmt
-Stmt::child_iterator ObjCAtCatchStmt::child_begin() { return &Body; }
-Stmt::child_iterator ObjCAtCatchStmt::child_end() { return &Body + 1; }
-
-// ObjCAtFinallyStmt
-Stmt::child_iterator ObjCAtFinallyStmt::child_begin() { return &AtFinallyStmt; }
-Stmt::child_iterator ObjCAtFinallyStmt::child_end() { return &AtFinallyStmt+1; }
-
-// ObjCAtTryStmt
-Stmt::child_iterator ObjCAtTryStmt::child_begin() { return getStmts(); }
-
-Stmt::child_iterator ObjCAtTryStmt::child_end() {
-  return getStmts() + 1 + NumCatchStmts + HasFinally;
-}
-
-// ObjCAtThrowStmt
-Stmt::child_iterator ObjCAtThrowStmt::child_begin() {
-  return &Throw;
-}
-
-Stmt::child_iterator ObjCAtThrowStmt::child_end() {
-  return &Throw+1;
-}
-
-// ObjCAtSynchronizedStmt
-Stmt::child_iterator ObjCAtSynchronizedStmt::child_begin() {
-  return &SubStmts[0];
-}
-
-Stmt::child_iterator ObjCAtSynchronizedStmt::child_end() {
-  return &SubStmts[0]+END_EXPR;
-}
-
-// CXXCatchStmt
-Stmt::child_iterator CXXCatchStmt::child_begin() {
-  return &HandlerBlock;
-}
-
-Stmt::child_iterator CXXCatchStmt::child_end() {
-  return &HandlerBlock + 1;
-}
-
-// CXXTryStmt
-Stmt::child_iterator CXXTryStmt::child_begin() {
-  return reinterpret_cast<Stmt **>(this + 1);
-}
-
-Stmt::child_iterator CXXTryStmt::child_end() {
-  return reinterpret_cast<Stmt **>(this + 1) + NumHandlers + 1;
-}
diff --git a/lib/AST/StmtDumper.cpp b/lib/AST/StmtDumper.cpp
index 5c236a45a690..846bd4ce1441 100644
--- a/lib/AST/StmtDumper.cpp
+++ b/lib/AST/StmtDumper.cpp
@@ -59,19 +59,12 @@ namespace  {
           Visit(S);
 
           // Print out children.
-          Stmt::child_iterator CI = S->child_begin(), CE = S->child_end();
-          if (CI != CE) {
-            while (CI != CE) {
+          Stmt::child_range CI = S->children();
+          if (CI) {
+            while (CI) {
               OS << '\n';
               DumpSubTree(*CI++);
             }
-            if (const ConditionalOperator *CO = 
-                  dyn_cast<ConditionalOperator>(S)) {
-              if (CO->getSAVE()) {
-                OS << '\n';
-                DumpSubTree(CO->getSAVE());
-              }
-            }
           }
         }
         OS << ')';
@@ -90,25 +83,44 @@ namespace  {
     }
 
     void DumpType(QualType T) {
-      OS << "'" << T.getAsString() << "'";
+      SplitQualType T_split = T.split();
+      OS << "'" << QualType::getAsString(T_split) << "'";
 
       if (!T.isNull()) {
         // If the type is sugared, also dump a (shallow) desugared type.
-        QualType Simplified = T.getDesugaredType();
-        if (Simplified != T)
-          OS << ":'" << Simplified.getAsString() << "'";
+        SplitQualType D_split = T.getSplitDesugaredType();
+        if (T_split != D_split)
+          OS << ":'" << QualType::getAsString(D_split) << "'";
       }
     }
+    void DumpDeclRef(Decl *node);
     void DumpStmt(const Stmt *Node) {
       Indent();
       OS << "(" << Node->getStmtClassName()
          << " " << (void*)Node;
       DumpSourceRange(Node);
     }
+    void DumpValueKind(ExprValueKind K) {
+      switch (K) {
+      case VK_RValue: break;
+      case VK_LValue: OS << " lvalue"; break;
+      case VK_XValue: OS << " xvalue"; break;
+      }
+    }
+    void DumpObjectKind(ExprObjectKind K) {
+      switch (K) {
+      case OK_Ordinary: break;
+      case OK_BitField: OS << " bitfield"; break;
+      case OK_ObjCProperty: OS << " objcproperty"; break;
+      case OK_VectorComponent: OS << " vectorcomponent"; break;
+      }
+    }
     void DumpExpr(const Expr *Node) {
       DumpStmt(Node);
       OS << ' ';
       DumpType(Node->getType());
+      DumpValueKind(Node->getValueKind());
+      DumpObjectKind(Node->getObjectKind());
     }
     void DumpSourceRange(const Stmt *Node);
     void DumpLocation(SourceLocation Loc);
@@ -122,7 +134,6 @@ namespace  {
     // Exprs
     void VisitExpr(Expr *Node);
     void VisitCastExpr(CastExpr *Node);
-    void VisitImplicitCastExpr(ImplicitCastExpr *Node);
     void VisitDeclRefExpr(DeclRefExpr *Node);
     void VisitPredefinedExpr(PredefinedExpr *Node);
     void VisitCharacterLiteral(CharacterLiteral *Node);
@@ -136,7 +147,7 @@ namespace  {
     void VisitBinaryOperator(BinaryOperator *Node);
     void VisitCompoundAssignOperator(CompoundAssignOperator *Node);
     void VisitAddrLabelExpr(AddrLabelExpr *Node);
-    void VisitTypesCompatibleExpr(TypesCompatibleExpr *Node);
+    void VisitBlockExpr(BlockExpr *Node);
 
     // C++
     void VisitCXXNamedCastExpr(CXXNamedCastExpr *Node);
@@ -145,7 +156,7 @@ namespace  {
     void VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *Node);
     void VisitCXXConstructExpr(CXXConstructExpr *Node);
     void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node);
-    void VisitCXXExprWithTemporaries(CXXExprWithTemporaries *Node);
+    void VisitExprWithCleanups(ExprWithCleanups *Node);
     void VisitUnresolvedLookupExpr(UnresolvedLookupExpr *Node);
     void DumpCXXTemporary(CXXTemporary *Temporary);
 
@@ -156,10 +167,7 @@ namespace  {
     void VisitObjCSelectorExpr(ObjCSelectorExpr *Node);
     void VisitObjCProtocolExpr(ObjCProtocolExpr *Node);
     void VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node);
-    void VisitObjCImplicitSetterGetterRefExpr(
-                                          ObjCImplicitSetterGetterRefExpr *Node);
     void VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node);
-    void VisitObjCSuperExpr(ObjCSuperExpr *Node);
   };
 }
 
@@ -170,15 +178,15 @@ namespace  {
 void StmtDumper::DumpLocation(SourceLocation Loc) {
   SourceLocation SpellingLoc = SM->getSpellingLoc(Loc);
 
-  if (SpellingLoc.isInvalid()) {
-    OS << "<invalid sloc>";
-    return;
-  }
-
   // The general format we print out is filename:line:col, but we drop pieces
   // that haven't changed since the last loc printed.
   PresumedLoc PLoc = SM->getPresumedLoc(SpellingLoc);
 
+  if (PLoc.isInvalid()) {
+    OS << "<invalid sloc>";
+    return;
+  }
+
   if (strcmp(PLoc.getFilename(), LastLocFilename) != 0) {
     OS << PLoc.getFilename() << ':' << PLoc.getLine()
        << ':' << PLoc.getColumn();
@@ -274,6 +282,8 @@ void StmtDumper::DumpDeclarator(Decl *D) {
     UD->getTargetNestedNameDecl()->print(OS,
         PrintingPolicy(UD->getASTContext().getLangOptions()));
     OS << ";\"";
+  } else if (LabelDecl *LD = dyn_cast<LabelDecl>(D)) {
+    OS << "label " << LD->getNameAsString();
   } else {
     assert(0 && "Unexpected decl");
   }
@@ -345,39 +355,25 @@ void StmtDumper::VisitCastExpr(CastExpr *Node) {
   OS << ">";
 }
 
-void StmtDumper::VisitImplicitCastExpr(ImplicitCastExpr *Node) {
-  VisitCastExpr(Node);
-  switch (Node->getValueKind()) {
-  case VK_LValue:
-    OS << " lvalue";
-    break;
-  case VK_XValue:
-    OS << " xvalue";
-    break;
-  case VK_RValue:
-    break;
-  }
-}
-
 void StmtDumper::VisitDeclRefExpr(DeclRefExpr *Node) {
   DumpExpr(Node);
 
   OS << " ";
-  switch (Node->getDecl()->getKind()) {
-  default: OS << "Decl"; break;
-  case Decl::Function: OS << "FunctionDecl"; break;
-  case Decl::Var: OS << "Var"; break;
-  case Decl::ParmVar: OS << "ParmVar"; break;
-  case Decl::EnumConstant: OS << "EnumConstant"; break;
-  case Decl::Typedef: OS << "Typedef"; break;
-  case Decl::Record: OS << "Record"; break;
-  case Decl::Enum: OS << "Enum"; break;
-  case Decl::CXXRecord: OS << "CXXRecord"; break;
-  case Decl::ObjCInterface: OS << "ObjCInterface"; break;
-  case Decl::ObjCClass: OS << "ObjCClass"; break;
+  DumpDeclRef(Node->getDecl());
+}
+
+void StmtDumper::DumpDeclRef(Decl *d) {
+  OS << d->getDeclKindName() << ' ' << (void*) d;
+
+  if (NamedDecl *nd = dyn_cast<NamedDecl>(d)) {
+    OS << " '";
+    nd->getDeclName().printName(OS);
+    OS << "'";
   }
 
-  OS << "='" << Node->getDecl() << "' " << (void*)Node->getDecl();
+  if (ValueDecl *vd = dyn_cast<ValueDecl>(d)) {
+    OS << ' '; DumpType(vd->getType());
+  }
 }
 
 void StmtDumper::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *Node) {
@@ -475,6 +471,30 @@ void StmtDumper::VisitCompoundAssignOperator(CompoundAssignOperator *Node) {
   DumpType(Node->getComputationResultType());
 }
 
+void StmtDumper::VisitBlockExpr(BlockExpr *Node) {
+  DumpExpr(Node);
+
+  IndentLevel++;
+  BlockDecl *block = Node->getBlockDecl();
+  if (block->capturesCXXThis()) {
+    OS << '\n'; Indent(); OS << "(capture this)";
+  }
+  for (BlockDecl::capture_iterator
+         i = block->capture_begin(), e = block->capture_end(); i != e; ++i) {
+    OS << '\n';
+    Indent();
+    OS << "(capture ";
+    if (i->isByRef()) OS << "byref ";
+    if (i->isNested()) OS << "nested ";
+    DumpDeclRef(i->getVariable());
+    if (i->hasCopyExpr()) DumpSubTree(i->getCopyExpr());
+    OS << ")";
+  }
+  IndentLevel--;
+
+  DumpSubTree(block->getBody());
+}
+
 // GNU extensions.
 
 void StmtDumper::VisitAddrLabelExpr(AddrLabelExpr *Node) {
@@ -483,14 +503,6 @@ void StmtDumper::VisitAddrLabelExpr(AddrLabelExpr *Node) {
      << " " << (void*)Node->getLabel();
 }
 
-void StmtDumper::VisitTypesCompatibleExpr(TypesCompatibleExpr *Node) {
-  DumpExpr(Node);
-  OS << " ";
-  DumpType(Node->getArgType1());
-  OS << " ";
-  DumpType(Node->getArgType2());
-}
-
 //===----------------------------------------------------------------------===//
 // C++ Expressions
 //===----------------------------------------------------------------------===//
@@ -535,7 +547,7 @@ void StmtDumper::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node) {
   DumpCXXTemporary(Node->getTemporary());
 }
 
-void StmtDumper::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *Node) {
+void StmtDumper::VisitExprWithCleanups(ExprWithCleanups *Node) {
   DumpExpr(Node);
   ++IndentLevel;
   for (unsigned i = 0, e = Node->getNumTemporaries(); i != e; ++i) {
@@ -606,29 +618,25 @@ void StmtDumper::VisitObjCProtocolExpr(ObjCProtocolExpr *Node) {
 
 void StmtDumper::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
   DumpExpr(Node);
+  if (Node->isImplicitProperty()) {
+    OS << " Kind=MethodRef Getter=\"";
+    if (Node->getImplicitPropertyGetter())
+      OS << Node->getImplicitPropertyGetter()->getSelector().getAsString();
+    else
+      OS << "(null)";
 
-  OS << " Kind=PropertyRef Property=\"" << Node->getProperty() << '"';
-}
-
-void StmtDumper::VisitObjCImplicitSetterGetterRefExpr(
-                                        ObjCImplicitSetterGetterRefExpr *Node) {
-  DumpExpr(Node);
-
-  ObjCMethodDecl *Getter = Node->getGetterMethod();
-  ObjCMethodDecl *Setter = Node->getSetterMethod();
-  OS << " Kind=MethodRef Getter=\""
-     << Getter->getSelector().getAsString()
-     << "\" Setter=\"";
-  if (Setter)
-    OS << Setter->getSelector().getAsString();
-  else
-    OS << "(null)";
-  OS << "\"";
-}
+    OS << "\" Setter=\"";
+    if (ObjCMethodDecl *Setter = Node->getImplicitPropertySetter())
+      OS << Setter->getSelector().getAsString();
+    else
+      OS << "(null)";
+    OS << "\"";
+  } else {
+    OS << " Kind=PropertyRef Property=\"" << Node->getExplicitProperty() << '"';
+  }
 
-void StmtDumper::VisitObjCSuperExpr(ObjCSuperExpr *Node) {
-  DumpExpr(Node);
-  OS << " super";
+  if (Node->isSuperReceiver())
+    OS << " super";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/StmtIterator.cpp b/lib/AST/StmtIterator.cpp
index 7fc7c96750de..9a7265a043f2 100644
--- a/lib/AST/StmtIterator.cpp
+++ b/lib/AST/StmtIterator.cpp
@@ -18,9 +18,9 @@ using namespace clang;
 
 // FIXME: Add support for dependent-sized array types in C++?
 // Does it even make sense to build a CFG for an uninstantiated template?
-static inline VariableArrayType* FindVA(Type* t) {
-  while (ArrayType* vt = dyn_cast<ArrayType>(t)) {
-    if (VariableArrayType* vat = dyn_cast<VariableArrayType>(vt))
+static inline const VariableArrayType *FindVA(const Type* t) {
+  while (const ArrayType *vt = dyn_cast<ArrayType>(t)) {
+    if (const VariableArrayType *vat = dyn_cast<VariableArrayType>(vt))
       if (vat->getSizeExpr())
         return vat;
 
@@ -33,7 +33,7 @@ static inline VariableArrayType* FindVA(Type* t) {
 void StmtIteratorBase::NextVA() {
   assert (getVAPtr());
 
-  VariableArrayType* p = getVAPtr();
+  const VariableArrayType *p = getVAPtr();
   p = FindVA(p->getElementType().getTypePtr());
   setVAPtr(p);
 
@@ -90,7 +90,7 @@ void StmtIteratorBase::NextDecl(bool ImmediateAdvance) {
 bool StmtIteratorBase::HandleDecl(Decl* D) {
 
   if (VarDecl* VD = dyn_cast<VarDecl>(D)) {
-    if (VariableArrayType* VAPtr = FindVA(VD->getType().getTypePtr())) {
+    if (const VariableArrayType* VAPtr = FindVA(VD->getType().getTypePtr())) {
       setVAPtr(VAPtr);
       return true;
     }
@@ -99,7 +99,7 @@ bool StmtIteratorBase::HandleDecl(Decl* D) {
       return true;
   }
   else if (TypedefDecl* TD = dyn_cast<TypedefDecl>(D)) {
-    if (VariableArrayType* VAPtr =
+    if (const VariableArrayType* VAPtr =
         FindVA(TD->getUnderlyingType().getTypePtr())) {
       setVAPtr(VAPtr);
       return true;
@@ -124,16 +124,16 @@ StmtIteratorBase::StmtIteratorBase(Decl** dgi, Decl** dge)
   NextDecl(false);
 }
 
-StmtIteratorBase::StmtIteratorBase(VariableArrayType* t)
+StmtIteratorBase::StmtIteratorBase(const VariableArrayType* t)
   : stmt(0), decl(0), RawVAPtr(SizeOfTypeVAMode) {
   RawVAPtr |= reinterpret_cast<uintptr_t>(t);
 }
 
 Stmt*& StmtIteratorBase::GetDeclExpr() const {
 
-  if (VariableArrayType* VAPtr = getVAPtr()) {
+  if (const VariableArrayType* VAPtr = getVAPtr()) {
     assert (VAPtr->SizeExpr);
-    return VAPtr->SizeExpr;
+    return const_cast<Stmt*&>(VAPtr->SizeExpr);
   }
 
   assert (inDecl() || inDeclGroup());
diff --git a/lib/AST/StmtPrinter.cpp b/lib/AST/StmtPrinter.cpp
index 5236a6672622..1cdd22088141 100644
--- a/lib/AST/StmtPrinter.cpp
+++ b/lib/AST/StmtPrinter.cpp
@@ -15,9 +15,11 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "llvm/Support/Format.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -63,6 +65,7 @@ namespace  {
     void PrintRawDeclStmt(DeclStmt *S);
     void PrintRawIfStmt(IfStmt *If);
     void PrintRawCXXCatchStmt(CXXCatchStmt *Catch);
+    void PrintCallArgs(CallExpr *E);
 
     void PrintExpr(Expr *E) {
       if (E)
@@ -83,10 +86,10 @@ namespace  {
       else StmtVisitor<StmtPrinter>::Visit(S);
     }
     
-    void VisitStmt(Stmt *Node) ATTRIBUTE_UNUSED {
+    void VisitStmt(Stmt *Node) LLVM_ATTRIBUTE_UNUSED {
       Indent() << "<<unknown stmt type>>\n";
     }
-    void VisitExpr(Expr *Node) ATTRIBUTE_UNUSED {
+    void VisitExpr(Expr *Node) LLVM_ATTRIBUTE_UNUSED {
       OS << "<<unknown expr type>>";
     }
     void VisitCXXNamedCastExpr(CXXNamedCastExpr *Node);
@@ -217,10 +220,6 @@ void StmtPrinter::VisitSwitchStmt(SwitchStmt *Node) {
   }
 }
 
-void StmtPrinter::VisitSwitchCase(SwitchCase*) {
-  assert(0 && "SwitchCase is an abstract class");
-}
-
 void StmtPrinter::VisitWhileStmt(WhileStmt *Node) {
   Indent() << "while (";
   PrintExpr(Node->getCond());
@@ -479,7 +478,8 @@ void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
 
 void StmtPrinter::VisitDependentScopeDeclRefExpr(
                                            DependentScopeDeclRefExpr *Node) {
-  Node->getQualifier()->print(OS, Policy);
+  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
+    Qualifier->print(OS, Policy);
   OS << Node->getNameInfo();
   if (Node->hasExplicitTemplateArgs())
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
@@ -508,22 +508,17 @@ void StmtPrinter::VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node) {
 }
 
 void StmtPrinter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
-  if (Node->getBase()) {
+  if (Node->isSuperReceiver())
+    OS << "super.";
+  else if (Node->getBase()) {
     PrintExpr(Node->getBase());
     OS << ".";
   }
-  OS << Node->getProperty()->getName();
-}
-
-void StmtPrinter::VisitObjCImplicitSetterGetterRefExpr(
-                                        ObjCImplicitSetterGetterRefExpr *Node) {
-  if (Node->getBase()) {
-    PrintExpr(Node->getBase());
-    OS << ".";
-  }
-  if (Node->getGetterMethod())
-    OS << Node->getGetterMethod();
 
+  if (Node->isImplicitProperty())
+    OS << Node->getImplicitPropertyGetter()->getSelector().getAsString();
+  else
+    OS << Node->getExplicitProperty()->getName();
 }
 
 void StmtPrinter::VisitPredefinedExpr(PredefinedExpr *Node) {
@@ -727,9 +722,7 @@ void StmtPrinter::VisitArraySubscriptExpr(ArraySubscriptExpr *Node) {
   OS << "]";
 }
 
-void StmtPrinter::VisitCallExpr(CallExpr *Call) {
-  PrintExpr(Call->getCallee());
-  OS << "(";
+void StmtPrinter::PrintCallArgs(CallExpr *Call) {
   for (unsigned i = 0, e = Call->getNumArgs(); i != e; ++i) {
     if (isa<CXXDefaultArgExpr>(Call->getArg(i))) {
       // Don't print any defaulted arguments
@@ -739,6 +732,12 @@ void StmtPrinter::VisitCallExpr(CallExpr *Call) {
     if (i) OS << ", ";
     PrintExpr(Call->getArg(i));
   }
+}
+
+void StmtPrinter::VisitCallExpr(CallExpr *Call) {
+  PrintExpr(Call->getCallee());
+  OS << "(";
+  PrintCallArgs(Call);
   OS << ")";
 }
 void StmtPrinter::VisitMemberExpr(MemberExpr *Node) {
@@ -793,21 +792,20 @@ void StmtPrinter::VisitCompoundAssignOperator(CompoundAssignOperator *Node) {
 }
 void StmtPrinter::VisitConditionalOperator(ConditionalOperator *Node) {
   PrintExpr(Node->getCond());
-
-  if (Node->getLHS()) {
-    OS << " ? ";
-    PrintExpr(Node->getLHS());
-    OS << " : ";
-  }
-  else { // Handle GCC extension where LHS can be NULL.
-    OS << " ?: ";
-  }
-
+  OS << " ? ";
+  PrintExpr(Node->getLHS());
+  OS << " : ";
   PrintExpr(Node->getRHS());
 }
 
 // GNU extensions.
 
+void
+StmtPrinter::VisitBinaryConditionalOperator(BinaryConditionalOperator *Node) {
+  PrintExpr(Node->getCommon());
+  OS << " ?: ";
+  PrintExpr(Node->getFalseExpr());
+}
 void StmtPrinter::VisitAddrLabelExpr(AddrLabelExpr *Node) {
   OS << "&&" << Node->getLabel()->getName();
 }
@@ -818,12 +816,6 @@ void StmtPrinter::VisitStmtExpr(StmtExpr *E) {
   OS << ")";
 }
 
-void StmtPrinter::VisitTypesCompatibleExpr(TypesCompatibleExpr *Node) {
-  OS << "__builtin_types_compatible_p(";
-  OS << Node->getArgType1().getAsString(Policy) << ",";
-  OS << Node->getArgType2().getAsString(Policy) << ")";
-}
-
 void StmtPrinter::VisitChooseExpr(ChooseExpr *Node) {
   OS << "__builtin_choose_expr(";
   PrintExpr(Node->getCond());
@@ -968,6 +960,15 @@ void StmtPrinter::VisitCXXMemberCallExpr(CXXMemberCallExpr *Node) {
   VisitCallExpr(cast<CallExpr>(Node));
 }
 
+void StmtPrinter::VisitCUDAKernelCallExpr(CUDAKernelCallExpr *Node) {
+  PrintExpr(Node->getCallee());
+  OS << "<<<";
+  PrintCallArgs(Node->getConfig());
+  OS << ">>>(";
+  PrintCallArgs(Node);
+  OS << ")";
+}
+
 void StmtPrinter::VisitCXXNamedCastExpr(CXXNamedCastExpr *Node) {
   OS << Node->getCastName() << '<';
   OS << Node->getTypeAsWritten().getAsString(Policy) << ">(";
@@ -1001,6 +1002,16 @@ void StmtPrinter::VisitCXXTypeidExpr(CXXTypeidExpr *Node) {
   OS << ")";
 }
 
+void StmtPrinter::VisitCXXUuidofExpr(CXXUuidofExpr *Node) {
+  OS << "__uuidof(";
+  if (Node->isTypeOperand()) {
+    OS << Node->getTypeOperand().getAsString(Policy);
+  } else {
+    PrintExpr(Node->getExprOperand());
+  }
+  OS << ")";
+}
+
 void StmtPrinter::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *Node) {
   OS << (Node->getValue() ? "true" : "false");
 }
@@ -1051,7 +1062,10 @@ void StmtPrinter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *Node) {
 }
 
 void StmtPrinter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *Node) {
-  OS << Node->getType().getAsString(Policy) << "()";
+  if (TypeSourceInfo *TSInfo = Node->getTypeSourceInfo())
+    OS << TSInfo->getType().getAsString(Policy) << "()";
+  else
+    OS << Node->getType().getAsString(Policy) << "()";
 }
 
 void StmtPrinter::VisitCXXNewExpr(CXXNewExpr *E) {
@@ -1123,17 +1137,18 @@ void StmtPrinter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
 }
 
 void StmtPrinter::VisitCXXConstructExpr(CXXConstructExpr *E) {
-  // FIXME. For now we just print a trivial constructor call expression,
-  // constructing its first argument object.
-  if (E->getNumArgs() == 1) {
-    CXXConstructorDecl *CD = E->getConstructor();
-    if (CD->isTrivial())
-      PrintExpr(E->getArg(0));
+  for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
+    if (isa<CXXDefaultArgExpr>(E->getArg(i))) {
+      // Don't print any defaulted arguments
+      break;
+    }
+
+    if (i) OS << ", ";
+    PrintExpr(E->getArg(i));
   }
-  // Nothing to print.
 }
 
-void StmtPrinter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
+void StmtPrinter::VisitExprWithCleanups(ExprWithCleanups *E) {
   // Just forward to the sub expression.
   PrintExpr(E->getSubExpr());
 }
@@ -1197,7 +1212,7 @@ void StmtPrinter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *Node) {
 
 static const char *getTypeTraitName(UnaryTypeTrait UTT) {
   switch (UTT) {
-  default: assert(false && "Unknown type trait");
+  default: llvm_unreachable("Unknown unary type trait");
   case UTT_HasNothrowAssign:      return "__has_nothrow_assign";
   case UTT_HasNothrowCopy:        return "__has_nothrow_copy";
   case UTT_HasNothrowConstructor: return "__has_nothrow_constructor";
@@ -1214,6 +1229,16 @@ static const char *getTypeTraitName(UnaryTypeTrait UTT) {
   case UTT_IsPolymorphic:         return "__is_polymorphic";
   case UTT_IsUnion:               return "__is_union";
   }
+  return "";
+}
+
+static const char *getTypeTraitName(BinaryTypeTrait BTT) {
+  switch (BTT) {
+  case BTT_IsBaseOf:         return "__is_base_of";
+  case BTT_TypeCompatible:   return "__builtin_types_compatible_p";
+  case BTT_IsConvertibleTo:  return "__is_convertible_to";
+  }
+  return "";
 }
 
 void StmtPrinter::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
@@ -1221,6 +1246,32 @@ void StmtPrinter::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
      << E->getQueriedType().getAsString(Policy) << ")";
 }
 
+void StmtPrinter::VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *E) {
+  OS << getTypeTraitName(E->getTrait()) << "("
+     << E->getLhsType().getAsString(Policy) << ","
+     << E->getRhsType().getAsString(Policy) << ")";
+}
+
+void StmtPrinter::VisitCXXNoexceptExpr(CXXNoexceptExpr *E) {
+  OS << "noexcept(";
+  PrintExpr(E->getOperand());
+  OS << ")";
+}
+
+void StmtPrinter::VisitPackExpansionExpr(PackExpansionExpr *E) {
+  PrintExpr(E->getPattern());
+  OS << "...";
+}
+
+void StmtPrinter::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
+  OS << "sizeof...(" << E->getPack()->getNameAsString() << ")";
+}
+
+void StmtPrinter::VisitSubstNonTypeTemplateParmPackExpr(
+                                       SubstNonTypeTemplateParmPackExpr *Node) {
+  OS << Node->getParameterPack()->getNameAsString();
+}
+
 // Obj-C
 
 void StmtPrinter::VisitObjCStringLiteral(ObjCStringLiteral *Node) {
@@ -1260,7 +1311,7 @@ void StmtPrinter::VisitObjCMessageExpr(ObjCMessageExpr *Mess) {
   OS << ' ';
   Selector selector = Mess->getSelector();
   if (selector.isUnarySelector()) {
-    OS << selector.getIdentifierInfoForSlot(0)->getName();
+    OS << selector.getNameForSlot(0);
   } else {
     for (unsigned i = 0, e = Mess->getNumArgs(); i != e; ++i) {
       if (i < selector.getNumArgs()) {
@@ -1278,9 +1329,6 @@ void StmtPrinter::VisitObjCMessageExpr(ObjCMessageExpr *Mess) {
   OS << "]";
 }
 
-void StmtPrinter::VisitObjCSuperExpr(ObjCSuperExpr *) {
-  OS << "super";
-}
 
 void StmtPrinter::VisitBlockExpr(BlockExpr *Node) {
   BlockDecl *BD = Node->getBlockDecl();
@@ -1313,6 +1361,9 @@ void StmtPrinter::VisitBlockExpr(BlockExpr *Node) {
 void StmtPrinter::VisitBlockDeclRefExpr(BlockDeclRefExpr *Node) {
   OS << Node->getDecl();
 }
+
+void StmtPrinter::VisitOpaqueValueExpr(OpaqueValueExpr *Node) {}
+
 //===----------------------------------------------------------------------===//
 // Stmt method implementations
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/StmtProfile.cpp b/lib/AST/StmtProfile.cpp
index 78a336d2bfbe..b54001167b42 100644
--- a/lib/AST/StmtProfile.cpp
+++ b/lib/AST/StmtProfile.cpp
@@ -25,11 +25,11 @@ using namespace clang;
 namespace {
   class StmtProfiler : public StmtVisitor<StmtProfiler> {
     llvm::FoldingSetNodeID &ID;
-    ASTContext &Context;
+    const ASTContext &Context;
     bool Canonical;
 
   public:
-    StmtProfiler(llvm::FoldingSetNodeID &ID, ASTContext &Context,
+    StmtProfiler(llvm::FoldingSetNodeID &ID, const ASTContext &Context,
                  bool Canonical)
       : ID(ID), Context(Context), Canonical(Canonical) { }
 
@@ -68,8 +68,7 @@ namespace {
 
 void StmtProfiler::VisitStmt(Stmt *S) {
   ID.AddInteger(S->getStmtClass());
-  for (Stmt::child_iterator C = S->child_begin(), CEnd = S->child_end();
-       C != CEnd; ++C)
+  for (Stmt::child_range C = S->children(); C; ++C)
     Visit(*C);
 }
 
@@ -102,7 +101,7 @@ void StmtProfiler::VisitDefaultStmt(DefaultStmt *S) {
 
 void StmtProfiler::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
-  VisitName(S->getID());
+  VisitDecl(S->getDecl());
 }
 
 void StmtProfiler::VisitIfStmt(IfStmt *S) {
@@ -130,7 +129,7 @@ void StmtProfiler::VisitForStmt(ForStmt *S) {
 
 void StmtProfiler::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
-  VisitName(S->getLabel()->getID());
+  VisitDecl(S->getLabel());
 }
 
 void StmtProfiler::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
@@ -350,19 +349,17 @@ void StmtProfiler::VisitConditionalOperator(ConditionalOperator *S) {
   VisitExpr(S);
 }
 
-void StmtProfiler::VisitAddrLabelExpr(AddrLabelExpr *S) {
+void StmtProfiler::VisitBinaryConditionalOperator(BinaryConditionalOperator *S){
   VisitExpr(S);
-  VisitName(S->getLabel()->getID());
 }
 
-void StmtProfiler::VisitStmtExpr(StmtExpr *S) {
+void StmtProfiler::VisitAddrLabelExpr(AddrLabelExpr *S) {
   VisitExpr(S);
+  VisitDecl(S->getLabel());
 }
 
-void StmtProfiler::VisitTypesCompatibleExpr(TypesCompatibleExpr *S) {
+void StmtProfiler::VisitStmtExpr(StmtExpr *S) {
   VisitExpr(S);
-  VisitType(S->getArgType1());
-  VisitType(S->getArgType2());
 }
 
 void StmtProfiler::VisitShuffleVectorExpr(ShuffleVectorExpr *S) {
@@ -431,8 +428,6 @@ void StmtProfiler::VisitBlockDeclRefExpr(BlockDeclRefExpr *S) {
   VisitDecl(S->getDecl());
   ID.AddBoolean(S->isByRef());
   ID.AddBoolean(S->isConstQualAdded());
-  if (S->getCopyConstructorExpr())
-    Visit(S->getCopyConstructorExpr());
 }
 
 static Stmt::StmtClass DecodeOperatorCall(CXXOperatorCallExpr *S,
@@ -652,6 +647,10 @@ void StmtProfiler::VisitCXXMemberCallExpr(CXXMemberCallExpr *S) {
   VisitCallExpr(S);
 }
 
+void StmtProfiler::VisitCUDAKernelCallExpr(CUDAKernelCallExpr *S) {
+  VisitCallExpr(S);
+}
+
 void StmtProfiler::VisitCXXNamedCastExpr(CXXNamedCastExpr *S) {
   VisitExplicitCastExpr(S);
 }
@@ -687,6 +686,12 @@ void StmtProfiler::VisitCXXTypeidExpr(CXXTypeidExpr *S) {
     VisitType(S->getTypeOperand());
 }
 
+void StmtProfiler::VisitCXXUuidofExpr(CXXUuidofExpr *S) {
+  VisitExpr(S);
+  if (S->isTypeOperand())
+    VisitType(S->getTypeOperand());
+}
+
 void StmtProfiler::VisitCXXThisExpr(CXXThisExpr *S) {
   VisitExpr(S);
 }
@@ -774,6 +779,13 @@ void StmtProfiler::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *S) {
   VisitType(S->getQueriedType());
 }
 
+void StmtProfiler::VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *S) {
+  VisitExpr(S);
+  ID.AddInteger(S->getTrait());
+  VisitType(S->getLhsType());
+  VisitType(S->getRhsType());
+}
+
 void
 StmtProfiler::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *S) {
   VisitExpr(S);
@@ -784,11 +796,8 @@ StmtProfiler::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *S) {
     VisitTemplateArguments(S->getTemplateArgs(), S->getNumTemplateArgs());
 }
 
-void StmtProfiler::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *S) {
+void StmtProfiler::VisitExprWithCleanups(ExprWithCleanups *S) {
   VisitExpr(S);
-  for (unsigned I = 0, N = S->getNumTemporaries(); I != N; ++I)
-    VisitDecl(
-      const_cast<CXXDestructorDecl *>(S->getTemporary(I)->getDestructor()));
 }
 
 void
@@ -824,6 +833,30 @@ void StmtProfiler::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *S) {
     VisitTemplateArguments(S->getTemplateArgs(), S->getNumTemplateArgs());
 }
 
+void StmtProfiler::VisitCXXNoexceptExpr(CXXNoexceptExpr *S) {
+  VisitExpr(S);
+}
+
+void StmtProfiler::VisitPackExpansionExpr(PackExpansionExpr *S) {
+  VisitExpr(S);
+}
+
+void StmtProfiler::VisitSizeOfPackExpr(SizeOfPackExpr *S) {
+  VisitExpr(S);
+  VisitDecl(S->getPack());
+}
+
+void StmtProfiler::VisitSubstNonTypeTemplateParmPackExpr(
+                                         SubstNonTypeTemplateParmPackExpr *S) {
+  VisitExpr(S);
+  VisitDecl(S->getParameterPack());
+  VisitTemplateArgument(S->getArgumentPack());
+}
+
+void StmtProfiler::VisitOpaqueValueExpr(OpaqueValueExpr *E) {
+  VisitExpr(E);  
+}
+
 void StmtProfiler::VisitObjCStringLiteral(ObjCStringLiteral *S) {
   VisitExpr(S);
 }
@@ -852,15 +885,16 @@ void StmtProfiler::VisitObjCIvarRefExpr(ObjCIvarRefExpr *S) {
 
 void StmtProfiler::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *S) {
   VisitExpr(S);
-  VisitDecl(S->getProperty());
-}
-
-void StmtProfiler::VisitObjCImplicitSetterGetterRefExpr(
-                                  ObjCImplicitSetterGetterRefExpr *S) {
-  VisitExpr(S);
-  VisitDecl(S->getGetterMethod());
-  VisitDecl(S->getSetterMethod());
-  VisitDecl(S->getInterfaceDecl());
+  if (S->isImplicitProperty()) {
+    VisitDecl(S->getImplicitPropertyGetter());
+    VisitDecl(S->getImplicitPropertySetter());
+  } else {
+    VisitDecl(S->getExplicitProperty());
+  }
+  if (S->isSuperReceiver()) {
+    ID.AddBoolean(S->isSuperReceiver());
+    VisitType(S->getSuperReceiverType());
+  }
 }
 
 void StmtProfiler::VisitObjCMessageExpr(ObjCMessageExpr *S) {
@@ -869,10 +903,6 @@ void StmtProfiler::VisitObjCMessageExpr(ObjCMessageExpr *S) {
   VisitDecl(S->getMethodDecl());
 }
 
-void StmtProfiler::VisitObjCSuperExpr(ObjCSuperExpr *S) {
-  VisitExpr(S);
-}
-
 void StmtProfiler::VisitObjCIsaExpr(ObjCIsaExpr *S) {
   VisitExpr(S);
   ID.AddBoolean(S->isArrow());
@@ -885,6 +915,7 @@ void StmtProfiler::VisitDecl(Decl *D) {
     if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(D)) {
       ID.AddInteger(NTTP->getDepth());
       ID.AddInteger(NTTP->getIndex());
+      ID.AddBoolean(NTTP->isParameterPack());
       VisitType(NTTP->getType());
       return;
     }
@@ -902,6 +933,7 @@ void StmtProfiler::VisitDecl(Decl *D) {
     if (TemplateTemplateParmDecl *TTP = dyn_cast<TemplateTemplateParmDecl>(D)) {
       ID.AddInteger(TTP->getDepth());
       ID.AddInteger(TTP->getIndex());
+      ID.AddBoolean(TTP->isParameterPack());
       return;
     }
   }
@@ -952,7 +984,8 @@ void StmtProfiler::VisitTemplateArgument(const TemplateArgument &Arg) {
     break;
 
   case TemplateArgument::Template:
-    VisitTemplateName(Arg.getAsTemplate());
+  case TemplateArgument::TemplateExpansion:
+    VisitTemplateName(Arg.getAsTemplateOrTemplatePattern());
     break;
       
   case TemplateArgument::Declaration:
@@ -976,7 +1009,7 @@ void StmtProfiler::VisitTemplateArgument(const TemplateArgument &Arg) {
   }
 }
 
-void Stmt::Profile(llvm::FoldingSetNodeID &ID, ASTContext &Context,
+void Stmt::Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Context,
                    bool Canonical) {
   StmtProfiler Profiler(ID, Context, Canonical);
   Profiler.Visit(this);
diff --git a/lib/AST/TemplateBase.cpp b/lib/AST/TemplateBase.cpp
index a3bf1459c283..5ab5f4644c88 100644
--- a/lib/AST/TemplateBase.cpp
+++ b/lib/AST/TemplateBase.cpp
@@ -12,41 +12,172 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/FoldingSet.h"
 #include "clang/AST/TemplateBase.h"
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/FoldingSet.h"
+#include <algorithm>
+#include <cctype>
+#include <iomanip>
+#include <sstream>
 
 using namespace clang;
 
+/// \brief Print a template integral argument value.
+///
+/// \param TemplArg the TemplateArgument instance to print.
+///
+/// \param Out the raw_ostream instance to use for printing.
+static void printIntegral(const TemplateArgument &TemplArg,
+                          llvm::raw_ostream &Out) {
+  const ::clang::Type *T = TemplArg.getIntegralType().getTypePtr();
+  const llvm::APSInt *Val = TemplArg.getAsIntegral();
+
+  if (T->isBooleanType()) {
+    Out << (Val->getBoolValue() ? "true" : "false");
+  } else if (T->isCharType()) {
+    char Ch = Val->getSExtValue();
+    if (std::isprint(Ch)) {
+      Out << "'";
+      if (Ch == '\'' || Ch == '\\')
+        Out << '\\';
+      Out << Ch << "'";
+    } else {
+      std::ostringstream Str;
+      Str << std::setw(2) << std::setfill('0') << std::hex << (int)Ch;
+      Out << "'\\x" << Str.str() << "'";
+    }
+  } else {
+    Out << Val->toString(10);
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // TemplateArgument Implementation
 //===----------------------------------------------------------------------===//
 
-/// \brief Construct a template argument pack.
-void TemplateArgument::setArgumentPack(TemplateArgument *args, unsigned NumArgs,
-                                       bool CopyArgs) {
-  assert(isNull() && "Must call setArgumentPack on a null argument");
+TemplateArgument TemplateArgument::CreatePackCopy(ASTContext &Context,
+                                                  const TemplateArgument *Args,
+                                                  unsigned NumArgs) {
+  if (NumArgs == 0)
+    return TemplateArgument(0, 0);
+  
+  TemplateArgument *Storage = new (Context) TemplateArgument [NumArgs];
+  std::copy(Args, Args + NumArgs, Storage);
+  return TemplateArgument(Storage, NumArgs);
+}
+
+bool TemplateArgument::isDependent() const {
+  switch (getKind()) {
+  case Null:
+    assert(false && "Should not have a NULL template argument");
+    return false;
+
+  case Type:
+    return getAsType()->isDependentType();
+
+  case Template:
+    return getAsTemplate().isDependent();
+
+  case TemplateExpansion:
+    return true;
+
+  case Declaration:
+    if (DeclContext *DC = dyn_cast<DeclContext>(getAsDecl()))
+      return DC->isDependentContext();
+    return getAsDecl()->getDeclContext()->isDependentContext();
+
+  case Integral:
+    // Never dependent
+    return false;
+
+  case Expression:
+    return (getAsExpr()->isTypeDependent() || getAsExpr()->isValueDependent());
+
+  case Pack:
+    for (pack_iterator P = pack_begin(), PEnd = pack_end(); P != PEnd; ++P) {
+      if (P->isDependent())
+        return true;
+    }
+
+    return false;
+  }
+
+  return false;
+}
+
+bool TemplateArgument::isPackExpansion() const {
+  switch (getKind()) {
+  case Null:
+  case Declaration:
+  case Integral:
+  case Pack:    
+  case Template:
+    return false;
+      
+  case TemplateExpansion:
+    return true;
+      
+  case Type:
+    return isa<PackExpansionType>(getAsType());
+          
+  case Expression:
+    return isa<PackExpansionExpr>(getAsExpr());
+  }
+  
+  return false;
+}
+
+bool TemplateArgument::containsUnexpandedParameterPack() const {
+  switch (getKind()) {
+  case Null:
+  case Declaration:
+  case Integral:
+  case TemplateExpansion:
+    break;
+
+  case Type:
+    if (getAsType()->containsUnexpandedParameterPack())
+      return true;
+    break;
+
+  case Template:
+    if (getAsTemplate().containsUnexpandedParameterPack())
+      return true;
+    break;
+        
+  case Expression:
+    if (getAsExpr()->containsUnexpandedParameterPack())
+      return true;
+    break;
 
-  Kind = Pack;
-  Args.NumArgs = NumArgs;
-  Args.CopyArgs = CopyArgs;
-  if (!Args.CopyArgs) {
-    Args.Args = args;
-    return;
+  case Pack:
+    for (pack_iterator P = pack_begin(), PEnd = pack_end(); P != PEnd; ++P)
+      if (P->containsUnexpandedParameterPack())
+        return true;
+
+    break;
   }
 
-  // FIXME: Allocate in ASTContext
-  Args.Args = new TemplateArgument[NumArgs];
-  for (unsigned I = 0; I != Args.NumArgs; ++I)
-    Args.Args[I] = args[I];
+  return false;
+}
+
+llvm::Optional<unsigned> TemplateArgument::getNumTemplateExpansions() const {
+  assert(Kind == TemplateExpansion);
+  if (TemplateArg.NumExpansions)
+    return TemplateArg.NumExpansions - 1;
+  
+  return llvm::Optional<unsigned>();
 }
 
 void TemplateArgument::Profile(llvm::FoldingSetNodeID &ID,
-                               ASTContext &Context) const {
+                               const ASTContext &Context) const {
   ID.AddInteger(Kind);
   switch (Kind) {
   case Null:
@@ -61,18 +192,22 @@ void TemplateArgument::Profile(llvm::FoldingSetNodeID &ID,
     break;
 
   case Template:
+  case TemplateExpansion: {
+    TemplateName Template = getAsTemplateOrTemplatePattern();
     if (TemplateTemplateParmDecl *TTP
           = dyn_cast_or_null<TemplateTemplateParmDecl>(
-                                       getAsTemplate().getAsTemplateDecl())) {
+                                                Template.getAsTemplateDecl())) {
       ID.AddBoolean(true);
       ID.AddInteger(TTP->getDepth());
       ID.AddInteger(TTP->getPosition());
+      ID.AddBoolean(TTP->isParameterPack());
     } else {
       ID.AddBoolean(false);
-      ID.AddPointer(Context.getCanonicalTemplateName(getAsTemplate())
-                      .getAsVoidPointer());
+      ID.AddPointer(Context.getCanonicalTemplateName(Template)
+                                                          .getAsVoidPointer());
     }
     break;
+  }
       
   case Integral:
     getAsIntegral()->Profile(ID);
@@ -97,8 +232,9 @@ bool TemplateArgument::structurallyEquals(const TemplateArgument &Other) const {
   case Null:
   case Type:
   case Declaration:
+  case Expression:      
   case Template:
-  case Expression:
+  case TemplateExpansion:
     return TypeOrValue == Other.TypeOrValue;
 
   case Integral:
@@ -117,10 +253,102 @@ bool TemplateArgument::structurallyEquals(const TemplateArgument &Other) const {
   return false;
 }
 
+TemplateArgument TemplateArgument::getPackExpansionPattern() const {
+  assert(isPackExpansion());
+  
+  switch (getKind()) {
+  case Type:
+    return getAsType()->getAs<PackExpansionType>()->getPattern();
+    
+  case Expression:
+    return cast<PackExpansionExpr>(getAsExpr())->getPattern();
+    
+  case TemplateExpansion:
+    return TemplateArgument(getAsTemplateOrTemplatePattern());
+    
+  case Declaration:
+  case Integral:
+  case Pack:
+  case Null:
+  case Template:
+    return TemplateArgument();
+  }
+  
+  return TemplateArgument();
+}
+
+void TemplateArgument::print(const PrintingPolicy &Policy, 
+                             llvm::raw_ostream &Out) const {
+  switch (getKind()) {
+  case Null:
+    Out << "<no value>";
+    break;
+    
+  case Type: {
+    std::string TypeStr;
+    getAsType().getAsStringInternal(TypeStr, Policy);
+    Out << TypeStr;
+    break;
+  }
+    
+  case Declaration: {
+    bool Unnamed = true;
+    if (NamedDecl *ND = dyn_cast_or_null<NamedDecl>(getAsDecl())) {
+      if (ND->getDeclName()) {
+        Unnamed = false;
+        Out << ND->getNameAsString();
+      }
+    }
+    
+    if (Unnamed) {
+      Out << "<anonymous>";
+    }
+    break;
+  }
+    
+  case Template:
+    getAsTemplate().print(Out, Policy);
+    break;
+
+  case TemplateExpansion:
+    getAsTemplateOrTemplatePattern().print(Out, Policy);
+    Out << "...";
+    break;
+      
+  case Integral: {
+    printIntegral(*this, Out);
+    break;
+  }
+    
+  case Expression:
+    getAsExpr()->printPretty(Out, 0, Policy);
+    break;
+    
+  case Pack:
+    Out << "<";
+    bool First = true;
+    for (TemplateArgument::pack_iterator P = pack_begin(), PEnd = pack_end();
+         P != PEnd; ++P) {
+      if (First)
+        First = false;
+      else
+        Out << ", ";
+      
+      P->print(Policy, Out);
+    }
+    Out << ">";
+    break;        
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // TemplateArgumentLoc Implementation
 //===----------------------------------------------------------------------===//
 
+TemplateArgumentLocInfo::TemplateArgumentLocInfo() {
+  memset(this, 0, sizeof(TemplateArgumentLocInfo));
+}
+
 SourceRange TemplateArgumentLoc::getSourceRange() const {
   switch (Argument.getKind()) {
   case TemplateArgument::Expression:
@@ -137,10 +365,16 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
 
   case TemplateArgument::Template:
     if (getTemplateQualifierRange().isValid())
-      return SourceRange(getTemplateQualifierRange().getBegin(),
+      return SourceRange(getTemplateQualifierRange().getBegin(), 
                          getTemplateNameLoc());
     return SourceRange(getTemplateNameLoc());
 
+  case TemplateArgument::TemplateExpansion:
+    if (getTemplateQualifierRange().isValid())
+      return SourceRange(getTemplateQualifierRange().getBegin(), 
+                         getTemplateEllipsisLoc());
+    return SourceRange(getTemplateNameLoc(), getTemplateEllipsisLoc());
+
   case TemplateArgument::Integral:
   case TemplateArgument::Pack:
   case TemplateArgument::Null:
@@ -151,6 +385,68 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
   return SourceRange();
 }
 
+TemplateArgumentLoc 
+TemplateArgumentLoc::getPackExpansionPattern(SourceLocation &Ellipsis,
+                                       llvm::Optional<unsigned> &NumExpansions,
+                                             ASTContext &Context) const {
+  assert(Argument.isPackExpansion());
+  
+  switch (Argument.getKind()) {
+  case TemplateArgument::Type: {
+    // FIXME: We shouldn't ever have to worry about missing
+    // type-source info!
+    TypeSourceInfo *ExpansionTSInfo = getTypeSourceInfo();
+    if (!ExpansionTSInfo)
+      ExpansionTSInfo = Context.getTrivialTypeSourceInfo(
+                                                     getArgument().getAsType(),
+                                                         Ellipsis);
+    PackExpansionTypeLoc Expansion
+      = cast<PackExpansionTypeLoc>(ExpansionTSInfo->getTypeLoc());
+    Ellipsis = Expansion.getEllipsisLoc();
+    
+    TypeLoc Pattern = Expansion.getPatternLoc();
+    NumExpansions = Expansion.getTypePtr()->getNumExpansions();
+    
+    // FIXME: This is horrible. We know where the source location data is for
+    // the pattern, and we have the pattern's type, but we are forced to copy
+    // them into an ASTContext because TypeSourceInfo bundles them together
+    // and TemplateArgumentLoc traffics in TypeSourceInfo pointers.
+    TypeSourceInfo *PatternTSInfo
+      = Context.CreateTypeSourceInfo(Pattern.getType(),
+                                     Pattern.getFullDataSize());
+    memcpy(PatternTSInfo->getTypeLoc().getOpaqueData(), 
+           Pattern.getOpaqueData(), Pattern.getFullDataSize());
+    return TemplateArgumentLoc(TemplateArgument(Pattern.getType()),
+                               PatternTSInfo);
+  }
+      
+  case TemplateArgument::Expression: {
+    PackExpansionExpr *Expansion
+      = cast<PackExpansionExpr>(Argument.getAsExpr());
+    Expr *Pattern = Expansion->getPattern();
+    Ellipsis = Expansion->getEllipsisLoc();
+    NumExpansions = Expansion->getNumExpansions();
+    return TemplateArgumentLoc(Pattern, Pattern);
+  }
+
+  case TemplateArgument::TemplateExpansion:
+    Ellipsis = getTemplateEllipsisLoc();
+    NumExpansions = Argument.getNumTemplateExpansions();
+    return TemplateArgumentLoc(Argument.getPackExpansionPattern(),
+                               getTemplateQualifierRange(),
+                               getTemplateNameLoc());
+    
+  case TemplateArgument::Declaration:
+  case TemplateArgument::Template:
+  case TemplateArgument::Integral:
+  case TemplateArgument::Pack:
+  case TemplateArgument::Null:
+    return TemplateArgumentLoc();
+  }
+  
+  return TemplateArgumentLoc();
+}
+
 const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
                                            const TemplateArgument &Arg) {
   switch (Arg.getKind()) {
@@ -170,7 +466,10 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
       
   case TemplateArgument::Template:
     return DB << Arg.getAsTemplate();
-      
+
+  case TemplateArgument::TemplateExpansion:
+    return DB << Arg.getAsTemplateOrTemplatePattern() << "...";
+
   case TemplateArgument::Expression: {
     // This shouldn't actually ever happen, so it's okay that we're
     // regurgitating an expression here.
@@ -184,9 +483,16 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
     return DB << OS.str();
   }
       
-  case TemplateArgument::Pack:
-    // FIXME: Format arguments in a list!
-    return DB << "<parameter pack>";
+  case TemplateArgument::Pack: {
+    // FIXME: We're guessing at LangOptions!
+    llvm::SmallString<32> Str;
+    llvm::raw_svector_ostream OS(Str);
+    LangOptions LangOpts;
+    LangOpts.CPlusPlus = true;
+    PrintingPolicy Policy(LangOpts);
+    Arg.print(Policy, OS);
+    return DB << OS.str();
+  }
   }
   
   return DB;
diff --git a/lib/AST/TemplateName.cpp b/lib/AST/TemplateName.cpp
index ef7b315314d4..6b378a001101 100644
--- a/lib/AST/TemplateName.cpp
+++ b/lib/AST/TemplateName.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/TemplateName.h"
+#include "clang/AST/TemplateBase.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/PrettyPrinter.h"
@@ -21,15 +22,33 @@
 using namespace clang;
 using namespace llvm;
 
+TemplateArgument 
+SubstTemplateTemplateParmPackStorage::getArgumentPack() const {
+  return TemplateArgument(Arguments, size());
+}
+
+void SubstTemplateTemplateParmPackStorage::Profile(llvm::FoldingSetNodeID &ID) {
+  Profile(ID, Context, Parameter, TemplateArgument(Arguments, size()));
+}
+
+void SubstTemplateTemplateParmPackStorage::Profile(llvm::FoldingSetNodeID &ID, 
+                                                   ASTContext &Context,
+                                           TemplateTemplateParmDecl *Parameter,
+                                             const TemplateArgument &ArgPack) {
+  ID.AddPointer(Parameter);
+  ArgPack.Profile(ID, Context);
+}
+
 TemplateName::NameKind TemplateName::getKind() const {
   if (Storage.is<TemplateDecl *>())
     return Template;
-  if (Storage.is<OverloadedTemplateStorage *>())
-    return OverloadedTemplate;
+  if (Storage.is<DependentTemplateName *>())
+    return DependentTemplate;
   if (Storage.is<QualifiedTemplateName *>())
     return QualifiedTemplate;
-  assert(Storage.is<DependentTemplateName *>() && "There's a case unhandled!");
-  return DependentTemplate;
+  
+  return getAsOverloadedTemplate()? OverloadedTemplate
+                                  : SubstTemplateTemplateParmPack;
 }
 
 TemplateDecl *TemplateName::getAsTemplateDecl() const {
@@ -44,8 +63,14 @@ TemplateDecl *TemplateName::getAsTemplateDecl() const {
 
 bool TemplateName::isDependent() const {
   if (TemplateDecl *Template = getAsTemplateDecl()) {
-    return isa<TemplateTemplateParmDecl>(Template) ||
-      Template->getDeclContext()->isDependentContext();
+    if (isa<TemplateTemplateParmDecl>(Template))
+      return true;
+    // FIXME: Hack, getDeclContext() can be null if Template is still
+    // initializing due to PCH reading, so we check it before using it.
+    // Should probably modify TemplateSpecializationType to allow constructing
+    // it without the isDependent() checking.
+    return Template->getDeclContext() &&
+           Template->getDeclContext()->isDependentContext();
   }
 
   assert(!getAsOverloadedTemplate() &&
@@ -54,6 +79,22 @@ bool TemplateName::isDependent() const {
   return true;
 }
 
+bool TemplateName::containsUnexpandedParameterPack() const {
+  if (TemplateDecl *Template = getAsTemplateDecl()) {
+    if (TemplateTemplateParmDecl *TTP 
+                                  = dyn_cast<TemplateTemplateParmDecl>(Template))
+      return TTP->isParameterPack();
+
+    return false;
+  }
+
+  if (DependentTemplateName *DTN = getAsDependentTemplateName())
+    return DTN->getQualifier() && 
+      DTN->getQualifier()->containsUnexpandedParameterPack();
+
+  return getAsSubstTemplateTemplateParmPack() != 0;
+}
+
 void
 TemplateName::print(llvm::raw_ostream &OS, const PrintingPolicy &Policy,
                     bool SuppressNNS) const {
@@ -74,7 +115,9 @@ TemplateName::print(llvm::raw_ostream &OS, const PrintingPolicy &Policy,
       OS << DTN->getIdentifier()->getName();
     else
       OS << "operator " << getOperatorSpelling(DTN->getOperator());
-  }
+  } else if (SubstTemplateTemplateParmPackStorage *SubstPack
+                                        = getAsSubstTemplateTemplateParmPack())
+    OS << SubstPack->getParameterPack()->getNameAsString();
 }
 
 const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
diff --git a/lib/AST/Type.cpp b/lib/AST/Type.cpp
index ca10532e729e..b03314e11d13 100644
--- a/lib/AST/Type.cpp
+++ b/lib/AST/Type.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/PrettyPrinter.h"
+#include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -35,14 +36,13 @@ bool QualType::isConstant(QualType T, ASTContext &Ctx) {
   return false;
 }
 
-Type::~Type() { }
-
 unsigned ConstantArrayType::getNumAddressingBits(ASTContext &Context,
                                                  QualType ElementType,
                                                const llvm::APInt &NumElements) {
   llvm::APSInt SizeExtended(NumElements, true);
   unsigned SizeTypeBits = Context.getTypeSize(Context.getSizeType());
-  SizeExtended.extend(std::max(SizeTypeBits, SizeExtended.getBitWidth()) * 2);
+  SizeExtended = SizeExtended.extend(std::max(SizeTypeBits,
+                                              SizeExtended.getBitWidth()) * 2);
 
   uint64_t ElementSize
     = Context.getTypeSizeInChars(ElementType).getQuantity();
@@ -63,8 +63,20 @@ unsigned ConstantArrayType::getMaxSizeBits(ASTContext &Context) {
   return Bits;
 }
 
+DependentSizedArrayType::DependentSizedArrayType(const ASTContext &Context, 
+                                                 QualType et, QualType can,
+                                                 Expr *e, ArraySizeModifier sm,
+                                                 unsigned tq,
+                                                 SourceRange brackets)
+    : ArrayType(DependentSizedArray, et, can, sm, tq, 
+                (et->containsUnexpandedParameterPack() ||
+                 (e && e->containsUnexpandedParameterPack()))),
+      Context(Context), SizeExpr((Stmt*) e), Brackets(brackets) 
+{
+}
+
 void DependentSizedArrayType::Profile(llvm::FoldingSetNodeID &ID,
-                                      ASTContext &Context,
+                                      const ASTContext &Context,
                                       QualType ET,
                                       ArraySizeModifier SizeMod,
                                       unsigned TypeQuals,
@@ -75,14 +87,51 @@ void DependentSizedArrayType::Profile(llvm::FoldingSetNodeID &ID,
   E->Profile(ID, Context, true);
 }
 
+DependentSizedExtVectorType::DependentSizedExtVectorType(const
+                                                         ASTContext &Context,
+                                                         QualType ElementType,
+                                                         QualType can, 
+                                                         Expr *SizeExpr, 
+                                                         SourceLocation loc)
+    : Type(DependentSizedExtVector, can, /*Dependent=*/true,
+           ElementType->isVariablyModifiedType(), 
+           (ElementType->containsUnexpandedParameterPack() ||
+            (SizeExpr && SizeExpr->containsUnexpandedParameterPack()))),
+      Context(Context), SizeExpr(SizeExpr), ElementType(ElementType),
+      loc(loc) 
+{
+}
+
 void
 DependentSizedExtVectorType::Profile(llvm::FoldingSetNodeID &ID,
-                                     ASTContext &Context,
+                                     const ASTContext &Context,
                                      QualType ElementType, Expr *SizeExpr) {
   ID.AddPointer(ElementType.getAsOpaquePtr());
   SizeExpr->Profile(ID, Context, true);
 }
 
+VectorType::VectorType(QualType vecType, unsigned nElements, QualType canonType,
+                       VectorKind vecKind)
+  : Type(Vector, canonType, vecType->isDependentType(),
+         vecType->isVariablyModifiedType(),
+         vecType->containsUnexpandedParameterPack()),
+    ElementType(vecType) 
+{
+  VectorTypeBits.VecKind = vecKind;
+  VectorTypeBits.NumElements = nElements;
+}
+
+VectorType::VectorType(TypeClass tc, QualType vecType, unsigned nElements,
+                       QualType canonType, VectorKind vecKind)
+  : Type(tc, canonType, vecType->isDependentType(),
+         vecType->isVariablyModifiedType(),
+         vecType->containsUnexpandedParameterPack()), 
+    ElementType(vecType) 
+{
+  VectorTypeBits.VecKind = vecKind;
+  VectorTypeBits.NumElements = nElements;
+}
+
 /// getArrayElementTypeNoTypeQual - If this is an array type, return the
 /// element type of the array, potentially with type qualifiers missing.
 /// This method should never be used when type qualifiers are meaningful.
@@ -101,51 +150,18 @@ const Type *Type::getArrayElementTypeNoTypeQual() const {
     ->getElementType().getTypePtr();
 }
 
-/// \brief Retrieve the unqualified variant of the given type, removing as
-/// little sugar as possible.
-///
-/// This routine looks through various kinds of sugar to find the 
-/// least-desuraged type that is unqualified. For example, given:
-///
-/// \code
-/// typedef int Integer;
-/// typedef const Integer CInteger;
-/// typedef CInteger DifferenceType;
-/// \endcode
-///
-/// Executing \c getUnqualifiedTypeSlow() on the type \c DifferenceType will
-/// desugar until we hit the type \c Integer, which has no qualifiers on it.
-QualType QualType::getUnqualifiedTypeSlow() const {
-  QualType Cur = *this;
-  while (true) {
-    if (!Cur.hasQualifiers())
-      return Cur;
-    
-    const Type *CurTy = Cur.getTypePtr();
-    switch (CurTy->getTypeClass()) {
-#define ABSTRACT_TYPE(Class, Parent)
-#define TYPE(Class, Parent)                                  \
-    case Type::Class: {                                      \
-      const Class##Type *Ty = cast<Class##Type>(CurTy);      \
-      if (!Ty->isSugared())                                  \
-        return Cur.getLocalUnqualifiedType();                \
-      Cur = Ty->desugar();                                   \
-      break;                                                 \
-    }
-#include "clang/AST/TypeNodes.def"
-    }
-  }
-  
-  return Cur.getUnqualifiedType();
-}
-
 /// getDesugaredType - Return the specified type with any "sugar" removed from
 /// the type.  This takes off typedefs, typeof's etc.  If the outer level of
 /// the type is already concrete, it returns it unmodified.  This is similar
 /// to getting the canonical type, but it doesn't remove *all* typedefs.  For
 /// example, it returns "T*" as "T*", (not as "int*"), because the pointer is
 /// concrete.
-QualType QualType::getDesugaredType(QualType T) {
+QualType QualType::getDesugaredType(QualType T, const ASTContext &Context) {
+  SplitQualType split = getSplitDesugaredType(T);
+  return Context.getQualifiedType(split.first, split.second);
+}
+
+SplitQualType QualType::getSplitDesugaredType(QualType T) {
   QualifierCollector Qs;
 
   QualType Cur = T;
@@ -157,7 +173,7 @@ QualType QualType::getDesugaredType(QualType T) {
     case Type::Class: { \
       const Class##Type *Ty = cast<Class##Type>(CurTy); \
       if (!Ty->isSugared()) \
-        return Qs.apply(Cur); \
+        return SplitQualType(Ty, Qs); \
       Cur = Ty->desugar(); \
       break; \
     }
@@ -166,6 +182,52 @@ QualType QualType::getDesugaredType(QualType T) {
   }
 }
 
+SplitQualType QualType::getSplitUnqualifiedTypeImpl(QualType type) {
+  SplitQualType split = type.split();
+
+  // All the qualifiers we've seen so far.
+  Qualifiers quals = split.second;
+
+  // The last type node we saw with any nodes inside it.
+  const Type *lastTypeWithQuals = split.first;
+
+  while (true) {
+    QualType next;
+
+    // Do a single-step desugar, aborting the loop if the type isn't
+    // sugared.
+    switch (split.first->getTypeClass()) {
+#define ABSTRACT_TYPE(Class, Parent)
+#define TYPE(Class, Parent) \
+    case Type::Class: { \
+      const Class##Type *ty = cast<Class##Type>(split.first); \
+      if (!ty->isSugared()) goto done; \
+      next = ty->desugar(); \
+      break; \
+    }
+#include "clang/AST/TypeNodes.def"
+    }
+
+    // Otherwise, split the underlying type.  If that yields qualifiers,
+    // update the information.
+    split = next.split();
+    if (!split.second.empty()) {
+      lastTypeWithQuals = split.first;
+      quals.addConsistentQualifiers(split.second);
+    }
+  }
+
+ done:
+  return SplitQualType(lastTypeWithQuals, quals);
+}
+
+QualType QualType::IgnoreParens(QualType T) {
+  // FIXME: this seems inherently un-qualifiers-safe.
+  while (const ParenType *PT = T->getAs<ParenType>())
+    T = PT->getInnerType();
+  return T;
+}
+
 /// getUnqualifiedDesugaredType - Pull any qualifiers and syntactic
 /// sugar off the given type.  This should produce an object of the
 /// same dynamic type as the canonical type.
@@ -268,42 +330,6 @@ QualType Type::getPointeeType() const {
   return QualType();
 }
 
-/// isVariablyModifiedType (C99 6.7.5p3) - Return true for variable length
-/// array types and types that contain variable array types in their
-/// declarator
-bool Type::isVariablyModifiedType() const {
-  // FIXME: We should really keep a "variably modified" bit in Type, rather
-  // than walking the type hierarchy to recompute it.
-  
-  // A VLA is a variably modified type.
-  if (isVariableArrayType())
-    return true;
-
-  // An array can contain a variably modified type
-  if (const Type *T = getArrayElementTypeNoTypeQual())
-    return T->isVariablyModifiedType();
-
-  // A pointer can point to a variably modified type.
-  // Also, C++ references and member pointers can point to a variably modified
-  // type, where VLAs appear as an extension to C++, and should be treated
-  // correctly.
-  if (const PointerType *PT = getAs<PointerType>())
-    return PT->getPointeeType()->isVariablyModifiedType();
-  if (const ReferenceType *RT = getAs<ReferenceType>())
-    return RT->getPointeeType()->isVariablyModifiedType();
-  if (const MemberPointerType *PT = getAs<MemberPointerType>())
-    return PT->getPointeeType()->isVariablyModifiedType();
-
-  // A function can return a variably modified type
-  // This one isn't completely obvious, but it follows from the
-  // definition in C99 6.7.5p3. Because of this rule, it's
-  // illegal to declare a function returning a variably modified type.
-  if (const FunctionType *FT = getAs<FunctionType>())
-    return FT->getResultType()->isVariablyModifiedType();
-
-  return false;
-}
-
 const RecordType *Type::getAsStructureType() const {
   // If this is directly a structure type, return it.
   if (const RecordType *RT = dyn_cast<RecordType>(this)) {
@@ -346,10 +372,11 @@ const RecordType *Type::getAsUnionType() const {
 ObjCObjectType::ObjCObjectType(QualType Canonical, QualType Base,
                                ObjCProtocolDecl * const *Protocols,
                                unsigned NumProtocols)
-  : Type(ObjCObject, Canonical, false),
-    NumProtocols(NumProtocols),
-    BaseType(Base) {
-  assert(this->NumProtocols == NumProtocols &&
+  : Type(ObjCObject, Canonical, false, false, false),
+    BaseType(Base) 
+{
+  ObjCObjectTypeBits.NumProtocols = NumProtocols;
+  assert(getNumProtocols() == NumProtocols &&
          "bitfield overflow in protocol count");
   if (NumProtocols)
     memcpy(getProtocolStorage(), Protocols,
@@ -405,15 +432,69 @@ CXXRecordDecl *Type::getAsCXXRecordDecl() const {
   return 0;
 }
 
+namespace {
+  class GetContainedAutoVisitor :
+    public TypeVisitor<GetContainedAutoVisitor, AutoType*> {
+  public:
+    using TypeVisitor<GetContainedAutoVisitor, AutoType*>::Visit;
+    AutoType *Visit(QualType T) {
+      if (T.isNull())
+        return 0;
+      return Visit(T.getTypePtr());
+    }
+
+    // The 'auto' type itself.
+    AutoType *VisitAutoType(const AutoType *AT) {
+      return const_cast<AutoType*>(AT);
+    }
+
+    // Only these types can contain the desired 'auto' type.
+    AutoType *VisitPointerType(const PointerType *T) {
+      return Visit(T->getPointeeType());
+    }
+    AutoType *VisitBlockPointerType(const BlockPointerType *T) {
+      return Visit(T->getPointeeType());
+    }
+    AutoType *VisitReferenceType(const ReferenceType *T) {
+      return Visit(T->getPointeeTypeAsWritten());
+    }
+    AutoType *VisitMemberPointerType(const MemberPointerType *T) {
+      return Visit(T->getPointeeType());
+    }
+    AutoType *VisitArrayType(const ArrayType *T) {
+      return Visit(T->getElementType());
+    }
+    AutoType *VisitDependentSizedExtVectorType(
+      const DependentSizedExtVectorType *T) {
+      return Visit(T->getElementType());
+    }
+    AutoType *VisitVectorType(const VectorType *T) {
+      return Visit(T->getElementType());
+    }
+    AutoType *VisitFunctionType(const FunctionType *T) {
+      return Visit(T->getResultType());
+    }
+    AutoType *VisitParenType(const ParenType *T) {
+      return Visit(T->getInnerType());
+    }
+    AutoType *VisitAttributedType(const AttributedType *T) {
+      return Visit(T->getModifiedType());
+    }
+  };
+}
+
+AutoType *Type::getContainedAutoType() const {
+  return GetContainedAutoVisitor().Visit(this);
+}
+
 bool Type::isIntegerType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::Int128;
-  if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     // Incomplete enum types are not treated as integer types.
     // FIXME: In C++, enum types are never integer types.
-    if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
-      return true;
+    return ET->getDecl()->isComplete();
   return false;
 }
 
@@ -449,9 +530,8 @@ bool Type::isIntegralType(ASTContext &Ctx) const {
     BT->getKind() <= BuiltinType::Int128;
   
   if (!Ctx.getLangOptions().CPlusPlus)
-    if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
-      if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
-        return true;  // Complete enum types are integral in C.
+    if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
+      return ET->getDecl()->isComplete(); // Complete enum types are integral in C.
   
   return false;
 }
@@ -463,19 +543,28 @@ bool Type::isIntegralOrEnumerationType() const {
 
   // Check for a complete enum type; incomplete enum types are not properly an
   // enumeration type in the sense required here.
-  if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
-    if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
-      return true;
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
+    return ET->getDecl()->isComplete();
 
   return false;  
 }
 
-bool Type::isEnumeralType() const {
-  if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
-    return TT->getDecl()->isEnum();
+bool Type::isIntegralOrUnscopedEnumerationType() const {
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
+    return BT->getKind() >= BuiltinType::Bool &&
+           BT->getKind() <= BuiltinType::Int128;
+
+  // Check for a complete enum type; incomplete enum types are not properly an
+  // enumeration type in the sense required here.
+  // C++0x: However, if the underlying type of the enum is fixed, it is
+  // considered complete.
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
+    return ET->getDecl()->isComplete() && !ET->getDecl()->isScoped();
+
   return false;
 }
 
+
 bool Type::isBooleanType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() == BuiltinType::Bool;
@@ -493,20 +582,28 @@ bool Type::isCharType() const {
 
 bool Type::isWideCharType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() == BuiltinType::WChar;
+    return BT->getKind() == BuiltinType::WChar_S ||
+           BT->getKind() == BuiltinType::WChar_U;
   return false;
 }
 
 /// \brief Determine whether this type is any of the built-in character
 /// types.
 bool Type::isAnyCharacterType() const {
-  if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return (BT->getKind() >= BuiltinType::Char_U &&
-            BT->getKind() <= BuiltinType::Char32) ||
-           (BT->getKind() >= BuiltinType::Char_S &&
-            BT->getKind() <= BuiltinType::WChar);
-  
-  return false;
+  const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType);
+  if (BT == 0) return false;
+  switch (BT->getKind()) {
+  default: return false;
+  case BuiltinType::Char_U:
+  case BuiltinType::UChar:
+  case BuiltinType::WChar_U:
+  case BuiltinType::Char16:
+  case BuiltinType::Char32:
+  case BuiltinType::Char_S:
+  case BuiltinType::SChar:
+  case BuiltinType::WChar_S:
+    return true;
+  }
 }
 
 /// isSignedIntegerType - Return true if this is an integer type that is
@@ -518,8 +615,12 @@ bool Type::isSignedIntegerType() const {
            BT->getKind() <= BuiltinType::Int128;
   }
 
-  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
-    return ET->getDecl()->getIntegerType()->isSignedIntegerType();
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType)) {
+    // Incomplete enum types are not treated as integer types.
+    // FIXME: In C++, enum types are never integer types.
+    if (ET->getDecl()->isComplete())
+      return ET->getDecl()->getIntegerType()->isSignedIntegerType();
+  }
 
   return false;
 }
@@ -540,8 +641,12 @@ bool Type::isUnsignedIntegerType() const {
            BT->getKind() <= BuiltinType::UInt128;
   }
 
-  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
-    return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType)) {
+    // Incomplete enum types are not treated as integer types.
+    // FIXME: In C++, enum types are never integer types.
+    if (ET->getDecl()->isComplete())
+      return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
+  }
 
   return false;
 }
@@ -579,8 +684,8 @@ bool Type::isRealType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::LongDouble;
-  if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
-    return TT->getDecl()->isEnum() && TT->getDecl()->isDefinition();
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
+      return ET->getDecl()->isComplete() && !ET->getDecl()->isScoped();
   return false;
 }
 
@@ -591,20 +696,22 @@ bool Type::isArithmeticType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     // GCC allows forward declaration of enum types (forbid by C99 6.7.2.3p2).
     // If a body isn't seen by the time we get here, return false.
-    return ET->getDecl()->isDefinition();
+    //
+    // C++0x: Enumerations are not arithmetic types. For now, just return
+    // false for scoped enumerations since that will disable any
+    // unwanted implicit conversions.
+    return !ET->getDecl()->isScoped() && ET->getDecl()->isComplete();
   return isa<ComplexType>(CanonicalType);
 }
 
 bool Type::isScalarType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
-    return BT->getKind() != BuiltinType::Void;
-  if (const TagType *TT = dyn_cast<TagType>(CanonicalType)) {
+    return BT->getKind() > BuiltinType::Void &&
+           BT->getKind() <= BuiltinType::NullPtr;
+  if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     // Enums are scalar types, but only if they are defined.  Incomplete enums
     // are not treated as scalar types.
-    if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
-      return true;
-    return false;
-  }
+    return ET->getDecl()->isComplete();
   return isa<PointerType>(CanonicalType) ||
          isa<BlockPointerType>(CanonicalType) ||
          isa<MemberPointerType>(CanonicalType) ||
@@ -612,6 +719,35 @@ bool Type::isScalarType() const {
          isa<ObjCObjectPointerType>(CanonicalType);
 }
 
+Type::ScalarTypeKind Type::getScalarTypeKind() const {
+  assert(isScalarType());
+
+  const Type *T = CanonicalType.getTypePtr();
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(T)) {
+    if (BT->getKind() == BuiltinType::Bool) return STK_Bool;
+    if (BT->getKind() == BuiltinType::NullPtr) return STK_Pointer;
+    if (BT->isInteger()) return STK_Integral;
+    if (BT->isFloatingPoint()) return STK_Floating;
+    llvm_unreachable("unknown scalar builtin type");
+  } else if (isa<PointerType>(T) ||
+             isa<BlockPointerType>(T) ||
+             isa<ObjCObjectPointerType>(T)) {
+    return STK_Pointer;
+  } else if (isa<MemberPointerType>(T)) {
+    return STK_MemberPointer;
+  } else if (isa<EnumType>(T)) {
+    assert(cast<EnumType>(T)->getDecl()->isComplete());
+    return STK_Integral;
+  } else if (const ComplexType *CT = dyn_cast<ComplexType>(T)) {
+    if (CT->getElementType()->isRealFloatingType())
+      return STK_FloatingComplex;
+    return STK_IntegralComplex;
+  }
+
+  llvm_unreachable("unknown scalar type");
+  return STK_Pointer;
+}
+
 /// \brief Determines whether the type is a C++ aggregate type or C
 /// aggregate or union type.
 ///
@@ -652,8 +788,12 @@ bool Type::isIncompleteType() const {
     // Void is the only incomplete builtin type.  Per C99 6.2.5p19, it can never
     // be completed.
     return isVoidType();
-  case Record:
   case Enum:
+    // An enumeration with fixed underlying type is complete (C++0x 7.2p3).
+    if (cast<EnumType>(CanonicalType)->getDecl()->isFixed())
+        return false;
+    // Fall through.
+  case Record:
     // A tagged type (struct/union/enum/class) is incomplete if the decl is a
     // forward declaration, but not a full definition (C99 6.2.5p22).
     return !cast<TagType>(CanonicalType)->getDecl()->isDefinition();
@@ -678,7 +818,11 @@ bool Type::isIncompleteType() const {
 /// isPODType - Return true if this is a plain-old-data type (C++ 3.9p10)
 bool Type::isPODType() const {
   // The compiler shouldn't query this for incomplete types, but the user might.
-  // We return false for that case.
+  // We return false for that case. Except for incomplete arrays of PODs, which
+  // are PODs according to the standard.
+  if (isIncompleteArrayType() &&
+      cast<ArrayType>(CanonicalType)->getElementType()->isPODType())
+    return true;
   if (isIncompleteType())
     return false;
 
@@ -687,7 +831,7 @@ bool Type::isPODType() const {
   default: return false;
   case VariableArray:
   case ConstantArray:
-    // IncompleteArray is caught by isIncompleteType() above.
+    // IncompleteArray is handled above.
     return cast<ArrayType>(CanonicalType)->getElementType()->isPODType();
 
   case Builtin:
@@ -765,7 +909,8 @@ bool Type::isPromotableIntegerType() const {
   // Enumerated types are promotable to their compatible integer types
   // (C99 6.3.1.1) a.k.a. its underlying type (C++ [conv.prom]p2).
   if (const EnumType *ET = getAs<EnumType>()){
-    if (this->isDependentType() || ET->getDecl()->getPromotionType().isNull())
+    if (this->isDependentType() || ET->getDecl()->getPromotionType().isNull()
+        || ET->getDecl()->isScoped())
       return false;
     
     const BuiltinType *BT
@@ -808,9 +953,6 @@ bool Type::isSpecifierType() const {
   }
 }
 
-TypeWithKeyword::~TypeWithKeyword() {
-}
-
 ElaboratedTypeKeyword
 TypeWithKeyword::getKeywordForTypeSpec(unsigned TypeSpec) {
   switch (TypeSpec) {
@@ -830,8 +972,10 @@ TypeWithKeyword::getTagTypeKindForTypeSpec(unsigned TypeSpec) {
   case TST_struct: return TTK_Struct;
   case TST_union: return TTK_Union;
   case TST_enum: return TTK_Enum;
-  default: llvm_unreachable("Type specifier is not a tag type kind.");
   }
+  
+  llvm_unreachable("Type specifier is not a tag type kind.");
+  return TTK_Union;
 }
 
 ElaboratedTypeKeyword
@@ -877,7 +1021,6 @@ TypeWithKeyword::KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword) {
 const char*
 TypeWithKeyword::getKeywordName(ElaboratedTypeKeyword Keyword) {
   switch (Keyword) {
-  default: llvm_unreachable("Unknown elaborated type keyword.");
   case ETK_None: return "";
   case ETK_Typename: return "typename";
   case ETK_Class:  return "class";
@@ -885,28 +1028,33 @@ TypeWithKeyword::getKeywordName(ElaboratedTypeKeyword Keyword) {
   case ETK_Union:  return "union";
   case ETK_Enum:   return "enum";
   }
-}
 
-ElaboratedType::~ElaboratedType() {}
-DependentNameType::~DependentNameType() {}
-DependentTemplateSpecializationType::~DependentTemplateSpecializationType() {}
+  llvm_unreachable("Unknown elaborated type keyword.");
+  return "";
+}
 
 DependentTemplateSpecializationType::DependentTemplateSpecializationType(
                          ElaboratedTypeKeyword Keyword,
                          NestedNameSpecifier *NNS, const IdentifierInfo *Name,
                          unsigned NumArgs, const TemplateArgument *Args,
                          QualType Canon)
-  : TypeWithKeyword(Keyword, DependentTemplateSpecialization, Canon, true),
+  : TypeWithKeyword(Keyword, DependentTemplateSpecialization, Canon, true,
+                    /*VariablyModified=*/false,
+                    NNS->containsUnexpandedParameterPack()),
     NNS(NNS), Name(Name), NumArgs(NumArgs) {
   assert(NNS && NNS->isDependent() &&
          "DependentTemplateSpecializatonType requires dependent qualifier");
-  for (unsigned I = 0; I != NumArgs; ++I)
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    if (Args[I].containsUnexpandedParameterPack())
+      setContainsUnexpandedParameterPack();
+
     new (&getArgBuffer()[I]) TemplateArgument(Args[I]);
+  }
 }
 
 void
 DependentTemplateSpecializationType::Profile(llvm::FoldingSetNodeID &ID,
-                                             ASTContext &Context,
+                                             const ASTContext &Context,
                                              ElaboratedTypeKeyword Keyword,
                                              NestedNameSpecifier *Qualifier,
                                              const IdentifierInfo *Name,
@@ -935,17 +1083,18 @@ bool Type::isElaboratedTypeSpecifier() const {
 }
 
 const char *Type::getTypeClassName() const {
-  switch (TC) {
-  default: assert(0 && "Type class not in TypeNodes.def!");
+  switch (TypeBits.TC) {
 #define ABSTRACT_TYPE(Derived, Base)
 #define TYPE(Derived, Base) case Derived: return #Derived;
 #include "clang/AST/TypeNodes.def"
   }
+  
+  llvm_unreachable("Invalid type class.");
+  return 0;
 }
 
 const char *BuiltinType::getName(const LangOptions &LO) const {
   switch (getKind()) {
-  default: assert(0 && "Unknown builtin type!");
   case Void:              return "void";
   case Bool:              return LO.Bool ? "bool" : "_Bool";
   case Char_S:            return "char";
@@ -965,21 +1114,22 @@ const char *BuiltinType::getName(const LangOptions &LO) const {
   case Float:             return "float";
   case Double:            return "double";
   case LongDouble:        return "long double";
-  case WChar:             return "wchar_t";
+  case WChar_S:
+  case WChar_U:           return "wchar_t";
   case Char16:            return "char16_t";
   case Char32:            return "char32_t";
   case NullPtr:           return "nullptr_t";
   case Overload:          return "<overloaded function type>";
   case Dependent:         return "<dependent type>";
-  case UndeducedAuto:     return "auto";
   case ObjCId:            return "id";
   case ObjCClass:         return "Class";
   case ObjCSel:           return "SEL";
   }
+  
+  llvm_unreachable("Invalid builtin type.");
+  return 0;
 }
 
-void FunctionType::ANCHOR() {} // Key function for FunctionType.
-
 QualType QualType::getNonLValueExprType(ASTContext &Context) const {
   if (const ReferenceType *RefType = getTypePtr()->getAs<ReferenceType>())
     return RefType->getPointeeType();
@@ -998,8 +1148,9 @@ QualType QualType::getNonLValueExprType(ASTContext &Context) const {
 
 llvm::StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   switch (CC) {
-  case CC_Default: llvm_unreachable("no name for default cc");
-  default: return "";
+  case CC_Default: 
+    llvm_unreachable("no name for default cc");
+    return "";
 
   case CC_C: return "cdecl";
   case CC_X86StdCall: return "stdcall";
@@ -1007,62 +1158,76 @@ llvm::StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   case CC_X86ThisCall: return "thiscall";
   case CC_X86Pascal: return "pascal";
   }
+
+  llvm_unreachable("Invalid calling convention.");
+  return "";
+}
+
+FunctionProtoType::FunctionProtoType(QualType result, const QualType *args,
+                                     unsigned numArgs, QualType canonical,
+                                     const ExtProtoInfo &epi)
+  : FunctionType(FunctionProto, result, epi.Variadic, epi.TypeQuals, 
+                 epi.RefQualifier, canonical,
+                 result->isDependentType(),
+                 result->isVariablyModifiedType(),
+                 result->containsUnexpandedParameterPack(),
+                 epi.ExtInfo),
+    NumArgs(numArgs), NumExceptions(epi.NumExceptions),
+    HasExceptionSpec(epi.HasExceptionSpec),
+    HasAnyExceptionSpec(epi.HasAnyExceptionSpec)
+{
+  // Fill in the trailing argument array.
+  QualType *argSlot = reinterpret_cast<QualType*>(this+1);
+  for (unsigned i = 0; i != numArgs; ++i) {
+    if (args[i]->isDependentType())
+      setDependent();
+
+    if (args[i]->containsUnexpandedParameterPack())
+      setContainsUnexpandedParameterPack();
+
+    argSlot[i] = args[i];
+  }
+  
+  // Fill in the exception array.
+  QualType *exnSlot = argSlot + numArgs;
+  for (unsigned i = 0, e = epi.NumExceptions; i != e; ++i) {
+    if (epi.Exceptions[i]->isDependentType())
+      setDependent();
+
+    if (epi.Exceptions[i]->containsUnexpandedParameterPack())
+      setContainsUnexpandedParameterPack();
+
+    exnSlot[i] = epi.Exceptions[i];
+  }
+}
+
+bool FunctionProtoType::isTemplateVariadic() const {
+  for (unsigned ArgIdx = getNumArgs(); ArgIdx; --ArgIdx)
+    if (isa<PackExpansionType>(getArgType(ArgIdx - 1)))
+      return true;
+  
+  return false;
 }
 
 void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID, QualType Result,
-                                arg_type_iterator ArgTys,
-                                unsigned NumArgs, bool isVariadic,
-                                unsigned TypeQuals, bool hasExceptionSpec,
-                                bool anyExceptionSpec, unsigned NumExceptions,
-                                exception_iterator Exs,
-                                const FunctionType::ExtInfo &Info) {
+                                const QualType *ArgTys, unsigned NumArgs,
+                                const ExtProtoInfo &epi) {
   ID.AddPointer(Result.getAsOpaquePtr());
   for (unsigned i = 0; i != NumArgs; ++i)
     ID.AddPointer(ArgTys[i].getAsOpaquePtr());
-  ID.AddInteger(isVariadic);
-  ID.AddInteger(TypeQuals);
-  ID.AddInteger(hasExceptionSpec);
-  if (hasExceptionSpec) {
-    ID.AddInteger(anyExceptionSpec);
-    for (unsigned i = 0; i != NumExceptions; ++i)
-      ID.AddPointer(Exs[i].getAsOpaquePtr());
+  ID.AddBoolean(epi.Variadic);
+  ID.AddInteger(epi.TypeQuals);
+  ID.AddInteger(epi.RefQualifier);
+  if (epi.HasExceptionSpec) {
+    ID.AddBoolean(epi.HasAnyExceptionSpec);
+    for (unsigned i = 0; i != epi.NumExceptions; ++i)
+      ID.AddPointer(epi.Exceptions[i].getAsOpaquePtr());
   }
-  ID.AddInteger(Info.getNoReturn());
-  ID.AddInteger(Info.getRegParm());
-  ID.AddInteger(Info.getCC());
+  epi.ExtInfo.Profile(ID);
 }
 
 void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID) {
-  Profile(ID, getResultType(), arg_type_begin(), NumArgs, isVariadic(),
-          getTypeQuals(), hasExceptionSpec(), hasAnyExceptionSpec(),
-          getNumExceptions(), exception_begin(),
-          getExtInfo());
-}
-
-/// LookThroughTypedefs - Return the ultimate type this typedef corresponds to
-/// potentially looking through *all* consequtive typedefs.  This returns the
-/// sum of the type qualifiers, so if you have:
-///   typedef const int A;
-///   typedef volatile A B;
-/// looking through the typedefs for B will give you "const volatile A".
-///
-QualType TypedefType::LookThroughTypedefs() const {
-  // Usually, there is only a single level of typedefs, be fast in that case.
-  QualType FirstType = getDecl()->getUnderlyingType();
-  if (!isa<TypedefType>(FirstType))
-    return FirstType;
-
-  // Otherwise, do the fully general loop.
-  QualifierCollector Qs;
-
-  QualType CurType;
-  const TypedefType *TDT = this;
-  do {
-    CurType = TDT->getDecl()->getUnderlyingType();
-    TDT = dyn_cast<TypedefType>(Qs.strip(CurType));
-  } while (TDT);
-
-  return Qs.apply(CurType);
+  Profile(ID, getResultType(), arg_type_begin(), NumArgs, getExtProtoInfo());
 }
 
 QualType TypedefType::desugar() const {
@@ -1070,7 +1235,10 @@ QualType TypedefType::desugar() const {
 }
 
 TypeOfExprType::TypeOfExprType(Expr *E, QualType can)
-  : Type(TypeOfExpr, can, E->isTypeDependent()), TOExpr(E) {
+  : Type(TypeOfExpr, can, E->isTypeDependent(), 
+         E->getType()->isVariablyModifiedType(),
+         E->containsUnexpandedParameterPack()), 
+    TOExpr(E) {
 }
 
 QualType TypeOfExprType::desugar() const {
@@ -1078,25 +1246,29 @@ QualType TypeOfExprType::desugar() const {
 }
 
 void DependentTypeOfExprType::Profile(llvm::FoldingSetNodeID &ID,
-                                      ASTContext &Context, Expr *E) {
+                                      const ASTContext &Context, Expr *E) {
   E->Profile(ID, Context, true);
 }
 
 DecltypeType::DecltypeType(Expr *E, QualType underlyingType, QualType can)
-  : Type(Decltype, can, E->isTypeDependent()), E(E),
+  : Type(Decltype, can, E->isTypeDependent(), 
+         E->getType()->isVariablyModifiedType(), 
+         E->containsUnexpandedParameterPack()), 
+    E(E),
   UnderlyingType(underlyingType) {
 }
 
-DependentDecltypeType::DependentDecltypeType(ASTContext &Context, Expr *E)
+DependentDecltypeType::DependentDecltypeType(const ASTContext &Context, Expr *E)
   : DecltypeType(E, Context.DependentTy), Context(Context) { }
 
 void DependentDecltypeType::Profile(llvm::FoldingSetNodeID &ID,
-                                    ASTContext &Context, Expr *E) {
+                                    const ASTContext &Context, Expr *E) {
   E->Profile(ID, Context, true);
 }
 
 TagType::TagType(TypeClass TC, const TagDecl *D, QualType can)
-  : Type(TC, can, D->isDependentType()),
+  : Type(TC, can, D->isDependentType(), /*VariablyModified=*/false, 
+         /*ContainsUnexpandedParameterPack=*/false),
     decl(const_cast<TagDecl*>(D)) {}
 
 static TagDecl *getInterestingTagDecl(TagDecl *decl) {
@@ -1130,43 +1302,32 @@ bool EnumType::classof(const TagType *TT) {
   return isa<EnumDecl>(TT->getDecl());
 }
 
-static bool isDependent(const TemplateArgument &Arg) {
-  switch (Arg.getKind()) {
-  case TemplateArgument::Null:
-    assert(false && "Should not have a NULL template argument");
-    return false;
-
-  case TemplateArgument::Type:
-    return Arg.getAsType()->isDependentType();
-
-  case TemplateArgument::Template:
-    return Arg.getAsTemplate().isDependent();
-      
-  case TemplateArgument::Declaration:
-    if (DeclContext *DC = dyn_cast<DeclContext>(Arg.getAsDecl()))
-      return DC->isDependentContext();
-    return Arg.getAsDecl()->getDeclContext()->isDependentContext();
-
-  case TemplateArgument::Integral:
-    // Never dependent
-    return false;
-
-  case TemplateArgument::Expression:
-    return (Arg.getAsExpr()->isTypeDependent() ||
-            Arg.getAsExpr()->isValueDependent());
+SubstTemplateTypeParmPackType::
+SubstTemplateTypeParmPackType(const TemplateTypeParmType *Param, 
+                              QualType Canon,
+                              const TemplateArgument &ArgPack)
+  : Type(SubstTemplateTypeParmPack, Canon, true, false, true), Replaced(Param), 
+    Arguments(ArgPack.pack_begin()), NumArguments(ArgPack.pack_size()) 
+{ 
+}
 
-  case TemplateArgument::Pack:
-    for (TemplateArgument::pack_iterator P = Arg.pack_begin(), 
-                                      PEnd = Arg.pack_end();
-         P != PEnd; ++P) {
-      if (isDependent(*P))
-        return true;
-    }
+TemplateArgument SubstTemplateTypeParmPackType::getArgumentPack() const {
+  return TemplateArgument(Arguments, NumArguments);
+}
 
-    return false;
-  }
+void SubstTemplateTypeParmPackType::Profile(llvm::FoldingSetNodeID &ID) {
+  Profile(ID, getReplacedParameter(), getArgumentPack());
+}
 
-  return false;
+void SubstTemplateTypeParmPackType::Profile(llvm::FoldingSetNodeID &ID,
+                                           const TemplateTypeParmType *Replaced,
+                                            const TemplateArgument &ArgPack) {
+  ID.AddPointer(Replaced);
+  ID.AddInteger(ArgPack.pack_size());
+  for (TemplateArgument::pack_iterator P = ArgPack.pack_begin(), 
+                                    PEnd = ArgPack.pack_end();
+       P != PEnd; ++P)
+    ID.AddPointer(P->getAsType().getAsOpaquePtr());
 }
 
 bool TemplateSpecializationType::
@@ -1177,7 +1338,7 @@ anyDependentTemplateArguments(const TemplateArgumentListInfo &Args) {
 bool TemplateSpecializationType::
 anyDependentTemplateArguments(const TemplateArgumentLoc *Args, unsigned N) {
   for (unsigned i = 0; i != N; ++i)
-    if (isDependent(Args[i].getArgument()))
+    if (Args[i].getArgument().isDependent())
       return true;
   return false;
 }
@@ -1185,7 +1346,7 @@ anyDependentTemplateArguments(const TemplateArgumentLoc *Args, unsigned N) {
 bool TemplateSpecializationType::
 anyDependentTemplateArguments(const TemplateArgument *Args, unsigned N) {
   for (unsigned i = 0; i != N; ++i)
-    if (isDependent(Args[i]))
+    if (Args[i].isDependent())
       return true;
   return false;
 }
@@ -1196,16 +1357,28 @@ TemplateSpecializationType(TemplateName T,
                            unsigned NumArgs, QualType Canon)
   : Type(TemplateSpecialization,
          Canon.isNull()? QualType(this, 0) : Canon,
-         T.isDependent() || anyDependentTemplateArguments(Args, NumArgs)),
-    Template(T), NumArgs(NumArgs) {
+         T.isDependent(), false,
+         T.containsUnexpandedParameterPack()),
+    Template(T), NumArgs(NumArgs) 
+{
   assert((!Canon.isNull() ||
           T.isDependent() || anyDependentTemplateArguments(Args, NumArgs)) &&
          "No canonical type for non-dependent class template specialization");
 
   TemplateArgument *TemplateArgs
     = reinterpret_cast<TemplateArgument *>(this + 1);
-  for (unsigned Arg = 0; Arg < NumArgs; ++Arg)
+  for (unsigned Arg = 0; Arg < NumArgs; ++Arg) {
+    // Update dependent and variably-modified bits.
+    if (Args[Arg].isDependent())
+      setDependent();
+    if (Args[Arg].getKind() == TemplateArgument::Type &&
+        Args[Arg].getAsType()->isVariablyModifiedType())
+      setVariablyModified();
+    if (Args[Arg].containsUnexpandedParameterPack())
+      setContainsUnexpandedParameterPack();
+
     new (&TemplateArgs[Arg]) TemplateArgument(Args[Arg]);
+  }
 }
 
 void
@@ -1213,26 +1386,26 @@ TemplateSpecializationType::Profile(llvm::FoldingSetNodeID &ID,
                                     TemplateName T,
                                     const TemplateArgument *Args,
                                     unsigned NumArgs,
-                                    ASTContext &Context) {
+                                    const ASTContext &Context) {
   T.Profile(ID);
   for (unsigned Idx = 0; Idx < NumArgs; ++Idx)
     Args[Idx].Profile(ID, Context);
 }
 
-QualType QualifierCollector::apply(QualType QT) const {
+QualType
+QualifierCollector::apply(const ASTContext &Context, QualType QT) const {
   if (!hasNonFastQualifiers())
     return QT.withFastQualifiers(getFastQualifiers());
 
-  assert(Context && "extended qualifiers but no context!");
-  return Context->getQualifiedType(QT, *this);
+  return Context.getQualifiedType(QT, *this);
 }
 
-QualType QualifierCollector::apply(const Type *T) const {
+QualType
+QualifierCollector::apply(const ASTContext &Context, const Type *T) const {
   if (!hasNonFastQualifiers())
     return QualType(T, getFastQualifiers());
 
-  assert(Context && "extended qualifiers but no context!");
-  return Context->getQualifiedType(T, *this);
+  return Context.getQualifiedType(T, *this);
 }
 
 void ObjCObjectTypeImpl::Profile(llvm::FoldingSetNodeID &ID,
@@ -1248,95 +1421,223 @@ void ObjCObjectTypeImpl::Profile(llvm::FoldingSetNodeID &ID) {
   Profile(ID, getBaseType(), qual_begin(), getNumProtocols());
 }
 
-/// \brief Determine the linkage of this type.
-Linkage Type::getLinkage() const {
-  if (this != CanonicalType.getTypePtr())
-    return CanonicalType->getLinkage();
+namespace {
+
+/// \brief The cached properties of a type.
+class CachedProperties {
+  char linkage;
+  char visibility;
+  bool local;
   
-  if (!LinkageKnown) {
-    CachedLinkage = getLinkageImpl();
-    LinkageKnown = true;
-  }
+public:
+  CachedProperties(Linkage linkage, Visibility visibility, bool local)
+    : linkage(linkage), visibility(visibility), local(local) {}
   
-  return static_cast<clang::Linkage>(CachedLinkage);
+  Linkage getLinkage() const { return (Linkage) linkage; }
+  Visibility getVisibility() const { return (Visibility) visibility; }
+  bool hasLocalOrUnnamedType() const { return local; }
+  
+  friend CachedProperties merge(CachedProperties L, CachedProperties R) {
+    return CachedProperties(minLinkage(L.getLinkage(), R.getLinkage()),
+                            minVisibility(L.getVisibility(), R.getVisibility()),
+                         L.hasLocalOrUnnamedType() | R.hasLocalOrUnnamedType());
+  }
+};
 }
 
-Linkage Type::getLinkageImpl() const { 
-  // C++ [basic.link]p8:
-  //   Names not covered by these rules have no linkage.
-  return NoLinkage; 
-}
+static CachedProperties computeCachedProperties(const Type *T);
 
-void Type::ClearLinkageCache() {
-  if (this != CanonicalType.getTypePtr())
-    CanonicalType->ClearLinkageCache();
-  else
-    LinkageKnown = false;
-}
+namespace clang {
+/// The type-property cache.  This is templated so as to be
+/// instantiated at an internal type to prevent unnecessary symbol
+/// leakage.
+template <class Private> class TypePropertyCache {
+public:
+  static CachedProperties get(QualType T) {
+    return get(T.getTypePtr());
+  }
 
-Linkage BuiltinType::getLinkageImpl() const {
-  // C++ [basic.link]p8:
-  //   A type is said to have linkage if and only if:
-  //     - it is a fundamental type (3.9.1); or
-  return ExternalLinkage;
-}
+  static CachedProperties get(const Type *T) {
+    ensure(T);
+    return CachedProperties(T->TypeBits.getLinkage(),
+                            T->TypeBits.getVisibility(),
+                            T->TypeBits.hasLocalOrUnnamedType());
+  }
 
-Linkage TagType::getLinkageImpl() const {
-  // C++ [basic.link]p8:
-  //     - it is a class or enumeration type that is named (or has a name for
-  //       linkage purposes (7.1.3)) and the name has linkage; or
-  //     -  it is a specialization of a class template (14); or
-  return getDecl()->getLinkage();
-}
+  static void ensure(const Type *T) {
+    // If the cache is valid, we're okay.
+    if (T->TypeBits.isCacheValid()) return;
+
+    // If this type is non-canonical, ask its canonical type for the
+    // relevant information.
+    if (!T->isCanonicalUnqualified()) {
+      const Type *CT = T->getCanonicalTypeInternal().getTypePtr();
+      ensure(CT);
+      T->TypeBits.CacheValidAndVisibility =
+        CT->TypeBits.CacheValidAndVisibility;
+      T->TypeBits.CachedLinkage = CT->TypeBits.CachedLinkage;
+      T->TypeBits.CachedLocalOrUnnamed = CT->TypeBits.CachedLocalOrUnnamed;
+      return;
+    }
 
-// C++ [basic.link]p8:
-//   - it is a compound type (3.9.2) other than a class or enumeration, 
-//     compounded exclusively from types that have linkage; or
-Linkage ComplexType::getLinkageImpl() const {
-  return ElementType->getLinkage();
+    // Compute the cached properties and then set the cache.
+    CachedProperties Result = computeCachedProperties(T);
+    T->TypeBits.CacheValidAndVisibility = Result.getVisibility() + 1U;
+    assert(T->TypeBits.isCacheValid() &&
+           T->TypeBits.getVisibility() == Result.getVisibility());
+    T->TypeBits.CachedLinkage = Result.getLinkage();
+    T->TypeBits.CachedLocalOrUnnamed = Result.hasLocalOrUnnamedType();
+  }
+};
 }
 
-Linkage PointerType::getLinkageImpl() const {
-  return PointeeType->getLinkage();
-}
+// Instantiate the friend template at a private class.  In a
+// reasonable implementation, these symbols will be internal.
+// It is terrible that this is the best way to accomplish this.
+namespace { class Private {}; }
+typedef TypePropertyCache<Private> Cache;
+
+static CachedProperties computeCachedProperties(const Type *T) {
+  switch (T->getTypeClass()) {
+#define TYPE(Class,Base)
+#define NON_CANONICAL_TYPE(Class,Base) case Type::Class:
+#include "clang/AST/TypeNodes.def"
+    llvm_unreachable("didn't expect a non-canonical type here");
+
+#define TYPE(Class,Base)
+#define DEPENDENT_TYPE(Class,Base) case Type::Class:
+#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class,Base) case Type::Class:
+#include "clang/AST/TypeNodes.def"
+    // Treat dependent types as external.
+    assert(T->isDependentType());
+    return CachedProperties(ExternalLinkage, DefaultVisibility, false);
+
+  case Type::Builtin:
+    // C++ [basic.link]p8:
+    //   A type is said to have linkage if and only if:
+    //     - it is a fundamental type (3.9.1); or
+    return CachedProperties(ExternalLinkage, DefaultVisibility, false);
+
+  case Type::Record:
+  case Type::Enum: {
+    const TagDecl *Tag = cast<TagType>(T)->getDecl();
+
+    // C++ [basic.link]p8:
+    //     - it is a class or enumeration type that is named (or has a name
+    //       for linkage purposes (7.1.3)) and the name has linkage; or
+    //     -  it is a specialization of a class template (14); or
+    NamedDecl::LinkageInfo LV = Tag->getLinkageAndVisibility();
+    bool IsLocalOrUnnamed =
+      Tag->getDeclContext()->isFunctionOrMethod() ||
+      (!Tag->getIdentifier() && !Tag->getTypedefForAnonDecl());
+    return CachedProperties(LV.linkage(), LV.visibility(), IsLocalOrUnnamed);
+  }
+
+    // C++ [basic.link]p8:
+    //   - it is a compound type (3.9.2) other than a class or enumeration, 
+    //     compounded exclusively from types that have linkage; or
+  case Type::Complex:
+    return Cache::get(cast<ComplexType>(T)->getElementType());
+  case Type::Pointer:
+    return Cache::get(cast<PointerType>(T)->getPointeeType());
+  case Type::BlockPointer:
+    return Cache::get(cast<BlockPointerType>(T)->getPointeeType());
+  case Type::LValueReference:
+  case Type::RValueReference:
+    return Cache::get(cast<ReferenceType>(T)->getPointeeType());
+  case Type::MemberPointer: {
+    const MemberPointerType *MPT = cast<MemberPointerType>(T);
+    return merge(Cache::get(MPT->getClass()),
+                 Cache::get(MPT->getPointeeType()));
+  }
+  case Type::ConstantArray:
+  case Type::IncompleteArray:
+  case Type::VariableArray:
+    return Cache::get(cast<ArrayType>(T)->getElementType());
+  case Type::Vector:
+  case Type::ExtVector:
+    return Cache::get(cast<VectorType>(T)->getElementType());
+  case Type::FunctionNoProto:
+    return Cache::get(cast<FunctionType>(T)->getResultType());
+  case Type::FunctionProto: {
+    const FunctionProtoType *FPT = cast<FunctionProtoType>(T);
+    CachedProperties result = Cache::get(FPT->getResultType());
+    for (FunctionProtoType::arg_type_iterator ai = FPT->arg_type_begin(),
+           ae = FPT->arg_type_end(); ai != ae; ++ai)
+      result = merge(result, Cache::get(*ai));
+    return result;
+  }
+  case Type::ObjCInterface: {
+    NamedDecl::LinkageInfo LV =
+      cast<ObjCInterfaceType>(T)->getDecl()->getLinkageAndVisibility();
+    return CachedProperties(LV.linkage(), LV.visibility(), false);
+  }
+  case Type::ObjCObject:
+    return Cache::get(cast<ObjCObjectType>(T)->getBaseType());
+  case Type::ObjCObjectPointer:
+    return Cache::get(cast<ObjCObjectPointerType>(T)->getPointeeType());
+  }
+
+  llvm_unreachable("unhandled type class");
 
-Linkage BlockPointerType::getLinkageImpl() const {
-  return PointeeType->getLinkage();
+  // C++ [basic.link]p8:
+  //   Names not covered by these rules have no linkage.
+  return CachedProperties(NoLinkage, DefaultVisibility, false);
 }
 
-Linkage ReferenceType::getLinkageImpl() const {
-  return PointeeType->getLinkage();
+/// \brief Determine the linkage of this type.
+Linkage Type::getLinkage() const {
+  Cache::ensure(this);
+  return TypeBits.getLinkage();
 }
 
-Linkage MemberPointerType::getLinkageImpl() const {
-  return minLinkage(Class->getLinkage(), PointeeType->getLinkage());
+/// \brief Determine the linkage of this type.
+Visibility Type::getVisibility() const {
+  Cache::ensure(this);
+  return TypeBits.getVisibility();
 }
 
-Linkage ArrayType::getLinkageImpl() const {
-  return ElementType->getLinkage();
+bool Type::hasUnnamedOrLocalType() const {
+  Cache::ensure(this);
+  return TypeBits.hasLocalOrUnnamedType();
 }
 
-Linkage VectorType::getLinkageImpl() const {
-  return ElementType->getLinkage();
+std::pair<Linkage,Visibility> Type::getLinkageAndVisibility() const {
+  Cache::ensure(this);
+  return std::make_pair(TypeBits.getLinkage(), TypeBits.getVisibility());
 }
 
-Linkage FunctionNoProtoType::getLinkageImpl() const {
-  return getResultType()->getLinkage();
+void Type::ClearLinkageCache() {
+  TypeBits.CacheValidAndVisibility = 0;
+  if (QualType(this, 0) != CanonicalType)
+    CanonicalType->TypeBits.CacheValidAndVisibility = 0;
 }
 
-Linkage FunctionProtoType::getLinkageImpl() const {
-  Linkage L = getResultType()->getLinkage();
-  for (arg_type_iterator A = arg_type_begin(), AEnd = arg_type_end();
-       A != AEnd; ++A)
-    L = minLinkage(L, (*A)->getLinkage());
+bool Type::hasSizedVLAType() const {
+  if (!isVariablyModifiedType()) return false;
 
-  return L;
-}
+  if (const PointerType *ptr = getAs<PointerType>())
+    return ptr->getPointeeType()->hasSizedVLAType();
+  if (const ReferenceType *ref = getAs<ReferenceType>())
+    return ref->getPointeeType()->hasSizedVLAType();
+  if (const ArrayType *arr = getAsArrayTypeUnsafe()) {
+    if (isa<VariableArrayType>(arr) && 
+        cast<VariableArrayType>(arr)->getSizeExpr())
+      return true;
+
+    return arr->getElementType()->hasSizedVLAType();
+  }
 
-Linkage ObjCObjectType::getLinkageImpl() const {
-  return ExternalLinkage;
+  return false;
 }
 
-Linkage ObjCObjectPointerType::getLinkageImpl() const {
-  return ExternalLinkage;
+QualType::DestructionKind QualType::isDestructedTypeImpl(QualType type) {
+  /// Currently, the only destruction kind we recognize is C++ objects
+  /// with non-trivial destructors.
+  const CXXRecordDecl *record =
+    type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
+  if (record && !record->hasTrivialDestructor())
+    return DK_cxx_destructor;
+
+  return DK_none;
 }
diff --git a/lib/AST/TypeLoc.cpp b/lib/AST/TypeLoc.cpp
index 66578fb3dc49..14db7f83c2d0 100644
--- a/lib/AST/TypeLoc.cpp
+++ b/lib/AST/TypeLoc.cpp
@@ -77,14 +77,15 @@ TypeLoc TypeLoc::getNextTypeLocImpl(TypeLoc TL) {
 /// \brief Initializes a type location, and all of its children
 /// recursively, as if the entire tree had been written in the
 /// given location.
-void TypeLoc::initializeImpl(TypeLoc TL, SourceLocation Loc) {
+void TypeLoc::initializeImpl(ASTContext &Context, TypeLoc TL, 
+                             SourceLocation Loc) {
   while (true) {
     switch (TL.getTypeLocClass()) {
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
 #define TYPELOC(CLASS, PARENT)        \
     case CLASS: {                     \
       CLASS##TypeLoc TLCasted = cast<CLASS##TypeLoc>(TL); \
-      TLCasted.initializeLocal(Loc);  \
+      TLCasted.initializeLocal(Context, Loc);  \
       TL = TLCasted.getNextTypeLoc(); \
       if (!TL) return;                \
       continue;                       \
@@ -187,11 +188,10 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
       return TST_char16;        
     case BuiltinType::Char32:
       return TST_char32;
-    case BuiltinType::WChar:
+    case BuiltinType::WChar_S:
+    case BuiltinType::WChar_U:
       return TST_wchar;
-    case BuiltinType::UndeducedAuto:
-      return TST_auto;
-        
+
     case BuiltinType::UChar:
     case BuiltinType::UShort:
     case BuiltinType::UInt:
@@ -222,3 +222,44 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
   
   return TST_unspecified;
 }
+
+TypeLoc TypeLoc::IgnoreParensImpl(TypeLoc TL) {
+  while (ParenTypeLoc* PTL = dyn_cast<ParenTypeLoc>(&TL))
+    TL = PTL->getInnerLoc();
+  return TL;
+}
+
+void TemplateSpecializationTypeLoc::initializeArgLocs(ASTContext &Context, 
+                                                      unsigned NumArgs,
+                                                  const TemplateArgument *Args,
+                                              TemplateArgumentLocInfo *ArgInfos,
+                                                      SourceLocation Loc) {
+  for (unsigned i = 0, e = NumArgs; i != e; ++i) {
+    switch (Args[i].getKind()) {
+    case TemplateArgument::Null: 
+    case TemplateArgument::Declaration:
+    case TemplateArgument::Integral:
+    case TemplateArgument::Pack:
+    case TemplateArgument::Expression:
+      // FIXME: Can we do better for declarations and integral values?
+      ArgInfos[i] = TemplateArgumentLocInfo();
+      break;
+      
+    case TemplateArgument::Type:
+      ArgInfos[i] = TemplateArgumentLocInfo(
+                          Context.getTrivialTypeSourceInfo(Args[i].getAsType(), 
+                                                           Loc));
+      break;
+        
+    case TemplateArgument::Template:
+      ArgInfos[i] = TemplateArgumentLocInfo(SourceRange(Loc), Loc, 
+                                            SourceLocation());
+      break;
+      
+    case TemplateArgument::TemplateExpansion:
+      ArgInfos[i] = TemplateArgumentLocInfo(SourceRange(Loc), Loc, Loc);
+      break;
+    }
+  }
+}
+
diff --git a/lib/AST/TypePrinter.cpp b/lib/AST/TypePrinter.cpp
index d3a6b645537c..139073987a0e 100644
--- a/lib/AST/TypePrinter.cpp
+++ b/lib/AST/TypePrinter.cpp
@@ -30,12 +30,13 @@ namespace {
   public:
     explicit TypePrinter(const PrintingPolicy &Policy) : Policy(Policy) { }
 
-    void Print(QualType T, std::string &S);
+    void print(const Type *ty, Qualifiers qs, std::string &buffer);
+    void print(QualType T, std::string &S);
     void AppendScope(DeclContext *DC, std::string &S);
-    void PrintTag(TagDecl *T, std::string &S);
+    void printTag(TagDecl *T, std::string &S);
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) \
-  void Print##CLASS(const CLASS##Type *T, std::string &S);
+    void print##CLASS(const CLASS##Type *T, std::string &S);
 #include "clang/AST/TypeNodes.def"
   };
 }
@@ -55,9 +56,14 @@ static void AppendTypeQualList(std::string &S, unsigned TypeQuals) {
   }
 }
 
-void TypePrinter::Print(QualType T, std::string &S) {
-  if (T.isNull()) {
-    S += "NULL TYPE";
+void TypePrinter::print(QualType t, std::string &buffer) {
+  SplitQualType split = t.split();
+  print(split.first, split.second, buffer);
+}
+
+void TypePrinter::print(const Type *T, Qualifiers Quals, std::string &buffer) {
+  if (!T) {
+    buffer += "NULL TYPE";
     return;
   }
   
@@ -65,28 +71,104 @@ void TypePrinter::Print(QualType T, std::string &S) {
     return;
   
   // Print qualifiers as appropriate.
-  Qualifiers Quals = T.getLocalQualifiers();
-  if (!Quals.empty()) {
-    std::string TQS;
-    Quals.getAsStringInternal(TQS, Policy);
+  
+  // CanPrefixQualifiers - We prefer to print type qualifiers before the type,
+  // so that we get "const int" instead of "int const", but we can't do this if
+  // the type is complex.  For example if the type is "int*", we *must* print
+  // "int * const", printing "const int *" is different.  Only do this when the
+  // type expands to a simple string.
+  bool CanPrefixQualifiers = false;
+  
+  Type::TypeClass TC = T->getTypeClass();
+  if (const AutoType *AT = dyn_cast<AutoType>(T))
+    TC = AT->desugar()->getTypeClass();
+  if (const SubstTemplateTypeParmType *Subst
+                                      = dyn_cast<SubstTemplateTypeParmType>(T))
+    TC = Subst->getReplacementType()->getTypeClass();
+  
+  switch (TC) {
+    case Type::Builtin:
+    case Type::Complex:
+    case Type::UnresolvedUsing:
+    case Type::Typedef:
+    case Type::TypeOfExpr:
+    case Type::TypeOf:
+    case Type::Decltype:
+    case Type::Record:
+    case Type::Enum:
+    case Type::Elaborated:
+    case Type::TemplateTypeParm:
+    case Type::SubstTemplateTypeParmPack:
+    case Type::TemplateSpecialization:
+    case Type::InjectedClassName:
+    case Type::DependentName:
+    case Type::DependentTemplateSpecialization:
+    case Type::ObjCObject:
+    case Type::ObjCInterface:
+      CanPrefixQualifiers = true;
+      break;
+      
+    case Type::ObjCObjectPointer:
+      CanPrefixQualifiers = T->isObjCIdType() || T->isObjCClassType() ||
+        T->isObjCQualifiedIdType() || T->isObjCQualifiedClassType();
+      break;
+      
+    case Type::Pointer:
+    case Type::BlockPointer:
+    case Type::LValueReference:
+    case Type::RValueReference:
+    case Type::MemberPointer:
+    case Type::ConstantArray:
+    case Type::IncompleteArray:
+    case Type::VariableArray:
+    case Type::DependentSizedArray:
+    case Type::DependentSizedExtVector:
+    case Type::Vector:
+    case Type::ExtVector:
+    case Type::FunctionProto:
+    case Type::FunctionNoProto:
+    case Type::Paren:
+    case Type::Attributed:
+    case Type::PackExpansion:
+    case Type::SubstTemplateTypeParm:
+    case Type::Auto:
+      CanPrefixQualifiers = false;
+      break;
+  }
+  
+  if (!CanPrefixQualifiers && !Quals.empty()) {
+    std::string qualsBuffer;
+    Quals.getAsStringInternal(qualsBuffer, Policy);
     
-    if (!S.empty()) {
-      TQS += ' ';
-      TQS += S;
+    if (!buffer.empty()) {
+      qualsBuffer += ' ';
+      qualsBuffer += buffer;
     }
-    std::swap(S, TQS);
+    std::swap(buffer, qualsBuffer);
   }
   
   switch (T->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
-#define TYPE(CLASS, PARENT) case Type::CLASS:                \
-    Print##CLASS(cast<CLASS##Type>(T.getTypePtr()), S);      \
+#define TYPE(CLASS, PARENT) case Type::CLASS: \
+    print##CLASS(cast<CLASS##Type>(T), buffer); \
     break;
 #include "clang/AST/TypeNodes.def"
   }
+  
+  // If we're adding the qualifiers as a prefix, do it now.
+  if (CanPrefixQualifiers && !Quals.empty()) {
+    std::string qualsBuffer;
+    Quals.getAsStringInternal(qualsBuffer, Policy);
+    
+    if (!buffer.empty()) {
+      qualsBuffer += ' ';
+      qualsBuffer += buffer;
+    }
+    std::swap(buffer, qualsBuffer);
+  }
 }
 
-void TypePrinter::PrintBuiltin(const BuiltinType *T, std::string &S) {
+void TypePrinter::printBuiltin(const BuiltinType *T, std::string &S) {
   if (S.empty()) {
     S = T->getName(Policy.LangOpts);
   } else {
@@ -96,12 +178,12 @@ void TypePrinter::PrintBuiltin(const BuiltinType *T, std::string &S) {
   }
 }
 
-void TypePrinter::PrintComplex(const ComplexType *T, std::string &S) {
-  Print(T->getElementType(), S);
+void TypePrinter::printComplex(const ComplexType *T, std::string &S) {
+  print(T->getElementType(), S);
   S = "_Complex " + S;
 }
 
-void TypePrinter::PrintPointer(const PointerType *T, std::string &S) { 
+void TypePrinter::printPointer(const PointerType *T, std::string &S) { 
   S = '*' + S;
   
   // Handle things like 'int (*A)[4];' correctly.
@@ -109,15 +191,15 @@ void TypePrinter::PrintPointer(const PointerType *T, std::string &S) {
   if (isa<ArrayType>(T->getPointeeType()))
     S = '(' + S + ')';
   
-  Print(T->getPointeeType(), S);
+  print(T->getPointeeType(), S);
 }
 
-void TypePrinter::PrintBlockPointer(const BlockPointerType *T, std::string &S) {
+void TypePrinter::printBlockPointer(const BlockPointerType *T, std::string &S) {
   S = '^' + S;
-  Print(T->getPointeeType(), S);
+  print(T->getPointeeType(), S);
 }
 
-void TypePrinter::PrintLValueReference(const LValueReferenceType *T, 
+void TypePrinter::printLValueReference(const LValueReferenceType *T, 
                                        std::string &S) { 
   S = '&' + S;
   
@@ -126,10 +208,10 @@ void TypePrinter::PrintLValueReference(const LValueReferenceType *T,
   if (isa<ArrayType>(T->getPointeeTypeAsWritten()))
     S = '(' + S + ')';
   
-  Print(T->getPointeeTypeAsWritten(), S);
+  print(T->getPointeeTypeAsWritten(), S);
 }
 
-void TypePrinter::PrintRValueReference(const RValueReferenceType *T, 
+void TypePrinter::printRValueReference(const RValueReferenceType *T, 
                                        std::string &S) { 
   S = "&&" + S;
   
@@ -138,13 +220,13 @@ void TypePrinter::PrintRValueReference(const RValueReferenceType *T,
   if (isa<ArrayType>(T->getPointeeTypeAsWritten()))
     S = '(' + S + ')';
   
-  Print(T->getPointeeTypeAsWritten(), S);
+  print(T->getPointeeTypeAsWritten(), S);
 }
 
-void TypePrinter::PrintMemberPointer(const MemberPointerType *T, 
+void TypePrinter::printMemberPointer(const MemberPointerType *T, 
                                      std::string &S) { 
   std::string C;
-  Print(QualType(T->getClass(), 0), C);
+  print(QualType(T->getClass(), 0), C);
   C += "::*";
   S = C + S;
   
@@ -153,25 +235,25 @@ void TypePrinter::PrintMemberPointer(const MemberPointerType *T,
   if (isa<ArrayType>(T->getPointeeType()))
     S = '(' + S + ')';
   
-  Print(T->getPointeeType(), S);
+  print(T->getPointeeType(), S);
 }
 
-void TypePrinter::PrintConstantArray(const ConstantArrayType *T, 
+void TypePrinter::printConstantArray(const ConstantArrayType *T, 
                                      std::string &S) {
   S += '[';
   S += llvm::utostr(T->getSize().getZExtValue());
   S += ']';
   
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
 }
 
-void TypePrinter::PrintIncompleteArray(const IncompleteArrayType *T, 
+void TypePrinter::printIncompleteArray(const IncompleteArrayType *T, 
                                        std::string &S) {
   S += "[]";
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
 }
 
-void TypePrinter::PrintVariableArray(const VariableArrayType *T, 
+void TypePrinter::printVariableArray(const VariableArrayType *T, 
                                      std::string &S) { 
   S += '[';
   
@@ -193,10 +275,10 @@ void TypePrinter::PrintVariableArray(const VariableArrayType *T,
   }
   S += ']';
   
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
 }
 
-void TypePrinter::PrintDependentSizedArray(const DependentSizedArrayType *T, 
+void TypePrinter::printDependentSizedArray(const DependentSizedArrayType *T, 
                                            std::string &S) {  
   S += '[';
   
@@ -208,13 +290,13 @@ void TypePrinter::PrintDependentSizedArray(const DependentSizedArrayType *T,
   }
   S += ']';
   
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
 }
 
-void TypePrinter::PrintDependentSizedExtVector(
+void TypePrinter::printDependentSizedExtVector(
                                           const DependentSizedExtVectorType *T, 
                                                std::string &S) { 
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
   
   S += " __attribute__((ext_vector_type(";
   if (T->getSizeExpr()) {
@@ -226,36 +308,52 @@ void TypePrinter::PrintDependentSizedExtVector(
   S += ")))";  
 }
 
-void TypePrinter::PrintVector(const VectorType *T, std::string &S) { 
-  if (T->getAltiVecSpecific() != VectorType::NotAltiVec) {
-    if (T->getAltiVecSpecific() == VectorType::Pixel)
-      S = "__vector __pixel " + S;
-    else {
-      Print(T->getElementType(), S);
-      S = ((T->getAltiVecSpecific() == VectorType::Bool)
-           ? "__vector __bool " : "__vector ") + S;
-    }
-  } else {
+void TypePrinter::printVector(const VectorType *T, std::string &S) { 
+  switch (T->getVectorKind()) {
+  case VectorType::AltiVecPixel:
+    S = "__vector __pixel " + S;
+    break;
+  case VectorType::AltiVecBool:
+    print(T->getElementType(), S);
+    S = "__vector __bool " + S;
+    break;
+  case VectorType::AltiVecVector:
+    print(T->getElementType(), S);
+    S = "__vector " + S;
+    break;
+  case VectorType::NeonVector:
+    print(T->getElementType(), S);
+    S = ("__attribute__((neon_vector_type(" +
+         llvm::utostr_32(T->getNumElements()) + "))) " + S);
+    break;
+  case VectorType::NeonPolyVector:
+    print(T->getElementType(), S);
+    S = ("__attribute__((neon_polyvector_type(" +
+         llvm::utostr_32(T->getNumElements()) + "))) " + S);
+    break;
+  case VectorType::GenericVector: {
     // FIXME: We prefer to print the size directly here, but have no way
     // to get the size of the type.
-    Print(T->getElementType(), S);
+    print(T->getElementType(), S);
     std::string V = "__attribute__((__vector_size__(";
     V += llvm::utostr_32(T->getNumElements()); // convert back to bytes.
     std::string ET;
-    Print(T->getElementType(), ET);
+    print(T->getElementType(), ET);
     V += " * sizeof(" + ET + ")))) ";
     S = V + S;
+    break;
+  }
   }
 }
 
-void TypePrinter::PrintExtVector(const ExtVectorType *T, std::string &S) { 
+void TypePrinter::printExtVector(const ExtVectorType *T, std::string &S) { 
   S += " __attribute__((ext_vector_type(";
   S += llvm::utostr_32(T->getNumElements());
   S += ")))";
-  Print(T->getElementType(), S);
+  print(T->getElementType(), S);
 }
 
-void TypePrinter::PrintFunctionProto(const FunctionProtoType *T, 
+void TypePrinter::printFunctionProto(const FunctionProtoType *T, 
                                      std::string &S) { 
   // If needed for precedence reasons, wrap the inner part in grouping parens.
   if (!S.empty())
@@ -267,7 +365,7 @@ void TypePrinter::PrintFunctionProto(const FunctionProtoType *T,
   ParamPolicy.SuppressSpecifiers = false;
   for (unsigned i = 0, e = T->getNumArgs(); i != e; ++i) {
     if (i) S += ", ";
-    Print(T->getArgType(i), Tmp);
+    print(T->getArgType(i), Tmp);
     S += Tmp;
     Tmp.clear();
   }
@@ -309,6 +407,21 @@ void TypePrinter::PrintFunctionProto(const FunctionProtoType *T,
     S += " __attribute__((regparm (" +
         llvm::utostr_32(Info.getRegParm()) + ")))";
   
+  AppendTypeQualList(S, T->getTypeQuals());
+
+  switch (T->getRefQualifier()) {
+  case RQ_None:
+    break;
+    
+  case RQ_LValue:
+    S += " &";
+    break;
+    
+  case RQ_RValue:
+    S += " &&";
+    break;
+  }
+  
   if (T->hasExceptionSpec()) {
     S += " throw(";
     if (T->hasAnyExceptionSpec())
@@ -319,18 +432,16 @@ void TypePrinter::PrintFunctionProto(const FunctionProtoType *T,
           S += ", ";
 
         std::string ExceptionType;
-        Print(T->getExceptionType(I), ExceptionType);
+        print(T->getExceptionType(I), ExceptionType);
         S += ExceptionType;
       }
     S += ")";
   }
 
-  AppendTypeQualList(S, T->getTypeQuals());
-  
-  Print(T->getResultType(), S);
+  print(T->getResultType(), S);
 }
 
-void TypePrinter::PrintFunctionNoProto(const FunctionNoProtoType *T, 
+void TypePrinter::printFunctionNoProto(const FunctionNoProtoType *T, 
                                        std::string &S) { 
   // If needed for precedence reasons, wrap the inner part in grouping parens.
   if (!S.empty())
@@ -339,10 +450,10 @@ void TypePrinter::PrintFunctionNoProto(const FunctionNoProtoType *T,
   S += "()";
   if (T->getNoReturnAttr())
     S += " __attribute__((noreturn))";
-  Print(T->getResultType(), S);
+  print(T->getResultType(), S);
 }
 
-static void PrintTypeSpec(const NamedDecl *D, std::string &S) {
+static void printTypeSpec(const NamedDecl *D, std::string &S) {
   IdentifierInfo *II = D->getIdentifier();
   if (S.empty())
     S = II->getName().str();
@@ -350,16 +461,16 @@ static void PrintTypeSpec(const NamedDecl *D, std::string &S) {
     S = II->getName().str() + ' ' + S;
 }
 
-void TypePrinter::PrintUnresolvedUsing(const UnresolvedUsingType *T,
+void TypePrinter::printUnresolvedUsing(const UnresolvedUsingType *T,
                                        std::string &S) {
-  PrintTypeSpec(T->getDecl(), S);
+  printTypeSpec(T->getDecl(), S);
 }
 
-void TypePrinter::PrintTypedef(const TypedefType *T, std::string &S) { 
-  PrintTypeSpec(T->getDecl(), S);
+void TypePrinter::printTypedef(const TypedefType *T, std::string &S) { 
+  printTypeSpec(T->getDecl(), S);
 }
 
-void TypePrinter::PrintTypeOfExpr(const TypeOfExprType *T, std::string &S) {
+void TypePrinter::printTypeOfExpr(const TypeOfExprType *T, std::string &S) {
   if (!S.empty())    // Prefix the basic type, e.g. 'typeof(e) X'.
     S = ' ' + S;
   std::string Str;
@@ -368,15 +479,15 @@ void TypePrinter::PrintTypeOfExpr(const TypeOfExprType *T, std::string &S) {
   S = "typeof " + s.str() + S;
 }
 
-void TypePrinter::PrintTypeOf(const TypeOfType *T, std::string &S) { 
+void TypePrinter::printTypeOf(const TypeOfType *T, std::string &S) { 
   if (!S.empty())    // Prefix the basic type, e.g. 'typeof(t) X'.
     S = ' ' + S;
   std::string Tmp;
-  Print(T->getUnderlyingType(), Tmp);
+  print(T->getUnderlyingType(), Tmp);
   S = "typeof(" + Tmp + ")" + S;
 }
 
-void TypePrinter::PrintDecltype(const DecltypeType *T, std::string &S) { 
+void TypePrinter::printDecltype(const DecltypeType *T, std::string &S) { 
   if (!S.empty())    // Prefix the basic type, e.g. 'decltype(t) X'.
     S = ' ' + S;
   std::string Str;
@@ -385,6 +496,17 @@ void TypePrinter::PrintDecltype(const DecltypeType *T, std::string &S) {
   S = "decltype(" + s.str() + ")" + S;
 }
 
+void TypePrinter::printAuto(const AutoType *T, std::string &S) { 
+  // If the type has been deduced, do not print 'auto'.
+  if (T->isDeduced()) {
+    print(T->getDeducedType(), S);
+  } else {
+    if (!S.empty())    // Prefix the basic type, e.g. 'auto X'.
+      S = ' ' + S;
+    S = "auto" + S;
+  }
+}
+
 /// Appends the given scope to the end of a string.
 void TypePrinter::AppendScope(DeclContext *DC, std::string &Buffer) {
   if (DC->isTranslationUnit()) return;
@@ -402,8 +524,8 @@ void TypePrinter::AppendScope(DeclContext *DC, std::string &Buffer) {
     const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
     std::string TemplateArgsStr
       = TemplateSpecializationType::PrintTemplateArgumentList(
-                                            TemplateArgs.getFlatArgumentList(),
-                                            TemplateArgs.flat_size(),
+                                            TemplateArgs.data(),
+                                            TemplateArgs.size(),
                                             Policy);
     Buffer += Spec->getIdentifier()->getName();
     Buffer += TemplateArgsStr;
@@ -418,7 +540,7 @@ void TypePrinter::AppendScope(DeclContext *DC, std::string &Buffer) {
     Buffer += "::";
 }
 
-void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
+void TypePrinter::printTag(TagDecl *D, std::string &InnerString) {
   if (Policy.SuppressTag)
     return;
 
@@ -457,9 +579,9 @@ void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
       if (!HasKindDecoration)
         OS << " " << D->getKindName();
 
-      if (D->getLocation().isValid()) {
-        PresumedLoc PLoc = D->getASTContext().getSourceManager().getPresumedLoc(
+      PresumedLoc PLoc = D->getASTContext().getSourceManager().getPresumedLoc(
           D->getLocation());
+      if (PLoc.isValid()) {
         OS << " at " << PLoc.getFilename()
            << ':' << PLoc.getLine()
            << ':' << PLoc.getColumn();
@@ -482,8 +604,8 @@ void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
       NumArgs = TST->getNumArgs();
     } else {
       const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-      Args = TemplateArgs.getFlatArgumentList();
-      NumArgs = TemplateArgs.flat_size();
+      Args = TemplateArgs.data();
+      NumArgs = TemplateArgs.size();
     }
     Buffer += TemplateSpecializationType::PrintTemplateArgumentList(Args,
                                                                     NumArgs,
@@ -498,15 +620,15 @@ void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
   std::swap(Buffer, InnerString);
 }
 
-void TypePrinter::PrintRecord(const RecordType *T, std::string &S) {
-  PrintTag(T->getDecl(), S);
+void TypePrinter::printRecord(const RecordType *T, std::string &S) {
+  printTag(T->getDecl(), S);
 }
 
-void TypePrinter::PrintEnum(const EnumType *T, std::string &S) { 
-  PrintTag(T->getDecl(), S);
+void TypePrinter::printEnum(const EnumType *T, std::string &S) { 
+  printTag(T->getDecl(), S);
 }
 
-void TypePrinter::PrintTemplateTypeParm(const TemplateTypeParmType *T, 
+void TypePrinter::printTemplateTypeParm(const TemplateTypeParmType *T, 
                                         std::string &S) { 
   if (!S.empty())    // Prefix the basic type, e.g. 'parmname X'.
     S = ' ' + S;
@@ -518,12 +640,18 @@ void TypePrinter::PrintTemplateTypeParm(const TemplateTypeParmType *T,
     S = T->getName()->getName().str() + S;  
 }
 
-void TypePrinter::PrintSubstTemplateTypeParm(const SubstTemplateTypeParmType *T, 
+void TypePrinter::printSubstTemplateTypeParm(const SubstTemplateTypeParmType *T, 
                                              std::string &S) { 
-  Print(T->getReplacementType(), S);
+  print(T->getReplacementType(), S);
 }
 
-void TypePrinter::PrintTemplateSpecialization(
+void TypePrinter::printSubstTemplateTypeParmPack(
+                                        const SubstTemplateTypeParmPackType *T, 
+                                             std::string &S) { 
+  printTemplateTypeParm(T->getReplacedParameter(), S);
+}
+
+void TypePrinter::printTemplateSpecialization(
                                             const TemplateSpecializationType *T, 
                                               std::string &S) { 
   std::string SpecString;
@@ -543,12 +671,12 @@ void TypePrinter::PrintTemplateSpecialization(
     S = SpecString + ' ' + S;
 }
 
-void TypePrinter::PrintInjectedClassName(const InjectedClassNameType *T,
+void TypePrinter::printInjectedClassName(const InjectedClassNameType *T,
                                          std::string &S) {
-  PrintTemplateSpecialization(T->getInjectedTST(), S);
+  printTemplateSpecialization(T->getInjectedTST(), S);
 }
 
-void TypePrinter::PrintElaborated(const ElaboratedType *T, std::string &S) {
+void TypePrinter::printElaborated(const ElaboratedType *T, std::string &S) {
   std::string MyString;
   
   {
@@ -564,7 +692,7 @@ void TypePrinter::PrintElaborated(const ElaboratedType *T, std::string &S) {
   std::string TypeStr;
   PrintingPolicy InnerPolicy(Policy);
   InnerPolicy.SuppressScope = true;
-  TypePrinter(InnerPolicy).Print(T->getNamedType(), TypeStr);
+  TypePrinter(InnerPolicy).print(T->getNamedType(), TypeStr);
   
   MyString += TypeStr;
   if (S.empty())
@@ -573,7 +701,13 @@ void TypePrinter::PrintElaborated(const ElaboratedType *T, std::string &S) {
     S = MyString + ' ' + S;  
 }
 
-void TypePrinter::PrintDependentName(const DependentNameType *T, std::string &S) { 
+void TypePrinter::printParen(const ParenType *T, std::string &S) {
+  if (!S.empty() && !isa<FunctionType>(T->getInnerType()))
+    S = '(' + S + ')';
+  print(T->getInnerType(), S);
+}
+
+void TypePrinter::printDependentName(const DependentNameType *T, std::string &S) { 
   std::string MyString;
   
   {
@@ -593,7 +727,7 @@ void TypePrinter::PrintDependentName(const DependentNameType *T, std::string &S)
     S = MyString + ' ' + S;
 }
 
-void TypePrinter::PrintDependentTemplateSpecialization(
+void TypePrinter::printDependentTemplateSpecialization(
         const DependentTemplateSpecializationType *T, std::string &S) { 
   std::string MyString;
   {
@@ -617,7 +751,91 @@ void TypePrinter::PrintDependentTemplateSpecialization(
     S = MyString + ' ' + S;
 }
 
-void TypePrinter::PrintObjCInterface(const ObjCInterfaceType *T, 
+void TypePrinter::printPackExpansion(const PackExpansionType *T, 
+                                     std::string &S) {
+  print(T->getPattern(), S);
+  S += "...";
+}
+
+void TypePrinter::printAttributed(const AttributedType *T,
+                                  std::string &S) {
+  print(T->getModifiedType(), S);
+
+  // TODO: not all attributes are GCC-style attributes.
+  S += "__attribute__((";
+  switch (T->getAttrKind()) {
+  case AttributedType::attr_address_space:
+    S += "address_space(";
+    S += T->getEquivalentType().getAddressSpace();
+    S += ")";
+    break;
+
+  case AttributedType::attr_vector_size: {
+    S += "__vector_size__(";
+    if (const VectorType *vector =T->getEquivalentType()->getAs<VectorType>()) {
+      S += vector->getNumElements();
+      S += " * sizeof(";
+
+      std::string tmp;
+      print(vector->getElementType(), tmp);
+      S += tmp;
+      S += ")";
+    }
+    S += ")";
+    break;
+  }
+
+  case AttributedType::attr_neon_vector_type:
+  case AttributedType::attr_neon_polyvector_type: {
+    if (T->getAttrKind() == AttributedType::attr_neon_vector_type)
+      S += "neon_vector_type(";
+    else
+      S += "neon_polyvector_type(";
+    const VectorType *vector = T->getEquivalentType()->getAs<VectorType>();
+    S += llvm::utostr_32(vector->getNumElements());
+    S += ")";
+    break;
+  }
+
+  case AttributedType::attr_regparm: {
+    S += "regparm(";
+    QualType t = T->getEquivalentType();
+    while (!t->isFunctionType())
+      t = t->getPointeeType();
+    S += t->getAs<FunctionType>()->getRegParmType();
+    S += ")";
+    break;
+  }
+
+  case AttributedType::attr_objc_gc: {
+    S += "objc_gc(";
+
+    QualType tmp = T->getEquivalentType();
+    while (tmp.getObjCGCAttr() == Qualifiers::GCNone) {
+      QualType next = tmp->getPointeeType();
+      if (next == tmp) break;
+      tmp = next;
+    }
+
+    if (tmp.isObjCGCWeak())
+      S += "weak";
+    else
+      S += "strong";
+    S += ")";
+    break;
+  }
+
+  case AttributedType::attr_noreturn: S += "noreturn"; break;
+  case AttributedType::attr_cdecl: S += "cdecl"; break;
+  case AttributedType::attr_fastcall: S += "fastcall"; break;
+  case AttributedType::attr_stdcall: S += "stdcall"; break;
+  case AttributedType::attr_thiscall: S += "thiscall"; break;
+  case AttributedType::attr_pascal: S += "pascal"; break;
+  }
+  S += "))";
+}
+
+void TypePrinter::printObjCInterface(const ObjCInterfaceType *T, 
                                      std::string &S) { 
   if (!S.empty())    // Prefix the basic type, e.g. 'typedefname X'.
     S = ' ' + S;
@@ -626,13 +844,13 @@ void TypePrinter::PrintObjCInterface(const ObjCInterfaceType *T,
   S = ObjCQIString + S;
 }
 
-void TypePrinter::PrintObjCObject(const ObjCObjectType *T,
+void TypePrinter::printObjCObject(const ObjCObjectType *T,
                                   std::string &S) {
   if (T->qual_empty())
-    return Print(T->getBaseType(), S);
+    return print(T->getBaseType(), S);
 
   std::string tmp;
-  Print(T->getBaseType(), tmp);
+  print(T->getBaseType(), tmp);
   tmp += '<';
   bool isFirst = true;
   for (ObjCObjectType::qual_iterator
@@ -652,18 +870,23 @@ void TypePrinter::PrintObjCObject(const ObjCObjectType *T,
   std::swap(tmp, S);
 }
 
-void TypePrinter::PrintObjCObjectPointer(const ObjCObjectPointerType *T, 
+void TypePrinter::printObjCObjectPointer(const ObjCObjectPointerType *T, 
                                          std::string &S) { 
   std::string ObjCQIString;
   
+  T->getPointeeType().getLocalQualifiers().getAsStringInternal(ObjCQIString, 
+                                                               Policy);
+  if (!ObjCQIString.empty())
+    ObjCQIString += ' ';
+    
   if (T->isObjCIdType() || T->isObjCQualifiedIdType())
-    ObjCQIString = "id";
+    ObjCQIString += "id";
   else if (T->isObjCClassType() || T->isObjCQualifiedClassType())
-    ObjCQIString = "Class";
+    ObjCQIString += "Class";
   else if (T->isObjCSelType())
-    ObjCQIString = "SEL";
+    ObjCQIString += "SEL";
   else
-    ObjCQIString = T->getInterfaceDecl()->getNameAsString();
+    ObjCQIString += T->getInterfaceDecl()->getNameAsString();
   
   if (!T->qual_empty()) {
     ObjCQIString += '<';
@@ -677,9 +900,6 @@ void TypePrinter::PrintObjCObjectPointer(const ObjCObjectPointerType *T,
     ObjCQIString += '>';
   }
   
-  T->getPointeeType().getLocalQualifiers().getAsStringInternal(ObjCQIString, 
-                                                               Policy);
-  
   if (!T->isObjCIdType() && !T->isObjCQualifiedIdType())
     ObjCQIString += " *"; // Don't forget the implicit pointer.
   else if (!S.empty()) // Prefix the basic type, e.g. 'typedefname X'.
@@ -688,44 +908,6 @@ void TypePrinter::PrintObjCObjectPointer(const ObjCObjectPointerType *T,
   S = ObjCQIString + S;  
 }
 
-static void PrintTemplateArgument(std::string &Buffer,
-                                  const TemplateArgument &Arg,
-                                  const PrintingPolicy &Policy) {
-  switch (Arg.getKind()) {
-    case TemplateArgument::Null:
-      assert(false && "Null template argument");
-      break;
-      
-    case TemplateArgument::Type:
-      Arg.getAsType().getAsStringInternal(Buffer, Policy);
-      break;
-      
-    case TemplateArgument::Declaration:
-      Buffer = cast<NamedDecl>(Arg.getAsDecl())->getNameAsString();
-      break;
-      
-    case TemplateArgument::Template: {
-      llvm::raw_string_ostream s(Buffer);
-      Arg.getAsTemplate().print(s, Policy);
-      break;
-    }
-      
-    case TemplateArgument::Integral:
-      Buffer = Arg.getAsIntegral()->toString(10, true);
-      break;
-      
-    case TemplateArgument::Expression: {
-      llvm::raw_string_ostream s(Buffer);
-      Arg.getAsExpr()->printPretty(s, 0, Policy);
-      break;
-    }
-      
-    case TemplateArgument::Pack:
-      assert(0 && "FIXME: Implement!");
-      break;
-  }
-}
-
 std::string TemplateSpecializationType::
   PrintTemplateArgumentList(const TemplateArgumentListInfo &Args,
                             const PrintingPolicy &Policy) {
@@ -738,17 +920,27 @@ std::string
 TemplateSpecializationType::PrintTemplateArgumentList(
                                                 const TemplateArgument *Args,
                                                 unsigned NumArgs,
-                                                const PrintingPolicy &Policy) {
+                                                  const PrintingPolicy &Policy,
+                                                      bool SkipBrackets) {
   std::string SpecString;
-  SpecString += '<';
+  if (!SkipBrackets)
+    SpecString += '<';
+  
   for (unsigned Arg = 0; Arg < NumArgs; ++Arg) {
-    if (Arg)
+    if (SpecString.size() > !SkipBrackets)
       SpecString += ", ";
     
     // Print the argument into a string.
     std::string ArgString;
-    PrintTemplateArgument(ArgString, Args[Arg], Policy);
-    
+    if (Args[Arg].getKind() == TemplateArgument::Pack) {
+      ArgString = PrintTemplateArgumentList(Args[Arg].pack_begin(), 
+                                            Args[Arg].pack_size(), 
+                                            Policy, true);
+    } else {
+      llvm::raw_string_ostream ArgOut(ArgString);
+      Args[Arg].print(Policy, ArgOut);
+    }
+   
     // If this is the first argument and its string representation
     // begins with the global scope specifier ('::foo'), add a space
     // to avoid printing the diagraph '<:'.
@@ -761,10 +953,11 @@ TemplateSpecializationType::PrintTemplateArgumentList(
   // If the last character of our string is '>', add another space to
   // keep the two '>''s separate tokens. We don't *have* to do this in
   // C++0x, but it's still good hygiene.
-  if (SpecString[SpecString.size() - 1] == '>')
+  if (!SpecString.empty() && SpecString[SpecString.size() - 1] == '>')
     SpecString += ' ';
   
-  SpecString += '>';
+  if (!SkipBrackets)
+    SpecString += '>';
   
   return SpecString;
 }
@@ -776,12 +969,20 @@ PrintTemplateArgumentList(const TemplateArgumentLoc *Args, unsigned NumArgs,
   std::string SpecString;
   SpecString += '<';
   for (unsigned Arg = 0; Arg < NumArgs; ++Arg) {
-    if (Arg)
+    if (SpecString.size() > 1)
       SpecString += ", ";
     
     // Print the argument into a string.
     std::string ArgString;
-    PrintTemplateArgument(ArgString, Args[Arg].getArgument(), Policy);
+    if (Args[Arg].getArgument().getKind() == TemplateArgument::Pack) {
+      ArgString = PrintTemplateArgumentList(
+                                           Args[Arg].getArgument().pack_begin(), 
+                                            Args[Arg].getArgument().pack_size(), 
+                                            Policy, true);
+    } else {
+      llvm::raw_string_ostream ArgOut(ArgString);
+      Args[Arg].getArgument().print(Policy, ArgOut);
+    }
     
     // If this is the first argument and its string representation
     // begins with the global scope specifier ('::foo'), add a space
@@ -847,15 +1048,15 @@ void Qualifiers::getAsStringInternal(std::string &S,
   }
 }
 
-std::string QualType::getAsString() const {
-  std::string S;
-  LangOptions LO;
-  getAsStringInternal(S, PrintingPolicy(LO));
-  return S;
+std::string QualType::getAsString(const Type *ty, Qualifiers qs) {
+  std::string buffer;
+  LangOptions options;
+  getAsStringInternal(ty, qs, buffer, PrintingPolicy(options));
+  return buffer;
 }
 
-void QualType::getAsStringInternal(std::string &S,
-                                   const PrintingPolicy &Policy) const {
-  TypePrinter Printer(Policy);
-  Printer.Print(*this, S);
+void QualType::getAsStringInternal(const Type *ty, Qualifiers qs,
+                                   std::string &buffer,
+                                   const PrintingPolicy &policy) {
+  TypePrinter(policy).print(ty, qs, buffer);
 }
diff --git a/lib/Analysis/AnalysisContext.cpp b/lib/Analysis/AnalysisContext.cpp
index bf9f96719001..5233d3b8f9f8 100644
--- a/lib/Analysis/AnalysisContext.cpp
+++ b/lib/Analysis/AnalysisContext.cpp
@@ -59,7 +59,11 @@ CFG *AnalysisContext::getCFG() {
     return getUnoptimizedCFG();
 
   if (!builtCFG) {
-    cfg = CFG::buildCFG(D, getBody(), &D->getASTContext(), true, AddEHEdges);
+    CFG::BuildOptions B;
+    B.AddEHEdges = AddEHEdges;
+    B.AddImplicitDtors = AddImplicitDtors;
+    B.AddInitializers = AddInitializers;
+    cfg = CFG::buildCFG(D, getBody(), &D->getASTContext(), B);
     // Even when the cfg is not successfully built, we don't
     // want to try building it again.
     builtCFG = true;
@@ -69,8 +73,12 @@ CFG *AnalysisContext::getCFG() {
 
 CFG *AnalysisContext::getUnoptimizedCFG() {
   if (!builtCompleteCFG) {
-    completeCFG = CFG::buildCFG(D, getBody(), &D->getASTContext(),
-                                false, AddEHEdges);
+    CFG::BuildOptions B;
+    B.PruneTriviallyFalseEdges = false;
+    B.AddEHEdges = AddEHEdges;
+    B.AddImplicitDtors = AddImplicitDtors;
+    B.AddInitializers = AddInitializers;
+    completeCFG = CFG::buildCFG(D, getBody(), &D->getASTContext(), B);
     // Even when the cfg is not successfully built, we don't
     // want to try building it again.
     builtCompleteCFG = true;
@@ -78,6 +86,10 @@ CFG *AnalysisContext::getUnoptimizedCFG() {
   return completeCFG;
 }
 
+void AnalysisContext::dumpCFG() {
+    getCFG()->dump(getASTContext().getLangOptions());
+}
+
 ParentMap &AnalysisContext::getParentMap() {
   if (!PM)
     PM = new ParentMap(getBody());
@@ -122,7 +134,8 @@ AnalysisContext *AnalysisContextManager::getContext(const Decl *D,
                                                     idx::TranslationUnit *TU) {
   AnalysisContext *&AC = Contexts[D];
   if (!AC)
-    AC = new AnalysisContext(D, TU, UseUnoptimizedCFG);
+    AC = new AnalysisContext(D, TU, UseUnoptimizedCFG, false,
+        AddImplicitDtors, AddInitializers);
 
   return AC;
 }
@@ -179,8 +192,8 @@ LocationContextManager::getLocationContext(AnalysisContext *ctx,
 const StackFrameContext*
 LocationContextManager::getStackFrame(AnalysisContext *ctx,
                                       const LocationContext *parent,
-                                      const Stmt *s, const CFGBlock *blk,
-                                      unsigned idx) {
+                                      const Stmt *s,
+                                      const CFGBlock *blk, unsigned idx) {
   llvm::FoldingSetNodeID ID;
   StackFrameContext::Profile(ID, ctx, parent, s, blk, idx);
   void *InsertPos;
@@ -260,7 +273,7 @@ public:
   }
 
   void VisitStmt(Stmt *S) {
-    for (Stmt::child_iterator I = S->child_begin(), E = S->child_end();I!=E;++I)
+    for (Stmt::child_range I = S->children(); I; ++I)
       if (Stmt *child = *I)
         Visit(child);
   }
diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp
index c97b9165bca2..a0ec5febbeff 100644
--- a/lib/Analysis/CFG.cpp
+++ b/lib/Analysis/CFG.cpp
@@ -32,24 +32,181 @@ static SourceLocation GetEndLoc(Decl* D) {
   if (VarDecl* VD = dyn_cast<VarDecl>(D))
     if (Expr* Ex = VD->getInit())
       return Ex->getSourceRange().getEnd();
-
   return D->getLocation();
 }
 
+/// The CFG builder uses a recursive algorithm to build the CFG.  When
+///  we process an expression, sometimes we know that we must add the
+///  subexpressions as block-level expressions.  For example:
+///
+///    exp1 || exp2
+///
+///  When processing the '||' expression, we know that exp1 and exp2
+///  need to be added as block-level expressions, even though they
+///  might not normally need to be.  AddStmtChoice records this
+///  contextual information.  If AddStmtChoice is 'NotAlwaysAdd', then
+///  the builder has an option not to add a subexpression as a
+///  block-level expression.
+///
 class AddStmtChoice {
 public:
-  enum Kind { NotAlwaysAdd = 0,
-              AlwaysAdd = 1,
-              AsLValueNotAlwaysAdd = 2,
-              AlwaysAddAsLValue = 3 };
+  enum Kind { NotAlwaysAdd = 0, AlwaysAdd = 1 };
 
-  AddStmtChoice(Kind kind) : k(kind) {}
+  AddStmtChoice(Kind a_kind = NotAlwaysAdd) : kind(a_kind) {}
 
-  bool alwaysAdd() const { return (unsigned)k & 0x1; }
-  bool asLValue() const { return k >= AsLValueNotAlwaysAdd; }
+  bool alwaysAdd() const { return kind & AlwaysAdd; }
+
+  /// Return a copy of this object, except with the 'always-add' bit
+  ///  set as specified.
+  AddStmtChoice withAlwaysAdd(bool alwaysAdd) const {
+    return AddStmtChoice(alwaysAdd ? Kind(kind | AlwaysAdd) :
+                                     Kind(kind & ~AlwaysAdd));
+  }
 
 private:
-  Kind k;
+  Kind kind;
+};
+
+/// LocalScope - Node in tree of local scopes created for C++ implicit
+/// destructor calls generation. It contains list of automatic variables
+/// declared in the scope and link to position in previous scope this scope
+/// began in.
+///
+/// The process of creating local scopes is as follows:
+/// - Init CFGBuilder::ScopePos with invalid position (equivalent for null),
+/// - Before processing statements in scope (e.g. CompoundStmt) create
+///   LocalScope object using CFGBuilder::ScopePos as link to previous scope
+///   and set CFGBuilder::ScopePos to the end of new scope,
+/// - On every occurrence of VarDecl increase CFGBuilder::ScopePos if it points
+///   at this VarDecl,
+/// - For every normal (without jump) end of scope add to CFGBlock destructors
+///   for objects in the current scope,
+/// - For every jump add to CFGBlock destructors for objects
+///   between CFGBuilder::ScopePos and local scope position saved for jump
+///   target. Thanks to C++ restrictions on goto jumps we can be sure that
+///   jump target position will be on the path to root from CFGBuilder::ScopePos
+///   (adding any variable that doesn't need constructor to be called to
+///   LocalScope can break this assumption),
+///
+class LocalScope {
+public:
+  typedef BumpVector<VarDecl*> AutomaticVarsTy;
+
+  /// const_iterator - Iterates local scope backwards and jumps to previous
+  /// scope on reaching the beginning of currently iterated scope.
+  class const_iterator {
+    const LocalScope* Scope;
+
+    /// VarIter is guaranteed to be greater then 0 for every valid iterator.
+    /// Invalid iterator (with null Scope) has VarIter equal to 0.
+    unsigned VarIter;
+
+  public:
+    /// Create invalid iterator. Dereferencing invalid iterator is not allowed.
+    /// Incrementing invalid iterator is allowed and will result in invalid
+    /// iterator.
+    const_iterator()
+        : Scope(NULL), VarIter(0) {}
+
+    /// Create valid iterator. In case when S.Prev is an invalid iterator and
+    /// I is equal to 0, this will create invalid iterator.
+    const_iterator(const LocalScope& S, unsigned I)
+        : Scope(&S), VarIter(I) {
+      // Iterator to "end" of scope is not allowed. Handle it by going up
+      // in scopes tree possibly up to invalid iterator in the root.
+      if (VarIter == 0 && Scope)
+        *this = Scope->Prev;
+    }
+
+    VarDecl* const* operator->() const {
+      assert (Scope && "Dereferencing invalid iterator is not allowed");
+      assert (VarIter != 0 && "Iterator has invalid value of VarIter member");
+      return &Scope->Vars[VarIter - 1];
+    }
+    VarDecl* operator*() const {
+      return *this->operator->();
+    }
+
+    const_iterator& operator++() {
+      if (!Scope)
+        return *this;
+
+      assert (VarIter != 0 && "Iterator has invalid value of VarIter member");
+      --VarIter;
+      if (VarIter == 0)
+        *this = Scope->Prev;
+      return *this;
+    }
+    const_iterator operator++(int) {
+      const_iterator P = *this;
+      ++*this;
+      return P;
+    }
+
+    bool operator==(const const_iterator& rhs) const {
+      return Scope == rhs.Scope && VarIter == rhs.VarIter;
+    }
+    bool operator!=(const const_iterator& rhs) const {
+      return !(*this == rhs);
+    }
+
+    operator bool() const {
+      return *this != const_iterator();
+    }
+
+    int distance(const_iterator L);
+  };
+
+  friend class const_iterator;
+
+private:
+  BumpVectorContext ctx;
+  
+  /// Automatic variables in order of declaration.
+  AutomaticVarsTy Vars;
+  /// Iterator to variable in previous scope that was declared just before
+  /// begin of this scope.
+  const_iterator Prev;
+
+public:
+  /// Constructs empty scope linked to previous scope in specified place.
+  LocalScope(BumpVectorContext &ctx, const_iterator P)
+      : ctx(ctx), Vars(ctx, 4), Prev(P) {}
+
+  /// Begin of scope in direction of CFG building (backwards).
+  const_iterator begin() const { return const_iterator(*this, Vars.size()); }
+
+  void addVar(VarDecl* VD) {
+    Vars.push_back(VD, ctx);
+  }
+};
+
+/// distance - Calculates distance from this to L. L must be reachable from this
+/// (with use of ++ operator). Cost of calculating the distance is linear w.r.t.
+/// number of scopes between this and L.
+int LocalScope::const_iterator::distance(LocalScope::const_iterator L) {
+  int D = 0;
+  const_iterator F = *this;
+  while (F.Scope != L.Scope) {
+    assert (F != const_iterator()
+        && "L iterator is not reachable from F iterator.");
+    D += F.VarIter;
+    F = F.Scope->Prev;
+  }
+  D += F.VarIter - L.VarIter;
+  return D;
+}
+
+/// BlockScopePosPair - Structure for specifying position in CFG during its
+/// build process. It consists of CFGBlock that specifies position in CFG graph
+/// and  LocalScope::const_iterator that specifies position in LocalScope graph.
+struct BlockScopePosPair {
+  BlockScopePosPair() : block(0) {}
+  BlockScopePosPair(CFGBlock* b, LocalScope::const_iterator scopePos)
+      : block(b), scopePosition(scopePos) {}
+
+  CFGBlock *block;
+  LocalScope::const_iterator scopePosition;
 };
 
 /// CFGBuilder - This class implements CFG construction from an AST.
@@ -67,41 +224,48 @@ private:
 ///  implicit fall-throughs without extra basic blocks.
 ///
 class CFGBuilder {
+  typedef BlockScopePosPair JumpTarget;
+  typedef BlockScopePosPair JumpSource;
+
   ASTContext *Context;
   llvm::OwningPtr<CFG> cfg;
 
   CFGBlock* Block;
   CFGBlock* Succ;
-  CFGBlock* ContinueTargetBlock;
-  CFGBlock* BreakTargetBlock;
+  JumpTarget ContinueJumpTarget;
+  JumpTarget BreakJumpTarget;
   CFGBlock* SwitchTerminatedBlock;
   CFGBlock* DefaultCaseBlock;
   CFGBlock* TryTerminatedBlock;
 
-  // LabelMap records the mapping from Label expressions to their blocks.
-  typedef llvm::DenseMap<LabelStmt*,CFGBlock*> LabelMapTy;
+  // Current position in local scope.
+  LocalScope::const_iterator ScopePos;
+
+  // LabelMap records the mapping from Label expressions to their jump targets.
+  typedef llvm::DenseMap<LabelDecl*, JumpTarget> LabelMapTy;
   LabelMapTy LabelMap;
 
   // A list of blocks that end with a "goto" that must be backpatched to their
   // resolved targets upon completion of CFG construction.
-  typedef std::vector<CFGBlock*> BackpatchBlocksTy;
+  typedef std::vector<JumpSource> BackpatchBlocksTy;
   BackpatchBlocksTy BackpatchBlocks;
 
   // A list of labels whose address has been taken (for indirect gotos).
-  typedef llvm::SmallPtrSet<LabelStmt*,5> LabelSetTy;
+  typedef llvm::SmallPtrSet<LabelDecl*, 5> LabelSetTy;
   LabelSetTy AddressTakenLabels;
 
+  bool badCFG;
+  CFG::BuildOptions BuildOpts;
+
 public:
   explicit CFGBuilder() : cfg(new CFG()), // crew a new CFG
                           Block(NULL), Succ(NULL),
-                          ContinueTargetBlock(NULL), BreakTargetBlock(NULL),
                           SwitchTerminatedBlock(NULL), DefaultCaseBlock(NULL),
-                          TryTerminatedBlock(NULL) {}
+                          TryTerminatedBlock(NULL), badCFG(false) {}
 
   // buildCFG - Used by external clients to construct the CFG.
   CFG* buildCFG(const Decl *D, Stmt *Statement, ASTContext *C,
-                bool pruneTriviallyFalseEdges, bool AddEHEdges,
-                bool AddScopes);
+      CFG::BuildOptions BO);
 
 private:
   // Visitors to walk an AST and construct the CFG.
@@ -110,22 +274,33 @@ private:
   CFGBlock *VisitBlockExpr(BlockExpr* E, AddStmtChoice asc);
   CFGBlock *VisitBreakStmt(BreakStmt *B);
   CFGBlock *VisitCXXCatchStmt(CXXCatchStmt *S);
+  CFGBlock *VisitExprWithCleanups(ExprWithCleanups *E,
+      AddStmtChoice asc);
   CFGBlock *VisitCXXThrowExpr(CXXThrowExpr *T);
   CFGBlock *VisitCXXTryStmt(CXXTryStmt *S);
+  CFGBlock *VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E, 
+                                      AddStmtChoice asc);
+  CFGBlock *VisitCXXConstructExpr(CXXConstructExpr *C, AddStmtChoice asc);
+  CFGBlock *VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E,
+                                       AddStmtChoice asc);
+  CFGBlock *VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *C, 
+                                        AddStmtChoice asc);
   CFGBlock *VisitCXXMemberCallExpr(CXXMemberCallExpr *C, AddStmtChoice asc);
   CFGBlock *VisitCallExpr(CallExpr *C, AddStmtChoice asc);
   CFGBlock *VisitCaseStmt(CaseStmt *C);
   CFGBlock *VisitChooseExpr(ChooseExpr *C, AddStmtChoice asc);
   CFGBlock *VisitCompoundStmt(CompoundStmt *C);
-  CFGBlock *VisitConditionalOperator(ConditionalOperator *C, AddStmtChoice asc);
+  CFGBlock *VisitConditionalOperator(AbstractConditionalOperator *C,
+                                     AddStmtChoice asc);
   CFGBlock *VisitContinueStmt(ContinueStmt *C);
   CFGBlock *VisitDeclStmt(DeclStmt *DS);
-  CFGBlock *VisitDeclSubExpr(Decl* D);
+  CFGBlock *VisitDeclSubExpr(DeclStmt* DS);
   CFGBlock *VisitDefaultStmt(DefaultStmt *D);
   CFGBlock *VisitDoStmt(DoStmt *D);
   CFGBlock *VisitForStmt(ForStmt *F);
   CFGBlock *VisitGotoStmt(GotoStmt* G);
   CFGBlock *VisitIfStmt(IfStmt *I);
+  CFGBlock *VisitImplicitCastExpr(ImplicitCastExpr *E, AddStmtChoice asc);
   CFGBlock *VisitIndirectGotoStmt(IndirectGotoStmt *I);
   CFGBlock *VisitLabelStmt(LabelStmt *L);
   CFGBlock *VisitMemberExpr(MemberExpr *M, AddStmtChoice asc);
@@ -138,55 +313,81 @@ private:
   CFGBlock *VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E, AddStmtChoice asc);
   CFGBlock *VisitStmtExpr(StmtExpr *S, AddStmtChoice asc);
   CFGBlock *VisitSwitchStmt(SwitchStmt *S);
+  CFGBlock *VisitUnaryOperator(UnaryOperator *U, AddStmtChoice asc);
   CFGBlock *VisitWhileStmt(WhileStmt *W);
 
   CFGBlock *Visit(Stmt *S, AddStmtChoice asc = AddStmtChoice::NotAlwaysAdd);
   CFGBlock *VisitStmt(Stmt *S, AddStmtChoice asc);
   CFGBlock *VisitChildren(Stmt* S);
 
+  // Visitors to walk an AST and generate destructors of temporaries in
+  // full expression.
+  CFGBlock *VisitForTemporaryDtors(Stmt *E, bool BindToTemporary = false);
+  CFGBlock *VisitChildrenForTemporaryDtors(Stmt *E);
+  CFGBlock *VisitBinaryOperatorForTemporaryDtors(BinaryOperator *E);
+  CFGBlock *VisitCXXBindTemporaryExprForTemporaryDtors(CXXBindTemporaryExpr *E,
+      bool BindToTemporary);
+  CFGBlock *
+  VisitConditionalOperatorForTemporaryDtors(AbstractConditionalOperator *E,
+                                            bool BindToTemporary);
+
   // NYS == Not Yet Supported
   CFGBlock* NYS() {
     badCFG = true;
     return Block;
   }
 
-  CFGBlock *StartScope(Stmt *S, CFGBlock *B) {
-    if (!AddScopes)
-      return B;
-
-    if (B == 0)
-      B = createBlock();
-    B->StartScope(S, cfg->getBumpVectorContext());
-    return B;
-  }
-
-  void EndScope(Stmt *S) {
-    if (!AddScopes)
-      return;
-
-    if (Block == 0)
-      Block = createBlock();
-    Block->EndScope(S, cfg->getBumpVectorContext());
-  }
-
   void autoCreateBlock() { if (!Block) Block = createBlock(); }
   CFGBlock *createBlock(bool add_successor = true);
-  bool FinishBlock(CFGBlock* B);
+
   CFGBlock *addStmt(Stmt *S) {
     return Visit(S, AddStmtChoice::AlwaysAdd);
   }
+  CFGBlock *addInitializer(CXXCtorInitializer *I);
+  void addAutomaticObjDtors(LocalScope::const_iterator B,
+                            LocalScope::const_iterator E, Stmt* S);
+  void addImplicitDtorsForDestructor(const CXXDestructorDecl *DD);
+
+  // Local scopes creation.
+  LocalScope* createOrReuseLocalScope(LocalScope* Scope);
+
+  void addLocalScopeForStmt(Stmt* S);
+  LocalScope* addLocalScopeForDeclStmt(DeclStmt* DS, LocalScope* Scope = NULL);
+  LocalScope* addLocalScopeForVarDecl(VarDecl* VD, LocalScope* Scope = NULL);
 
-  void AppendStmt(CFGBlock *B, Stmt *S,
+  void addLocalScopeAndDtors(Stmt* S);
+
+  // Interface to CFGBlock - adding CFGElements.
+  void appendStmt(CFGBlock *B, Stmt *S,
                   AddStmtChoice asc = AddStmtChoice::AlwaysAdd) {
-    B->appendStmt(S, cfg->getBumpVectorContext(), asc.asLValue());
+    B->appendStmt(S, cfg->getBumpVectorContext());
+  }
+  void appendInitializer(CFGBlock *B, CXXCtorInitializer *I) {
+    B->appendInitializer(I, cfg->getBumpVectorContext());
+  }
+  void appendBaseDtor(CFGBlock *B, const CXXBaseSpecifier *BS) {
+    B->appendBaseDtor(BS, cfg->getBumpVectorContext());
+  }
+  void appendMemberDtor(CFGBlock *B, FieldDecl *FD) {
+    B->appendMemberDtor(FD, cfg->getBumpVectorContext());
   }
+  void appendTemporaryDtor(CFGBlock *B, CXXBindTemporaryExpr *E) {
+    B->appendTemporaryDtor(E, cfg->getBumpVectorContext());
+  }
+
+  void insertAutomaticObjDtors(CFGBlock* Blk, CFGBlock::iterator I,
+    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S);
+  void appendAutomaticObjDtors(CFGBlock* Blk, LocalScope::const_iterator B,
+      LocalScope::const_iterator E, Stmt* S);
+  void prependAutomaticObjDtorsWithTerminator(CFGBlock* Blk,
+      LocalScope::const_iterator B, LocalScope::const_iterator E);
 
-  void AddSuccessor(CFGBlock *B, CFGBlock *S) {
+  void addSuccessor(CFGBlock *B, CFGBlock *S) {
     B->addSuccessor(S, cfg->getBumpVectorContext());
   }
 
   /// TryResult - a class representing a variant over the values
-  ///  'true', 'false', or 'unknown'.  This is returned by TryEvaluateBool,
+  ///  'true', 'false', or 'unknown'.  This is returned by tryEvaluateBool,
   ///  and is used by the CFGBuilder to decide if a branch condition
   ///  can be decided up front during CFG construction.
   class TryResult {
@@ -204,10 +405,10 @@ private:
     }
   };
 
-  /// TryEvaluateBool - Try and evaluate the Stmt and return 0 or 1
+  /// tryEvaluateBool - Try and evaluate the Stmt and return 0 or 1
   /// if we can evaluate to a known value, otherwise return -1.
-  TryResult TryEvaluateBool(Expr *S) {
-    if (!PruneTriviallyFalseEdges)
+  TryResult tryEvaluateBool(Expr *S) {
+    if (!BuildOpts.PruneTriviallyFalseEdges)
       return TryResult();
 
     Expr::EvalResult Result;
@@ -217,24 +418,13 @@ private:
 
     return TryResult();
   }
-
-  bool badCFG;
-
-  // True iff trivially false edges should be pruned from the CFG.
-  bool PruneTriviallyFalseEdges;
-
-  // True iff EH edges on CallExprs should be added to the CFG.
-  bool AddEHEdges;
-
-  // True iff scope start and scope end notes should be added to the CFG.
-  bool AddScopes;
 };
 
 // FIXME: Add support for dependent-sized array types in C++?
 // Does it even make sense to build a CFG for an uninstantiated template?
-static VariableArrayType* FindVA(Type* t) {
-  while (ArrayType* vt = dyn_cast<ArrayType>(t)) {
-    if (VariableArrayType* vat = dyn_cast<VariableArrayType>(vt))
+static const VariableArrayType *FindVA(const Type *t) {
+  while (const ArrayType *vt = dyn_cast<ArrayType>(t)) {
+    if (const VariableArrayType *vat = dyn_cast<VariableArrayType>(vt))
       if (vat->getSizeExpr())
         return vat;
 
@@ -250,19 +440,14 @@ static VariableArrayType* FindVA(Type* t) {
 ///  transferred to the caller.  If CFG construction fails, this method returns
 ///  NULL.
 CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement, ASTContext* C,
-                          bool pruneTriviallyFalseEdges,
-                          bool addehedges, bool AddScopes) {
-
-  AddEHEdges = addehedges;
-  PruneTriviallyFalseEdges = pruneTriviallyFalseEdges;
+    CFG::BuildOptions BO) {
 
   Context = C;
   assert(cfg.get());
   if (!Statement)
     return NULL;
 
-  this->AddScopes = AddScopes;
-  badCFG = false;
+  BuildOpts = BO;
 
   // Create an empty block that will serve as the exit block for the CFG.  Since
   // this is the first block added to the CFG, it will be implicitly registered
@@ -271,59 +456,67 @@ CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement, ASTContext* C,
   assert(Succ == &cfg->getExit());
   Block = NULL;  // the EXIT block is empty.  Create all other blocks lazily.
 
+  if (BuildOpts.AddImplicitDtors)
+    if (const CXXDestructorDecl *DD = dyn_cast_or_null<CXXDestructorDecl>(D))
+      addImplicitDtorsForDestructor(DD);
+
   // Visit the statements and create the CFG.
-  CFGBlock* B = addStmt(Statement);
+  CFGBlock *B = addStmt(Statement);
 
+  if (badCFG)
+    return NULL;
+
+  // For C++ constructor add initializers to CFG.
   if (const CXXConstructorDecl *CD = dyn_cast_or_null<CXXConstructorDecl>(D)) {
-    // FIXME: Add code for base initializers and member initializers.
-    (void)CD;
+    for (CXXConstructorDecl::init_const_reverse_iterator I = CD->init_rbegin(),
+        E = CD->init_rend(); I != E; ++I) {
+      B = addInitializer(*I);
+      if (badCFG)
+        return NULL;
+    }
   }
-  if (!B)
-    B = Succ;
-
-  if (B) {
-    // Finalize the last constructed block.  This usually involves reversing the
-    // order of the statements in the block.
-    if (Block) FinishBlock(B);
-
-    // Backpatch the gotos whose label -> block mappings we didn't know when we
-    // encountered them.
-    for (BackpatchBlocksTy::iterator I = BackpatchBlocks.begin(),
-         E = BackpatchBlocks.end(); I != E; ++I ) {
 
-      CFGBlock* B = *I;
-      GotoStmt* G = cast<GotoStmt>(B->getTerminator());
-      LabelMapTy::iterator LI = LabelMap.find(G->getLabel());
-
-      // If there is no target for the goto, then we are looking at an
-      // incomplete AST.  Handle this by not registering a successor.
-      if (LI == LabelMap.end()) continue;
+  if (B)
+    Succ = B;
 
-      AddSuccessor(B, LI->second);
-    }
+  // Backpatch the gotos whose label -> block mappings we didn't know when we
+  // encountered them.
+  for (BackpatchBlocksTy::iterator I = BackpatchBlocks.begin(),
+                                   E = BackpatchBlocks.end(); I != E; ++I ) {
 
-    // Add successors to the Indirect Goto Dispatch block (if we have one).
-    if (CFGBlock* B = cfg->getIndirectGotoBlock())
-      for (LabelSetTy::iterator I = AddressTakenLabels.begin(),
-           E = AddressTakenLabels.end(); I != E; ++I ) {
+    CFGBlock* B = I->block;
+    GotoStmt* G = cast<GotoStmt>(B->getTerminator());
+    LabelMapTy::iterator LI = LabelMap.find(G->getLabel());
 
-        // Lookup the target block.
-        LabelMapTy::iterator LI = LabelMap.find(*I);
+    // If there is no target for the goto, then we are looking at an
+    // incomplete AST.  Handle this by not registering a successor.
+    if (LI == LabelMap.end()) continue;
 
-        // If there is no target block that contains label, then we are looking
-        // at an incomplete AST.  Handle this by not registering a successor.
-        if (LI == LabelMap.end()) continue;
+    JumpTarget JT = LI->second;
+    prependAutomaticObjDtorsWithTerminator(B, I->scopePosition,
+                                           JT.scopePosition);
+    addSuccessor(B, JT.block);
+  }
 
-        AddSuccessor(B, LI->second);
-      }
+  // Add successors to the Indirect Goto Dispatch block (if we have one).
+  if (CFGBlock* B = cfg->getIndirectGotoBlock())
+    for (LabelSetTy::iterator I = AddressTakenLabels.begin(),
+                              E = AddressTakenLabels.end(); I != E; ++I ) {
+      
+      // Lookup the target block.
+      LabelMapTy::iterator LI = LabelMap.find(*I);
 
-    Succ = B;
-  }
+      // If there is no target block that contains label, then we are looking
+      // at an incomplete AST.  Handle this by not registering a successor.
+      if (LI == LabelMap.end()) continue;
+      
+      addSuccessor(B, LI->second.block);
+    }
 
   // Create an empty entry block that has no predecessors.
   cfg->setEntry(createBlock());
 
-  return badCFG ? NULL : cfg.take();
+  return cfg.take();
 }
 
 /// createBlock - Used to lazily create blocks that are connected
@@ -331,17 +524,251 @@ CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement, ASTContext* C,
 CFGBlock* CFGBuilder::createBlock(bool add_successor) {
   CFGBlock* B = cfg->createBlock();
   if (add_successor && Succ)
-    AddSuccessor(B, Succ);
+    addSuccessor(B, Succ);
   return B;
 }
 
-/// FinishBlock - "Finalize" the block by checking if we have a bad CFG.
-bool CFGBuilder::FinishBlock(CFGBlock* B) {
-  if (badCFG)
-    return false;
+/// addInitializer - Add C++ base or member initializer element to CFG.
+CFGBlock *CFGBuilder::addInitializer(CXXCtorInitializer *I) {
+  if (!BuildOpts.AddInitializers)
+    return Block;
 
-  assert(B);
-  return true;
+  bool IsReference = false;
+  bool HasTemporaries = false;
+
+  // Destructors of temporaries in initialization expression should be called
+  // after initialization finishes.
+  Expr *Init = I->getInit();
+  if (Init) {
+    if (FieldDecl *FD = I->getAnyMember())
+      IsReference = FD->getType()->isReferenceType();
+    HasTemporaries = isa<ExprWithCleanups>(Init);
+
+    if (BuildOpts.AddImplicitDtors && HasTemporaries) {
+      // Generate destructors for temporaries in initialization expression.
+      VisitForTemporaryDtors(cast<ExprWithCleanups>(Init)->getSubExpr(),
+          IsReference);
+    }
+  }
+
+  autoCreateBlock();
+  appendInitializer(Block, I);
+
+  if (Init) {
+    if (HasTemporaries) {
+      // For expression with temporaries go directly to subexpression to omit
+      // generating destructors for the second time.
+      return Visit(cast<ExprWithCleanups>(Init)->getSubExpr());
+    }
+    return Visit(Init);
+  }
+
+  return Block;
+}
+
+/// addAutomaticObjDtors - Add to current block automatic objects destructors
+/// for objects in range of local scope positions. Use S as trigger statement
+/// for destructors.
+void CFGBuilder::addAutomaticObjDtors(LocalScope::const_iterator B,
+                                      LocalScope::const_iterator E, Stmt* S) {
+  if (!BuildOpts.AddImplicitDtors)
+    return;
+
+  if (B == E)
+    return;
+
+  autoCreateBlock();
+  appendAutomaticObjDtors(Block, B, E, S);
+}
+
+/// addImplicitDtorsForDestructor - Add implicit destructors generated for
+/// base and member objects in destructor.
+void CFGBuilder::addImplicitDtorsForDestructor(const CXXDestructorDecl *DD) {
+  assert (BuildOpts.AddImplicitDtors
+      && "Can be called only when dtors should be added");
+  const CXXRecordDecl *RD = DD->getParent();
+
+  // At the end destroy virtual base objects.
+  for (CXXRecordDecl::base_class_const_iterator VI = RD->vbases_begin(),
+      VE = RD->vbases_end(); VI != VE; ++VI) {
+    const CXXRecordDecl *CD = VI->getType()->getAsCXXRecordDecl();
+    if (!CD->hasTrivialDestructor()) {
+      autoCreateBlock();
+      appendBaseDtor(Block, VI);
+    }
+  }
+
+  // Before virtual bases destroy direct base objects.
+  for (CXXRecordDecl::base_class_const_iterator BI = RD->bases_begin(),
+      BE = RD->bases_end(); BI != BE; ++BI) {
+    if (!BI->isVirtual()) {
+      const CXXRecordDecl *CD = BI->getType()->getAsCXXRecordDecl();
+      if (!CD->hasTrivialDestructor()) {
+        autoCreateBlock();
+        appendBaseDtor(Block, BI);
+      }
+    }
+  }
+
+  // First destroy member objects.
+  for (CXXRecordDecl::field_iterator FI = RD->field_begin(),
+      FE = RD->field_end(); FI != FE; ++FI) {
+    // Check for constant size array. Set type to array element type.
+    QualType QT = FI->getType();
+    if (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) {
+      if (AT->getSize() == 0)
+        continue;
+      QT = AT->getElementType();
+    }
+
+    if (const CXXRecordDecl *CD = QT->getAsCXXRecordDecl())
+      if (!CD->hasTrivialDestructor()) {
+        autoCreateBlock();
+        appendMemberDtor(Block, *FI);
+      }
+  }
+}
+
+/// createOrReuseLocalScope - If Scope is NULL create new LocalScope. Either
+/// way return valid LocalScope object.
+LocalScope* CFGBuilder::createOrReuseLocalScope(LocalScope* Scope) {
+  if (!Scope) {
+    llvm::BumpPtrAllocator &alloc = cfg->getAllocator();
+    Scope = alloc.Allocate<LocalScope>();
+    BumpVectorContext ctx(alloc);
+    new (Scope) LocalScope(ctx, ScopePos);
+  }
+  return Scope;
+}
+
+/// addLocalScopeForStmt - Add LocalScope to local scopes tree for statement
+/// that should create implicit scope (e.g. if/else substatements). 
+void CFGBuilder::addLocalScopeForStmt(Stmt* S) {
+  if (!BuildOpts.AddImplicitDtors)
+    return;
+
+  LocalScope *Scope = 0;
+
+  // For compound statement we will be creating explicit scope.
+  if (CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
+    for (CompoundStmt::body_iterator BI = CS->body_begin(), BE = CS->body_end()
+        ; BI != BE; ++BI) {
+      Stmt *SI = *BI;
+      if (LabelStmt *LS = dyn_cast<LabelStmt>(SI))
+        SI = LS->getSubStmt();
+      if (DeclStmt *DS = dyn_cast<DeclStmt>(SI))
+        Scope = addLocalScopeForDeclStmt(DS, Scope);
+    }
+    return;
+  }
+
+  // For any other statement scope will be implicit and as such will be
+  // interesting only for DeclStmt.
+  if (LabelStmt *LS = dyn_cast<LabelStmt>(S))
+    S = LS->getSubStmt();
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S))
+    addLocalScopeForDeclStmt(DS);
+}
+
+/// addLocalScopeForDeclStmt - Add LocalScope for declaration statement. Will
+/// reuse Scope if not NULL.
+LocalScope* CFGBuilder::addLocalScopeForDeclStmt(DeclStmt* DS,
+                                                 LocalScope* Scope) {
+  if (!BuildOpts.AddImplicitDtors)
+    return Scope;
+
+  for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end()
+      ; DI != DE; ++DI) {
+    if (VarDecl* VD = dyn_cast<VarDecl>(*DI))
+      Scope = addLocalScopeForVarDecl(VD, Scope);
+  }
+  return Scope;
+}
+
+/// addLocalScopeForVarDecl - Add LocalScope for variable declaration. It will
+/// create add scope for automatic objects and temporary objects bound to
+/// const reference. Will reuse Scope if not NULL.
+LocalScope* CFGBuilder::addLocalScopeForVarDecl(VarDecl* VD,
+                                                LocalScope* Scope) {
+  if (!BuildOpts.AddImplicitDtors)
+    return Scope;
+
+  // Check if variable is local.
+  switch (VD->getStorageClass()) {
+  case SC_None:
+  case SC_Auto:
+  case SC_Register:
+    break;
+  default: return Scope;
+  }
+
+  // Check for const references bound to temporary. Set type to pointee.
+  QualType QT = VD->getType();
+  if (const ReferenceType* RT = QT.getTypePtr()->getAs<ReferenceType>()) {
+    QT = RT->getPointeeType();
+    if (!QT.isConstQualified())
+      return Scope;
+    if (!VD->getInit() || !VD->getInit()->Classify(*Context).isRValue())
+      return Scope;
+  }
+
+  // Check for constant size array. Set type to array element type.
+  if (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) {
+    if (AT->getSize() == 0)
+      return Scope;
+    QT = AT->getElementType();
+  }
+
+  // Check if type is a C++ class with non-trivial destructor.
+  if (const CXXRecordDecl* CD = QT->getAsCXXRecordDecl())
+    if (!CD->hasTrivialDestructor()) {
+      // Add the variable to scope
+      Scope = createOrReuseLocalScope(Scope);
+      Scope->addVar(VD);
+      ScopePos = Scope->begin();
+    }
+  return Scope;
+}
+
+/// addLocalScopeAndDtors - For given statement add local scope for it and
+/// add destructors that will cleanup the scope. Will reuse Scope if not NULL.
+void CFGBuilder::addLocalScopeAndDtors(Stmt* S) {
+  if (!BuildOpts.AddImplicitDtors)
+    return;
+
+  LocalScope::const_iterator scopeBeginPos = ScopePos;
+  addLocalScopeForStmt(S);
+  addAutomaticObjDtors(ScopePos, scopeBeginPos, S);
+}
+
+/// insertAutomaticObjDtors - Insert destructor CFGElements for variables with
+/// automatic storage duration to CFGBlock's elements vector. Insertion will be
+/// performed in place specified with iterator.
+void CFGBuilder::insertAutomaticObjDtors(CFGBlock* Blk, CFGBlock::iterator I,
+    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S) {
+  BumpVectorContext& C = cfg->getBumpVectorContext();
+  I = Blk->beginAutomaticObjDtorsInsert(I, B.distance(E), C);
+  while (B != E)
+    I = Blk->insertAutomaticObjDtor(I, *B++, S);
+}
+
+/// appendAutomaticObjDtors - Append destructor CFGElements for variables with
+/// automatic storage duration to CFGBlock's elements vector. Elements will be
+/// appended to physical end of the vector which happens to be logical
+/// beginning.
+void CFGBuilder::appendAutomaticObjDtors(CFGBlock* Blk,
+    LocalScope::const_iterator B, LocalScope::const_iterator E, Stmt* S) {
+  insertAutomaticObjDtors(Blk, Blk->begin(), B, E, S);
+}
+
+/// prependAutomaticObjDtorsWithTerminator - Prepend destructor CFGElements for
+/// variables with automatic storage duration to CFGBlock's elements vector.
+/// Elements will be prepended to physical beginning of the vector which
+/// happens to be logical end. Use blocks terminator as statement that specifies
+/// destructors call site.
+void CFGBuilder::prependAutomaticObjDtorsWithTerminator(CFGBlock* Blk,
+    LocalScope::const_iterator B, LocalScope::const_iterator E) {
+  insertAutomaticObjDtors(Blk, Blk->end(), B, E, Blk->getTerminator());
 }
 
 /// Visit - Walk the subtree of a statement and add extra
@@ -360,6 +787,9 @@ tryAgain:
     case Stmt::AddrLabelExprClass:
       return VisitAddrLabelExpr(cast<AddrLabelExpr>(S), asc);
 
+    case Stmt::BinaryConditionalOperatorClass:
+      return VisitConditionalOperator(cast<BinaryConditionalOperator>(S), asc);
+
     case Stmt::BinaryOperatorClass:
       return VisitBinaryOperator(cast<BinaryOperator>(S), asc);
 
@@ -391,11 +821,20 @@ tryAgain:
     case Stmt::CXXCatchStmtClass:
       return VisitCXXCatchStmt(cast<CXXCatchStmt>(S));
 
-    case Stmt::CXXExprWithTemporariesClass: {
-      // FIXME: Handle temporaries.  For now, just visit the subexpression
-      // so we don't artificially create extra blocks.
-      return Visit(cast<CXXExprWithTemporaries>(S)->getSubExpr(), asc);
-    }
+    case Stmt::ExprWithCleanupsClass:
+      return VisitExprWithCleanups(cast<ExprWithCleanups>(S), asc);
+
+    case Stmt::CXXBindTemporaryExprClass:
+      return VisitCXXBindTemporaryExpr(cast<CXXBindTemporaryExpr>(S), asc);
+
+    case Stmt::CXXConstructExprClass:
+      return VisitCXXConstructExpr(cast<CXXConstructExpr>(S), asc);
+
+    case Stmt::CXXFunctionalCastExprClass:
+      return VisitCXXFunctionalCastExpr(cast<CXXFunctionalCastExpr>(S), asc);
+
+    case Stmt::CXXTemporaryObjectExprClass:
+      return VisitCXXTemporaryObjectExpr(cast<CXXTemporaryObjectExpr>(S), asc);
 
     case Stmt::CXXMemberCallExprClass:
       return VisitCXXMemberCallExpr(cast<CXXMemberCallExpr>(S), asc);
@@ -424,6 +863,9 @@ tryAgain:
     case Stmt::IfStmtClass:
       return VisitIfStmt(cast<IfStmt>(S));
 
+    case Stmt::ImplicitCastExprClass:
+      return VisitImplicitCastExpr(cast<ImplicitCastExpr>(S), asc);
+
     case Stmt::IndirectGotoStmtClass:
       return VisitIndirectGotoStmt(cast<IndirectGotoStmt>(S));
 
@@ -467,6 +909,9 @@ tryAgain:
     case Stmt::SwitchStmtClass:
       return VisitSwitchStmt(cast<SwitchStmt>(S));
 
+    case Stmt::UnaryOperatorClass:
+      return VisitUnaryOperator(cast<UnaryOperator>(S), asc);
+
     case Stmt::WhileStmtClass:
       return VisitWhileStmt(cast<WhileStmt>(S));
   }
@@ -475,7 +920,7 @@ tryAgain:
 CFGBlock *CFGBuilder::VisitStmt(Stmt *S, AddStmtChoice asc) {
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, S, asc);
+    appendStmt(Block, S, asc);
   }
 
   return VisitChildren(S);
@@ -484,8 +929,7 @@ CFGBlock *CFGBuilder::VisitStmt(Stmt *S, AddStmtChoice asc) {
 /// VisitChildren - Visit the children of a Stmt.
 CFGBlock *CFGBuilder::VisitChildren(Stmt* Terminator) {
   CFGBlock *B = Block;
-  for (Stmt::child_iterator I = Terminator->child_begin(),
-         E = Terminator->child_end(); I != E; ++I) {
+  for (Stmt::child_range I = Terminator->children(); I; ++I) {
     if (*I) B = Visit(*I);
   }
   return B;
@@ -497,19 +941,29 @@ CFGBlock *CFGBuilder::VisitAddrLabelExpr(AddrLabelExpr *A,
 
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, A, asc);
+    appendStmt(Block, A, asc);
   }
 
   return Block;
 }
 
+CFGBlock *CFGBuilder::VisitUnaryOperator(UnaryOperator *U,
+           AddStmtChoice asc) {
+  if (asc.alwaysAdd()) {
+    autoCreateBlock();
+    appendStmt(Block, U, asc);
+  }
+
+  return Visit(U->getSubExpr(), AddStmtChoice());
+}
+
 CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
                                           AddStmtChoice asc) {
   if (B->isLogicalOp()) { // && or ||
     CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
-    AppendStmt(ConfluenceBlock, B, asc);
+    appendStmt(ConfluenceBlock, B, asc);
 
-    if (!FinishBlock(ConfluenceBlock))
+    if (badCFG)
       return 0;
 
     // create the block evaluating the LHS
@@ -522,57 +976,67 @@ CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
     CFGBlock* RHSBlock = addStmt(B->getRHS());
 
     if (RHSBlock) {
-      if (!FinishBlock(RHSBlock))
+      if (badCFG)
         return 0;
-    }
-    else {
+    } else {
       // Create an empty block for cases where the RHS doesn't require
       // any explicit statements in the CFG.
       RHSBlock = createBlock();
     }
 
     // See if this is a known constant.
-    TryResult KnownVal = TryEvaluateBool(B->getLHS());
+    TryResult KnownVal = tryEvaluateBool(B->getLHS());
     if (KnownVal.isKnown() && (B->getOpcode() == BO_LOr))
       KnownVal.negate();
 
     // Now link the LHSBlock with RHSBlock.
     if (B->getOpcode() == BO_LOr) {
-      AddSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
-      AddSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
+      addSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
+      addSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
     } else {
       assert(B->getOpcode() == BO_LAnd);
-      AddSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
-      AddSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
+      addSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
+      addSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
     }
 
     // Generate the blocks for evaluating the LHS.
     Block = LHSBlock;
     return addStmt(B->getLHS());
   }
-  else if (B->getOpcode() == BO_Comma) { // ,
+
+  if (B->getOpcode() == BO_Comma) { // ,
     autoCreateBlock();
-    AppendStmt(Block, B, asc);
+    appendStmt(Block, B, asc);
     addStmt(B->getRHS());
     return addStmt(B->getLHS());
   }
-  else if (B->isAssignmentOp()) {
+
+  if (B->isAssignmentOp()) {
     if (asc.alwaysAdd()) {
       autoCreateBlock();
-      AppendStmt(Block, B, asc);
+      appendStmt(Block, B, asc);
     }
+    Visit(B->getLHS());
+    return Visit(B->getRHS());
+  }
 
-    Visit(B->getRHS());
-    return Visit(B->getLHS(), AddStmtChoice::AsLValueNotAlwaysAdd);
+  if (asc.alwaysAdd()) {
+    autoCreateBlock();
+    appendStmt(Block, B, asc);
   }
 
-  return VisitStmt(B, asc);
+  CFGBlock *RBlock = Visit(B->getRHS());
+  CFGBlock *LBlock = Visit(B->getLHS());
+  // If visiting RHS causes us to finish 'Block', e.g. the RHS is a StmtExpr
+  // containing a DoStmt, and the LHS doesn't create a new block, then we should
+  // return RBlock.  Otherwise we'll incorrectly return NULL.
+  return (LBlock ? LBlock : RBlock);
 }
 
 CFGBlock *CFGBuilder::VisitBlockExpr(BlockExpr *E, AddStmtChoice asc) {
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, E, asc);
+    appendStmt(Block, E, asc);
   }
   return Block;
 }
@@ -580,8 +1044,8 @@ CFGBlock *CFGBuilder::VisitBlockExpr(BlockExpr *E, AddStmtChoice asc) {
 CFGBlock *CFGBuilder::VisitBreakStmt(BreakStmt *B) {
   // "break" is a control-flow statement.  Thus we stop processing the current
   // block.
-  if (Block && !FinishBlock(Block))
-      return 0;
+  if (badCFG)
+    return 0;
 
   // Now create a new block that ends with the break statement.
   Block = createBlock(false);
@@ -589,9 +1053,10 @@ CFGBlock *CFGBuilder::VisitBreakStmt(BreakStmt *B) {
 
   // If there is no target for the break, then we are looking at an incomplete
   // AST.  This means that the CFG cannot be constructed.
-  if (BreakTargetBlock)
-    AddSuccessor(Block, BreakTargetBlock);
-  else
+  if (BreakJumpTarget.block) {
+    addAutomaticObjDtors(ScopePos, BreakJumpTarget.scopePosition, B);
+    addSuccessor(Block, BreakJumpTarget.block);
+  } else
     badCFG = true;
 
 
@@ -624,8 +1089,8 @@ CFGBlock *CFGBuilder::VisitCallExpr(CallExpr *C, AddStmtChoice asc) {
   bool AddEHEdge = false;
 
   // Languages without exceptions are assumed to not throw.
-  if (Context->getLangOptions().Exceptions) {
-    if (AddEHEdges)
+  if (Context->getLangOptions().areExceptionsEnabled()) {
+    if (BuildOpts.AddEHEdges)
       AddEHEdge = true;
   }
 
@@ -639,32 +1104,28 @@ CFGBlock *CFGBuilder::VisitCallExpr(CallExpr *C, AddStmtChoice asc) {
   if (!CanThrow(C->getCallee()))
     AddEHEdge = false;
 
-  if (!NoReturn && !AddEHEdge) {
-    if (asc.asLValue())
-      return VisitStmt(C, AddStmtChoice::AlwaysAddAsLValue);
-    else
-      return VisitStmt(C, AddStmtChoice::AlwaysAdd);
-  }
+  if (!NoReturn && !AddEHEdge)
+    return VisitStmt(C, asc.withAlwaysAdd(true));
 
   if (Block) {
     Succ = Block;
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
   }
 
   Block = createBlock(!NoReturn);
-  AppendStmt(Block, C, asc);
+  appendStmt(Block, C, asc);
 
   if (NoReturn) {
     // Wire this to the exit block directly.
-    AddSuccessor(Block, &cfg->getExit());
+    addSuccessor(Block, &cfg->getExit());
   }
   if (AddEHEdge) {
     // Add exceptional edges.
     if (TryTerminatedBlock)
-      AddSuccessor(Block, TryTerminatedBlock);
+      addSuccessor(Block, TryTerminatedBlock);
     else
-      AddSuccessor(Block, &cfg->getExit());
+      addSuccessor(Block, &cfg->getExit());
   }
 
   return VisitChildren(C);
@@ -673,38 +1134,35 @@ CFGBlock *CFGBuilder::VisitCallExpr(CallExpr *C, AddStmtChoice asc) {
 CFGBlock *CFGBuilder::VisitChooseExpr(ChooseExpr *C,
                                       AddStmtChoice asc) {
   CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
-  AppendStmt(ConfluenceBlock, C, asc);
-  if (!FinishBlock(ConfluenceBlock))
+  appendStmt(ConfluenceBlock, C, asc);
+  if (badCFG)
     return 0;
 
-  asc = asc.asLValue() ? AddStmtChoice::AlwaysAddAsLValue
-                       : AddStmtChoice::AlwaysAdd;
-
+  AddStmtChoice alwaysAdd = asc.withAlwaysAdd(true);
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* LHSBlock = Visit(C->getLHS(), asc);
-  if (!FinishBlock(LHSBlock))
+  CFGBlock* LHSBlock = Visit(C->getLHS(), alwaysAdd);
+  if (badCFG)
     return 0;
 
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* RHSBlock = Visit(C->getRHS(), asc);
-  if (!FinishBlock(RHSBlock))
+  CFGBlock* RHSBlock = Visit(C->getRHS(), alwaysAdd);
+  if (badCFG)
     return 0;
 
   Block = createBlock(false);
   // See if this is a known constant.
-  const TryResult& KnownVal = TryEvaluateBool(C->getCond());
-  AddSuccessor(Block, KnownVal.isFalse() ? NULL : LHSBlock);
-  AddSuccessor(Block, KnownVal.isTrue() ? NULL : RHSBlock);
+  const TryResult& KnownVal = tryEvaluateBool(C->getCond());
+  addSuccessor(Block, KnownVal.isFalse() ? NULL : LHSBlock);
+  addSuccessor(Block, KnownVal.isTrue() ? NULL : RHSBlock);
   Block->setTerminator(C);
   return addStmt(C->getCond());
 }
 
 
 CFGBlock* CFGBuilder::VisitCompoundStmt(CompoundStmt* C) {
-  EndScope(C);
-
+  addLocalScopeAndDtors(C);
   CFGBlock* LastBlock = Block;
 
   for (CompoundStmt::reverse_body_iterator I=C->body_rbegin(), E=C->body_rend();
@@ -718,22 +1176,22 @@ CFGBlock* CFGBuilder::VisitCompoundStmt(CompoundStmt* C) {
       return NULL;
   }
 
-  LastBlock = StartScope(C, LastBlock);
-
   return LastBlock;
 }
 
-CFGBlock *CFGBuilder::VisitConditionalOperator(ConditionalOperator *C,
+CFGBlock *CFGBuilder::VisitConditionalOperator(AbstractConditionalOperator *C,
                                                AddStmtChoice asc) {
+  const BinaryConditionalOperator *BCO = dyn_cast<BinaryConditionalOperator>(C);
+  const OpaqueValueExpr *opaqueValue = (BCO ? BCO->getOpaqueValue() : NULL);
+
   // Create the confluence block that will "merge" the results of the ternary
   // expression.
   CFGBlock* ConfluenceBlock = Block ? Block : createBlock();
-  AppendStmt(ConfluenceBlock, C, asc);
-  if (!FinishBlock(ConfluenceBlock))
+  appendStmt(ConfluenceBlock, C, asc);
+  if (badCFG)
     return 0;
 
-  asc = asc.asLValue() ? AddStmtChoice::AlwaysAddAsLValue
-                       : AddStmtChoice::AlwaysAdd;
+  AddStmtChoice alwaysAdd = asc.withAlwaysAdd(true);
 
   // Create a block for the LHS expression if there is an LHS expression.  A
   // GCC extension allows LHS to be NULL, causing the condition to be the
@@ -741,60 +1199,48 @@ CFGBlock *CFGBuilder::VisitConditionalOperator(ConditionalOperator *C,
   //  e.g: x ?: y is shorthand for: x ? x : y;
   Succ = ConfluenceBlock;
   Block = NULL;
-  CFGBlock* LHSBlock = NULL;
-  if (C->getLHS()) {
-    LHSBlock = Visit(C->getLHS(), asc);
-    if (!FinishBlock(LHSBlock))
+  CFGBlock* LHSBlock = 0;
+  const Expr *trueExpr = C->getTrueExpr();
+  if (trueExpr != opaqueValue) {
+    LHSBlock = Visit(C->getTrueExpr(), alwaysAdd);
+    if (badCFG)
       return 0;
     Block = NULL;
   }
 
   // Create the block for the RHS expression.
   Succ = ConfluenceBlock;
-  CFGBlock* RHSBlock = Visit(C->getRHS(), asc);
-  if (!FinishBlock(RHSBlock))
+  CFGBlock* RHSBlock = Visit(C->getFalseExpr(), alwaysAdd);
+  if (badCFG)
     return 0;
 
   // Create the block that will contain the condition.
   Block = createBlock(false);
 
   // See if this is a known constant.
-  const TryResult& KnownVal = TryEvaluateBool(C->getCond());
-  if (LHSBlock) {
-    AddSuccessor(Block, KnownVal.isFalse() ? NULL : LHSBlock);
-  } else {
-    if (KnownVal.isFalse()) {
-      // If we know the condition is false, add NULL as the successor for
-      // the block containing the condition.  In this case, the confluence
-      // block will have just one predecessor.
-      AddSuccessor(Block, 0);
-      assert(ConfluenceBlock->pred_size() == 1);
-    } else {
-      // If we have no LHS expression, add the ConfluenceBlock as a direct
-      // successor for the block containing the condition.  Moreover, we need to
-      // reverse the order of the predecessors in the ConfluenceBlock because
-      // the RHSBlock will have been added to the succcessors already, and we
-      // want the first predecessor to the the block containing the expression
-      // for the case when the ternary expression evaluates to true.
-      AddSuccessor(Block, ConfluenceBlock);
-      assert(ConfluenceBlock->pred_size() == 2);
-      std::reverse(ConfluenceBlock->pred_begin(),
-                   ConfluenceBlock->pred_end());
-    }
-  }
-
-  AddSuccessor(Block, KnownVal.isTrue() ? NULL : RHSBlock);
+  const TryResult& KnownVal = tryEvaluateBool(C->getCond());
+  if (LHSBlock)
+    addSuccessor(Block, KnownVal.isFalse() ? NULL : LHSBlock);
+  addSuccessor(Block, KnownVal.isTrue() ? NULL : RHSBlock);
   Block->setTerminator(C);
-  return addStmt(C->getCond());
+  Expr *condExpr = C->getCond();
+
+  CFGBlock *result = 0;
+
+  // Run the condition expression if it's not trivially expressed in
+  // terms of the opaque value (or if there is no opaque value).
+  if (condExpr != opaqueValue) result = addStmt(condExpr);
+
+  // Before that, run the common subexpression if there was one.
+  // At least one of this or the above will be run.
+  if (opaqueValue) result = addStmt(BCO->getCommon());
+
+  return result;
 }
 
 CFGBlock *CFGBuilder::VisitDeclStmt(DeclStmt *DS) {
-  autoCreateBlock();
-
-  if (DS->isSingleDecl()) {
-    AppendStmt(Block, DS);
-    return VisitDeclSubExpr(DS->getSingleDecl());
-  }
+  if (DS->isSingleDecl())
+    return VisitDeclSubExpr(DS);
 
   CFGBlock *B = 0;
 
@@ -815,37 +1261,63 @@ CFGBlock *CFGBuilder::VisitDeclStmt(DeclStmt *DS) {
     DeclStmt *DSNew = new (Mem) DeclStmt(DG, D->getLocation(), GetEndLoc(D));
 
     // Append the fake DeclStmt to block.
-    AppendStmt(Block, DSNew);
-    B = VisitDeclSubExpr(D);
+    B = VisitDeclSubExpr(DSNew);
   }
 
   return B;
 }
 
 /// VisitDeclSubExpr - Utility method to add block-level expressions for
-///  initializers in Decls.
-CFGBlock *CFGBuilder::VisitDeclSubExpr(Decl* D) {
-  assert(Block);
+/// DeclStmts and initializers in them.
+CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt* DS) {
+  assert(DS->isSingleDecl() && "Can handle single declarations only.");
 
-  VarDecl *VD = dyn_cast<VarDecl>(D);
+  VarDecl *VD = dyn_cast<VarDecl>(DS->getSingleDecl());
 
-  if (!VD)
+  if (!VD) {
+    autoCreateBlock();
+    appendStmt(Block, DS);
     return Block;
+  }
 
+  bool IsReference = false;
+  bool HasTemporaries = false;
+
+  // Destructors of temporaries in initialization expression should be called
+  // after initialization finishes.
   Expr *Init = VD->getInit();
+  if (Init) {
+    IsReference = VD->getType()->isReferenceType();
+    HasTemporaries = isa<ExprWithCleanups>(Init);
+
+    if (BuildOpts.AddImplicitDtors && HasTemporaries) {
+      // Generate destructors for temporaries in initialization expression.
+      VisitForTemporaryDtors(cast<ExprWithCleanups>(Init)->getSubExpr(),
+          IsReference);
+    }
+  }
+
+  autoCreateBlock();
+  appendStmt(Block, DS);
 
   if (Init) {
-    AddStmtChoice::Kind k =
-      VD->getType()->isReferenceType() ? AddStmtChoice::AsLValueNotAlwaysAdd
-                                       : AddStmtChoice::NotAlwaysAdd;
-    Visit(Init, AddStmtChoice(k));
+    if (HasTemporaries)
+      // For expression with temporaries go directly to subexpression to omit
+      // generating destructors for the second time.
+      Visit(cast<ExprWithCleanups>(Init)->getSubExpr());
+    else
+      Visit(Init);
   }
 
   // If the type of VD is a VLA, then we must process its size expressions.
-  for (VariableArrayType* VA = FindVA(VD->getType().getTypePtr()); VA != 0;
-       VA = FindVA(VA->getElementType().getTypePtr()))
+  for (const VariableArrayType* VA = FindVA(VD->getType().getTypePtr());
+       VA != 0; VA = FindVA(VA->getElementType().getTypePtr()))
     Block = addStmt(VA->getSizeExpr());
 
+  // Remove variable from local scope.
+  if (ScopePos && VD == *ScopePos)
+    ++ScopePos;
+
   return Block;
 }
 
@@ -857,11 +1329,23 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   // middle of a block, we stop processing that block.  That block is then the
   // implicit successor for the "then" and "else" clauses.
 
+  // Save local scope position because in case of condition variable ScopePos
+  // won't be restored when traversing AST.
+  SaveAndRestore<LocalScope::const_iterator> save_scope_pos(ScopePos);
+
+  // Create local scope for possible condition variable.
+  // Store scope position. Add implicit destructor.
+  if (VarDecl* VD = I->getConditionVariable()) {
+    LocalScope::const_iterator BeginScopePos = ScopePos;
+    addLocalScopeForVarDecl(VD);
+    addAutomaticObjDtors(ScopePos, BeginScopePos, I);
+  }
+
   // The block we were proccessing is now finished.  Make it the successor
   // block.
   if (Block) {
     Succ = Block;
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
   }
 
@@ -874,12 +1358,18 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
     // NULL out Block so that the recursive call to Visit will
     // create a new basic block.
     Block = NULL;
+
+    // If branch is not a compound statement create implicit scope
+    // and add destructors.
+    if (!isa<CompoundStmt>(Else))
+      addLocalScopeAndDtors(Else);
+
     ElseBlock = addStmt(Else);
 
     if (!ElseBlock) // Can occur when the Else body has all NullStmts.
       ElseBlock = sv.get();
     else if (Block) {
-      if (!FinishBlock(ElseBlock))
+      if (badCFG)
         return 0;
     }
   }
@@ -891,6 +1381,12 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
     assert(Then);
     SaveAndRestore<CFGBlock*> sv(Succ);
     Block = NULL;
+
+    // If branch is not a compound statement create implicit scope
+    // and add destructors.
+    if (!isa<CompoundStmt>(Then))
+      addLocalScopeAndDtors(Then);
+
     ThenBlock = addStmt(Then);
 
     if (!ThenBlock) {
@@ -898,9 +1394,9 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
       // Create an empty block so we can distinguish between true and false
       // branches in path-sensitive analyses.
       ThenBlock = createBlock(false);
-      AddSuccessor(ThenBlock, sv.get());
+      addSuccessor(ThenBlock, sv.get());
     } else if (Block) {
-      if (!FinishBlock(ThenBlock))
+      if (badCFG)
         return 0;
     }
   }
@@ -912,11 +1408,11 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   Block->setTerminator(I);
 
   // See if this is a known constant.
-  const TryResult &KnownVal = TryEvaluateBool(I->getCond());
+  const TryResult &KnownVal = tryEvaluateBool(I->getCond());
 
   // Now add the successors.
-  AddSuccessor(Block, KnownVal.isFalse() ? NULL : ThenBlock);
-  AddSuccessor(Block, KnownVal.isTrue()? NULL : ElseBlock);
+  addSuccessor(Block, KnownVal.isFalse() ? NULL : ThenBlock);
+  addSuccessor(Block, KnownVal.isTrue()? NULL : ElseBlock);
 
   // Add the condition as the last statement in the new block.  This may create
   // new blocks as the condition may contain control-flow.  Any newly created
@@ -928,7 +1424,7 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   if (VarDecl *VD = I->getConditionVariable()) {
     if (Expr *Init = VD->getInit()) {
       autoCreateBlock();
-      AppendStmt(Block, I, AddStmtChoice::AlwaysAdd);
+      appendStmt(Block, I, AddStmtChoice::AlwaysAdd);
       addStmt(Init);
     }
   }
@@ -944,37 +1440,37 @@ CFGBlock* CFGBuilder::VisitReturnStmt(ReturnStmt* R) {
   //       code afterwards is DEAD (unreachable).  We still keep a basic block
   //       for that code; a simple "mark-and-sweep" from the entry block will be
   //       able to report such dead blocks.
-  if (Block)
-    FinishBlock(Block);
 
   // Create the new block.
   Block = createBlock(false);
 
   // The Exit block is the only successor.
-  AddSuccessor(Block, &cfg->getExit());
+  addAutomaticObjDtors(ScopePos, LocalScope::const_iterator(), R);
+  addSuccessor(Block, &cfg->getExit());
 
   // Add the return statement to the block.  This may create new blocks if R
   // contains control-flow (short-circuit operations).
   return VisitStmt(R, AddStmtChoice::AlwaysAdd);
 }
 
-CFGBlock* CFGBuilder::VisitLabelStmt(LabelStmt* L) {
+CFGBlock* CFGBuilder::VisitLabelStmt(LabelStmt *L) {
   // Get the block of the labeled statement.  Add it to our map.
   addStmt(L->getSubStmt());
-  CFGBlock* LabelBlock = Block;
+  CFGBlock *LabelBlock = Block;
 
   if (!LabelBlock)              // This can happen when the body is empty, i.e.
     LabelBlock = createBlock(); // scopes that only contains NullStmts.
 
-  assert(LabelMap.find(L) == LabelMap.end() && "label already in map");
-  LabelMap[ L ] = LabelBlock;
+  assert(LabelMap.find(L->getDecl()) == LabelMap.end() &&
+         "label already in map");
+  LabelMap[L->getDecl()] = JumpTarget(LabelBlock, ScopePos);
 
   // Labels partition blocks, so this is the end of the basic block we were
   // processing (L is the block's label).  Because this is label (and we have
   // already processed the substatement) there is no extra control-flow to worry
   // about.
   LabelBlock->setLabel(L);
-  if (!FinishBlock(LabelBlock))
+  if (badCFG)
     return 0;
 
   // We set Block to NULL to allow lazy creation of a new block (if necessary);
@@ -989,8 +1485,6 @@ CFGBlock* CFGBuilder::VisitLabelStmt(LabelStmt* L) {
 CFGBlock* CFGBuilder::VisitGotoStmt(GotoStmt* G) {
   // Goto is a control-flow statement.  Thus we stop processing the current
   // block and create a new one.
-  if (Block)
-    FinishBlock(Block);
 
   Block = createBlock(false);
   Block->setTerminator(G);
@@ -1000,9 +1494,12 @@ CFGBlock* CFGBuilder::VisitGotoStmt(GotoStmt* G) {
 
   if (I == LabelMap.end())
     // We will need to backpatch this block later.
-    BackpatchBlocks.push_back(Block);
-  else
-    AddSuccessor(Block, I->second);
+    BackpatchBlocks.push_back(JumpSource(Block, ScopePos));
+  else {
+    JumpTarget JT = I->second;
+    addAutomaticObjDtors(ScopePos, JT.scopePosition, G);
+    addSuccessor(Block, JT.block);
+  }
 
   return Block;
 }
@@ -1010,10 +1507,27 @@ CFGBlock* CFGBuilder::VisitGotoStmt(GotoStmt* G) {
 CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
   CFGBlock* LoopSuccessor = NULL;
 
+  // Save local scope position because in case of condition variable ScopePos
+  // won't be restored when traversing AST.
+  SaveAndRestore<LocalScope::const_iterator> save_scope_pos(ScopePos);
+
+  // Create local scope for init statement and possible condition variable.
+  // Add destructor for init statement and condition variable.
+  // Store scope position for continue statement.
+  if (Stmt* Init = F->getInit())
+    addLocalScopeForStmt(Init);
+  LocalScope::const_iterator LoopBeginScopePos = ScopePos;
+
+  if (VarDecl* VD = F->getConditionVariable())
+    addLocalScopeForVarDecl(VD);
+  LocalScope::const_iterator ContinueScopePos = ScopePos;
+
+  addAutomaticObjDtors(ScopePos, save_scope_pos.get(), F);
+
   // "for" is a control-flow statement.  Thus we stop processing the current
   // block.
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     LoopSuccessor = Block;
   } else
@@ -1021,8 +1535,8 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
 
   // Save the current value for the break targets.
   // All breaks should go to the code following the loop.
-  SaveAndRestore<CFGBlock*> save_break(BreakTargetBlock);
-  BreakTargetBlock = LoopSuccessor;
+  SaveAndRestore<JumpTarget> save_break(BreakJumpTarget);
+  BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
 
   // Because of short-circuit evaluation, the condition of the loop can span
   // multiple basic blocks.  Thus we need the "Entry" and "Exit" blocks that
@@ -1038,21 +1552,24 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
   if (Stmt* C = F->getCond()) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
-    assert(Block == EntryConditionBlock);
+    if (badCFG)
+      return 0;
+    assert(Block == EntryConditionBlock ||
+           (Block == 0 && EntryConditionBlock == Succ));
 
     // If this block contains a condition variable, add both the condition
     // variable and initializer to the CFG.
     if (VarDecl *VD = F->getConditionVariable()) {
       if (Expr *Init = VD->getInit()) {
         autoCreateBlock();
-        AppendStmt(Block, F, AddStmtChoice::AlwaysAdd);
+        appendStmt(Block, F, AddStmtChoice::AlwaysAdd);
         EntryConditionBlock = addStmt(Init);
         assert(Block == EntryConditionBlock);
       }
     }
 
     if (Block) {
-      if (!FinishBlock(EntryConditionBlock))
+      if (badCFG)
         return 0;
     }
   }
@@ -1065,18 +1582,21 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
   TryResult KnownVal(true);
 
   if (F->getCond())
-    KnownVal = TryEvaluateBool(F->getCond());
+    KnownVal = tryEvaluateBool(F->getCond());
 
   // Now create the loop body.
   {
     assert(F->getBody());
 
    // Save the current values for Block, Succ, and continue targets.
-   SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ),
-      save_continue(ContinueTargetBlock);
+   SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ);
+   SaveAndRestore<JumpTarget> save_continue(ContinueJumpTarget);
 
     // Create a new block to contain the (bottom) of the loop body.
     Block = NULL;
+    
+    // Loop body should end with destructor of Condition variable (if any).
+    addAutomaticObjDtors(ScopePos, LoopBeginScopePos, F);
 
     if (Stmt* I = F->getInc()) {
       // Generate increment code in its own basic block.  This is the target of
@@ -1086,62 +1606,65 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
       // No increment code.  Create a special, empty, block that is used as the
       // target block for "looping back" to the start of the loop.
       assert(Succ == EntryConditionBlock);
-      Succ = createBlock();
+      Succ = Block ? Block : createBlock();
     }
 
     // Finish up the increment (or empty) block if it hasn't been already.
     if (Block) {
       assert(Block == Succ);
-      if (!FinishBlock(Block))
+      if (badCFG)
         return 0;
       Block = 0;
     }
 
-    ContinueTargetBlock = Succ;
+    ContinueJumpTarget = JumpTarget(Succ, ContinueScopePos);
 
     // The starting block for the loop increment is the block that should
     // represent the 'loop target' for looping back to the start of the loop.
-    ContinueTargetBlock->setLoopTarget(F);
+    ContinueJumpTarget.block->setLoopTarget(F);
+
+    // If body is not a compound statement create implicit scope
+    // and add destructors.
+    if (!isa<CompoundStmt>(F->getBody()))
+      addLocalScopeAndDtors(F->getBody());
 
     // Now populate the body block, and in the process create new blocks as we
     // walk the body of the loop.
     CFGBlock* BodyBlock = addStmt(F->getBody());
 
     if (!BodyBlock)
-      BodyBlock = ContinueTargetBlock; // can happen for "for (...;...;...) ;"
-    else if (Block && !FinishBlock(BodyBlock))
+      BodyBlock = ContinueJumpTarget.block;//can happen for "for (...;...;...);"
+    else if (badCFG)
       return 0;
 
     // This new body block is a successor to our "exit" condition block.
-    AddSuccessor(ExitConditionBlock, KnownVal.isFalse() ? NULL : BodyBlock);
+    addSuccessor(ExitConditionBlock, KnownVal.isFalse() ? NULL : BodyBlock);
   }
 
   // Link up the condition block with the code that follows the loop.  (the
   // false branch).
-  AddSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
+  addSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
 
   // If the loop contains initialization, create a new block for those
   // statements.  This block can also contain statements that precede the loop.
   if (Stmt* I = F->getInit()) {
     Block = createBlock();
     return addStmt(I);
-  } else {
-    // There is no loop initialization.  We are thus basically a while loop.
-    // NULL out Block to force lazy block construction.
-    Block = NULL;
-    Succ = EntryConditionBlock;
-    return EntryConditionBlock;
   }
+
+  // There is no loop initialization.  We are thus basically a while loop.
+  // NULL out Block to force lazy block construction.
+  Block = NULL;
+  Succ = EntryConditionBlock;
+  return EntryConditionBlock;
 }
 
 CFGBlock *CFGBuilder::VisitMemberExpr(MemberExpr *M, AddStmtChoice asc) {
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, M, asc);
+    appendStmt(Block, M, asc);
   }
-  return Visit(M->getBase(),
-               M->isArrow() ? AddStmtChoice::NotAlwaysAdd
-                            : AddStmtChoice::AsLValueNotAlwaysAdd);
+  return Visit(M->getBase());
 }
 
 CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
@@ -1180,7 +1703,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   CFGBlock* LoopSuccessor = 0;
 
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     LoopSuccessor = Block;
     Block = 0;
@@ -1197,7 +1720,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   // The last statement in the block should be the ObjCForCollectionStmt, which
   // performs the actual binding to 'element' and determines if there are any
   // more items in the collection.
-  AppendStmt(ExitConditionBlock, S);
+  appendStmt(ExitConditionBlock, S);
   Block = ExitConditionBlock;
 
   // Walk the 'element' expression to see if there are any side-effects.  We
@@ -1205,7 +1728,7 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   // the CFG unless it contains control-flow.
   EntryConditionBlock = Visit(S->getElement(), AddStmtChoice::NotAlwaysAdd);
   if (Block) {
-    if (!FinishBlock(EntryConditionBlock))
+    if (badCFG)
       return 0;
     Block = 0;
   }
@@ -1217,28 +1740,29 @@ CFGBlock* CFGBuilder::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
   // Now create the true branch.
   {
     // Save the current values for Succ, continue and break targets.
-    SaveAndRestore<CFGBlock*> save_Succ(Succ),
-      save_continue(ContinueTargetBlock), save_break(BreakTargetBlock);
+    SaveAndRestore<CFGBlock*> save_Succ(Succ);
+    SaveAndRestore<JumpTarget> save_continue(ContinueJumpTarget),
+        save_break(BreakJumpTarget);
 
-    BreakTargetBlock = LoopSuccessor;
-    ContinueTargetBlock = EntryConditionBlock;
+    BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
+    ContinueJumpTarget = JumpTarget(EntryConditionBlock, ScopePos);
 
     CFGBlock* BodyBlock = addStmt(S->getBody());
 
     if (!BodyBlock)
       BodyBlock = EntryConditionBlock; // can happen for "for (X in Y) ;"
     else if (Block) {
-      if (!FinishBlock(BodyBlock))
+      if (badCFG)
         return 0;
     }
 
     // This new body block is a successor to our "exit" condition block.
-    AddSuccessor(ExitConditionBlock, BodyBlock);
+    addSuccessor(ExitConditionBlock, BodyBlock);
   }
 
   // Link up the condition block with the code that follows the loop.
   // (the false branch).
-  AddSuccessor(ExitConditionBlock, LoopSuccessor);
+  addSuccessor(ExitConditionBlock, LoopSuccessor);
 
   // Now create a prologue block to contain the collection expression.
   Block = createBlock();
@@ -1254,13 +1778,17 @@ CFGBlock* CFGBuilder::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt* S) {
   // The sync body starts its own basic block.  This makes it a little easier
   // for diagnostic clients.
   if (SyncBlock) {
-    if (!FinishBlock(SyncBlock))
+    if (badCFG)
       return 0;
 
     Block = 0;
     Succ = SyncBlock;
   }
 
+  // Add the @synchronized to the CFG.
+  autoCreateBlock();
+  appendStmt(Block, S, AddStmtChoice::AlwaysAdd);
+
   // Inline the sync expression.
   return addStmt(S->getSynchExpr());
 }
@@ -1273,10 +1801,22 @@ CFGBlock* CFGBuilder::VisitObjCAtTryStmt(ObjCAtTryStmt* S) {
 CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   CFGBlock* LoopSuccessor = NULL;
 
+  // Save local scope position because in case of condition variable ScopePos
+  // won't be restored when traversing AST.
+  SaveAndRestore<LocalScope::const_iterator> save_scope_pos(ScopePos);
+
+  // Create local scope for possible condition variable.
+  // Store scope position for continue statement.
+  LocalScope::const_iterator LoopBeginScopePos = ScopePos;
+  if (VarDecl* VD = W->getConditionVariable()) {
+    addLocalScopeForVarDecl(VD);
+    addAutomaticObjDtors(ScopePos, LoopBeginScopePos, W);
+  }
+
   // "while" is a control-flow statement.  Thus we stop processing the current
   // block.
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     LoopSuccessor = Block;
   } else
@@ -1297,21 +1837,22 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   if (Stmt* C = W->getCond()) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
-    assert(Block == EntryConditionBlock);
+    // The condition might finish the current 'Block'.
+    Block = EntryConditionBlock;
 
     // If this block contains a condition variable, add both the condition
     // variable and initializer to the CFG.
     if (VarDecl *VD = W->getConditionVariable()) {
       if (Expr *Init = VD->getInit()) {
         autoCreateBlock();
-        AppendStmt(Block, W, AddStmtChoice::AlwaysAdd);
+        appendStmt(Block, W, AddStmtChoice::AlwaysAdd);
         EntryConditionBlock = addStmt(Init);
         assert(Block == EntryConditionBlock);
       }
     }
 
     if (Block) {
-      if (!FinishBlock(EntryConditionBlock))
+      if (badCFG)
         return 0;
     }
   }
@@ -1321,16 +1862,16 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
   Succ = EntryConditionBlock;
 
   // See if this is a known constant.
-  const TryResult& KnownVal = TryEvaluateBool(W->getCond());
+  const TryResult& KnownVal = tryEvaluateBool(W->getCond());
 
   // Process the loop body.
   {
     assert(W->getBody());
 
     // Save the current values for Block, Succ, and continue and break targets
-    SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ),
-                              save_continue(ContinueTargetBlock),
-                              save_break(BreakTargetBlock);
+    SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ);
+    SaveAndRestore<JumpTarget> save_continue(ContinueJumpTarget),
+        save_break(BreakJumpTarget);
 
     // Create an empty block to represent the transition block for looping back
     // to the head of the loop.
@@ -1338,31 +1879,39 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
     assert(Succ == EntryConditionBlock);
     Succ = createBlock();
     Succ->setLoopTarget(W);
-    ContinueTargetBlock = Succ;
+    ContinueJumpTarget = JumpTarget(Succ, LoopBeginScopePos);
 
     // All breaks should go to the code following the loop.
-    BreakTargetBlock = LoopSuccessor;
+    BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
 
     // NULL out Block to force lazy instantiation of blocks for the body.
     Block = NULL;
 
+    // Loop body should end with destructor of Condition variable (if any).
+    addAutomaticObjDtors(ScopePos, LoopBeginScopePos, W);
+
+    // If body is not a compound statement create implicit scope
+    // and add destructors.
+    if (!isa<CompoundStmt>(W->getBody()))
+      addLocalScopeAndDtors(W->getBody());
+
     // Create the body.  The returned block is the entry to the loop body.
     CFGBlock* BodyBlock = addStmt(W->getBody());
 
     if (!BodyBlock)
-      BodyBlock = ContinueTargetBlock; // can happen for "while(...) ;"
+      BodyBlock = ContinueJumpTarget.block; // can happen for "while(...) ;"
     else if (Block) {
-      if (!FinishBlock(BodyBlock))
+      if (badCFG)
         return 0;
     }
 
     // Add the loop body entry as a successor to the condition.
-    AddSuccessor(ExitConditionBlock, KnownVal.isFalse() ? NULL : BodyBlock);
+    addSuccessor(ExitConditionBlock, KnownVal.isFalse() ? NULL : BodyBlock);
   }
 
   // Link up the condition block with the code that follows the loop.  (the
   // false branch).
-  AddSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
+  addSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
 
   // There can be no more statements in the condition block since we loop back
   // to this block.  NULL out Block to force lazy creation of another block.
@@ -1385,14 +1934,14 @@ CFGBlock* CFGBuilder::VisitObjCAtThrowStmt(ObjCAtThrowStmt* S) {
   //  statement.
 
   // If we were in the middle of a block we stop processing that block.
-  if (Block && !FinishBlock(Block))
+  if (badCFG)
     return 0;
 
   // Create the new block.
   Block = createBlock(false);
 
   // The Exit block is the only successor.
-  AddSuccessor(Block, &cfg->getExit());
+  addSuccessor(Block, &cfg->getExit());
 
   // Add the statement to the block.  This may create new blocks if S contains
   // control-flow (short-circuit operations).
@@ -1401,7 +1950,7 @@ CFGBlock* CFGBuilder::VisitObjCAtThrowStmt(ObjCAtThrowStmt* S) {
 
 CFGBlock* CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr* T) {
   // If we were in the middle of a block we stop processing that block.
-  if (Block && !FinishBlock(Block))
+  if (badCFG)
     return 0;
 
   // Create the new block.
@@ -1409,10 +1958,10 @@ CFGBlock* CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr* T) {
 
   if (TryTerminatedBlock)
     // The current try statement is the only successor.
-    AddSuccessor(Block, TryTerminatedBlock);
+    addSuccessor(Block, TryTerminatedBlock);
   else
     // otherwise the Exit block is the only successor.
-    AddSuccessor(Block, &cfg->getExit());
+    addSuccessor(Block, &cfg->getExit());
 
   // Add the statement to the block.  This may create new blocks if S contains
   // control-flow (short-circuit operations).
@@ -1425,7 +1974,7 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
   // "do...while" is a control-flow statement.  Thus we stop processing the
   // current block.
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     LoopSuccessor = Block;
   } else
@@ -1446,7 +1995,7 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
     if (Block) {
-      if (!FinishBlock(EntryConditionBlock))
+      if (badCFG)
         return 0;
     }
   }
@@ -1455,7 +2004,7 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
   Succ = EntryConditionBlock;
 
   // See if this is a known constant.
-  const TryResult &KnownVal = TryEvaluateBool(D->getCond());
+  const TryResult &KnownVal = tryEvaluateBool(D->getCond());
 
   // Process the loop body.
   CFGBlock* BodyBlock = NULL;
@@ -1463,26 +2012,31 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
     assert(D->getBody());
 
     // Save the current values for Block, Succ, and continue and break targets
-    SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ),
-      save_continue(ContinueTargetBlock),
-      save_break(BreakTargetBlock);
+    SaveAndRestore<CFGBlock*> save_Block(Block), save_Succ(Succ);
+    SaveAndRestore<JumpTarget> save_continue(ContinueJumpTarget),
+        save_break(BreakJumpTarget);
 
     // All continues within this loop should go to the condition block
-    ContinueTargetBlock = EntryConditionBlock;
+    ContinueJumpTarget = JumpTarget(EntryConditionBlock, ScopePos);
 
     // All breaks should go to the code following the loop.
-    BreakTargetBlock = LoopSuccessor;
+    BreakJumpTarget = JumpTarget(LoopSuccessor, ScopePos);
 
     // NULL out Block to force lazy instantiation of blocks for the body.
     Block = NULL;
 
+    // If body is not a compound statement create implicit scope
+    // and add destructors.
+    if (!isa<CompoundStmt>(D->getBody()))
+      addLocalScopeAndDtors(D->getBody());
+
     // Create the body.  The returned block is the entry to the loop body.
     BodyBlock = addStmt(D->getBody());
 
     if (!BodyBlock)
       BodyBlock = EntryConditionBlock; // can happen for "do ; while(...)"
     else if (Block) {
-      if (!FinishBlock(BodyBlock))
+      if (badCFG)
         return 0;
     }
 
@@ -1498,15 +2052,15 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
       LoopBackBlock->setLoopTarget(D);
 
       // Add the loop body entry as a successor to the condition.
-      AddSuccessor(ExitConditionBlock, LoopBackBlock);
+      addSuccessor(ExitConditionBlock, LoopBackBlock);
     }
     else
-      AddSuccessor(ExitConditionBlock, NULL);
+      addSuccessor(ExitConditionBlock, NULL);
   }
 
   // Link up the condition block with the code that follows the loop.
   // (the false branch).
-  AddSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
+  addSuccessor(ExitConditionBlock, KnownVal.isTrue() ? NULL : LoopSuccessor);
 
   // There can be no more statements in the body block(s) since we loop back to
   // the body.  NULL out Block to force lazy creation of another block.
@@ -1520,8 +2074,8 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
 CFGBlock* CFGBuilder::VisitContinueStmt(ContinueStmt* C) {
   // "continue" is a control-flow statement.  Thus we stop processing the
   // current block.
-  if (Block && !FinishBlock(Block))
-      return 0;
+  if (badCFG)
+    return 0;
 
   // Now create a new block that ends with the continue statement.
   Block = createBlock(false);
@@ -1529,9 +2083,10 @@ CFGBlock* CFGBuilder::VisitContinueStmt(ContinueStmt* C) {
 
   // If there is no target for the continue, then we are looking at an
   // incomplete AST.  This means the CFG cannot be constructed.
-  if (ContinueTargetBlock)
-    AddSuccessor(Block, ContinueTargetBlock);
-  else
+  if (ContinueJumpTarget.block) {
+    addAutomaticObjDtors(ScopePos, ContinueJumpTarget.scopePosition, C);
+    addSuccessor(Block, ContinueJumpTarget.block);
+  } else
     badCFG = true;
 
   return Block;
@@ -1542,12 +2097,12 @@ CFGBlock *CFGBuilder::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E,
 
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, E);
+    appendStmt(Block, E);
   }
 
   // VLA types have expressions that must be evaluated.
   if (E->isArgumentType()) {
-    for (VariableArrayType* VA = FindVA(E->getArgumentType().getTypePtr());
+    for (const VariableArrayType *VA =FindVA(E->getArgumentType().getTypePtr());
          VA != 0; VA = FindVA(VA->getElementType().getTypePtr()))
       addStmt(VA->getSizeExpr());
   }
@@ -1560,7 +2115,7 @@ CFGBlock *CFGBuilder::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E,
 CFGBlock* CFGBuilder::VisitStmtExpr(StmtExpr *SE, AddStmtChoice asc) {
   if (asc.alwaysAdd()) {
     autoCreateBlock();
-    AppendStmt(Block, SE);
+    appendStmt(Block, SE);
   }
   return VisitCompoundStmt(SE->getSubStmt());
 }
@@ -1570,16 +2125,28 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
   // block.
   CFGBlock* SwitchSuccessor = NULL;
 
+  // Save local scope position because in case of condition variable ScopePos
+  // won't be restored when traversing AST.
+  SaveAndRestore<LocalScope::const_iterator> save_scope_pos(ScopePos);
+
+  // Create local scope for possible condition variable.
+  // Store scope position. Add implicit destructor.
+  if (VarDecl* VD = Terminator->getConditionVariable()) {
+    LocalScope::const_iterator SwitchBeginScopePos = ScopePos;
+    addLocalScopeForVarDecl(VD);
+    addAutomaticObjDtors(ScopePos, SwitchBeginScopePos, Terminator);
+  }
+
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     SwitchSuccessor = Block;
   } else SwitchSuccessor = Succ;
 
   // Save the current "switch" context.
   SaveAndRestore<CFGBlock*> save_switch(SwitchTerminatedBlock),
-                            save_break(BreakTargetBlock),
                             save_default(DefaultCaseBlock);
+  SaveAndRestore<JumpTarget> save_break(BreakJumpTarget);
 
   // Set the "default" case to be the block after the switch statement.  If the
   // switch statement contains a "default:", this value will be overwritten with
@@ -1592,22 +2159,28 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
   // Now process the switch body.  The code after the switch is the implicit
   // successor.
   Succ = SwitchSuccessor;
-  BreakTargetBlock = SwitchSuccessor;
+  BreakJumpTarget = JumpTarget(SwitchSuccessor, ScopePos);
 
   // When visiting the body, the case statements should automatically get linked
   // up to the switch.  We also don't keep a pointer to the body, since all
   // control-flow from the switch goes to case/default statements.
   assert(Terminator->getBody() && "switch must contain a non-NULL body");
   Block = NULL;
-  CFGBlock *BodyBlock = addStmt(Terminator->getBody());
+
+  // If body is not a compound statement create implicit scope
+  // and add destructors.
+  if (!isa<CompoundStmt>(Terminator->getBody()))
+    addLocalScopeAndDtors(Terminator->getBody());
+
+  addStmt(Terminator->getBody());
   if (Block) {
-    if (!FinishBlock(BodyBlock))
+    if (badCFG)
       return 0;
   }
 
   // If we have no "default:" case, the default transition is to the code
   // following the switch body.
-  AddSuccessor(SwitchTerminatedBlock, DefaultCaseBlock);
+  addSuccessor(SwitchTerminatedBlock, DefaultCaseBlock);
 
   // Add the terminator and condition in the switch block.
   SwitchTerminatedBlock->setTerminator(Terminator);
@@ -1620,7 +2193,7 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
   if (VarDecl *VD = Terminator->getConditionVariable()) {
     if (Expr *Init = VD->getInit()) {
       autoCreateBlock();
-      AppendStmt(Block, Terminator, AddStmtChoice::AlwaysAdd);
+      appendStmt(Block, Terminator, AddStmtChoice::AlwaysAdd);
       addStmt(Init);
     }
   }
@@ -1638,16 +2211,16 @@ CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
     // (which can blow out the stack), manually unroll and create blocks
     // along the way.
     while (isa<CaseStmt>(Sub)) {
-      CFGBlock *CurrentBlock = createBlock(false);
-      CurrentBlock->setLabel(CS);
+      CFGBlock *currentBlock = createBlock(false);
+      currentBlock->setLabel(CS);
 
       if (TopBlock)
-        AddSuccessor(LastBlock, CurrentBlock);
+        addSuccessor(LastBlock, currentBlock);
       else
-        TopBlock = CurrentBlock;
+        TopBlock = currentBlock;
 
-      AddSuccessor(SwitchTerminatedBlock, CurrentBlock);
-      LastBlock = CurrentBlock;
+      addSuccessor(SwitchTerminatedBlock, currentBlock);
+      LastBlock = currentBlock;
 
       CS = cast<CaseStmt>(Sub);
       Sub = CS->getSubStmt();
@@ -1664,22 +2237,21 @@ CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
   // were processing (the "case XXX:" is the label).
   CaseBlock->setLabel(CS);
 
-  if (!FinishBlock(CaseBlock))
+  if (badCFG)
     return 0;
 
   // Add this block to the list of successors for the block with the switch
   // statement.
   assert(SwitchTerminatedBlock);
-  AddSuccessor(SwitchTerminatedBlock, CaseBlock);
+  addSuccessor(SwitchTerminatedBlock, CaseBlock);
 
   // We set Block to NULL to allow lazy creation of a new block (if necessary)
   Block = NULL;
 
   if (TopBlock) {
-    AddSuccessor(LastBlock, CaseBlock);
+    addSuccessor(LastBlock, CaseBlock);
     Succ = TopBlock;
-  }
-  else {
+  } else {
     // This block is now the implicit successor of other blocks.
     Succ = CaseBlock;
   }
@@ -1700,7 +2272,7 @@ CFGBlock* CFGBuilder::VisitDefaultStmt(DefaultStmt* Terminator) {
   // we were processing (the "default:" is the label).
   DefaultCaseBlock->setLabel(Terminator);
 
-  if (!FinishBlock(DefaultCaseBlock))
+  if (badCFG)
     return 0;
 
   // Unlike case statements, we don't add the default block to the successors
@@ -1724,7 +2296,7 @@ CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
   CFGBlock* TrySuccessor = NULL;
 
   if (Block) {
-    if (!FinishBlock(Block))
+    if (badCFG)
       return 0;
     TrySuccessor = Block;
   } else TrySuccessor = Succ;
@@ -1750,13 +2322,13 @@ CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
       return 0;
     // Add this block to the list of successors for the block with the try
     // statement.
-    AddSuccessor(NewTryTerminatedBlock, CatchBlock);
+    addSuccessor(NewTryTerminatedBlock, CatchBlock);
   }
   if (!HasCatchAll) {
     if (PrevTryTerminatedBlock)
-      AddSuccessor(NewTryTerminatedBlock, PrevTryTerminatedBlock);
+      addSuccessor(NewTryTerminatedBlock, PrevTryTerminatedBlock);
     else
-      AddSuccessor(NewTryTerminatedBlock, &cfg->getExit());
+      addSuccessor(NewTryTerminatedBlock, &cfg->getExit());
   }
 
   // The code after the try is the implicit successor.
@@ -1776,6 +2348,18 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
   // CXXCatchStmt are treated like labels, so they are the first statement in a
   // block.
 
+  // Save local scope position because in case of exception variable ScopePos
+  // won't be restored when traversing AST.
+  SaveAndRestore<LocalScope::const_iterator> save_scope_pos(ScopePos);
+
+  // Create local scope for possible exception variable.
+  // Store scope position. Add implicit destructor.
+  if (VarDecl* VD = CS->getExceptionDecl()) {
+    LocalScope::const_iterator BeginScopePos = ScopePos;
+    addLocalScopeForVarDecl(VD);
+    addAutomaticObjDtors(ScopePos, BeginScopePos, CS);
+  }
+
   if (CS->getHandlerBlock())
     addStmt(CS->getHandlerBlock());
 
@@ -1785,7 +2369,7 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
 
   CatchBlock->setLabel(CS);
 
-  if (!FinishBlock(CatchBlock))
+  if (badCFG)
     return 0;
 
   // We set Block to NULL to allow lazy creation of a new block (if necessary)
@@ -1794,15 +2378,75 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
   return CatchBlock;
 }
 
+CFGBlock *CFGBuilder::VisitExprWithCleanups(ExprWithCleanups *E,
+    AddStmtChoice asc) {
+  if (BuildOpts.AddImplicitDtors) {
+    // If adding implicit destructors visit the full expression for adding
+    // destructors of temporaries.
+    VisitForTemporaryDtors(E->getSubExpr());
+
+    // Full expression has to be added as CFGStmt so it will be sequenced
+    // before destructors of it's temporaries.
+    asc = asc.withAlwaysAdd(true);
+  }
+  return Visit(E->getSubExpr(), asc);
+}
+
+CFGBlock *CFGBuilder::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E,
+                                                AddStmtChoice asc) {
+  if (asc.alwaysAdd()) {
+    autoCreateBlock();
+    appendStmt(Block, E, asc);
+
+    // We do not want to propagate the AlwaysAdd property.
+    asc = asc.withAlwaysAdd(false);
+  }
+  return Visit(E->getSubExpr(), asc);
+}
+
+CFGBlock *CFGBuilder::VisitCXXConstructExpr(CXXConstructExpr *C,
+                                            AddStmtChoice asc) {
+  autoCreateBlock();
+  if (!C->isElidable())
+    appendStmt(Block, C, asc.withAlwaysAdd(true));
+
+  return VisitChildren(C);
+}
+
+CFGBlock *CFGBuilder::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E,
+                                                 AddStmtChoice asc) {
+  if (asc.alwaysAdd()) {
+    autoCreateBlock();
+    appendStmt(Block, E, asc);
+    // We do not want to propagate the AlwaysAdd property.
+    asc = asc.withAlwaysAdd(false);
+  }
+  return Visit(E->getSubExpr(), asc);
+}
+
+CFGBlock *CFGBuilder::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *C,
+                                                  AddStmtChoice asc) {
+  autoCreateBlock();
+  appendStmt(Block, C, asc.withAlwaysAdd(true));
+  return VisitChildren(C);
+}
+
 CFGBlock *CFGBuilder::VisitCXXMemberCallExpr(CXXMemberCallExpr *C,
                                              AddStmtChoice asc) {
-  AddStmtChoice::Kind K = asc.asLValue() ? AddStmtChoice::AlwaysAddAsLValue
-                                         : AddStmtChoice::AlwaysAdd;
   autoCreateBlock();
-  AppendStmt(Block, C, AddStmtChoice(K));
+  appendStmt(Block, C, asc.withAlwaysAdd(true));
   return VisitChildren(C);
 }
 
+CFGBlock *CFGBuilder::VisitImplicitCastExpr(ImplicitCastExpr *E,
+                                            AddStmtChoice asc) {
+  if (asc.alwaysAdd()) {
+    autoCreateBlock();
+    appendStmt(Block, E, asc);
+  }
+  return Visit(E->getSubExpr(), AddStmtChoice());
+}
+
 CFGBlock* CFGBuilder::VisitIndirectGotoStmt(IndirectGotoStmt* I) {
   // Lazily create the indirect-goto dispatch block if there isn't one already.
   CFGBlock* IBlock = cfg->getIndirectGotoBlock();
@@ -1814,15 +2458,210 @@ CFGBlock* CFGBuilder::VisitIndirectGotoStmt(IndirectGotoStmt* I) {
 
   // IndirectGoto is a control-flow statement.  Thus we stop processing the
   // current block and create a new one.
-  if (Block && !FinishBlock(Block))
+  if (badCFG)
     return 0;
 
   Block = createBlock(false);
   Block->setTerminator(I);
-  AddSuccessor(Block, IBlock);
+  addSuccessor(Block, IBlock);
   return addStmt(I->getTarget());
 }
 
+CFGBlock *CFGBuilder::VisitForTemporaryDtors(Stmt *E, bool BindToTemporary) {
+tryAgain:
+  if (!E) {
+    badCFG = true;
+    return NULL;
+  }
+  switch (E->getStmtClass()) {
+    default:
+      return VisitChildrenForTemporaryDtors(E);
+
+    case Stmt::BinaryOperatorClass:
+      return VisitBinaryOperatorForTemporaryDtors(cast<BinaryOperator>(E));
+
+    case Stmt::CXXBindTemporaryExprClass:
+      return VisitCXXBindTemporaryExprForTemporaryDtors(
+          cast<CXXBindTemporaryExpr>(E), BindToTemporary);
+
+    case Stmt::BinaryConditionalOperatorClass:
+    case Stmt::ConditionalOperatorClass:
+      return VisitConditionalOperatorForTemporaryDtors(
+          cast<AbstractConditionalOperator>(E), BindToTemporary);
+
+    case Stmt::ImplicitCastExprClass:
+      // For implicit cast we want BindToTemporary to be passed further.
+      E = cast<CastExpr>(E)->getSubExpr();
+      goto tryAgain;
+
+    case Stmt::ParenExprClass:
+      E = cast<ParenExpr>(E)->getSubExpr();
+      goto tryAgain;
+  }
+}
+
+CFGBlock *CFGBuilder::VisitChildrenForTemporaryDtors(Stmt *E) {
+  // When visiting children for destructors we want to visit them in reverse
+  // order. Because there's no reverse iterator for children must to reverse
+  // them in helper vector.
+  typedef llvm::SmallVector<Stmt *, 4> ChildrenVect;
+  ChildrenVect ChildrenRev;
+  for (Stmt::child_range I = E->children(); I; ++I) {
+    if (*I) ChildrenRev.push_back(*I);
+  }
+
+  CFGBlock *B = Block;
+  for (ChildrenVect::reverse_iterator I = ChildrenRev.rbegin(),
+      L = ChildrenRev.rend(); I != L; ++I) {
+    if (CFGBlock *R = VisitForTemporaryDtors(*I))
+      B = R;
+  }
+  return B;
+}
+
+CFGBlock *CFGBuilder::VisitBinaryOperatorForTemporaryDtors(BinaryOperator *E) {
+  if (E->isLogicalOp()) {
+    // Destructors for temporaries in LHS expression should be called after
+    // those for RHS expression. Even if this will unnecessarily create a block,
+    // this block will be used at least by the full expression.
+    autoCreateBlock();
+    CFGBlock *ConfluenceBlock = VisitForTemporaryDtors(E->getLHS());
+    if (badCFG)
+      return NULL;
+
+    Succ = ConfluenceBlock;
+    Block = NULL;
+    CFGBlock *RHSBlock = VisitForTemporaryDtors(E->getRHS());
+
+    if (RHSBlock) {
+      if (badCFG)
+        return NULL;
+
+      // If RHS expression did produce destructors we need to connect created
+      // blocks to CFG in same manner as for binary operator itself.
+      CFGBlock *LHSBlock = createBlock(false);
+      LHSBlock->setTerminator(CFGTerminator(E, true));
+
+      // For binary operator LHS block is before RHS in list of predecessors
+      // of ConfluenceBlock.
+      std::reverse(ConfluenceBlock->pred_begin(),
+          ConfluenceBlock->pred_end());
+
+      // See if this is a known constant.
+      TryResult KnownVal = tryEvaluateBool(E->getLHS());
+      if (KnownVal.isKnown() && (E->getOpcode() == BO_LOr))
+        KnownVal.negate();
+
+      // Link LHSBlock with RHSBlock exactly the same way as for binary operator
+      // itself.
+      if (E->getOpcode() == BO_LOr) {
+        addSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
+        addSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
+      } else {
+        assert (E->getOpcode() == BO_LAnd);
+        addSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
+        addSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
+      }
+
+      Block = LHSBlock;
+      return LHSBlock;
+    }
+
+    Block = ConfluenceBlock;
+    return ConfluenceBlock;
+  }
+
+  if (E->isAssignmentOp()) {
+    // For assignment operator (=) LHS expression is visited
+    // before RHS expression. For destructors visit them in reverse order.
+    CFGBlock *RHSBlock = VisitForTemporaryDtors(E->getRHS());
+    CFGBlock *LHSBlock = VisitForTemporaryDtors(E->getLHS());
+    return LHSBlock ? LHSBlock : RHSBlock;
+  }
+
+  // For any other binary operator RHS expression is visited before
+  // LHS expression (order of children). For destructors visit them in reverse
+  // order.
+  CFGBlock *LHSBlock = VisitForTemporaryDtors(E->getLHS());
+  CFGBlock *RHSBlock = VisitForTemporaryDtors(E->getRHS());
+  return RHSBlock ? RHSBlock : LHSBlock;
+}
+
+CFGBlock *CFGBuilder::VisitCXXBindTemporaryExprForTemporaryDtors(
+    CXXBindTemporaryExpr *E, bool BindToTemporary) {
+  // First add destructors for temporaries in subexpression.
+  CFGBlock *B = VisitForTemporaryDtors(E->getSubExpr());
+  if (!BindToTemporary) {
+    // If lifetime of temporary is not prolonged (by assigning to constant
+    // reference) add destructor for it.
+    autoCreateBlock();
+    appendTemporaryDtor(Block, E);
+    B = Block;
+  }
+  return B;
+}
+
+CFGBlock *CFGBuilder::VisitConditionalOperatorForTemporaryDtors(
+    AbstractConditionalOperator *E, bool BindToTemporary) {
+  // First add destructors for condition expression.  Even if this will
+  // unnecessarily create a block, this block will be used at least by the full
+  // expression.
+  autoCreateBlock();
+  CFGBlock *ConfluenceBlock = VisitForTemporaryDtors(E->getCond());
+  if (badCFG)
+    return NULL;
+  if (BinaryConditionalOperator *BCO
+        = dyn_cast<BinaryConditionalOperator>(E)) {
+    ConfluenceBlock = VisitForTemporaryDtors(BCO->getCommon());
+    if (badCFG)
+      return NULL;
+  }
+
+  // Try to add block with destructors for LHS expression.
+  CFGBlock *LHSBlock = NULL;
+  Succ = ConfluenceBlock;
+  Block = NULL;
+  LHSBlock = VisitForTemporaryDtors(E->getTrueExpr(), BindToTemporary);
+  if (badCFG)
+    return NULL;
+
+  // Try to add block with destructors for RHS expression;
+  Succ = ConfluenceBlock;
+  Block = NULL;
+  CFGBlock *RHSBlock = VisitForTemporaryDtors(E->getFalseExpr(),
+                                              BindToTemporary);
+  if (badCFG)
+    return NULL;
+
+  if (!RHSBlock && !LHSBlock) {
+    // If neither LHS nor RHS expression had temporaries to destroy don't create
+    // more blocks.
+    Block = ConfluenceBlock;
+    return Block;
+  }
+
+  Block = createBlock(false);
+  Block->setTerminator(CFGTerminator(E, true));
+
+  // See if this is a known constant.
+  const TryResult &KnownVal = tryEvaluateBool(E->getCond());
+
+  if (LHSBlock) {
+    addSuccessor(Block, KnownVal.isFalse() ? NULL : LHSBlock);
+  } else if (KnownVal.isFalse()) {
+    addSuccessor(Block, NULL);
+  } else {
+    addSuccessor(Block, ConfluenceBlock);
+    std::reverse(ConfluenceBlock->pred_begin(), ConfluenceBlock->pred_end());
+  }
+
+  if (!RHSBlock)
+    RHSBlock = ConfluenceBlock;
+  addSuccessor(Block, KnownVal.isTrue() ? NULL : RHSBlock);
+
+  return Block;
+}
+
 } // end anonymous namespace
 
 /// createBlock - Constructs and adds a new CFGBlock to the CFG.  The block has
@@ -1847,11 +2686,9 @@ CFGBlock* CFG::createBlock() {
 /// buildCFG - Constructs a CFG from an AST.  Ownership of the returned
 ///  CFG is returned to the caller.
 CFG* CFG::buildCFG(const Decl *D, Stmt* Statement, ASTContext *C,
-                   bool PruneTriviallyFalse,
-                   bool AddEHEdges, bool AddScopes) {
+    BuildOptions BO) {
   CFGBuilder Builder;
-  return Builder.buildCFG(D, Statement, C, PruneTriviallyFalse,
-                          AddEHEdges, AddScopes);
+  return Builder.buildCFG(D, Statement, C, BO);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1867,7 +2704,7 @@ static void FindSubExprAssignments(Stmt *S,
   if (!S)
     return;
 
-  for (Stmt::child_iterator I=S->child_begin(), E=S->child_end(); I!=E; ++I) {
+  for (Stmt::child_range I = S->children(); I; ++I) {
     Stmt *child = *I;
     if (!child)
       continue;
@@ -1890,15 +2727,19 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
 
   for (CFG::iterator I=cfg.begin(), E=cfg.end(); I != E; ++I)
     for (CFGBlock::iterator BI=(*I)->begin(), EI=(*I)->end(); BI != EI; ++BI)
-      FindSubExprAssignments(*BI, SubExprAssignments);
+      if (CFGStmt S = BI->getAs<CFGStmt>())
+        FindSubExprAssignments(S, SubExprAssignments);
 
   for (CFG::iterator I=cfg.begin(), E=cfg.end(); I != E; ++I) {
 
     // Iterate over the statements again on identify the Expr* and Stmt* at the
     // block-level that are block-level expressions.
 
-    for (CFGBlock::iterator BI=(*I)->begin(), EI=(*I)->end(); BI != EI; ++BI)
-      if (Expr* Exp = dyn_cast<Expr>(*BI)) {
+    for (CFGBlock::iterator BI=(*I)->begin(), EI=(*I)->end(); BI != EI; ++BI) {
+      CFGStmt CS = BI->getAs<CFGStmt>();
+      if (!CS.isValid())
+        continue;
+      if (Expr* Exp = dyn_cast<Expr>(CS.getStmt())) {
 
         if (BinaryOperator* B = dyn_cast<BinaryOperator>(Exp)) {
           // Assignment expressions that are not nested within another
@@ -1919,6 +2760,7 @@ static BlkExprMapTy* PopulateBlkExprMap(CFG& cfg) {
         unsigned x = M->size();
         (*M)[Exp] = x;
       }
+    }
 
     // Look at terminators.  The condition is a block-level expression.
 
@@ -1945,12 +2787,34 @@ CFG::BlkExprNumTy CFG::getBlkExprNum(const Stmt* S) {
 unsigned CFG::getNumBlkExprs() {
   if (const BlkExprMapTy* M = reinterpret_cast<const BlkExprMapTy*>(BlkExprMap))
     return M->size();
-  else {
-    // We assume callers interested in the number of BlkExprs will want
-    // the map constructed if it doesn't already exist.
-    BlkExprMap = (void*) PopulateBlkExprMap(*this);
-    return reinterpret_cast<BlkExprMapTy*>(BlkExprMap)->size();
+
+  // We assume callers interested in the number of BlkExprs will want
+  // the map constructed if it doesn't already exist.
+  BlkExprMap = (void*) PopulateBlkExprMap(*this);
+  return reinterpret_cast<BlkExprMapTy*>(BlkExprMap)->size();
+}
+
+//===----------------------------------------------------------------------===//
+// Filtered walking of the CFG.
+//===----------------------------------------------------------------------===//
+
+bool CFGBlock::FilterEdge(const CFGBlock::FilterOptions &F,
+        const CFGBlock *From, const CFGBlock *To) {
+
+  if (F.IgnoreDefaultsWithCoveredEnums) {
+    // If the 'To' has no label or is labeled but the label isn't a
+    // CaseStmt then filter this edge.
+    if (const SwitchStmt *S =
+  dyn_cast_or_null<SwitchStmt>(From->getTerminator().getStmt())) {
+      if (S->isAllEnumCasesCovered()) {
+  const Stmt *L = To->getLabel();
+  if (!L || !isa<CaseStmt>(L))
+    return true;
+      }
+    }
   }
+
+  return false;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1969,37 +2833,81 @@ namespace {
 
 class StmtPrinterHelper : public PrinterHelper  {
   typedef llvm::DenseMap<Stmt*,std::pair<unsigned,unsigned> > StmtMapTy;
+  typedef llvm::DenseMap<Decl*,std::pair<unsigned,unsigned> > DeclMapTy;
   StmtMapTy StmtMap;
-  signed CurrentBlock;
-  unsigned CurrentStmt;
+  DeclMapTy DeclMap;
+  signed currentBlock;
+  unsigned currentStmt;
   const LangOptions &LangOpts;
 public:
 
   StmtPrinterHelper(const CFG* cfg, const LangOptions &LO)
-    : CurrentBlock(0), CurrentStmt(0), LangOpts(LO) {
+    : currentBlock(0), currentStmt(0), LangOpts(LO) {
     for (CFG::const_iterator I = cfg->begin(), E = cfg->end(); I != E; ++I ) {
       unsigned j = 1;
       for (CFGBlock::const_iterator BI = (*I)->begin(), BEnd = (*I)->end() ;
-           BI != BEnd; ++BI, ++j )
-        StmtMap[*BI] = std::make_pair((*I)->getBlockID(),j);
+           BI != BEnd; ++BI, ++j ) {        
+        if (CFGStmt SE = BI->getAs<CFGStmt>()) {
+          std::pair<unsigned, unsigned> P((*I)->getBlockID(), j);
+          StmtMap[SE] = P;
+
+          if (DeclStmt* DS = dyn_cast<DeclStmt>(SE.getStmt())) {
+              DeclMap[DS->getSingleDecl()] = P;
+
+          } else if (IfStmt* IS = dyn_cast<IfStmt>(SE.getStmt())) {
+            if (VarDecl* VD = IS->getConditionVariable())
+              DeclMap[VD] = P;
+
+          } else if (ForStmt* FS = dyn_cast<ForStmt>(SE.getStmt())) {
+            if (VarDecl* VD = FS->getConditionVariable())
+              DeclMap[VD] = P;
+
+          } else if (WhileStmt* WS = dyn_cast<WhileStmt>(SE.getStmt())) {
+            if (VarDecl* VD = WS->getConditionVariable())
+              DeclMap[VD] = P;
+
+          } else if (SwitchStmt* SS = dyn_cast<SwitchStmt>(SE.getStmt())) {
+            if (VarDecl* VD = SS->getConditionVariable())
+              DeclMap[VD] = P;
+
+          } else if (CXXCatchStmt* CS = dyn_cast<CXXCatchStmt>(SE.getStmt())) {
+            if (VarDecl* VD = CS->getExceptionDecl())
+              DeclMap[VD] = P;
+          }
+        }
       }
+    }
   }
 
   virtual ~StmtPrinterHelper() {}
 
   const LangOptions &getLangOpts() const { return LangOpts; }
-  void setBlockID(signed i) { CurrentBlock = i; }
-  void setStmtID(unsigned i) { CurrentStmt = i; }
-
-  virtual bool handledStmt(Stmt* Terminator, llvm::raw_ostream& OS) {
+  void setBlockID(signed i) { currentBlock = i; }
+  void setStmtID(unsigned i) { currentStmt = i; }
 
-    StmtMapTy::iterator I = StmtMap.find(Terminator);
+  virtual bool handledStmt(Stmt* S, llvm::raw_ostream& OS) {
+    StmtMapTy::iterator I = StmtMap.find(S);
 
     if (I == StmtMap.end())
       return false;
 
-    if (CurrentBlock >= 0 && I->second.first == (unsigned) CurrentBlock
-                          && I->second.second == CurrentStmt) {
+    if (currentBlock >= 0 && I->second.first == (unsigned) currentBlock
+                          && I->second.second == currentStmt) {
+      return false;
+    }
+
+    OS << "[B" << I->second.first << "." << I->second.second << "]";
+    return true;
+  }
+
+  bool handleDecl(Decl* D, llvm::raw_ostream& OS) {
+    DeclMapTy::iterator I = DeclMap.find(D);
+
+    if (I == DeclMap.end())
+      return false;
+
+    if (currentBlock >= 0 && I->second.first == (unsigned) currentBlock
+                          && I->second.second == currentStmt) {
       return false;
     }
 
@@ -2066,7 +2974,7 @@ public:
     OS << "try ...";
   }
 
-  void VisitConditionalOperator(ConditionalOperator* C) {
+  void VisitAbstractConditionalOperator(AbstractConditionalOperator* C) {
     C->getCond()->printPretty(OS, Helper, Policy);
     OS << " ? ... : ...";
   }
@@ -2108,58 +3016,95 @@ public:
 };
 } // end anonymous namespace
 
-
-static void print_stmt(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
+static void print_elem(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
                        const CFGElement &E) {
-
-  if (E.asStartScope()) {
-    OS << "start scope\n";
-    return;
-  }
-  if (E.asEndScope()) {
-    OS << "end scope\n";
-    return;
-  }
-
-  Stmt *S = E;
-  
-  if (Helper) {
-    // special printing for statement-expressions.
-    if (StmtExpr* SE = dyn_cast<StmtExpr>(S)) {
-      CompoundStmt* Sub = SE->getSubStmt();
-
-      if (Sub->child_begin() != Sub->child_end()) {
-        OS << "({ ... ; ";
-        Helper->handledStmt(*SE->getSubStmt()->body_rbegin(),OS);
-        OS << " })\n";
-        return;
+  if (CFGStmt CS = E.getAs<CFGStmt>()) {
+    Stmt *S = CS;
+    
+    if (Helper) {
+
+      // special printing for statement-expressions.
+      if (StmtExpr* SE = dyn_cast<StmtExpr>(S)) {
+        CompoundStmt* Sub = SE->getSubStmt();
+
+        if (Sub->children()) {
+          OS << "({ ... ; ";
+          Helper->handledStmt(*SE->getSubStmt()->body_rbegin(),OS);
+          OS << " })\n";
+          return;
+        }
+      }
+      // special printing for comma expressions.
+      if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
+        if (B->getOpcode() == BO_Comma) {
+          OS << "... , ";
+          Helper->handledStmt(B->getRHS(),OS);
+          OS << '\n';
+          return;
+        }
       }
     }
+    S->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
 
-    // special printing for comma expressions.
-    if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
-      if (B->getOpcode() == BO_Comma) {
-        OS << "... , ";
-        Helper->handledStmt(B->getRHS(),OS);
-        OS << '\n';
-        return;
-      }
+    if (isa<CXXOperatorCallExpr>(S)) {
+      OS << " (OperatorCall)";
+    } else if (isa<CXXBindTemporaryExpr>(S)) {
+      OS << " (BindTemporary)";
     }
-  }
 
-  S->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
-  
-  if (isa<CXXOperatorCallExpr>(S)) {
-    OS << " (OperatorCall)";    
-  }
-  else if (isa<CXXBindTemporaryExpr>(S)) {
-    OS << " (BindTemporary)";    
-  }
+    // Expressions need a newline.
+    if (isa<Expr>(S))
+      OS << '\n';
 
+  } else if (CFGInitializer IE = E.getAs<CFGInitializer>()) {
+    CXXCtorInitializer* I = IE;
+    if (I->isBaseInitializer())
+      OS << I->getBaseClass()->getAsCXXRecordDecl()->getName();
+    else OS << I->getAnyMember()->getName();
 
-  // Expressions need a newline.
-  if (isa<Expr>(S))
-    OS << '\n';
+    OS << "(";
+    if (Expr* IE = I->getInit())
+      IE->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
+    OS << ")";
+
+    if (I->isBaseInitializer())
+      OS << " (Base initializer)\n";
+    else OS << " (Member initializer)\n";
+
+  } else if (CFGAutomaticObjDtor DE = E.getAs<CFGAutomaticObjDtor>()){
+    VarDecl* VD = DE.getVarDecl();
+    Helper->handleDecl(VD, OS);
+
+    const Type* T = VD->getType().getTypePtr();
+    if (const ReferenceType* RT = T->getAs<ReferenceType>())
+      T = RT->getPointeeType().getTypePtr();
+    else if (const Type *ET = T->getArrayElementTypeNoTypeQual())
+      T = ET;
+
+    OS << ".~" << T->getAsCXXRecordDecl()->getName().str() << "()";
+    OS << " (Implicit destructor)\n";
+
+  } else if (CFGBaseDtor BE = E.getAs<CFGBaseDtor>()) {
+    const CXXBaseSpecifier *BS = BE.getBaseSpecifier();
+    OS << "~" << BS->getType()->getAsCXXRecordDecl()->getName() << "()";
+    OS << " (Base object destructor)\n";
+
+  } else if (CFGMemberDtor ME = E.getAs<CFGMemberDtor>()) {
+    FieldDecl *FD = ME.getFieldDecl();
+
+    const Type *T = FD->getType().getTypePtr();
+    if (const Type *ET = T->getArrayElementTypeNoTypeQual())
+      T = ET;
+
+    OS << "this->" << FD->getName();
+    OS << ".~" << T->getAsCXXRecordDecl()->getName() << "()";
+    OS << " (Member object destructor)\n";
+
+  } else if (CFGTemporaryDtor TE = E.getAs<CFGTemporaryDtor>()) {
+    CXXBindTemporaryExpr *BT = TE.getBindTemporaryExpr();
+    OS << "~" << BT->getType()->getAsCXXRecordDecl()->getName() << "()";
+    OS << " (Temporary object destructor)\n";
+  }
 }
 
 static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
@@ -2229,7 +3174,7 @@ static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
     if (Helper)
       Helper->setStmtID(j);
 
-    print_stmt(OS,Helper,*I);
+    print_elem(OS,Helper,*I);
   }
 
   // Print the terminator of this block.
@@ -2243,7 +3188,7 @@ static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
 
     CFGBlockTerminatorPrint TPrinter(OS, Helper,
                                      PrintingPolicy(Helper->getLangOpts()));
-    TPrinter.Visit(const_cast<Stmt*>(B.getTerminator()));
+    TPrinter.Visit(const_cast<Stmt*>(B.getTerminator().getStmt()));
     OS << '\n';
   }
 
@@ -2325,11 +3270,11 @@ void CFGBlock::print(llvm::raw_ostream& OS, const CFG* cfg,
 void CFGBlock::printTerminator(llvm::raw_ostream &OS,
                                const LangOptions &LO) const {
   CFGBlockTerminatorPrint TPrinter(OS, NULL, PrintingPolicy(LO));
-  TPrinter.Visit(const_cast<Stmt*>(getTerminator()));
+  TPrinter.Visit(const_cast<Stmt*>(getTerminator().getStmt()));
 }
 
 Stmt* CFGBlock::getTerminatorCondition() {
-
+  Stmt *Terminator = this->Terminator;
   if (!Terminator)
     return NULL;
 
@@ -2367,6 +3312,10 @@ Stmt* CFGBlock::getTerminatorCondition() {
       E = cast<SwitchStmt>(Terminator)->getCond();
       break;
 
+    case Stmt::BinaryConditionalOperatorClass:
+      E = cast<BinaryConditionalOperator>(Terminator)->getCond();
+      break;
+
     case Stmt::ConditionalOperatorClass:
       E = cast<ConditionalOperator>(Terminator)->getCond();
       break;
@@ -2383,7 +3332,7 @@ Stmt* CFGBlock::getTerminatorCondition() {
 }
 
 bool CFGBlock::hasBinaryBranchTerminator() const {
-
+  const Stmt *Terminator = this->Terminator;
   if (!Terminator)
     return false;
 
@@ -2398,6 +3347,7 @@ bool CFGBlock::hasBinaryBranchTerminator() const {
     case Stmt::DoStmtClass:
     case Stmt::IfStmtClass:
     case Stmt::ChooseExprClass:
+    case Stmt::BinaryConditionalOperatorClass:
     case Stmt::ConditionalOperatorClass:
     case Stmt::BinaryOperatorClass:
       return true;
diff --git a/lib/Analysis/CFGStmtMap.cpp b/lib/Analysis/CFGStmtMap.cpp
index 965eca1b3c90..3a030f9bdd1e 100644
--- a/lib/Analysis/CFGStmtMap.cpp
+++ b/lib/Analysis/CFGStmtMap.cpp
@@ -50,15 +50,18 @@ static void Accumulate(SMap &SM, CFGBlock *B) {
   // First walk the block-level expressions.
   for (CFGBlock::iterator I = B->begin(), E = B->end(); I != E; ++I) {
     const CFGElement &CE = *I;
-    if (Stmt *S = CE.getStmt()) {
-      CFGBlock *&Entry = SM[S];
-      // If 'Entry' is already initialized (e.g., a terminator was already),
-      // skip.
-      if (Entry)
-        continue;
+    CFGStmt CS = CE.getAs<CFGStmt>();
+    if (!CS.isValid())
+      continue;
+    
+    CFGBlock *&Entry = SM[CS];
+    // If 'Entry' is already initialized (e.g., a terminator was already),
+    // skip.
+    if (Entry)
+      continue;
       
-      Entry = B;
-    }
+    Entry = B;
+    
   }
   
   // Look at the label of the block.
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index 850e9b468100..0912f3c87448 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -1,9 +1,10 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST clangIndex)
 
 add_clang_library(clangAnalysis
   AnalysisContext.cpp
   CFG.cpp
   CFGStmtMap.cpp
+  CocoaConventions.cpp
   FormatString.cpp
   LiveVariables.cpp
   PrintfFormatString.cpp
@@ -11,6 +12,7 @@ add_clang_library(clangAnalysis
   ReachableCode.cpp
   ScanfFormatString.cpp
   UninitializedValues.cpp
+  UninitializedValuesV2.cpp
   )
 
 add_dependencies(clangAnalysis ClangAttrClasses ClangAttrList
diff --git a/lib/Checker/CocoaConventions.cpp b/lib/Analysis/CocoaConventions.cpp
index b446a048d488..22b6c1aaf021 100644
--- a/lib/Checker/CocoaConventions.cpp
+++ b/lib/Analysis/CocoaConventions.cpp
@@ -11,13 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/DomainSpecific/CocoaConventions.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
 #include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
+using namespace ento;
 
 using llvm::StringRef;
 
@@ -53,7 +54,8 @@ static const char* parseWord(const char* s) {
   return s;
 }
 
-cocoa::NamingConvention cocoa::deriveNamingConvention(Selector S) {
+cocoa::NamingConvention cocoa::deriveNamingConvention(Selector S,
+                                                      bool ignorePrefix) {
   IdentifierInfo *II = S.getIdentifierInfoForSlot(0);
 
   if (!II)
@@ -61,80 +63,62 @@ cocoa::NamingConvention cocoa::deriveNamingConvention(Selector S) {
 
   const char *s = II->getNameStart();
 
+  const char *orig = s;
   // A method/function name may contain a prefix.  We don't know it is there,
   // however, until we encounter the first '_'.
-  bool InPossiblePrefix = true;
-  bool AtBeginning = true;
-  NamingConvention C = NoConvention;
-
   while (*s != '\0') {
-    // Skip '_'.
-    if (*s == '_') {
-      if (InPossiblePrefix) {
-        // If we already have a convention, return it.  Otherwise, skip
-        // the prefix as if it wasn't there.
-        if (C != NoConvention)
-          break;
-        
-        InPossiblePrefix = false;
-        AtBeginning = true;
-        assert(C == NoConvention);
-      }
+    // Skip '_', numbers, ':', etc.
+    if (*s == '_' || !isalpha(*s)) {      
       ++s;
       continue;
     }
+    break;
+  }
 
-    // Skip numbers, ':', etc.
-    if (!isalpha(*s)) {
-      ++s;
-      continue;
-    }
+  if (!ignorePrefix && s != orig)
+    return NoConvention;
 
-    const char *wordEnd = parseWord(s);
-    assert(wordEnd > s);
-    unsigned len = wordEnd - s;
+  // Parse the first word, and look for specific keywords.
+  const char *wordEnd = parseWord(s);
+  assert(wordEnd > s);
+  unsigned len = wordEnd - s;
 
-    switch (len) {
+  switch (len) {
     default:
-      break;
+      return NoConvention;
     case 3:
       // Methods starting with 'new' follow the create rule.
-      if (AtBeginning && StringRef(s, len).equals_lower("new"))
-        C = CreateRule;
-      break;
+      return (memcmp(s, "new", 3) == 0) ? CreateRule : NoConvention;
     case 4:
-      // Methods starting with 'alloc' or contain 'copy' follow the
-      // create rule
-      if (C == NoConvention && StringRef(s, len).equals_lower("copy"))
-        C = CreateRule;
-      else // Methods starting with 'init' follow the init rule.
-        if (AtBeginning && StringRef(s, len).equals_lower("init"))
-          C = InitRule;
-      break;
+      // Methods starting with 'copy' follow the create rule.
+      if (memcmp(s, "copy", 4) == 0)
+        return CreateRule;
+      // Methods starting with 'init' follow the init rule.
+      if (memcmp(s, "init", 4) == 0)
+        return InitRule;
+      return NoConvention;
     case 5:
-      if (AtBeginning && StringRef(s, len).equals_lower("alloc"))
-        C = CreateRule;
-      break;
-    }
-
-    // If we aren't in the prefix and have a derived convention then just
-    // return it now.
-    if (!InPossiblePrefix && C != NoConvention)
-      return C;
-
-    AtBeginning = false;
-    s = wordEnd;
+      return (memcmp(s, "alloc", 5) == 0) ? CreateRule : NoConvention;
+    case 7:
+      // Methods starting with 'mutableCopy' follow the create rule.
+      if (memcmp(s, "mutable", 7) == 0) {
+        // Look at the next word to see if it is "Copy".
+        s = wordEnd;
+        if (*s != '\0') {
+          wordEnd = parseWord(s);
+          len = wordEnd - s;
+          if (len == 4 && memcmp(s, "Copy", 4) == 0)
+            return CreateRule;
+        }
+      }
+      return NoConvention;
   }
-
-  // We will get here if there wasn't more than one word
-  // after the prefix.
-  return C;
 }
 
 bool cocoa::isRefType(QualType RetTy, llvm::StringRef Prefix,
                       llvm::StringRef Name) {
   // Recursively walk the typedef stack, allowing typedefs of reference types.
-  while (TypedefType* TD = dyn_cast<TypedefType>(RetTy.getTypePtr())) {
+  while (const TypedefType *TD = dyn_cast<TypedefType>(RetTy.getTypePtr())) {
     llvm::StringRef TDName = TD->getDecl()->getIdentifier()->getName();
     if (TDName.startswith(Prefix) && TDName.endswith("Ref"))
       return true;
diff --git a/lib/Analysis/FormatString.cpp b/lib/Analysis/FormatString.cpp
index 388b9d34a238..c1b5ea8a652a 100644
--- a/lib/Analysis/FormatString.cpp
+++ b/lib/Analysis/FormatString.cpp
@@ -296,8 +296,8 @@ bool ArgTypeResult::matchesType(ASTContext &C, QualType argTy) const {
     }
 
     case CPointerTy:
-      return argTy->getAs<PointerType>() != NULL ||
-             argTy->getAs<ObjCObjectPointerType>() != NULL;
+      return argTy->isPointerType() || argTy->isObjCObjectPointerType() ||
+        argTy->isNullPtrType();
 
     case ObjCPointerTy:
       return argTy->getAs<ObjCObjectPointerType>() != NULL;
diff --git a/lib/Analysis/LiveVariables.cpp b/lib/Analysis/LiveVariables.cpp
index 47b2e3d6040f..303dc0f604d7 100644
--- a/lib/Analysis/LiveVariables.cpp
+++ b/lib/Analysis/LiveVariables.cpp
@@ -104,8 +104,9 @@ namespace {
 class TransferFuncs : public CFGRecStmtVisitor<TransferFuncs>{
   LiveVariables::AnalysisDataTy& AD;
   LiveVariables::ValTy LiveState;
+  const CFGBlock *currentBlock;
 public:
-  TransferFuncs(LiveVariables::AnalysisDataTy& ad) : AD(ad) {}
+  TransferFuncs(LiveVariables::AnalysisDataTy& ad) : AD(ad), currentBlock(0) {}
 
   LiveVariables::ValTy& getVal() { return LiveState; }
   CFG& getCFG() { return AD.getCFG(); }
@@ -128,7 +129,10 @@ public:
   void SetTopValue(LiveVariables::ValTy& V) {
     V = AD.AlwaysLive;
   }
-
+  
+  void setCurrentBlock(const CFGBlock *block) {
+    currentBlock = block;
+  }
 };
 
 void TransferFuncs::Visit(Stmt *S) {
@@ -136,7 +140,7 @@ void TransferFuncs::Visit(Stmt *S) {
   if (S == getCurrentBlkStmt()) {
 
     if (AD.Observer)
-      AD.Observer->ObserveStmt(S,AD,LiveState);
+      AD.Observer->ObserveStmt(S, currentBlock, AD, LiveState);
 
     if (getCFG().isBlkExpr(S))
       LiveState(S, AD) = Dead;
@@ -146,7 +150,7 @@ void TransferFuncs::Visit(Stmt *S) {
   else if (!getCFG().isBlkExpr(S)) {
 
     if (AD.Observer)
-      AD.Observer->ObserveStmt(S,AD,LiveState);
+      AD.Observer->ObserveStmt(S, currentBlock, AD, LiveState);
 
     StmtVisitor<TransferFuncs,void>::Visit(S);
 
diff --git a/lib/Analysis/PrintfFormatString.cpp b/lib/Analysis/PrintfFormatString.cpp
index b8c327cdeba6..db9f7f2c834e 100644
--- a/lib/Analysis/PrintfFormatString.cpp
+++ b/lib/Analysis/PrintfFormatString.cpp
@@ -100,6 +100,10 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
   for ( ; I != E; ++I) {
     switch (*I) {
       default: hasMore = false; break;
+      case '\'':
+        // FIXME: POSIX specific.  Always accept?
+        FS.setHasThousandsGrouping(I);
+        break;
       case '-': FS.setIsLeftJustified(I); break;
       case '+': FS.setHasPlusPrefix(I); break;
       case ' ': FS.setHasSpacePrefix(I); break;
@@ -185,7 +189,7 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
     case 's': k = ConversionSpecifier::sArg;      break;
     case 'u': k = ConversionSpecifier::uArg; break;
     case 'x': k = ConversionSpecifier::xArg; break;
-    // Mac OS X (unicode) specific
+    // POSIX specific.
     case 'C': k = ConversionSpecifier::CArg; break;
     case 'S': k = ConversionSpecifier::SArg; break;
     // Objective-C.
@@ -200,7 +204,7 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
 
   if (k == ConversionSpecifier::InvalidSpecifier) {
     // Assume the conversion takes one argument.
-    return !H.HandleInvalidPrintfConversionSpecifier(FS, Beg, I - Beg);
+    return !H.HandleInvalidPrintfConversionSpecifier(FS, Start, I - Start);
   }
   return PrintfSpecifierResult(Start, FS);
 }
@@ -277,7 +281,7 @@ const char *ConversionSpecifier::toString() const {
 
 ArgTypeResult PrintfSpecifier::getArgType(ASTContext &Ctx) const {
   const PrintfConversionSpecifier &CS = getConversionSpecifier();
-  
+
   if (!CS.consumesDataArgument())
     return ArgTypeResult::Invalid();
 
@@ -288,7 +292,7 @@ ArgTypeResult PrintfSpecifier::getArgType(ASTContext &Ctx) const {
       default:
         return ArgTypeResult::Invalid();
     }
-  
+
   if (CS.isIntArg())
     switch (LM.getKind()) {
       case LengthModifier::AsLongDouble:
@@ -382,7 +386,20 @@ bool PrintfSpecifier::fixType(QualType QT) {
     LM.setKind(LengthModifier::None);
     break;
 
-  case BuiltinType::WChar:
+  case BuiltinType::Char_U:
+  case BuiltinType::UChar:
+  case BuiltinType::Char_S:
+  case BuiltinType::SChar:
+    LM.setKind(LengthModifier::AsChar);
+    break;
+
+  case BuiltinType::Short:
+  case BuiltinType::UShort:
+    LM.setKind(LengthModifier::AsShort);
+    break;
+
+  case BuiltinType::WChar_S:
+  case BuiltinType::WChar_U:
   case BuiltinType::Long:
   case BuiltinType::ULong:
     LM.setKind(LengthModifier::AsLong);
@@ -399,8 +416,10 @@ bool PrintfSpecifier::fixType(QualType QT) {
   }
 
   // Set conversion specifier and disable any flags which do not apply to it.
-  if (QT->isAnyCharacterType()) {
+  // Let typedefs to char fall through to int, as %c is silly for uint8_t.
+  if (isa<TypedefType>(QT) && QT->isAnyCharacterType()) {
     CS.setKind(ConversionSpecifier::cArg);
+    LM.setKind(LengthModifier::None);
     Precision.setHowSpecified(OptionalAmount::NotSpecified);
     HasAlternativeForm = 0;
     HasLeadingZeroes = 0;
@@ -435,7 +454,7 @@ bool PrintfSpecifier::fixType(QualType QT) {
 
 void PrintfSpecifier::toString(llvm::raw_ostream &os) const {
   // Whilst some features have no defined order, we are using the order
-  // appearing in the C99 standard (ISO/IEC 9899:1999 (E) ¤7.19.6.1)
+  // appearing in the C99 standard (ISO/IEC 9899:1999 (E) 7.19.6.1)
   os << "%";
 
   // Positional args
@@ -487,10 +506,11 @@ bool PrintfSpecifier::hasValidAlternativeForm() const {
   if (!HasAlternativeForm)
     return true;
 
-  // Alternate form flag only valid with the oxaAeEfFgG conversions
+  // Alternate form flag only valid with the oxXaAeEfFgG conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::oArg:
   case ConversionSpecifier::xArg:
+  case ConversionSpecifier::XArg:
   case ConversionSpecifier::aArg:
   case ConversionSpecifier::AArg:
   case ConversionSpecifier::eArg:
@@ -570,6 +590,24 @@ bool PrintfSpecifier::hasValidLeftJustified() const {
   }
 }
 
+bool PrintfSpecifier::hasValidThousandsGroupingPrefix() const {
+  if (!HasThousandsGrouping)
+    return true;
+
+  switch (CS.getKind()) {
+    case ConversionSpecifier::dArg:
+    case ConversionSpecifier::iArg:
+    case ConversionSpecifier::uArg:
+    case ConversionSpecifier::fArg:
+    case ConversionSpecifier::FArg:
+    case ConversionSpecifier::gArg:
+    case ConversionSpecifier::GArg:
+      return true;
+    default:
+      return false;
+  }
+}
+
 bool PrintfSpecifier::hasValidPrecision() const {
   if (Precision.getHowSpecified() == OptionalAmount::NotSpecified)
     return true;
diff --git a/lib/Analysis/PseudoConstantAnalysis.cpp b/lib/Analysis/PseudoConstantAnalysis.cpp
index ff43fc252aa3..ff96eb4a0a73 100644
--- a/lib/Analysis/PseudoConstantAnalysis.cpp
+++ b/lib/Analysis/PseudoConstantAnalysis.cpp
@@ -86,6 +86,9 @@ void PseudoConstantAnalysis::RunAnalysis() {
     const Stmt* Head = WorkList.front();
     WorkList.pop_front();
 
+    if (const Expr *Ex = dyn_cast<Expr>(Head))
+      Head = Ex->IgnoreParenCasts();
+
     switch (Head->getStmtClass()) {
     // Case 1: Assignment operators modifying VarDecls
     case Stmt::BinaryOperatorClass: {
@@ -225,13 +228,12 @@ void PseudoConstantAnalysis::RunAnalysis() {
       continue;
     }
 
-      default:
-        break;
+    default:
+      break;
     } // switch (head->getStmtClass())
 
     // Add all substatements to the worklist
-    for (Stmt::const_child_iterator I = Head->child_begin(),
-        E = Head->child_end(); I != E; ++I)
+    for (Stmt::const_child_range I = Head->children(); I; ++I)
       if (*I)
         WorkList.push_back(*I);
   } // while (!WorkList.empty())
diff --git a/lib/Analysis/ReachableCode.cpp b/lib/Analysis/ReachableCode.cpp
index 05439392f916..7afa586479b3 100644
--- a/lib/Analysis/ReachableCode.cpp
+++ b/lib/Analysis/ReachableCode.cpp
@@ -30,20 +30,26 @@ static SourceLocation GetUnreachableLoc(const CFGBlock &b, SourceRange &R1,
   unsigned sn = 0;
   R1 = R2 = SourceRange();
 
-top:
-  if (sn < b.size())
-    S = b[sn].getStmt();
-  else if (b.getTerminator())
+  if (sn < b.size()) {
+    CFGStmt CS = b[sn].getAs<CFGStmt>();
+    if (!CS)
+      return SourceLocation();
+
+    S = CS.getStmt(); 
+  } else if (b.getTerminator())
     S = b.getTerminator();
   else
     return SourceLocation();
 
+  if (const Expr *Ex = dyn_cast<Expr>(S))
+    S = Ex->IgnoreParenImpCasts();
+
   switch (S->getStmtClass()) {
     case Expr::BinaryOperatorClass: {
       const BinaryOperator *BO = cast<BinaryOperator>(S);
       if (BO->getOpcode() == BO_Comma) {
         if (sn+1 < b.size())
-          return b[sn+1].getStmt()->getLocStart();
+          return b[sn+1].getAs<CFGStmt>().getStmt()->getLocStart();
         const CFGBlock *n = &b;
         while (1) {
           if (n->getTerminator())
@@ -54,7 +60,7 @@ top:
           if (n->pred_size() != 1)
             return SourceLocation();
           if (!n->empty())
-            return n[0][0].getStmt()->getLocStart();
+            return n[0][0].getAs<CFGStmt>().getStmt()->getLocStart();
         }
       }
       R1 = BO->getLHS()->getSourceRange();
@@ -72,8 +78,10 @@ top:
       R2 = CAO->getRHS()->getSourceRange();
       return CAO->getOperatorLoc();
     }
+    case Expr::BinaryConditionalOperatorClass:
     case Expr::ConditionalOperatorClass: {
-      const ConditionalOperator *CO = cast<ConditionalOperator>(S);
+      const AbstractConditionalOperator *CO =
+        cast<AbstractConditionalOperator>(S);
       return CO->getQuestionLoc();
     }
     case Expr::MemberExprClass: {
@@ -97,9 +105,6 @@ top:
       R1 = CE->getSubExpr()->getSourceRange();
       return CE->getTypeBeginLoc();
     }
-    case Expr::ImplicitCastExprClass:
-      ++sn;
-      goto top;
     case Stmt::CXXTryStmtClass: {
       return cast<CXXTryStmt>(S)->getHandler(0)->getCatchLoc();
     }
@@ -131,6 +136,9 @@ static SourceLocation MarkLiveTop(const CFGBlock *Start,
   }
 
   // Solve
+  CFGBlock::FilterOptions FO;
+  FO.IgnoreDefaultsWithCoveredEnums = 1;
+
   while (!WL.empty()) {
     const CFGBlock *item = WL.back();
     WL.pop_back();
@@ -147,8 +155,8 @@ static SourceLocation MarkLiveTop(const CFGBlock *Start,
         }
 
     reachable.set(item->getBlockID());
-    for (CFGBlock::const_succ_iterator I=item->succ_begin(), E=item->succ_end();
-         I != E; ++I)
+    for (CFGBlock::filtered_succ_iterator I =
+	   item->filtered_succ_start_end(FO); I.hasMore(); ++I)
       if (const CFGBlock *B = *I) {
         unsigned blockID = B->getBlockID();
         if (!reachable[blockID]) {
@@ -190,14 +198,17 @@ unsigned ScanReachableFromBlock(const CFGBlock &Start,
   ++count;
   WL.push_back(&Start);
 
-    // Find the reachable blocks from 'Start'.
+  // Find the reachable blocks from 'Start'.
+  CFGBlock::FilterOptions FO;
+  FO.IgnoreDefaultsWithCoveredEnums = 1;
+
   while (!WL.empty()) {
     const CFGBlock *item = WL.back();
     WL.pop_back();
 
       // Look at the successors and mark then reachable.
-    for (CFGBlock::const_succ_iterator I=item->succ_begin(), E=item->succ_end();
-         I != E; ++I)
+    for (CFGBlock::filtered_succ_iterator I= item->filtered_succ_start_end(FO);
+         I.hasMore(); ++I)
       if (const CFGBlock *B = *I) {
         unsigned blockID = B->getBlockID();
         if (!Reachable[blockID]) {
@@ -234,7 +245,8 @@ void FindUnreachableCode(AnalysisContext &AC, Callback &CB) {
     CFGBlock &b = **I;
     if (!reachable[b.getBlockID()]) {
       if (b.pred_empty()) {
-        if (!AddEHEdges && dyn_cast_or_null<CXXTryStmt>(b.getTerminator())) {
+        if (!AddEHEdges
+        && dyn_cast_or_null<CXXTryStmt>(b.getTerminator().getStmt())) {
             // When not adding EH edges from calls, catch clauses
             // can otherwise seem dead.  Avoid noting them as dead.
           numReachable += ScanReachableFromBlock(b, reachable);
diff --git a/lib/Analysis/UninitializedValues.cpp b/lib/Analysis/UninitializedValues.cpp
index 0f43efa58cc0..c08cbedf4b94 100644
--- a/lib/Analysis/UninitializedValues.cpp
+++ b/lib/Analysis/UninitializedValues.cpp
@@ -72,13 +72,15 @@ public:
   bool VisitStmt(Stmt* S);
   bool VisitCallExpr(CallExpr* C);
   bool VisitDeclStmt(DeclStmt* D);
-  bool VisitConditionalOperator(ConditionalOperator* C);
+  bool VisitAbstractConditionalOperator(AbstractConditionalOperator* C);
   bool BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt* S);
 
   bool Visit(Stmt *S);
   bool BlockStmt_VisitExpr(Expr* E);
 
   void VisitTerminator(CFGBlock* B) { }
+    
+  void setCurrentBlock(const CFGBlock *block) {}
 };
 
 static const bool Initialized = false;
@@ -87,7 +89,7 @@ static const bool Uninitialized = true;
 bool TransferFuncs::VisitDeclRefExpr(DeclRefExpr* DR) {
 
   if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl()))
-    if (VD->isBlockVarDecl()) {
+    if (VD->isLocalVarDecl()) {
 
       if (AD.Observer)
         AD.Observer->ObserveDeclRefExpr(V, AD, DR, VD);
@@ -112,7 +114,7 @@ static VarDecl* FindBlockVarDecl(Expr* E) {
 
   if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(E->IgnoreParenCasts()))
     if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl()))
-      if (VD->isBlockVarDecl()) return VD;
+      if (VD->isLocalVarDecl()) return VD;
 
   return NULL;
 }
@@ -133,7 +135,7 @@ bool TransferFuncs::VisitBinaryOperator(BinaryOperator* B) {
 bool TransferFuncs::VisitDeclStmt(DeclStmt* S) {
   for (DeclStmt::decl_iterator I=S->decl_begin(), E=S->decl_end(); I!=E; ++I) {
     VarDecl *VD = dyn_cast<VarDecl>(*I);
-    if (VD && VD->isBlockVarDecl()) {
+    if (VD && VD->isLocalVarDecl()) {
       if (Stmt* I = VD->getInit()) {
         // Visit the subexpression to check for uses of uninitialized values,
         // even if we don't propagate that value.
@@ -170,7 +172,7 @@ bool TransferFuncs::VisitUnaryOperator(UnaryOperator* U) {
   switch (U->getOpcode()) {
     case UO_AddrOf: {
       VarDecl* VD = FindBlockVarDecl(U->getSubExpr());
-      if (VD && VD->isBlockVarDecl())
+      if (VD && VD->isLocalVarDecl())
         return V(VD,AD) = Initialized;
       break;
     }
@@ -211,13 +213,14 @@ TransferFuncs::BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt* S) {
 }
 
 
-bool TransferFuncs::VisitConditionalOperator(ConditionalOperator* C) {
+bool TransferFuncs::
+VisitAbstractConditionalOperator(AbstractConditionalOperator* C) {
   Visit(C->getCond());
 
-  bool rhsResult = Visit(C->getRHS());
+  bool rhsResult = Visit(C->getFalseExpr());
   // Handle the GNU extension for missing LHS.
-  if (Expr *lhs = C->getLHS())
-    return Visit(lhs) & rhsResult; // Yes: we want &, not &&.
+  if (isa<ConditionalOperator>(C))
+    return Visit(C->getTrueExpr()) & rhsResult; // Yes: we want &, not &&.
   else
     return rhsResult;
 }
@@ -228,7 +231,7 @@ bool TransferFuncs::VisitStmt(Stmt* S) {
   // We don't stop at the first subexpression that is Uninitialized because
   // evaluating some subexpressions may result in propogating "Uninitialized"
   // or "Initialized" to variables referenced in the other subexpressions.
-  for (Stmt::child_iterator I=S->child_begin(), E=S->child_end(); I!=E; ++I)
+  for (Stmt::child_range I = S->children(); I; ++I)
     if (*I && Visit(*I) == Uninitialized) x = Uninitialized;
 
   return x;
diff --git a/lib/Analysis/UninitializedValuesV2.cpp b/lib/Analysis/UninitializedValuesV2.cpp
new file mode 100644
index 000000000000..75eccbf7a3c8
--- /dev/null
+++ b/lib/Analysis/UninitializedValuesV2.cpp
@@ -0,0 +1,610 @@
+//==- UninitializedValuesV2.cpp - Find Uninitialized Values -----*- C++ --*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements uninitialized values analysis for source-level CFGs.
+//
+//===----------------------------------------------------------------------===//
+
+#include <utility>
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/DenseMap.h"
+#include "clang/AST/Decl.h"
+#include "clang/Analysis/CFG.h"
+#include "clang/Analysis/AnalysisContext.h"
+#include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
+#include "clang/Analysis/Analyses/UninitializedValuesV2.h"
+#include "clang/Analysis/Support/SaveAndRestore.h"
+
+using namespace clang;
+
+static bool isTrackedVar(const VarDecl *vd, const DeclContext *dc) {
+  return vd->isLocalVarDecl() && !vd->hasGlobalStorage() && 
+         vd->getType()->isScalarType() &&
+         vd->getDeclContext() == dc;
+}
+
+//------------------------------------------------------------------------====//
+// DeclToBit: a mapping from Decls we track to bitvector indices.
+//====------------------------------------------------------------------------//
+
+namespace {
+class DeclToBit {
+  llvm::DenseMap<const VarDecl *, unsigned> map;
+public:
+  DeclToBit() {}
+  
+  /// Compute the actual mapping from declarations to bits.
+  void computeMap(const DeclContext &dc);
+  
+  /// Return the number of declarations in the map.
+  unsigned size() const { return map.size(); }
+  
+  /// Returns the bit vector index for a given declaration.
+  llvm::Optional<unsigned> getBitVectorIndex(const VarDecl *d);
+};
+}
+
+void DeclToBit::computeMap(const DeclContext &dc) {
+  unsigned count = 0;
+  DeclContext::specific_decl_iterator<VarDecl> I(dc.decls_begin()),
+                                               E(dc.decls_end());
+  for ( ; I != E; ++I) {
+    const VarDecl *vd = *I;
+    if (isTrackedVar(vd, &dc))
+      map[vd] = count++;
+  }
+}
+
+llvm::Optional<unsigned> DeclToBit::getBitVectorIndex(const VarDecl *d) {
+  llvm::DenseMap<const VarDecl *, unsigned>::iterator I = map.find(d);
+  if (I == map.end())
+    return llvm::Optional<unsigned>();
+  return I->second;
+}
+
+//------------------------------------------------------------------------====//
+// CFGBlockValues: dataflow values for CFG blocks.
+//====------------------------------------------------------------------------//
+
+typedef std::pair<llvm::BitVector *, llvm::BitVector *> BVPair;
+
+namespace {
+class CFGBlockValues {
+  const CFG &cfg;
+  BVPair *vals;
+  llvm::BitVector scratch;
+  DeclToBit declToBit;
+  
+  llvm::BitVector &lazyCreate(llvm::BitVector *&bv);
+public:
+  CFGBlockValues(const CFG &cfg);
+  ~CFGBlockValues();
+  
+  void computeSetOfDeclarations(const DeclContext &dc);  
+  llvm::BitVector &getBitVector(const CFGBlock *block,
+                                const CFGBlock *dstBlock);
+
+  BVPair &getBitVectors(const CFGBlock *block, bool shouldLazyCreate);
+
+  void mergeIntoScratch(llvm::BitVector const &source, bool isFirst);
+  bool updateBitVectorWithScratch(const CFGBlock *block);
+  bool updateBitVectors(const CFGBlock *block, const BVPair &newVals);
+  
+  bool hasNoDeclarations() const {
+    return declToBit.size() == 0;
+  }
+  
+  void resetScratch();
+  llvm::BitVector &getScratch() { return scratch; }
+  
+  llvm::BitVector::reference operator[](const VarDecl *vd);
+};  
+}
+
+CFGBlockValues::CFGBlockValues(const CFG &c) : cfg(c), vals(0) {
+  unsigned n = cfg.getNumBlockIDs();
+  if (!n)
+    return;
+  vals = new std::pair<llvm::BitVector*, llvm::BitVector*>[n];
+  memset(vals, 0, sizeof(*vals) * n);
+}
+
+CFGBlockValues::~CFGBlockValues() {
+  unsigned n = cfg.getNumBlockIDs();
+  if (n == 0)
+    return;
+  for (unsigned i = 0; i < n; ++i) {
+    delete vals[i].first;
+    delete vals[i].second;
+  }
+  delete [] vals;
+}
+
+void CFGBlockValues::computeSetOfDeclarations(const DeclContext &dc) {
+  declToBit.computeMap(dc);
+  scratch.resize(declToBit.size());
+}
+
+llvm::BitVector &CFGBlockValues::lazyCreate(llvm::BitVector *&bv) {
+  if (!bv)
+    bv = new llvm::BitVector(declToBit.size());
+  return *bv;
+}
+
+/// This function pattern matches for a '&&' or '||' that appears at
+/// the beginning of a CFGBlock that also (1) has a terminator and 
+/// (2) has no other elements.  If such an expression is found, it is returned.
+static BinaryOperator *getLogicalOperatorInChain(const CFGBlock *block) {
+  if (block->empty())
+    return 0;
+
+  CFGStmt cstmt = block->front().getAs<CFGStmt>();
+  BinaryOperator *b = llvm::dyn_cast_or_null<BinaryOperator>(cstmt.getStmt());
+  
+  if (!b || !b->isLogicalOp())
+    return 0;
+  
+  if (block->pred_size() == 2 &&
+      ((block->succ_size() == 2 && block->getTerminatorCondition() == b) ||
+       block->size() == 1))
+    return b;
+  
+  return 0;
+}
+
+llvm::BitVector &CFGBlockValues::getBitVector(const CFGBlock *block,
+                                              const CFGBlock *dstBlock) {
+  unsigned idx = block->getBlockID();
+  if (dstBlock && getLogicalOperatorInChain(block)) {
+    if (*block->succ_begin() == dstBlock)
+      return lazyCreate(vals[idx].first);
+    assert(*(block->succ_begin()+1) == dstBlock);
+    return lazyCreate(vals[idx].second);
+  }
+
+  assert(vals[idx].second == 0);
+  return lazyCreate(vals[idx].first);
+}
+
+BVPair &CFGBlockValues::getBitVectors(const clang::CFGBlock *block,
+                                      bool shouldLazyCreate) {
+  unsigned idx = block->getBlockID();
+  lazyCreate(vals[idx].first);
+  if (shouldLazyCreate)
+    lazyCreate(vals[idx].second);
+  return vals[idx];
+}
+
+void CFGBlockValues::mergeIntoScratch(llvm::BitVector const &source,
+                                      bool isFirst) {
+  if (isFirst)
+    scratch = source;
+  else
+    scratch |= source;  
+}
+#if 0
+static void printVector(const CFGBlock *block, llvm::BitVector &bv,
+                        unsigned num) {
+  
+  llvm::errs() << block->getBlockID() << " :";
+  for (unsigned i = 0; i < bv.size(); ++i) {
+    llvm::errs() << ' ' << bv[i];
+  }
+  llvm::errs() << " : " << num << '\n';
+}
+#endif
+
+bool CFGBlockValues::updateBitVectorWithScratch(const CFGBlock *block) {
+  llvm::BitVector &dst = getBitVector(block, 0);
+  bool changed = (dst != scratch);
+  if (changed)
+    dst = scratch;
+#if 0
+  printVector(block, scratch, 0);
+#endif
+  return changed;
+}
+
+bool CFGBlockValues::updateBitVectors(const CFGBlock *block,
+                                      const BVPair &newVals) {
+  BVPair &vals = getBitVectors(block, true);
+  bool changed = *newVals.first != *vals.first ||
+                 *newVals.second != *vals.second;
+  *vals.first = *newVals.first;
+  *vals.second = *newVals.second;
+#if 0
+  printVector(block, *vals.first, 1);
+  printVector(block, *vals.second, 2);
+#endif
+  return changed;
+}
+
+void CFGBlockValues::resetScratch() {
+  scratch.reset();
+}
+
+llvm::BitVector::reference CFGBlockValues::operator[](const VarDecl *vd) {
+  const llvm::Optional<unsigned> &idx = declToBit.getBitVectorIndex(vd);
+  assert(idx.hasValue());
+  return scratch[idx.getValue()];
+}
+
+//------------------------------------------------------------------------====//
+// Worklist: worklist for dataflow analysis.
+//====------------------------------------------------------------------------//
+
+namespace {
+class DataflowWorklist {
+  llvm::SmallVector<const CFGBlock *, 20> worklist;
+  llvm::BitVector enqueuedBlocks;
+public:
+  DataflowWorklist(const CFG &cfg) : enqueuedBlocks(cfg.getNumBlockIDs()) {}
+  
+  void enqueue(const CFGBlock *block);
+  void enqueueSuccessors(const CFGBlock *block);
+  const CFGBlock *dequeue();
+  
+};
+}
+
+void DataflowWorklist::enqueue(const CFGBlock *block) {
+  if (!block)
+    return;
+  unsigned idx = block->getBlockID();
+  if (enqueuedBlocks[idx])
+    return;
+  worklist.push_back(block);
+  enqueuedBlocks[idx] = true;
+}
+
+void DataflowWorklist::enqueueSuccessors(const clang::CFGBlock *block) {
+  for (CFGBlock::const_succ_iterator I = block->succ_begin(),
+       E = block->succ_end(); I != E; ++I) {
+    enqueue(*I);
+  }
+}
+
+const CFGBlock *DataflowWorklist::dequeue() {
+  if (worklist.empty())
+    return 0;
+  const CFGBlock *b = worklist.back();
+  worklist.pop_back();
+  enqueuedBlocks[b->getBlockID()] = false;
+  return b;
+}
+
+//------------------------------------------------------------------------====//
+// Transfer function for uninitialized values analysis.
+//====------------------------------------------------------------------------//
+
+static const bool Initialized = false;
+static const bool Uninitialized = true;
+
+namespace {
+class FindVarResult {
+  const VarDecl *vd;
+  const DeclRefExpr *dr;
+public:
+  FindVarResult(VarDecl *vd, DeclRefExpr *dr) : vd(vd), dr(dr) {}
+  
+  const DeclRefExpr *getDeclRefExpr() const { return dr; }
+  const VarDecl *getDecl() const { return vd; }
+};
+  
+class TransferFunctions : public CFGRecStmtVisitor<TransferFunctions> {
+  CFGBlockValues &vals;
+  const CFG &cfg;
+  AnalysisContext &ac;
+  UninitVariablesHandler *handler;
+  const DeclRefExpr *currentDR;
+  const Expr *currentVoidCast;
+  const bool flagBlockUses;
+public:
+  TransferFunctions(CFGBlockValues &vals, const CFG &cfg,
+                    AnalysisContext &ac,
+                    UninitVariablesHandler *handler,
+                    bool flagBlockUses)
+    : vals(vals), cfg(cfg), ac(ac), handler(handler), currentDR(0),
+      currentVoidCast(0), flagBlockUses(flagBlockUses) {}
+  
+  const CFG &getCFG() { return cfg; }
+  void reportUninit(const DeclRefExpr *ex, const VarDecl *vd);
+
+  void VisitBlockExpr(BlockExpr *be);
+  void VisitDeclStmt(DeclStmt *ds);
+  void VisitDeclRefExpr(DeclRefExpr *dr);
+  void VisitUnaryOperator(UnaryOperator *uo);
+  void VisitBinaryOperator(BinaryOperator *bo);
+  void VisitCastExpr(CastExpr *ce);
+  void VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *se);
+  void BlockStmt_VisitObjCForCollectionStmt(ObjCForCollectionStmt *fs);
+  
+  bool isTrackedVar(const VarDecl *vd) {
+    return ::isTrackedVar(vd, cast<DeclContext>(ac.getDecl()));
+  }
+  
+  FindVarResult findBlockVarDecl(Expr *ex);
+};
+}
+
+void TransferFunctions::reportUninit(const DeclRefExpr *ex,
+                                     const VarDecl *vd) {
+  if (handler) handler->handleUseOfUninitVariable(ex, vd);
+}
+
+FindVarResult TransferFunctions::findBlockVarDecl(Expr* ex) {
+  if (DeclRefExpr* dr = dyn_cast<DeclRefExpr>(ex->IgnoreParenCasts()))
+    if (VarDecl *vd = dyn_cast<VarDecl>(dr->getDecl()))
+      if (isTrackedVar(vd))
+        return FindVarResult(vd, dr);  
+  return FindVarResult(0, 0);
+}
+
+void TransferFunctions::BlockStmt_VisitObjCForCollectionStmt(
+    ObjCForCollectionStmt *fs) {
+  
+  Visit(fs->getCollection());
+  
+  // This represents an initialization of the 'element' value.
+  Stmt *element = fs->getElement();
+  const VarDecl* vd = 0;
+  
+  if (DeclStmt* ds = dyn_cast<DeclStmt>(element)) {
+    vd = cast<VarDecl>(ds->getSingleDecl());
+    if (!isTrackedVar(vd))
+      vd = 0;
+  }
+  else {
+    // Initialize the value of the reference variable.
+    const FindVarResult &res = findBlockVarDecl(cast<Expr>(element));
+    vd = res.getDecl();
+    if (!vd) {
+      Visit(element);
+      return;
+    }
+  }
+  
+  if (vd)
+    vals[vd] = Initialized;
+}
+
+void TransferFunctions::VisitBlockExpr(BlockExpr *be) {
+  if (!flagBlockUses || !handler)
+    return;
+  AnalysisContext::referenced_decls_iterator i, e;
+  llvm::tie(i, e) = ac.getReferencedBlockVars(be->getBlockDecl());
+  for ( ; i != e; ++i) {
+    const VarDecl *vd = *i;
+    if (vd->getAttr<BlocksAttr>() || !vd->hasLocalStorage() || 
+        !isTrackedVar(vd))
+      continue;
+    if (vals[vd] == Uninitialized)
+      handler->handleUseOfUninitVariable(be, vd);
+  }
+}
+
+void TransferFunctions::VisitDeclStmt(DeclStmt *ds) {
+  for (DeclStmt::decl_iterator DI = ds->decl_begin(), DE = ds->decl_end();
+       DI != DE; ++DI) {
+    if (VarDecl *vd = dyn_cast<VarDecl>(*DI)) {
+      if (isTrackedVar(vd)) {
+        vals[vd] = Uninitialized;
+        if (Stmt *init = vd->getInit()) {
+          Visit(init);
+          vals[vd] = Initialized;
+        }
+      }
+      else if (Stmt *init = vd->getInit()) {
+        Visit(init);
+      }
+    }
+  }
+}
+
+void TransferFunctions::VisitDeclRefExpr(DeclRefExpr *dr) {
+  // We assume that DeclRefExprs wrapped in an lvalue-to-rvalue cast
+  // cannot be block-level expressions.  Therefore, we determine if
+  // a DeclRefExpr is involved in a "load" by comparing it to the current
+  // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
+  // If a DeclRefExpr is not involved in a load, we are essentially computing
+  // its address, either for assignment to a reference or via the '&' operator.
+  // In such cases, treat the variable as being initialized, since this
+  // analysis isn't powerful enough to do alias tracking.
+  if (dr != currentDR)
+    if (const VarDecl *vd = dyn_cast<VarDecl>(dr->getDecl()))
+      if (isTrackedVar(vd))
+        vals[vd] = Initialized;
+}
+
+void TransferFunctions::VisitBinaryOperator(clang::BinaryOperator *bo) {
+  if (bo->isAssignmentOp()) {
+    const FindVarResult &res = findBlockVarDecl(bo->getLHS());
+    if (const VarDecl* vd = res.getDecl()) {
+      // We assume that DeclRefExprs wrapped in a BinaryOperator "assignment"
+      // cannot be block-level expressions.  Therefore, we determine if
+      // a DeclRefExpr is involved in a "load" by comparing it to the current
+      // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
+      SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
+                                                res.getDeclRefExpr());
+      Visit(bo->getRHS());
+      Visit(bo->getLHS());
+
+      llvm::BitVector::reference bit = vals[vd];
+      if (bit == Uninitialized) {
+        if (bo->getOpcode() != BO_Assign)
+          reportUninit(res.getDeclRefExpr(), vd);
+        bit = Initialized;
+      }
+      return;
+    }
+  }
+  Visit(bo->getRHS());
+  Visit(bo->getLHS());
+}
+
+void TransferFunctions::VisitUnaryOperator(clang::UnaryOperator *uo) {
+  switch (uo->getOpcode()) {
+    case clang::UO_PostDec:
+    case clang::UO_PostInc:
+    case clang::UO_PreDec:
+    case clang::UO_PreInc: {
+      const FindVarResult &res = findBlockVarDecl(uo->getSubExpr());
+      if (const VarDecl *vd = res.getDecl()) {
+        // We assume that DeclRefExprs wrapped in a unary operator ++/--
+        // cannot be block-level expressions.  Therefore, we determine if
+        // a DeclRefExpr is involved in a "load" by comparing it to the current
+        // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
+        SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
+                                                  res.getDeclRefExpr());
+        Visit(uo->getSubExpr());
+
+        llvm::BitVector::reference bit = vals[vd];
+        if (bit == Uninitialized) {
+          reportUninit(res.getDeclRefExpr(), vd);
+          bit = Initialized;
+        }
+        return;
+      }
+      break;
+    }
+    default:
+      break;
+  }
+  Visit(uo->getSubExpr());
+}
+
+void TransferFunctions::VisitCastExpr(clang::CastExpr *ce) {
+  if (ce->getCastKind() == CK_LValueToRValue) {
+    const FindVarResult &res = findBlockVarDecl(ce->getSubExpr());
+    if (const VarDecl *vd = res.getDecl()) {
+      // We assume that DeclRefExprs wrapped in an lvalue-to-rvalue cast
+      // cannot be block-level expressions.  Therefore, we determine if
+      // a DeclRefExpr is involved in a "load" by comparing it to the current
+      // DeclRefExpr found when analyzing the last lvalue-to-rvalue CastExpr.
+      // Here we update 'currentDR' to be the one associated with this
+      // lvalue-to-rvalue cast.  Then, when we analyze the DeclRefExpr, we
+      // will know that we are not computing its lvalue for other purposes
+      // than to perform a load.
+      SaveAndRestore<const DeclRefExpr*> lastDR(currentDR, 
+                                                res.getDeclRefExpr());
+      Visit(ce->getSubExpr());
+      if (currentVoidCast != ce && vals[vd] == Uninitialized) {
+        reportUninit(res.getDeclRefExpr(), vd);
+        // Don't cascade warnings.
+        vals[vd] = Initialized;
+      }
+      return;
+    }
+  }
+  else if (CStyleCastExpr *cse = dyn_cast<CStyleCastExpr>(ce)) {
+    if (cse->getType()->isVoidType()) {
+      // e.g. (void) x;
+      SaveAndRestore<const Expr *>
+        lastVoidCast(currentVoidCast, cse->getSubExpr()->IgnoreParens());
+      Visit(cse->getSubExpr());
+      return;
+    }
+  }
+  Visit(ce->getSubExpr());
+}
+
+void TransferFunctions::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *se) {
+  if (se->isSizeOf()) {
+    if (se->getType()->isConstantSizeType())
+      return;
+    // Handle VLAs.
+    Visit(se->getArgumentExpr());
+  }
+}
+
+//------------------------------------------------------------------------====//
+// High-level "driver" logic for uninitialized values analysis.
+//====------------------------------------------------------------------------//
+
+static bool runOnBlock(const CFGBlock *block, const CFG &cfg,
+                       AnalysisContext &ac, CFGBlockValues &vals,
+                       UninitVariablesHandler *handler = 0,
+                       bool flagBlockUses = false) {
+  
+  if (const BinaryOperator *b = getLogicalOperatorInChain(block)) {
+    CFGBlock::const_pred_iterator itr = block->pred_begin();
+    BVPair vA = vals.getBitVectors(*itr, false);
+    ++itr;
+    BVPair vB = vals.getBitVectors(*itr, false);
+
+    BVPair valsAB;
+    
+    if (b->getOpcode() == BO_LAnd) {
+      // Merge the 'F' bits from the first and second.
+      vals.mergeIntoScratch(*(vA.second ? vA.second : vA.first), true);
+      vals.mergeIntoScratch(*(vB.second ? vB.second : vB.first), false);
+      valsAB.first = vA.first;
+      valsAB.second = &vals.getScratch();
+    }
+    else {
+      // Merge the 'T' bits from the first and second.
+      assert(b->getOpcode() == BO_LOr);
+      vals.mergeIntoScratch(*vA.first, true);
+      vals.mergeIntoScratch(*vB.first, false);
+      valsAB.first = &vals.getScratch();
+      valsAB.second = vA.second ? vA.second : vA.first;
+    }
+    return vals.updateBitVectors(block, valsAB);
+  }
+
+  // Default behavior: merge in values of predecessor blocks.
+  vals.resetScratch();
+  bool isFirst = true;
+  for (CFGBlock::const_pred_iterator I = block->pred_begin(),
+       E = block->pred_end(); I != E; ++I) {
+    vals.mergeIntoScratch(vals.getBitVector(*I, block), isFirst);
+    isFirst = false;
+  }
+  // Apply the transfer function.
+  TransferFunctions tf(vals, cfg, ac, handler, flagBlockUses);
+  for (CFGBlock::const_iterator I = block->begin(), E = block->end(); 
+       I != E; ++I) {
+    if (const CFGStmt *cs = dyn_cast<CFGStmt>(&*I)) {
+      tf.BlockStmt_Visit(cs->getStmt());
+    }
+  }
+  return vals.updateBitVectorWithScratch(block);
+}
+
+void clang::runUninitializedVariablesAnalysis(const DeclContext &dc,
+                                              const CFG &cfg,
+                                              AnalysisContext &ac,
+                                              UninitVariablesHandler &handler) {
+  CFGBlockValues vals(cfg);
+  vals.computeSetOfDeclarations(dc);
+  if (vals.hasNoDeclarations())
+    return;
+  DataflowWorklist worklist(cfg);
+  llvm::BitVector previouslyVisited(cfg.getNumBlockIDs());
+  
+  worklist.enqueueSuccessors(&cfg.getEntry());
+
+  while (const CFGBlock *block = worklist.dequeue()) {
+    // Did the block change?
+    bool changed = runOnBlock(block, cfg, ac, vals);    
+    if (changed || !previouslyVisited[block->getBlockID()])
+      worklist.enqueueSuccessors(block);    
+    previouslyVisited[block->getBlockID()] = true;
+  }
+  
+  // Run through the blocks one more time, and report uninitialized variabes.
+  for (CFG::const_iterator BI = cfg.begin(), BE = cfg.end(); BI != BE; ++BI) {
+    runOnBlock(*BI, cfg, ac, vals, &handler, /* flagBlockUses */ true);
+  }
+}
+
+UninitVariablesHandler::~UninitVariablesHandler() {}
+
diff --git a/lib/Basic/Builtins.cpp b/lib/Basic/Builtins.cpp
index 040cdb5d55f3..845ae81517da 100644
--- a/lib/Basic/Builtins.cpp
+++ b/lib/Basic/Builtins.cpp
@@ -14,12 +14,14 @@
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/LangOptions.h"
 using namespace clang;
 
 static const Builtin::Info BuiltinInfo[] = {
-  { "not a builtin function", 0, 0, 0, false },
-#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, false },
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER, false },
+  { "not a builtin function", 0, 0, 0, ALL_LANGUAGES, false },
+#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, ALL_LANGUAGES, false },
+#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER, BUILTIN_LANG) { #ID, TYPE, ATTRS, HEADER,\
+                                                            BUILTIN_LANG, false },
 #include "clang/Basic/Builtins.def"
 };
 
@@ -41,17 +43,20 @@ Builtin::Context::Context(const TargetInfo &Target) {
 /// appropriate builtin ID # and mark any non-portable builtin identifiers as
 /// such.
 void Builtin::Context::InitializeBuiltins(IdentifierTable &Table,
-                                          bool NoBuiltins) {
+                                          const LangOptions& LangOpts) {
   // Step #1: mark all target-independent builtins with their ID's.
   for (unsigned i = Builtin::NotBuiltin+1; i != Builtin::FirstTSBuiltin; ++i)
     if (!BuiltinInfo[i].Suppressed &&
-        (!NoBuiltins || !strchr(BuiltinInfo[i].Attributes, 'f')))
-      Table.get(BuiltinInfo[i].Name).setBuiltinID(i);
+        (!LangOpts.NoBuiltin || !strchr(BuiltinInfo[i].Attributes, 'f'))) {
+      if (LangOpts.ObjC1 || 
+          BuiltinInfo[i].builtin_lang != clang::OBJC_LANG)
+        Table.get(BuiltinInfo[i].Name).setBuiltinID(i);
+    }
 
   // Step #2: Register target-specific builtins.
   for (unsigned i = 0, e = NumTSRecords; i != e; ++i)
     if (!TSRecords[i].Suppressed &&
-        (!NoBuiltins ||
+        (!LangOpts.NoBuiltin ||
          (TSRecords[i].Attributes &&
           !strchr(TSRecords[i].Attributes, 'f'))))
       Table.get(TSRecords[i].Name).setBuiltinID(i+Builtin::FirstTSBuiltin);
@@ -75,6 +80,10 @@ Builtin::Context::GetBuiltinNames(llvm::SmallVectorImpl<const char *> &Names,
       Names.push_back(TSRecords[i].Name);
 }
 
+void Builtin::Context::ForgetBuiltin(unsigned ID, IdentifierTable &Table) {
+  Table.get(GetRecord(ID).Name).setBuiltinID(0);
+}
+
 bool
 Builtin::Context::isPrintfLike(unsigned ID, unsigned &FormatIdx,
                                bool &HasVAListArg) {
@@ -112,4 +121,3 @@ Builtin::Context::isScanfLike(unsigned ID, unsigned &FormatIdx,
   return true;
 }
 
-
diff --git a/lib/Basic/CMakeLists.txt b/lib/Basic/CMakeLists.txt
index 87bf834c2752..91e7deb078ad 100644
--- a/lib/Basic/CMakeLists.txt
+++ b/lib/Basic/CMakeLists.txt
@@ -1,10 +1,12 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_LINK_COMPONENTS mc)
 
 add_clang_library(clangBasic
   Builtins.cpp
   ConvertUTF.c
   Diagnostic.cpp
+  DiagnosticIDs.cpp
   FileManager.cpp
+  FileSystemStatCache.cpp
   IdentifierTable.cpp
   SourceLocation.cpp
   SourceManager.cpp
@@ -17,7 +19,9 @@ add_clang_library(clangBasic
 # Determine Subversion revision.
 # FIXME: This only gets updated when CMake is run, so this revision number
 # may be out-of-date!
-find_package(Subversion)
+if( NOT IS_SYMLINK "${CLANG_SOURCE_DIR}" )  # See PR 8437
+  find_package(Subversion)
+endif()
 if (Subversion_FOUND AND EXISTS "${CLANG_SOURCE_DIR}/.svn")
   Subversion_WC_INFO(${CLANG_SOURCE_DIR} CLANG)
   set_source_files_properties(Version.cpp
diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
index d8095f4f6d54..31e33315cce2 100644
--- a/lib/Basic/Diagnostic.cpp
+++ b/lib/Basic/Diagnostic.cpp
@@ -11,227 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/ASTDiagnostic.h"
-#include "clang/Analysis/AnalysisDiagnostic.h"
 #include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/FileManager.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/PartialDiagnostic.h"
-#include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Driver/DriverDiagnostic.h"
-#include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Lex/LexDiagnostic.h"
-#include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Sema/SemaDiagnostic.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-
-#include <vector>
-#include <map>
-#include <cstring>
 using namespace clang;
 
-//===----------------------------------------------------------------------===//
-// Builtin Diagnostic information
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-// Diagnostic classes.
-enum {
-  CLASS_NOTE       = 0x01,
-  CLASS_WARNING    = 0x02,
-  CLASS_EXTENSION  = 0x03,
-  CLASS_ERROR      = 0x04
-};
-
-struct StaticDiagInfoRec {
-  unsigned short DiagID;
-  unsigned Mapping : 3;
-  unsigned Class : 3;
-  bool SFINAE : 1;
-  unsigned Category : 5;
-  
-  const char *Description;
-  const char *OptionGroup;
-
-  bool operator<(const StaticDiagInfoRec &RHS) const {
-    return DiagID < RHS.DiagID;
-  }
-};
-
-}
-
-static const StaticDiagInfoRec StaticDiagInfo[] = {
-#define DIAG(ENUM,CLASS,DEFAULT_MAPPING,DESC,GROUP,SFINAE, CATEGORY)    \
-  { diag::ENUM, DEFAULT_MAPPING, CLASS, SFINAE, CATEGORY, DESC, GROUP },
-#include "clang/Basic/DiagnosticCommonKinds.inc"
-#include "clang/Basic/DiagnosticDriverKinds.inc"
-#include "clang/Basic/DiagnosticFrontendKinds.inc"
-#include "clang/Basic/DiagnosticLexKinds.inc"
-#include "clang/Basic/DiagnosticParseKinds.inc"
-#include "clang/Basic/DiagnosticASTKinds.inc"
-#include "clang/Basic/DiagnosticSemaKinds.inc"
-#include "clang/Basic/DiagnosticAnalysisKinds.inc"
-  { 0, 0, 0, 0, 0, 0, 0}
-};
-#undef DIAG
-
-/// GetDiagInfo - Return the StaticDiagInfoRec entry for the specified DiagID,
-/// or null if the ID is invalid.
-static const StaticDiagInfoRec *GetDiagInfo(unsigned DiagID) {
-  unsigned NumDiagEntries = sizeof(StaticDiagInfo)/sizeof(StaticDiagInfo[0])-1;
-
-  // If assertions are enabled, verify that the StaticDiagInfo array is sorted.
-#ifndef NDEBUG
-  static bool IsFirst = true;
-  if (IsFirst) {
-    for (unsigned i = 1; i != NumDiagEntries; ++i) {
-      assert(StaticDiagInfo[i-1].DiagID != StaticDiagInfo[i].DiagID &&
-             "Diag ID conflict, the enums at the start of clang::diag (in "
-             "Diagnostic.h) probably need to be increased");
-
-      assert(StaticDiagInfo[i-1] < StaticDiagInfo[i] &&
-             "Improperly sorted diag info");
-    }
-    IsFirst = false;
-  }
-#endif
-
-  // Search the diagnostic table with a binary search.
-  StaticDiagInfoRec Find = { DiagID, 0, 0, 0, 0, 0, 0 };
-
-  const StaticDiagInfoRec *Found =
-    std::lower_bound(StaticDiagInfo, StaticDiagInfo + NumDiagEntries, Find);
-  if (Found == StaticDiagInfo + NumDiagEntries ||
-      Found->DiagID != DiagID)
-    return 0;
-
-  return Found;
-}
-
-static unsigned GetDefaultDiagMapping(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Mapping;
-  return diag::MAP_FATAL;
-}
-
-/// getWarningOptionForDiag - Return the lowest-level warning option that
-/// enables the specified diagnostic.  If there is no -Wfoo flag that controls
-/// the diagnostic, this returns null.
-const char *Diagnostic::getWarningOptionForDiag(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->OptionGroup;
-  return 0;
-}
-
-/// getWarningOptionForDiag - Return the category number that a specified
-/// DiagID belongs to, or 0 if no category.
-unsigned Diagnostic::getCategoryNumberForDiag(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Category;
-  return 0;
-}
-
-/// getCategoryNameFromID - Given a category ID, return the name of the
-/// category, an empty string if CategoryID is zero, or null if CategoryID is
-/// invalid.
-const char *Diagnostic::getCategoryNameFromID(unsigned CategoryID) {
-  // Second the table of options, sorted by name for fast binary lookup.
-  static const char *CategoryNameTable[] = {
-#define GET_CATEGORY_TABLE
-#define CATEGORY(X) X,
-#include "clang/Basic/DiagnosticGroups.inc"
-#undef GET_CATEGORY_TABLE
-    "<<END>>"
-  };
-  static const size_t CategoryNameTableSize =
-    sizeof(CategoryNameTable) / sizeof(CategoryNameTable[0])-1;
-  
-  if (CategoryID >= CategoryNameTableSize) return 0;
-  return CategoryNameTable[CategoryID];
-}
-
-
-
-Diagnostic::SFINAEResponse 
-Diagnostic::getDiagnosticSFINAEResponse(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID)) {
-    if (!Info->SFINAE)
-      return SFINAE_Report;
-
-    if (Info->Class == CLASS_ERROR)
-      return SFINAE_SubstitutionFailure;
-    
-    // Suppress notes, warnings, and extensions;
-    return SFINAE_Suppress;
-  }
-  
-  return SFINAE_Report;
-}
-
-/// getDiagClass - Return the class field of the diagnostic.
-///
-static unsigned getBuiltinDiagClass(unsigned DiagID) {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Class;
-  return ~0U;
-}
-
-//===----------------------------------------------------------------------===//
-// Custom Diagnostic information
-//===----------------------------------------------------------------------===//
-
-namespace clang {
-  namespace diag {
-    class CustomDiagInfo {
-      typedef std::pair<Diagnostic::Level, std::string> DiagDesc;
-      std::vector<DiagDesc> DiagInfo;
-      std::map<DiagDesc, unsigned> DiagIDs;
-    public:
-
-      /// getDescription - Return the description of the specified custom
-      /// diagnostic.
-      const char *getDescription(unsigned DiagID) const {
-        assert(this && DiagID-DIAG_UPPER_LIMIT < DiagInfo.size() &&
-               "Invalid diagnosic ID");
-        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].second.c_str();
-      }
-
-      /// getLevel - Return the level of the specified custom diagnostic.
-      Diagnostic::Level getLevel(unsigned DiagID) const {
-        assert(this && DiagID-DIAG_UPPER_LIMIT < DiagInfo.size() &&
-               "Invalid diagnosic ID");
-        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].first;
-      }
-
-      unsigned getOrCreateDiagID(Diagnostic::Level L, llvm::StringRef Message,
-                                 Diagnostic &Diags) {
-        DiagDesc D(L, Message);
-        // Check to see if it already exists.
-        std::map<DiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
-        if (I != DiagIDs.end() && I->first == D)
-          return I->second;
-
-        // If not, assign a new ID.
-        unsigned ID = DiagInfo.size()+DIAG_UPPER_LIMIT;
-        DiagIDs.insert(std::make_pair(D, ID));
-        DiagInfo.push_back(D);
-        return ID;
-      }
-    };
-
-  } // end diag namespace
-} // end clang namespace
-
-
-//===----------------------------------------------------------------------===//
-// Common Diagnostic implementation
-//===----------------------------------------------------------------------===//
-
 static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT,
                                const char *Modifier, unsigned ML,
                                const char *Argument, unsigned ArgLen,
@@ -244,7 +30,10 @@ static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT,
 }
 
 
-Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) {
+Diagnostic::Diagnostic(const llvm::IntrusiveRefCntPtr<DiagnosticIDs> &diags,
+                       DiagnosticClient *client, bool ShouldOwnClient)
+  : Diags(diags), Client(client), OwnsDiagClient(ShouldOwnClient),
+    SourceMgr(0) {
   ArgToStringFn = DummyArgToStringFn;
   ArgToStringCookie = 0;
 
@@ -259,72 +48,41 @@ Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) {
 
   ErrorLimit = 0;
   TemplateBacktraceLimit = 0;
-  CustomDiagInfo = 0;
 
-  // Set all mappings to 'unset'.
-  DiagMappingsStack.clear();
-  DiagMappingsStack.push_back(DiagMappings());
+  // Create a DiagState and DiagStatePoint representing diagnostic changes
+  // through command-line.
+  DiagStates.push_back(DiagState());
+  PushDiagStatePoint(&DiagStates.back(), SourceLocation());
 
   Reset();
 }
 
 Diagnostic::~Diagnostic() {
-  delete CustomDiagInfo;
-}
-
-
-void Diagnostic::pushMappings() {
-  // Avoids undefined behavior when the stack has to resize.
-  DiagMappingsStack.reserve(DiagMappingsStack.size() + 1);
-  DiagMappingsStack.push_back(DiagMappingsStack.back());
-}
-
-bool Diagnostic::popMappings() {
-  if (DiagMappingsStack.size() == 1)
-    return false;
-
-  DiagMappingsStack.pop_back();
-  return true;
-}
-
-/// getCustomDiagID - Return an ID for a diagnostic with the specified message
-/// and level.  If this is the first request for this diagnosic, it is
-/// registered and created, otherwise the existing ID is returned.
-unsigned Diagnostic::getCustomDiagID(Level L, llvm::StringRef Message) {
-  if (CustomDiagInfo == 0)
-    CustomDiagInfo = new diag::CustomDiagInfo();
-  return CustomDiagInfo->getOrCreateDiagID(L, Message, *this);
+  if (OwnsDiagClient)
+    delete Client;
 }
 
-
-/// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic
-/// level of the specified diagnostic ID is a Warning or Extension.
-/// This only works on builtin diagnostics, not custom ones, and is not legal to
-/// call on NOTEs.
-bool Diagnostic::isBuiltinWarningOrExtension(unsigned DiagID) {
-  return DiagID < diag::DIAG_UPPER_LIMIT &&
-         getBuiltinDiagClass(DiagID) != CLASS_ERROR;
+void Diagnostic::setClient(DiagnosticClient *client, bool ShouldOwnClient) {
+  if (OwnsDiagClient && Client)
+    delete Client;
+  
+  Client = client;
+  OwnsDiagClient = ShouldOwnClient;
 }
 
-/// \brief Determine whether the given built-in diagnostic ID is a
-/// Note.
-bool Diagnostic::isBuiltinNote(unsigned DiagID) {
-  return DiagID < diag::DIAG_UPPER_LIMIT &&
-    getBuiltinDiagClass(DiagID) == CLASS_NOTE;
+void Diagnostic::pushMappings(SourceLocation Loc) {
+  DiagStateOnPushStack.push_back(GetCurDiagState());
 }
 
-/// isBuiltinExtensionDiag - Determine whether the given built-in diagnostic
-/// ID is for an extension of some sort.  This also returns EnabledByDefault,
-/// which is set to indicate whether the diagnostic is ignored by default (in
-/// which case -pedantic enables it) or treated as a warning/error by default.
-///
-bool Diagnostic::isBuiltinExtensionDiag(unsigned DiagID,
-                                        bool &EnabledByDefault) {
-  if (DiagID >= diag::DIAG_UPPER_LIMIT ||
-      getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
+bool Diagnostic::popMappings(SourceLocation Loc) {
+  if (DiagStateOnPushStack.empty())
     return false;
-  
-  EnabledByDefault = GetDefaultDiagMapping(DiagID) != diag::MAP_IGNORE;
+
+  if (DiagStateOnPushStack.back() != GetCurDiagState()) {
+    // State changed at some point between push/pop.
+    PushDiagStatePoint(DiagStateOnPushStack.back(), Loc);
+  }
+  DiagStateOnPushStack.pop_back();
   return true;
 }
 
@@ -336,18 +94,14 @@ void Diagnostic::Reset() {
   NumErrors = 0;
   NumErrorsSuppressed = 0;
   CurDiagID = ~0U;
-  LastDiagLevel = Ignored;
+  // Set LastDiagLevel to an "unset" state. If we set it to 'Ignored', notes
+  // using a Diagnostic associated to a translation unit that follow
+  // diagnostics from a Diagnostic associated to anoter t.u. will not be
+  // displayed.
+  LastDiagLevel = (DiagnosticIDs::Level)-1;
   DelayedDiagID = 0;
 }
 
-/// getDescription - Given a diagnostic ID, return a description of the
-/// issue.
-const char *Diagnostic::getDescription(unsigned DiagID) const {
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Info->Description;
-  return CustomDiagInfo->getDescription(DiagID);
-}
-
 void Diagnostic::SetDelayedDiagnostic(unsigned DiagID, llvm::StringRef Arg1,
                                       llvm::StringRef Arg2) {
   if (DelayedDiagID)
@@ -365,264 +119,96 @@ void Diagnostic::ReportDelayed() {
   DelayedDiagArg2.clear();
 }
 
-/// getDiagnosticLevel - Based on the way the client configured the Diagnostic
-/// object, classify the specified diagnostic ID into a Level, consumable by
-/// the DiagnosticClient.
-Diagnostic::Level Diagnostic::getDiagnosticLevel(unsigned DiagID) const {
-  // Handle custom diagnostics, which cannot be mapped.
-  if (DiagID >= diag::DIAG_UPPER_LIMIT)
-    return CustomDiagInfo->getLevel(DiagID);
-
-  unsigned DiagClass = getBuiltinDiagClass(DiagID);
-  assert(DiagClass != CLASS_NOTE && "Cannot get diagnostic level of a note!");
-  return getDiagnosticLevel(DiagID, DiagClass);
-}
-
-/// getDiagnosticLevel - Based on the way the client configured the Diagnostic
-/// object, classify the specified diagnostic ID into a Level, consumable by
-/// the DiagnosticClient.
-Diagnostic::Level
-Diagnostic::getDiagnosticLevel(unsigned DiagID, unsigned DiagClass) const {
-  // Specific non-error diagnostics may be mapped to various levels from ignored
-  // to error.  Errors can only be mapped to fatal.
-  Diagnostic::Level Result = Diagnostic::Fatal;
-
-  // Get the mapping information, if unset, compute it lazily.
-  unsigned MappingInfo = getDiagnosticMappingInfo((diag::kind)DiagID);
-  if (MappingInfo == 0) {
-    MappingInfo = GetDefaultDiagMapping(DiagID);
-    setDiagnosticMappingInternal(DiagID, MappingInfo, false);
-  }
-
-  switch (MappingInfo & 7) {
-  default: assert(0 && "Unknown mapping!");
-  case diag::MAP_IGNORE:
-    // Ignore this, unless this is an extension diagnostic and we're mapping
-    // them onto warnings or errors.
-    if (!isBuiltinExtensionDiag(DiagID) ||  // Not an extension
-        ExtBehavior == Ext_Ignore ||        // Extensions ignored anyway
-        (MappingInfo & 8) != 0)             // User explicitly mapped it.
-      return Diagnostic::Ignored;
-    Result = Diagnostic::Warning;
-    if (ExtBehavior == Ext_Error) Result = Diagnostic::Error;
-    if (Result == Diagnostic::Error && ErrorsAsFatal)
-      Result = Diagnostic::Fatal;
-    break;
-  case diag::MAP_ERROR:
-    Result = Diagnostic::Error;
-    if (ErrorsAsFatal)
-      Result = Diagnostic::Fatal;
-    break;
-  case diag::MAP_FATAL:
-    Result = Diagnostic::Fatal;
-    break;
-  case diag::MAP_WARNING:
-    // If warnings are globally mapped to ignore or error, do it.
-    if (IgnoreAllWarnings)
-      return Diagnostic::Ignored;
-
-    Result = Diagnostic::Warning;
-
-    // If this is an extension diagnostic and we're in -pedantic-error mode, and
-    // if the user didn't explicitly map it, upgrade to an error.
-    if (ExtBehavior == Ext_Error &&
-        (MappingInfo & 8) == 0 &&
-        isBuiltinExtensionDiag(DiagID))
-      Result = Diagnostic::Error;
-
-    if (WarningsAsErrors)
-      Result = Diagnostic::Error;
-    if (Result == Diagnostic::Error && ErrorsAsFatal)
-      Result = Diagnostic::Fatal;
-    break;
-
-  case diag::MAP_WARNING_NO_WERROR:
-    // Diagnostics specified with -Wno-error=foo should be set to warnings, but
-    // not be adjusted by -Werror or -pedantic-errors.
-    Result = Diagnostic::Warning;
-
-    // If warnings are globally mapped to ignore or error, do it.
-    if (IgnoreAllWarnings)
-      return Diagnostic::Ignored;
-
-    break;
-
-  case diag::MAP_ERROR_NO_WFATAL:
-    // Diagnostics specified as -Wno-fatal-error=foo should be errors, but
-    // unaffected by -Wfatal-errors.
-    Result = Diagnostic::Error;
-    break;
-  }
-
-  // Okay, we're about to return this as a "diagnostic to emit" one last check:
-  // if this is any sort of extension warning, and if we're in an __extension__
-  // block, silence it.
-  if (AllExtensionsSilenced && isBuiltinExtensionDiag(DiagID))
-    return Diagnostic::Ignored;
-
-  return Result;
-}
-
-struct WarningOption {
-  const char  *Name;
-  const short *Members;
-  const short *SubGroups;
-};
-
-#define GET_DIAG_ARRAYS
-#include "clang/Basic/DiagnosticGroups.inc"
-#undef GET_DIAG_ARRAYS
-
-// Second the table of options, sorted by name for fast binary lookup.
-static const WarningOption OptionTable[] = {
-#define GET_DIAG_TABLE
-#include "clang/Basic/DiagnosticGroups.inc"
-#undef GET_DIAG_TABLE
-};
-static const size_t OptionTableSize =
-sizeof(OptionTable) / sizeof(OptionTable[0]);
-
-static bool WarningOptionCompare(const WarningOption &LHS,
-                                 const WarningOption &RHS) {
-  return strcmp(LHS.Name, RHS.Name) < 0;
-}
-
-static void MapGroupMembers(const WarningOption *Group, diag::Mapping Mapping,
-                            Diagnostic &Diags) {
-  // Option exists, poke all the members of its diagnostic set.
-  if (const short *Member = Group->Members) {
-    for (; *Member != -1; ++Member)
-      Diags.setDiagnosticMapping(*Member, Mapping);
-  }
+Diagnostic::DiagStatePointsTy::iterator
+Diagnostic::GetDiagStatePointForLoc(SourceLocation L) const {
+  assert(!DiagStatePoints.empty());
+  assert(DiagStatePoints.front().Loc.isInvalid() &&
+         "Should have created a DiagStatePoint for command-line");
 
-  // Enable/disable all subgroups along with this one.
-  if (const short *SubGroups = Group->SubGroups) {
-    for (; *SubGroups != (short)-1; ++SubGroups)
-      MapGroupMembers(&OptionTable[(short)*SubGroups], Mapping, Diags);
-  }
-}
+  FullSourceLoc Loc(L, *SourceMgr);
+  if (Loc.isInvalid())
+    return DiagStatePoints.end() - 1;
 
-/// setDiagnosticGroupMapping - Change an entire diagnostic group (e.g.
-/// "unknown-pragmas" to have the specified mapping.  This returns true and
-/// ignores the request if "Group" was unknown, false otherwise.
-bool Diagnostic::setDiagnosticGroupMapping(const char *Group,
-                                           diag::Mapping Map) {
-
-  WarningOption Key = { Group, 0, 0 };
-  const WarningOption *Found =
-  std::lower_bound(OptionTable, OptionTable + OptionTableSize, Key,
-                   WarningOptionCompare);
-  if (Found == OptionTable + OptionTableSize ||
-      strcmp(Found->Name, Group) != 0)
-    return true;  // Option not found.
-
-  MapGroupMembers(Found, Map, *this);
-  return false;
+  DiagStatePointsTy::iterator Pos = DiagStatePoints.end();
+  FullSourceLoc LastStateChangePos = DiagStatePoints.back().Loc;
+  if (LastStateChangePos.isValid() &&
+      Loc.isBeforeInTranslationUnitThan(LastStateChangePos))
+    Pos = std::upper_bound(DiagStatePoints.begin(), DiagStatePoints.end(),
+                           DiagStatePoint(0, Loc));
+  --Pos;
+  return Pos;
 }
 
-
-/// ProcessDiag - This is the method used to report a diagnostic that is
-/// finally fully formed.
-bool Diagnostic::ProcessDiag() {
-  DiagnosticInfo Info(this);
-
-  if (SuppressAllDiagnostics)
-    return false;
-  
-  // Figure out the diagnostic level of this message.
-  Diagnostic::Level DiagLevel;
-  unsigned DiagID = Info.getID();
-
-  // ShouldEmitInSystemHeader - True if this diagnostic should be produced even
-  // in a system header.
-  bool ShouldEmitInSystemHeader;
-
-  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
-    // Handle custom diagnostics, which cannot be mapped.
-    DiagLevel = CustomDiagInfo->getLevel(DiagID);
-
-    // Custom diagnostics always are emitted in system headers.
-    ShouldEmitInSystemHeader = true;
-  } else {
-    // Get the class of the diagnostic.  If this is a NOTE, map it onto whatever
-    // the diagnostic level was for the previous diagnostic so that it is
-    // filtered the same as the previous diagnostic.
-    unsigned DiagClass = getBuiltinDiagClass(DiagID);
-    if (DiagClass == CLASS_NOTE) {
-      DiagLevel = Diagnostic::Note;
-      ShouldEmitInSystemHeader = false;  // extra consideration is needed
-    } else {
-      // If this is not an error and we are in a system header, we ignore it.
-      // Check the original Diag ID here, because we also want to ignore
-      // extensions and warnings in -Werror and -pedantic-errors modes, which
-      // *map* warnings/extensions to errors.
-      ShouldEmitInSystemHeader = DiagClass == CLASS_ERROR;
-
-      DiagLevel = getDiagnosticLevel(DiagID, DiagClass);
-    }
-  }
-
-  if (DiagLevel != Diagnostic::Note) {
-    // Record that a fatal error occurred only when we see a second
-    // non-note diagnostic. This allows notes to be attached to the
-    // fatal error, but suppresses any diagnostics that follow those
-    // notes.
-    if (LastDiagLevel == Diagnostic::Fatal)
-      FatalErrorOccurred = true;
-
-    LastDiagLevel = DiagLevel;
+/// \brief This allows the client to specify that certain
+/// warnings are ignored.  Notes can never be mapped, errors can only be
+/// mapped to fatal, and WARNINGs and EXTENSIONs can be mapped arbitrarily.
+///
+/// \param The source location that this change of diagnostic state should
+/// take affect. It can be null if we are setting the latest state.
+void Diagnostic::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
+                                      SourceLocation L) {
+  assert(Diag < diag::DIAG_UPPER_LIMIT &&
+         "Can only map builtin diagnostics");
+  assert((Diags->isBuiltinWarningOrExtension(Diag) ||
+          (Map == diag::MAP_FATAL || Map == diag::MAP_ERROR)) &&
+         "Cannot map errors into warnings!");
+  assert(!DiagStatePoints.empty());
+
+  bool isPragma = L.isValid();
+  FullSourceLoc Loc(L, *SourceMgr);
+  FullSourceLoc LastStateChangePos = DiagStatePoints.back().Loc;
+
+  // Common case; setting all the diagnostics of a group in one place.
+  if (Loc.isInvalid() || Loc == LastStateChangePos) {
+    setDiagnosticMappingInternal(Diag, Map, GetCurDiagState(), true, isPragma);
+    return;
   }
 
-  // If a fatal error has already been emitted, silence all subsequent
-  // diagnostics.
-  if (FatalErrorOccurred) {
-    if (DiagLevel >= Diagnostic::Error && Client->IncludeInDiagnosticCounts()) {
-      ++NumErrors;
-      ++NumErrorsSuppressed;
-    }
-
-    return false;
+  // Another common case; modifying diagnostic state in a source location
+  // after the previous one.
+  if ((Loc.isValid() && LastStateChangePos.isInvalid()) ||
+      LastStateChangePos.isBeforeInTranslationUnitThan(Loc)) {
+    // A diagnostic pragma occured, create a new DiagState initialized with
+    // the current one and a new DiagStatePoint to record at which location
+    // the new state became active.
+    DiagStates.push_back(*GetCurDiagState());
+    PushDiagStatePoint(&DiagStates.back(), Loc);
+    setDiagnosticMappingInternal(Diag, Map, GetCurDiagState(), true, isPragma);
+    return;
   }
 
-  // If the client doesn't care about this message, don't issue it.  If this is
-  // a note and the last real diagnostic was ignored, ignore it too.
-  if (DiagLevel == Diagnostic::Ignored ||
-      (DiagLevel == Diagnostic::Note && LastDiagLevel == Diagnostic::Ignored))
-    return false;
-
-  // If this diagnostic is in a system header and is not a clang error, suppress
-  // it.
-  if (SuppressSystemWarnings && !ShouldEmitInSystemHeader &&
-      Info.getLocation().isValid() &&
-      Info.getLocation().getInstantiationLoc().isInSystemHeader() &&
-      (DiagLevel != Diagnostic::Note || LastDiagLevel == Diagnostic::Ignored)) {
-    LastDiagLevel = Diagnostic::Ignored;
-    return false;
-  }
+  // We allow setting the diagnostic state in random source order for
+  // completeness but it should not be actually happening in normal practice.
 
-  if (DiagLevel >= Diagnostic::Error) {
-    if (Client->IncludeInDiagnosticCounts()) {
-      ErrorOccurred = true;
-      ++NumErrors;
-    }
+  DiagStatePointsTy::iterator Pos = GetDiagStatePointForLoc(Loc);
+  assert(Pos != DiagStatePoints.end());
 
-    // If we've emitted a lot of errors, emit a fatal error after it to stop a
-    // flood of bogus errors.
-    if (ErrorLimit && NumErrors >= ErrorLimit &&
-        DiagLevel == Diagnostic::Error)
-      SetDelayedDiagnostic(diag::fatal_too_many_errors);
+  // Update all diagnostic states that are active after the given location.
+  for (DiagStatePointsTy::iterator
+         I = Pos+1, E = DiagStatePoints.end(); I != E; ++I) {
+    setDiagnosticMappingInternal(Diag, Map, I->State, true, isPragma);
   }
 
-  // Finally, report it.
-  Client->HandleDiagnostic(DiagLevel, Info);
-  if (Client->IncludeInDiagnosticCounts()) {
-    if (DiagLevel == Diagnostic::Warning)
-      ++NumWarnings;
+  // If the location corresponds to an existing point, just update its state.
+  if (Pos->Loc == Loc) {
+    setDiagnosticMappingInternal(Diag, Map, Pos->State, true, isPragma);
+    return;
   }
 
-  CurDiagID = ~0U;
+  // Create a new state/point and fit it into the vector of DiagStatePoints
+  // so that the vector is always ordered according to location.
+  Pos->Loc.isBeforeInTranslationUnitThan(Loc);
+  DiagStates.push_back(*Pos->State);
+  DiagState *NewState = &DiagStates.back();
+  setDiagnosticMappingInternal(Diag, Map, NewState, true, isPragma);
+  DiagStatePoints.insert(Pos+1, DiagStatePoint(NewState,
+                                               FullSourceLoc(Loc, *SourceMgr)));
+}
 
-  return true;
+void DiagnosticBuilder::FlushCounts() {
+  DiagObj->NumDiagArgs = NumArgs;
+  DiagObj->NumDiagRanges = NumRanges;
+  DiagObj->NumFixItHints = NumFixItHints;
 }
 
 bool DiagnosticBuilder::Emit() {
@@ -632,9 +218,7 @@ bool DiagnosticBuilder::Emit() {
 
   // When emitting diagnostics, we set the final argument count into
   // the Diagnostic object.
-  DiagObj->NumDiagArgs = NumArgs;
-  DiagObj->NumDiagRanges = NumRanges;
-  DiagObj->NumFixItHints = NumFixItHints;
+  FlushCounts();
 
   // Process the diagnostic, sending the accumulated information to the
   // DiagnosticClient.
@@ -657,6 +241,16 @@ bool DiagnosticBuilder::Emit() {
 
 DiagnosticClient::~DiagnosticClient() {}
 
+void DiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
+                                        const DiagnosticInfo &Info) {
+  if (!IncludeInDiagnosticCounts())
+    return;
+
+  if (DiagLevel == Diagnostic::Warning)
+    ++NumWarnings;
+  else if (DiagLevel >= Diagnostic::Error)
+    ++NumErrors;
+}
 
 /// ModifierIs - Return true if the specified modifier matches specified string.
 template <std::size_t StrLen>
@@ -855,7 +449,7 @@ static bool EvalPluralExpr(unsigned ValNo, const char *Start, const char *End) {
 /// {1:form0|[2,4]:form1|:form2}
 /// Polish (requires repeated form):
 /// {1:form0|%100=[10,20]:form2|%10=[2,4]:form1|:form2}
-static void HandlePluralModifier(unsigned ValNo,
+static void HandlePluralModifier(const DiagnosticInfo &DInfo, unsigned ValNo,
                                  const char *Argument, unsigned ArgumentLen,
                                  llvm::SmallVectorImpl<char> &OutStr) {
   const char *ArgumentEnd = Argument + ArgumentLen;
@@ -869,7 +463,10 @@ static void HandlePluralModifier(unsigned ValNo,
     if (EvalPluralExpr(ValNo, Argument, ExprEnd)) {
       Argument = ExprEnd + 1;
       ExprEnd = ScanFormat(Argument, ArgumentEnd, '|');
-      OutStr.append(Argument, ExprEnd);
+
+      // Recursively format the result of the plural clause into the
+      // output string.
+      DInfo.FormatDiagnostic(Argument, ExprEnd, OutStr);
       return;
     }
     Argument = ScanFormat(Argument, ArgumentEnd - 1, '|') + 1;
@@ -882,7 +479,7 @@ static void HandlePluralModifier(unsigned ValNo,
 /// array.
 void DiagnosticInfo::
 FormatDiagnostic(llvm::SmallVectorImpl<char> &OutStr) const {
-  const char *DiagStr = getDiags()->getDescription(getID());
+  const char *DiagStr = getDiags()->getDiagnosticIDs()->getDescription(getID());
   const char *DiagEnd = DiagStr+strlen(DiagStr);
 
   FormatDiagnostic(DiagStr, DiagEnd, OutStr);
@@ -971,11 +568,13 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       int Val = getArgSInt(ArgNo);
 
       if (ModifierIs(Modifier, ModifierLen, "select")) {
-        HandleSelectModifier(*this, (unsigned)Val, Argument, ArgumentLen, OutStr);
+        HandleSelectModifier(*this, (unsigned)Val, Argument, ArgumentLen,
+                             OutStr);
       } else if (ModifierIs(Modifier, ModifierLen, "s")) {
         HandleIntegerSModifier(Val, OutStr);
       } else if (ModifierIs(Modifier, ModifierLen, "plural")) {
-        HandlePluralModifier((unsigned)Val, Argument, ArgumentLen, OutStr);
+        HandlePluralModifier(*this, (unsigned)Val, Argument, ArgumentLen,
+                             OutStr);
       } else if (ModifierIs(Modifier, ModifierLen, "ordinal")) {
         HandleOrdinalModifier((unsigned)Val, OutStr);
       } else {
@@ -992,7 +591,8 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
       } else if (ModifierIs(Modifier, ModifierLen, "s")) {
         HandleIntegerSModifier(Val, OutStr);
       } else if (ModifierIs(Modifier, ModifierLen, "plural")) {
-        HandlePluralModifier((unsigned)Val, Argument, ArgumentLen, OutStr);
+        HandlePluralModifier(*this, (unsigned)Val, Argument, ArgumentLen,
+                             OutStr);
       } else if (ModifierIs(Modifier, ModifierLen, "ordinal")) {
         HandleOrdinalModifier(Val, OutStr);
       } else {
@@ -1043,13 +643,18 @@ FormatDiagnostic(const char *DiagStr, const char *DiagEnd,
 
 StoredDiagnostic::StoredDiagnostic() { }
 
-StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, 
+StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, unsigned ID,
                                    llvm::StringRef Message)
-  : Level(Level), Loc(), Message(Message) { }
+  : ID(ID), Level(Level), Loc(), Message(Message) { }
 
 StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, 
                                    const DiagnosticInfo &Info)
-  : Level(Level), Loc(Info.getLocation()) {
+  : ID(Info.getID()), Level(Level) 
+{
+  assert((Info.getLocation().isInvalid() || Info.hasSourceManager()) &&
+       "Valid source location without setting a source manager for diagnostic");
+  if (Info.getLocation().isValid())
+    Loc = FullSourceLoc(Info.getLocation(), Info.getSourceManager());
   llvm::SmallString<64> Message;
   Info.FormatDiagnostic(Message);
   this->Message.assign(Message.begin(), Message.end());
@@ -1065,251 +670,6 @@ StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level,
 
 StoredDiagnostic::~StoredDiagnostic() { }
 
-static void WriteUnsigned(llvm::raw_ostream &OS, unsigned Value) {
-  OS.write((const char *)&Value, sizeof(unsigned));
-}
-
-static void WriteString(llvm::raw_ostream &OS, llvm::StringRef String) {
-  WriteUnsigned(OS, String.size());
-  OS.write(String.data(), String.size());
-}
-
-static void WriteSourceLocation(llvm::raw_ostream &OS, 
-                                SourceManager *SM,
-                                SourceLocation Location) {
-  if (!SM || Location.isInvalid()) {
-    // If we don't have a source manager or this location is invalid,
-    // just write an invalid location.
-    WriteUnsigned(OS, 0);
-    WriteUnsigned(OS, 0);
-    WriteUnsigned(OS, 0);
-    return;
-  }
-
-  Location = SM->getInstantiationLoc(Location);
-  std::pair<FileID, unsigned> Decomposed = SM->getDecomposedLoc(Location);
-
-  const FileEntry *FE = SM->getFileEntryForID(Decomposed.first);
-  if (FE)
-    WriteString(OS, FE->getName());
-  else {
-    // Fallback to using the buffer name when there is no entry.
-    WriteString(OS, SM->getBuffer(Decomposed.first)->getBufferIdentifier());
-  }
-
-  WriteUnsigned(OS, SM->getLineNumber(Decomposed.first, Decomposed.second));
-  WriteUnsigned(OS, SM->getColumnNumber(Decomposed.first, Decomposed.second));
-}
-
-void StoredDiagnostic::Serialize(llvm::raw_ostream &OS) const {
-  SourceManager *SM = 0;
-  if (getLocation().isValid())
-    SM = &const_cast<SourceManager &>(getLocation().getManager());
-
-  // Write a short header to help identify diagnostics.
-  OS << (char)0x06 << (char)0x07;
-  
-  // Write the diagnostic level and location.
-  WriteUnsigned(OS, (unsigned)Level);
-  WriteSourceLocation(OS, SM, getLocation());
-
-  // Write the diagnostic message.
-  llvm::SmallString<64> Message;
-  WriteString(OS, getMessage());
-  
-  // Count the number of ranges that don't point into macros, since
-  // only simple file ranges serialize well.
-  unsigned NumNonMacroRanges = 0;
-  for (range_iterator R = range_begin(), REnd = range_end(); R != REnd; ++R) {
-    if (R->getBegin().isMacroID() || R->getEnd().isMacroID())
-      continue;
-
-    ++NumNonMacroRanges;
-  }
-
-  // Write the ranges.
-  WriteUnsigned(OS, NumNonMacroRanges);
-  if (NumNonMacroRanges) {
-    for (range_iterator R = range_begin(), REnd = range_end(); R != REnd; ++R) {
-      if (R->getBegin().isMacroID() || R->getEnd().isMacroID())
-        continue;
-      
-      WriteSourceLocation(OS, SM, R->getBegin());
-      WriteSourceLocation(OS, SM, R->getEnd());
-      WriteUnsigned(OS, R->isTokenRange());
-    }
-  }
-
-  // Determine if all of the fix-its involve rewrites with simple file
-  // locations (not in macro instantiations). If so, we can write
-  // fix-it information.
-  unsigned NumFixIts = 0;
-  for (fixit_iterator F = fixit_begin(), FEnd = fixit_end(); F != FEnd; ++F) {
-    if (F->RemoveRange.isValid() &&
-        (F->RemoveRange.getBegin().isMacroID() ||
-         F->RemoveRange.getEnd().isMacroID())) {
-      NumFixIts = 0;
-      break;
-    }
-
-    ++NumFixIts;
-  }
-
-  // Write the fix-its.
-  WriteUnsigned(OS, NumFixIts);
-  for (fixit_iterator F = fixit_begin(), FEnd = fixit_end(); F != FEnd; ++F) {
-    WriteSourceLocation(OS, SM, F->RemoveRange.getBegin());
-    WriteSourceLocation(OS, SM, F->RemoveRange.getEnd());
-    WriteUnsigned(OS, F->RemoveRange.isTokenRange());
-    WriteString(OS, F->CodeToInsert);
-  }
-}
-
-static bool ReadUnsigned(const char *&Memory, const char *MemoryEnd,
-                         unsigned &Value) {
-  if (Memory + sizeof(unsigned) > MemoryEnd)
-    return true;
-
-  memmove(&Value, Memory, sizeof(unsigned));
-  Memory += sizeof(unsigned);
-  return false;
-}
-
-static bool ReadSourceLocation(FileManager &FM, SourceManager &SM,
-                               const char *&Memory, const char *MemoryEnd,
-                               SourceLocation &Location) {
-  // Read the filename.
-  unsigned FileNameLen = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, FileNameLen) || 
-      Memory + FileNameLen > MemoryEnd)
-    return true;
-
-  llvm::StringRef FileName(Memory, FileNameLen);
-  Memory += FileNameLen;
-
-  // Read the line, column.
-  unsigned Line = 0, Column = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, Line) ||
-      ReadUnsigned(Memory, MemoryEnd, Column))
-    return true;
-
-  if (FileName.empty()) {
-    Location = SourceLocation();
-    return false;
-  }
-
-  const FileEntry *File = FM.getFile(FileName);
-  if (!File)
-    return true;
-
-  // Make sure that this file has an entry in the source manager.
-  if (!SM.hasFileInfo(File))
-    SM.createFileID(File, SourceLocation(), SrcMgr::C_User);
-
-  Location = SM.getLocation(File, Line, Column);
-  return false;
-}
-
-StoredDiagnostic 
-StoredDiagnostic::Deserialize(FileManager &FM, SourceManager &SM, 
-                              const char *&Memory, const char *MemoryEnd) {
-  while (true) {
-    if (Memory == MemoryEnd)
-      return StoredDiagnostic();
-    
-    if (*Memory != 0x06) {
-      ++Memory;
-      continue;
-    }
-    
-    ++Memory;
-    if (Memory == MemoryEnd)
-      return StoredDiagnostic();
-  
-    if (*Memory != 0x07) {
-      ++Memory;
-      continue;
-    }
-    
-    // We found the header. We're done.
-    ++Memory;
-    break;
-  }
-  
-  // Read the severity level.
-  unsigned Level = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, Level) || Level > Diagnostic::Fatal)
-    return StoredDiagnostic();
-
-  // Read the source location.
-  SourceLocation Location;
-  if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, Location))
-    return StoredDiagnostic();
-
-  // Read the diagnostic text.
-  if (Memory == MemoryEnd)
-    return StoredDiagnostic();
-
-  unsigned MessageLen = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, MessageLen) ||
-      Memory + MessageLen > MemoryEnd)
-    return StoredDiagnostic();
-  
-  llvm::StringRef Message(Memory, MessageLen);
-  Memory += MessageLen;
-
-
-  // At this point, we have enough information to form a diagnostic. Do so.
-  StoredDiagnostic Diag;
-  Diag.Level = (Diagnostic::Level)Level;
-  Diag.Loc = FullSourceLoc(Location, SM);
-  Diag.Message = Message;
-  if (Memory == MemoryEnd)
-    return Diag;
-
-  // Read the source ranges.
-  unsigned NumSourceRanges = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, NumSourceRanges))
-    return Diag;
-  for (unsigned I = 0; I != NumSourceRanges; ++I) {
-    SourceLocation Begin, End;
-    unsigned IsTokenRange;
-    if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, Begin) ||
-        ReadSourceLocation(FM, SM, Memory, MemoryEnd, End) ||
-        ReadUnsigned(Memory, MemoryEnd, IsTokenRange))
-      return Diag;
-
-    Diag.Ranges.push_back(CharSourceRange(SourceRange(Begin, End),
-                                          IsTokenRange));
-  }
-
-  // Read the fix-it hints.
-  unsigned NumFixIts = 0;
-  if (ReadUnsigned(Memory, MemoryEnd, NumFixIts))
-    return Diag;
-  for (unsigned I = 0; I != NumFixIts; ++I) {
-    SourceLocation RemoveBegin, RemoveEnd;
-    unsigned InsertLen = 0, RemoveIsTokenRange;
-    if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveBegin) ||
-        ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveEnd) ||
-        ReadUnsigned(Memory, MemoryEnd, RemoveIsTokenRange) ||
-        ReadUnsigned(Memory, MemoryEnd, InsertLen) ||
-        Memory + InsertLen > MemoryEnd) {
-      Diag.FixIts.clear();
-      return Diag;
-    }
-
-    FixItHint Hint;
-    Hint.RemoveRange = CharSourceRange(SourceRange(RemoveBegin, RemoveEnd),
-                                       RemoveIsTokenRange);
-    Hint.CodeToInsert.assign(Memory, Memory + InsertLen);
-    Memory += InsertLen;
-    Diag.FixIts.push_back(Hint);
-  }
-
-  return Diag;
-}
-
 /// IncludeInDiagnosticCounts - This method (whose default implementation
 ///  returns true) indicates whether the diagnostics handled by this
 ///  DiagnosticClient should be included in the number of diagnostics
diff --git a/lib/Basic/DiagnosticIDs.cpp b/lib/Basic/DiagnosticIDs.cpp
new file mode 100644
index 000000000000..8725e7f9c0cc
--- /dev/null
+++ b/lib/Basic/DiagnosticIDs.cpp
@@ -0,0 +1,586 @@
+//===--- DiagnosticIDs.cpp - Diagnostic IDs Handling ----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Diagnostic IDs-related interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTDiagnostic.h"
+#include "clang/Analysis/AnalysisDiagnostic.h"
+#include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Driver/DriverDiagnostic.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Parse/ParseDiagnostic.h"
+#include "clang/Sema/SemaDiagnostic.h"
+
+#include <map>
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Builtin Diagnostic information
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+// Diagnostic classes.
+enum {
+  CLASS_NOTE       = 0x01,
+  CLASS_WARNING    = 0x02,
+  CLASS_EXTENSION  = 0x03,
+  CLASS_ERROR      = 0x04
+};
+
+struct StaticDiagInfoRec {
+  unsigned short DiagID;
+  unsigned Mapping : 3;
+  unsigned Class : 3;
+  unsigned SFINAE : 1;
+  unsigned AccessControl : 1;
+  unsigned Category : 5;
+  
+  const char *Description;
+  const char *OptionGroup;
+
+  bool operator<(const StaticDiagInfoRec &RHS) const {
+    return DiagID < RHS.DiagID;
+  }
+};
+
+}
+
+static const StaticDiagInfoRec StaticDiagInfo[] = {
+#define DIAG(ENUM,CLASS,DEFAULT_MAPPING,DESC,GROUP,SFINAE,ACCESS,CATEGORY)    \
+  { diag::ENUM, DEFAULT_MAPPING, CLASS, SFINAE, ACCESS, CATEGORY, DESC, GROUP },
+#include "clang/Basic/DiagnosticCommonKinds.inc"
+#include "clang/Basic/DiagnosticDriverKinds.inc"
+#include "clang/Basic/DiagnosticFrontendKinds.inc"
+#include "clang/Basic/DiagnosticLexKinds.inc"
+#include "clang/Basic/DiagnosticParseKinds.inc"
+#include "clang/Basic/DiagnosticASTKinds.inc"
+#include "clang/Basic/DiagnosticSemaKinds.inc"
+#include "clang/Basic/DiagnosticAnalysisKinds.inc"
+  { 0, 0, 0, 0, 0, 0, 0, 0}
+};
+#undef DIAG
+
+/// GetDiagInfo - Return the StaticDiagInfoRec entry for the specified DiagID,
+/// or null if the ID is invalid.
+static const StaticDiagInfoRec *GetDiagInfo(unsigned DiagID) {
+  unsigned NumDiagEntries = sizeof(StaticDiagInfo)/sizeof(StaticDiagInfo[0])-1;
+
+  // If assertions are enabled, verify that the StaticDiagInfo array is sorted.
+#ifndef NDEBUG
+  static bool IsFirst = true;
+  if (IsFirst) {
+    for (unsigned i = 1; i != NumDiagEntries; ++i) {
+      assert(StaticDiagInfo[i-1].DiagID != StaticDiagInfo[i].DiagID &&
+             "Diag ID conflict, the enums at the start of clang::diag (in "
+             "DiagnosticIDs.h) probably need to be increased");
+
+      assert(StaticDiagInfo[i-1] < StaticDiagInfo[i] &&
+             "Improperly sorted diag info");
+    }
+    IsFirst = false;
+  }
+#endif
+
+  // Search the diagnostic table with a binary search.
+  StaticDiagInfoRec Find = { DiagID, 0, 0, 0, 0, 0, 0, 0 };
+
+  const StaticDiagInfoRec *Found =
+    std::lower_bound(StaticDiagInfo, StaticDiagInfo + NumDiagEntries, Find);
+  if (Found == StaticDiagInfo + NumDiagEntries ||
+      Found->DiagID != DiagID)
+    return 0;
+
+  return Found;
+}
+
+static unsigned GetDefaultDiagMapping(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->Mapping;
+  return diag::MAP_FATAL;
+}
+
+/// getWarningOptionForDiag - Return the lowest-level warning option that
+/// enables the specified diagnostic.  If there is no -Wfoo flag that controls
+/// the diagnostic, this returns null.
+const char *DiagnosticIDs::getWarningOptionForDiag(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->OptionGroup;
+  return 0;
+}
+
+/// getWarningOptionForDiag - Return the category number that a specified
+/// DiagID belongs to, or 0 if no category.
+unsigned DiagnosticIDs::getCategoryNumberForDiag(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->Category;
+  return 0;
+}
+
+/// getCategoryNameFromID - Given a category ID, return the name of the
+/// category, an empty string if CategoryID is zero, or null if CategoryID is
+/// invalid.
+const char *DiagnosticIDs::getCategoryNameFromID(unsigned CategoryID) {
+  // Second the table of options, sorted by name for fast binary lookup.
+  static const char *CategoryNameTable[] = {
+#define GET_CATEGORY_TABLE
+#define CATEGORY(X) X,
+#include "clang/Basic/DiagnosticGroups.inc"
+#undef GET_CATEGORY_TABLE
+    "<<END>>"
+  };
+  static const size_t CategoryNameTableSize =
+    sizeof(CategoryNameTable) / sizeof(CategoryNameTable[0])-1;
+  
+  if (CategoryID >= CategoryNameTableSize) return 0;
+  return CategoryNameTable[CategoryID];
+}
+
+
+
+DiagnosticIDs::SFINAEResponse 
+DiagnosticIDs::getDiagnosticSFINAEResponse(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID)) {
+    if (Info->AccessControl)
+      return SFINAE_AccessControl;
+    
+    if (!Info->SFINAE)
+      return SFINAE_Report;
+
+    if (Info->Class == CLASS_ERROR)
+      return SFINAE_SubstitutionFailure;
+    
+    // Suppress notes, warnings, and extensions;
+    return SFINAE_Suppress;
+  }
+  
+  return SFINAE_Report;
+}
+
+/// getDiagClass - Return the class field of the diagnostic.
+///
+static unsigned getBuiltinDiagClass(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->Class;
+  return ~0U;
+}
+
+//===----------------------------------------------------------------------===//
+// Custom Diagnostic information
+//===----------------------------------------------------------------------===//
+
+namespace clang {
+  namespace diag {
+    class CustomDiagInfo {
+      typedef std::pair<DiagnosticIDs::Level, std::string> DiagDesc;
+      std::vector<DiagDesc> DiagInfo;
+      std::map<DiagDesc, unsigned> DiagIDs;
+    public:
+
+      /// getDescription - Return the description of the specified custom
+      /// diagnostic.
+      const char *getDescription(unsigned DiagID) const {
+        assert(this && DiagID-DIAG_UPPER_LIMIT < DiagInfo.size() &&
+               "Invalid diagnosic ID");
+        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].second.c_str();
+      }
+
+      /// getLevel - Return the level of the specified custom diagnostic.
+      DiagnosticIDs::Level getLevel(unsigned DiagID) const {
+        assert(this && DiagID-DIAG_UPPER_LIMIT < DiagInfo.size() &&
+               "Invalid diagnosic ID");
+        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].first;
+      }
+
+      unsigned getOrCreateDiagID(DiagnosticIDs::Level L, llvm::StringRef Message,
+                                 DiagnosticIDs &Diags) {
+        DiagDesc D(L, Message);
+        // Check to see if it already exists.
+        std::map<DiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
+        if (I != DiagIDs.end() && I->first == D)
+          return I->second;
+
+        // If not, assign a new ID.
+        unsigned ID = DiagInfo.size()+DIAG_UPPER_LIMIT;
+        DiagIDs.insert(std::make_pair(D, ID));
+        DiagInfo.push_back(D);
+        return ID;
+      }
+    };
+
+  } // end diag namespace
+} // end clang namespace
+
+
+//===----------------------------------------------------------------------===//
+// Common Diagnostic implementation
+//===----------------------------------------------------------------------===//
+
+DiagnosticIDs::DiagnosticIDs() {
+  CustomDiagInfo = 0;
+}
+
+DiagnosticIDs::~DiagnosticIDs() {
+  delete CustomDiagInfo;
+}
+
+/// getCustomDiagID - Return an ID for a diagnostic with the specified message
+/// and level.  If this is the first request for this diagnosic, it is
+/// registered and created, otherwise the existing ID is returned.
+unsigned DiagnosticIDs::getCustomDiagID(Level L, llvm::StringRef Message) {
+  if (CustomDiagInfo == 0)
+    CustomDiagInfo = new diag::CustomDiagInfo();
+  return CustomDiagInfo->getOrCreateDiagID(L, Message, *this);
+}
+
+
+/// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic
+/// level of the specified diagnostic ID is a Warning or Extension.
+/// This only works on builtin diagnostics, not custom ones, and is not legal to
+/// call on NOTEs.
+bool DiagnosticIDs::isBuiltinWarningOrExtension(unsigned DiagID) {
+  return DiagID < diag::DIAG_UPPER_LIMIT &&
+         getBuiltinDiagClass(DiagID) != CLASS_ERROR;
+}
+
+/// \brief Determine whether the given built-in diagnostic ID is a
+/// Note.
+bool DiagnosticIDs::isBuiltinNote(unsigned DiagID) {
+  return DiagID < diag::DIAG_UPPER_LIMIT &&
+    getBuiltinDiagClass(DiagID) == CLASS_NOTE;
+}
+
+/// isBuiltinExtensionDiag - Determine whether the given built-in diagnostic
+/// ID is for an extension of some sort.  This also returns EnabledByDefault,
+/// which is set to indicate whether the diagnostic is ignored by default (in
+/// which case -pedantic enables it) or treated as a warning/error by default.
+///
+bool DiagnosticIDs::isBuiltinExtensionDiag(unsigned DiagID,
+                                        bool &EnabledByDefault) {
+  if (DiagID >= diag::DIAG_UPPER_LIMIT ||
+      getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
+    return false;
+  
+  EnabledByDefault = GetDefaultDiagMapping(DiagID) != diag::MAP_IGNORE;
+  return true;
+}
+
+/// getDescription - Given a diagnostic ID, return a description of the
+/// issue.
+const char *DiagnosticIDs::getDescription(unsigned DiagID) const {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->Description;
+  return CustomDiagInfo->getDescription(DiagID);
+}
+
+/// getDiagnosticLevel - Based on the way the client configured the Diagnostic
+/// object, classify the specified diagnostic ID into a Level, consumable by
+/// the DiagnosticClient.
+DiagnosticIDs::Level
+DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
+                                  const Diagnostic &Diag) const {
+  // Handle custom diagnostics, which cannot be mapped.
+  if (DiagID >= diag::DIAG_UPPER_LIMIT)
+    return CustomDiagInfo->getLevel(DiagID);
+
+  unsigned DiagClass = getBuiltinDiagClass(DiagID);
+  assert(DiagClass != CLASS_NOTE && "Cannot get diagnostic level of a note!");
+  return getDiagnosticLevel(DiagID, DiagClass, Loc, Diag);
+}
+
+/// \brief Based on the way the client configured the Diagnostic
+/// object, classify the specified diagnostic ID into a Level, consumable by
+/// the DiagnosticClient.
+///
+/// \param Loc The source location we are interested in finding out the
+/// diagnostic state. Can be null in order to query the latest state.
+DiagnosticIDs::Level
+DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, unsigned DiagClass,
+                                  SourceLocation Loc,
+                                  const Diagnostic &Diag) const {
+  // Specific non-error diagnostics may be mapped to various levels from ignored
+  // to error.  Errors can only be mapped to fatal.
+  DiagnosticIDs::Level Result = DiagnosticIDs::Fatal;
+
+  Diagnostic::DiagStatePointsTy::iterator
+    Pos = Diag.GetDiagStatePointForLoc(Loc);
+  Diagnostic::DiagState *State = Pos->State;
+
+  // Get the mapping information, if unset, compute it lazily.
+  unsigned MappingInfo = Diag.getDiagnosticMappingInfo((diag::kind)DiagID,
+                                                       State);
+  if (MappingInfo == 0) {
+    MappingInfo = GetDefaultDiagMapping(DiagID);
+    Diag.setDiagnosticMappingInternal(DiagID, MappingInfo, State, false, false);
+  }
+
+  switch (MappingInfo & 7) {
+  default: assert(0 && "Unknown mapping!");
+  case diag::MAP_IGNORE:
+    // Ignore this, unless this is an extension diagnostic and we're mapping
+    // them onto warnings or errors.
+    if (!isBuiltinExtensionDiag(DiagID) ||  // Not an extension
+        Diag.ExtBehavior == Diagnostic::Ext_Ignore || // Ext ignored
+        (MappingInfo & 8) != 0)             // User explicitly mapped it.
+      return DiagnosticIDs::Ignored;
+    Result = DiagnosticIDs::Warning;
+    if (Diag.ExtBehavior == Diagnostic::Ext_Error) Result = DiagnosticIDs::Error;
+    if (Result == DiagnosticIDs::Error && Diag.ErrorsAsFatal)
+      Result = DiagnosticIDs::Fatal;
+    break;
+  case diag::MAP_ERROR:
+    Result = DiagnosticIDs::Error;
+    if (Diag.ErrorsAsFatal)
+      Result = DiagnosticIDs::Fatal;
+    break;
+  case diag::MAP_FATAL:
+    Result = DiagnosticIDs::Fatal;
+    break;
+  case diag::MAP_WARNING:
+    // If warnings are globally mapped to ignore or error, do it.
+    if (Diag.IgnoreAllWarnings)
+      return DiagnosticIDs::Ignored;
+
+    Result = DiagnosticIDs::Warning;
+
+    // If this is an extension diagnostic and we're in -pedantic-error mode, and
+    // if the user didn't explicitly map it, upgrade to an error.
+    if (Diag.ExtBehavior == Diagnostic::Ext_Error &&
+        (MappingInfo & 8) == 0 &&
+        isBuiltinExtensionDiag(DiagID))
+      Result = DiagnosticIDs::Error;
+
+    if (Diag.WarningsAsErrors)
+      Result = DiagnosticIDs::Error;
+    if (Result == DiagnosticIDs::Error && Diag.ErrorsAsFatal)
+      Result = DiagnosticIDs::Fatal;
+    break;
+
+  case diag::MAP_WARNING_NO_WERROR:
+    // Diagnostics specified with -Wno-error=foo should be set to warnings, but
+    // not be adjusted by -Werror or -pedantic-errors.
+    Result = DiagnosticIDs::Warning;
+
+    // If warnings are globally mapped to ignore or error, do it.
+    if (Diag.IgnoreAllWarnings)
+      return DiagnosticIDs::Ignored;
+
+    break;
+
+  case diag::MAP_ERROR_NO_WFATAL:
+    // Diagnostics specified as -Wno-fatal-error=foo should be errors, but
+    // unaffected by -Wfatal-errors.
+    Result = DiagnosticIDs::Error;
+    break;
+  }
+
+  // Okay, we're about to return this as a "diagnostic to emit" one last check:
+  // if this is any sort of extension warning, and if we're in an __extension__
+  // block, silence it.
+  if (Diag.AllExtensionsSilenced && isBuiltinExtensionDiag(DiagID))
+    return DiagnosticIDs::Ignored;
+
+  return Result;
+}
+
+struct WarningOption {
+  const char  *Name;
+  const short *Members;
+  const short *SubGroups;
+};
+
+#define GET_DIAG_ARRAYS
+#include "clang/Basic/DiagnosticGroups.inc"
+#undef GET_DIAG_ARRAYS
+
+// Second the table of options, sorted by name for fast binary lookup.
+static const WarningOption OptionTable[] = {
+#define GET_DIAG_TABLE
+#include "clang/Basic/DiagnosticGroups.inc"
+#undef GET_DIAG_TABLE
+};
+static const size_t OptionTableSize =
+sizeof(OptionTable) / sizeof(OptionTable[0]);
+
+static bool WarningOptionCompare(const WarningOption &LHS,
+                                 const WarningOption &RHS) {
+  return strcmp(LHS.Name, RHS.Name) < 0;
+}
+
+static void MapGroupMembers(const WarningOption *Group, diag::Mapping Mapping,
+                            SourceLocation Loc, Diagnostic &Diag) {
+  // Option exists, poke all the members of its diagnostic set.
+  if (const short *Member = Group->Members) {
+    for (; *Member != -1; ++Member)
+      Diag.setDiagnosticMapping(*Member, Mapping, Loc);
+  }
+
+  // Enable/disable all subgroups along with this one.
+  if (const short *SubGroups = Group->SubGroups) {
+    for (; *SubGroups != (short)-1; ++SubGroups)
+      MapGroupMembers(&OptionTable[(short)*SubGroups], Mapping, Loc, Diag);
+  }
+}
+
+/// setDiagnosticGroupMapping - Change an entire diagnostic group (e.g.
+/// "unknown-pragmas" to have the specified mapping.  This returns true and
+/// ignores the request if "Group" was unknown, false otherwise.
+bool DiagnosticIDs::setDiagnosticGroupMapping(const char *Group,
+                                           diag::Mapping Map,
+                                           SourceLocation Loc,
+                                           Diagnostic &Diag) const {
+  assert((Loc.isValid() ||
+          Diag.DiagStatePoints.empty() ||
+          Diag.DiagStatePoints.back().Loc.isInvalid()) &&
+         "Loc should be invalid only when the mapping comes from command-line");
+  assert((Loc.isInvalid() || Diag.DiagStatePoints.empty() ||
+          Diag.DiagStatePoints.back().Loc.isInvalid() ||
+          !Diag.SourceMgr->isBeforeInTranslationUnit(Loc,
+                                            Diag.DiagStatePoints.back().Loc)) &&
+         "Source location of new mapping is before the previous one!");
+
+  WarningOption Key = { Group, 0, 0 };
+  const WarningOption *Found =
+  std::lower_bound(OptionTable, OptionTable + OptionTableSize, Key,
+                   WarningOptionCompare);
+  if (Found == OptionTable + OptionTableSize ||
+      strcmp(Found->Name, Group) != 0)
+    return true;  // Option not found.
+
+  MapGroupMembers(Found, Map, Loc, Diag);
+  return false;
+}
+
+/// ProcessDiag - This is the method used to report a diagnostic that is
+/// finally fully formed.
+bool DiagnosticIDs::ProcessDiag(Diagnostic &Diag) const {
+  DiagnosticInfo Info(&Diag);
+
+  if (Diag.SuppressAllDiagnostics)
+    return false;
+
+  assert(Diag.getClient() && "DiagnosticClient not set!");
+
+  // Figure out the diagnostic level of this message.
+  DiagnosticIDs::Level DiagLevel;
+  unsigned DiagID = Info.getID();
+
+  // ShouldEmitInSystemHeader - True if this diagnostic should be produced even
+  // in a system header.
+  bool ShouldEmitInSystemHeader;
+
+  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
+    // Handle custom diagnostics, which cannot be mapped.
+    DiagLevel = CustomDiagInfo->getLevel(DiagID);
+
+    // Custom diagnostics always are emitted in system headers.
+    ShouldEmitInSystemHeader = true;
+  } else {
+    // Get the class of the diagnostic.  If this is a NOTE, map it onto whatever
+    // the diagnostic level was for the previous diagnostic so that it is
+    // filtered the same as the previous diagnostic.
+    unsigned DiagClass = getBuiltinDiagClass(DiagID);
+    if (DiagClass == CLASS_NOTE) {
+      DiagLevel = DiagnosticIDs::Note;
+      ShouldEmitInSystemHeader = false;  // extra consideration is needed
+    } else {
+      // If this is not an error and we are in a system header, we ignore it.
+      // Check the original Diag ID here, because we also want to ignore
+      // extensions and warnings in -Werror and -pedantic-errors modes, which
+      // *map* warnings/extensions to errors.
+      ShouldEmitInSystemHeader = DiagClass == CLASS_ERROR;
+
+      DiagLevel = getDiagnosticLevel(DiagID, DiagClass, Info.getLocation(),
+                                     Diag);
+    }
+  }
+
+  if (DiagLevel != DiagnosticIDs::Note) {
+    // Record that a fatal error occurred only when we see a second
+    // non-note diagnostic. This allows notes to be attached to the
+    // fatal error, but suppresses any diagnostics that follow those
+    // notes.
+    if (Diag.LastDiagLevel == DiagnosticIDs::Fatal)
+      Diag.FatalErrorOccurred = true;
+
+    Diag.LastDiagLevel = DiagLevel;
+  }
+
+  // If a fatal error has already been emitted, silence all subsequent
+  // diagnostics.
+  if (Diag.FatalErrorOccurred) {
+    if (DiagLevel >= DiagnosticIDs::Error &&
+        Diag.Client->IncludeInDiagnosticCounts()) {
+      ++Diag.NumErrors;
+      ++Diag.NumErrorsSuppressed;
+    }
+
+    return false;
+  }
+
+  // If the client doesn't care about this message, don't issue it.  If this is
+  // a note and the last real diagnostic was ignored, ignore it too.
+  if (DiagLevel == DiagnosticIDs::Ignored ||
+      (DiagLevel == DiagnosticIDs::Note &&
+       Diag.LastDiagLevel == DiagnosticIDs::Ignored))
+    return false;
+
+  // If this diagnostic is in a system header and is not a clang error, suppress
+  // it.
+  if (Diag.SuppressSystemWarnings && !ShouldEmitInSystemHeader &&
+      Info.getLocation().isValid() &&
+      Diag.getSourceManager().isInSystemHeader(
+          Diag.getSourceManager().getInstantiationLoc(Info.getLocation())) &&
+      (DiagLevel != DiagnosticIDs::Note ||
+       Diag.LastDiagLevel == DiagnosticIDs::Ignored)) {
+    Diag.LastDiagLevel = DiagnosticIDs::Ignored;
+    return false;
+  }
+
+  if (DiagLevel >= DiagnosticIDs::Error) {
+    if (Diag.Client->IncludeInDiagnosticCounts()) {
+      Diag.ErrorOccurred = true;
+      ++Diag.NumErrors;
+    }
+
+    // If we've emitted a lot of errors, emit a fatal error after it to stop a
+    // flood of bogus errors.
+    if (Diag.ErrorLimit && Diag.NumErrors >= Diag.ErrorLimit &&
+        DiagLevel == DiagnosticIDs::Error)
+      Diag.SetDelayedDiagnostic(diag::fatal_too_many_errors);
+  }
+
+  // If we have any Fix-Its, make sure that all of the Fix-Its point into
+  // source locations that aren't macro instantiations. If any point into
+  // macro instantiations, remove all of the Fix-Its.
+  for (unsigned I = 0, N = Diag.NumFixItHints; I != N; ++I) {
+    const FixItHint &FixIt = Diag.FixItHints[I];
+    if (FixIt.RemoveRange.isInvalid() ||
+        FixIt.RemoveRange.getBegin().isMacroID() ||
+        FixIt.RemoveRange.getEnd().isMacroID()) {
+      Diag.NumFixItHints = 0;
+      break;
+    }    
+  }
+  
+  // Finally, report it.
+  Diag.Client->HandleDiagnostic((Diagnostic::Level)DiagLevel, Info);
+  if (Diag.Client->IncludeInDiagnosticCounts()) {
+    if (DiagLevel == DiagnosticIDs::Warning)
+      ++Diag.NumWarnings;
+  }
+
+  Diag.CurDiagID = ~0U;
+
+  return true;
+}
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
index 565f8a61dee6..342413d7da52 100644
--- a/lib/Basic/FileManager.cpp
+++ b/lib/Basic/FileManager.cpp
@@ -1,4 +1,4 @@
-///===--- FileManager.cpp - File System Probing and Caching ----------------===//
+//===--- FileManager.cpp - File System Probing and Caching ----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -18,35 +18,52 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/system_error.h"
 #include "llvm/Config/config.h"
 #include <map>
 #include <set>
 #include <string>
+
+// FIXME: This is terrible, we need this for ::close.
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#include <sys/uio.h>
+#else
+#include <io.h>
+#endif
 using namespace clang;
 
 // FIXME: Enhance libsystem to support inode and other fields.
 #include <sys/stat.h>
 
-#if defined(_MSC_VER)
-#define S_ISDIR(s) (_S_IFDIR & s)
-#endif
-
 /// NON_EXISTENT_DIR - A special value distinct from null that is used to
 /// represent a dir name that doesn't exist on the disk.
 #define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1)
 
+/// NON_EXISTENT_FILE - A special value distinct from null that is used to
+/// represent a filename that doesn't exist on the disk.
+#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1)
+
+
+FileEntry::~FileEntry() {
+  // If this FileEntry owns an open file descriptor that never got used, close
+  // it.
+  if (FD != -1) ::close(FD);
+}
+
 //===----------------------------------------------------------------------===//
 // Windows.
 //===----------------------------------------------------------------------===//
 
 #ifdef LLVM_ON_WIN32
 
-#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/' || (x) == '\\')
-
 namespace {
   static std::string GetFullPath(const char *relPath) {
     char *absPathStrPtr = _fullpath(NULL, relPath, 0);
@@ -65,15 +82,16 @@ class FileManager::UniqueDirContainer {
   llvm::StringMap<DirectoryEntry> UniqueDirs;
 
 public:
-  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
+  /// getDirectory - Return an existing DirectoryEntry with the given
+  /// name if there is already one; otherwise create and return a
+  /// default-constructed DirectoryEntry.
+  DirectoryEntry &getDirectory(const char *Name,
+                               const struct stat & /*StatBuf*/) {
     std::string FullPath(GetFullPath(Name));
-    return UniqueDirs.GetOrCreateValue(
-                              FullPath.c_str(),
-                              FullPath.c_str() + FullPath.size()
-                                                                ).getValue();
+    return UniqueDirs.GetOrCreateValue(FullPath).getValue();
   }
 
-  size_t size() { return UniqueDirs.size(); }
+  size_t size() const { return UniqueDirs.size(); }
 };
 
 class FileManager::UniqueFileContainer {
@@ -82,18 +100,18 @@ class FileManager::UniqueFileContainer {
   llvm::StringMap<FileEntry, llvm::BumpPtrAllocator> UniqueFiles;
 
 public:
-  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
+  /// getFile - Return an existing FileEntry with the given name if
+  /// there is already one; otherwise create and return a
+  /// default-constructed FileEntry.
+  FileEntry &getFile(const char *Name, const struct stat & /*StatBuf*/) {
     std::string FullPath(GetFullPath(Name));
-    
+
     // LowercaseString because Windows filesystem is case insensitive.
     FullPath = llvm::LowercaseString(FullPath);
-    return UniqueFiles.GetOrCreateValue(
-                               FullPath.c_str(),
-                               FullPath.c_str() + FullPath.size()
-                                                                 ).getValue();
+    return UniqueFiles.GetOrCreateValue(FullPath).getValue();
   }
 
-  size_t size() { return UniqueFiles.size(); }
+  size_t size() const { return UniqueFiles.size(); }
 };
 
 //===----------------------------------------------------------------------===//
@@ -102,28 +120,31 @@ public:
 
 #else
 
-#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/')
-
 class FileManager::UniqueDirContainer {
   /// UniqueDirs - Cache from ID's to existing directories/files.
-  ///
   std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs;
 
 public:
-  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
+  /// getDirectory - Return an existing DirectoryEntry with the given
+  /// ID's if there is already one; otherwise create and return a
+  /// default-constructed DirectoryEntry.
+  DirectoryEntry &getDirectory(const char * /*Name*/,
+                               const struct stat &StatBuf) {
     return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)];
   }
 
-  size_t size() { return UniqueDirs.size(); }
+  size_t size() const { return UniqueDirs.size(); }
 };
 
 class FileManager::UniqueFileContainer {
   /// UniqueFiles - Cache from ID's to existing directories/files.
-  ///
   std::set<FileEntry> UniqueFiles;
 
 public:
-  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
+  /// getFile - Return an existing FileEntry with the given ID's if
+  /// there is already one; otherwise create and return a
+  /// default-constructed FileEntry.
+  FileEntry &getFile(const char * /*Name*/, const struct stat &StatBuf) {
     return
       const_cast<FileEntry&>(
                     *UniqueFiles.insert(FileEntry(StatBuf.st_dev,
@@ -131,7 +152,7 @@ public:
                                                   StatBuf.st_mode)).first);
   }
 
-  size_t size() { return UniqueFiles.size(); }
+  size_t size() const { return UniqueFiles.size(); }
 };
 
 #endif
@@ -140,26 +161,26 @@ public:
 // Common logic.
 //===----------------------------------------------------------------------===//
 
-FileManager::FileManager()
-  : UniqueDirs(*new UniqueDirContainer),
-    UniqueFiles(*new UniqueFileContainer),
-    DirEntries(64), FileEntries(64), NextFileUID(0) {
+FileManager::FileManager(const FileSystemOptions &FSO)
+  : FileSystemOpts(FSO),
+    UniqueRealDirs(*new UniqueDirContainer()),
+    UniqueRealFiles(*new UniqueFileContainer()),
+    SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) {
   NumDirLookups = NumFileLookups = 0;
   NumDirCacheMisses = NumFileCacheMisses = 0;
 }
 
 FileManager::~FileManager() {
-  delete &UniqueDirs;
-  delete &UniqueFiles;
-  for (llvm::SmallVectorImpl<FileEntry *>::iterator
-         V = VirtualFileEntries.begin(),
-         VEnd = VirtualFileEntries.end();
-       V != VEnd; 
-       ++V)
-    delete *V;
+  delete &UniqueRealDirs;
+  delete &UniqueRealFiles;
+  for (unsigned i = 0, e = VirtualFileEntries.size(); i != e; ++i)
+    delete VirtualFileEntries[i];
+  for (unsigned i = 0, e = VirtualDirectoryEntries.size(); i != e; ++i)
+    delete VirtualDirectoryEntries[i];
 }
 
-void FileManager::addStatCache(StatSysCallCache *statCache, bool AtBeginning) {
+void FileManager::addStatCache(FileSystemStatCache *statCache,
+                               bool AtBeginning) {
   assert(statCache && "No stat cache provided?");
   if (AtBeginning || StatCache.get() == 0) {
     statCache->setNextStatCache(StatCache.take());
@@ -167,14 +188,14 @@ void FileManager::addStatCache(StatSysCallCache *statCache, bool AtBeginning) {
     return;
   }
   
-  StatSysCallCache *LastCache = StatCache.get();
+  FileSystemStatCache *LastCache = StatCache.get();
   while (LastCache->getNextStatCache())
     LastCache = LastCache->getNextStatCache();
   
   LastCache->setNextStatCache(statCache);
 }
 
-void FileManager::removeStatCache(StatSysCallCache *statCache) {
+void FileManager::removeStatCache(FileSystemStatCache *statCache) {
   if (!statCache)
     return;
   
@@ -185,54 +206,74 @@ void FileManager::removeStatCache(StatSysCallCache *statCache) {
   }
   
   // Find the stat cache in the list.
-  StatSysCallCache *PrevCache = StatCache.get();
+  FileSystemStatCache *PrevCache = StatCache.get();
   while (PrevCache && PrevCache->getNextStatCache() != statCache)
     PrevCache = PrevCache->getNextStatCache();
-  if (PrevCache)
-    PrevCache->setNextStatCache(statCache->getNextStatCache());
-  else
-    assert(false && "Stat cache not found for removal");
+  
+  assert(PrevCache && "Stat cache not found for removal");
+  PrevCache->setNextStatCache(statCache->getNextStatCache());
 }
 
 /// \brief Retrieve the directory that the given file name resides in.
+/// Filename can point to either a real file or a virtual file.
 static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr,
-                                                  const char *NameStart,
-                                                  const char *NameEnd) {
-  // Figure out what directory it is in.   If the string contains a / in it,
-  // strip off everything after it.
-  // FIXME: this logic should be in sys::Path.
-  const char *SlashPos = NameEnd-1;
-  while (SlashPos >= NameStart && !IS_DIR_SEPARATOR_CHAR(SlashPos[0]))
-    --SlashPos;
-  // Ignore duplicate //'s.
-  while (SlashPos > NameStart && IS_DIR_SEPARATOR_CHAR(SlashPos[-1]))
-    --SlashPos;
-
-  if (SlashPos < NameStart) {
-    // Use the current directory if file has no path component.
-    const char *Name = ".";
-    return FileMgr.getDirectory(Name, Name+1);
-  } else if (SlashPos == NameEnd-1)
-    return 0;       // If filename ends with a /, it's a directory.
-  else
-    return FileMgr.getDirectory(NameStart, SlashPos);
+                                                  llvm::StringRef Filename) {
+  if (Filename.empty())
+    return NULL;
+
+  if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))
+    return NULL;  // If Filename is a directory.
+
+  llvm::StringRef DirName = llvm::sys::path::parent_path(Filename);
+  // Use the current directory if file has no path component.
+  if (DirName.empty())
+    DirName = ".";
+
+  return FileMgr.getDirectory(DirName);
 }
 
-/// getDirectory - Lookup, cache, and verify the specified directory.  This
-/// returns null if the directory doesn't exist.
+/// Add all ancestors of the given path (pointing to either a file or
+/// a directory) as virtual directories.
+void FileManager::addAncestorsAsVirtualDirs(llvm::StringRef Path) {
+  llvm::StringRef DirName = llvm::sys::path::parent_path(Path);
+  if (DirName.empty())
+    return;
+
+  llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt =
+    SeenDirEntries.GetOrCreateValue(DirName);
+
+  // When caching a virtual directory, we always cache its ancestors
+  // at the same time.  Therefore, if DirName is already in the cache,
+  // we don't need to recurse as its ancestors must also already be in
+  // the cache.
+  if (NamedDirEnt.getValue())
+    return;
+
+  // Add the virtual directory to the cache.
+  DirectoryEntry *UDE = new DirectoryEntry;
+  UDE->Name = NamedDirEnt.getKeyData();
+  NamedDirEnt.setValue(UDE);
+  VirtualDirectoryEntries.push_back(UDE);
+
+  // Recursively add the other ancestors.
+  addAncestorsAsVirtualDirs(DirName);
+}
+
+/// getDirectory - Lookup, cache, and verify the specified directory
+/// (real or virtual).  This returns NULL if the directory doesn't
+/// exist.
 ///
-const DirectoryEntry *FileManager::getDirectory(const char *NameStart,
-                                                const char *NameEnd) {
+const DirectoryEntry *FileManager::getDirectory(llvm::StringRef DirName) {
   // stat doesn't like trailing separators (at least on Windows).
-  if (((NameEnd - NameStart) > 1) &&
-      ((*(NameEnd - 1) == '/') || (*(NameEnd - 1) == '\\')))
-    NameEnd--;
+  if (DirName.size() > 1 && llvm::sys::path::is_separator(DirName.back()))
+    DirName = DirName.substr(0, DirName.size()-1);
 
   ++NumDirLookups;
   llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt =
-    DirEntries.GetOrCreateValue(NameStart, NameEnd);
+    SeenDirEntries.GetOrCreateValue(DirName);
 
-  // See if there is already an entry in the map.
+  // See if there was already an entry in the map.  Note that the map
+  // contains both virtual and real directories.
   if (NamedDirEnt.getValue())
     return NamedDirEnt.getValue() == NON_EXISTENT_DIR
               ? 0 : NamedDirEnt.getValue();
@@ -243,43 +284,41 @@ const DirectoryEntry *FileManager::getDirectory(const char *NameStart,
   NamedDirEnt.setValue(NON_EXISTENT_DIR);
 
   // Get the null-terminated directory name as stored as the key of the
-  // DirEntries map.
+  // SeenDirEntries map.
   const char *InterndDirName = NamedDirEnt.getKeyData();
 
   // Check to see if the directory exists.
   struct stat StatBuf;
-  if (stat_cached(InterndDirName, &StatBuf) ||   // Error stat'ing.
-      !S_ISDIR(StatBuf.st_mode))          // Not a directory?
+  if (getStatValue(InterndDirName, StatBuf, 0/*directory lookup*/)) {
+    // There's no real directory at the given path.
     return 0;
+  }
 
-  // It exists.  See if we have already opened a directory with the same inode.
-  // This occurs when one dir is symlinked to another, for example.
-  DirectoryEntry &UDE = UniqueDirs.getDirectory(InterndDirName, StatBuf);
+  // It exists.  See if we have already opened a directory with the
+  // same inode (this occurs on Unix-like systems when one dir is
+  // symlinked to another, for example) or the same path (on
+  // Windows).
+  DirectoryEntry &UDE = UniqueRealDirs.getDirectory(InterndDirName, StatBuf);
 
   NamedDirEnt.setValue(&UDE);
-  if (UDE.getName()) // Already have an entry with this inode, return it.
-    return &UDE;
+  if (!UDE.getName()) {
+    // We don't have this directory yet, add it.  We use the string
+    // key from the SeenDirEntries map as the string.
+    UDE.Name  = InterndDirName;
+  }
 
-  // Otherwise, we don't have this directory yet, add it.  We use the string
-  // key from the DirEntries map as the string.
-  UDE.Name  = InterndDirName;
   return &UDE;
 }
 
-/// NON_EXISTENT_FILE - A special value distinct from null that is used to
-/// represent a filename that doesn't exist on the disk.
-#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1)
-
-/// getFile - Lookup, cache, and verify the specified file.  This returns null
-/// if the file doesn't exist.
+/// getFile - Lookup, cache, and verify the specified file (real or
+/// virtual).  This returns NULL if the file doesn't exist.
 ///
-const FileEntry *FileManager::getFile(const char *NameStart,
-                                      const char *NameEnd) {
+const FileEntry *FileManager::getFile(llvm::StringRef Filename) {
   ++NumFileLookups;
 
   // See if there is already an entry in the map.
   llvm::StringMapEntry<FileEntry *> &NamedFileEnt =
-    FileEntries.GetOrCreateValue(NameStart, NameEnd);
+    SeenFileEntries.GetOrCreateValue(Filename);
 
   // See if there is already an entry in the map.
   if (NamedFileEnt.getValue())
@@ -291,13 +330,16 @@ const FileEntry *FileManager::getFile(const char *NameStart,
   // By default, initialize it to invalid.
   NamedFileEnt.setValue(NON_EXISTENT_FILE);
 
-
   // Get the null-terminated file name as stored as the key of the
-  // FileEntries map.
+  // SeenFileEntries map.
   const char *InterndFileName = NamedFileEnt.getKeyData();
 
-  const DirectoryEntry *DirInfo
-    = getDirectoryFromFile(*this, NameStart, NameEnd);
+  // Look up the directory for the file.  When looking up something like
+  // sys/foo.h we'll discover all of the search directories that have a 'sys'
+  // subdirectory.  This will let us avoid having to waste time on known-to-fail
+  // searches when we go to find sys/bar.h, because all the search directories
+  // without a 'sys' subdir will get a cached failure result.
+  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
   if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
     return 0;
 
@@ -305,89 +347,218 @@ const FileEntry *FileManager::getFile(const char *NameStart,
   // FIXME: This will reduce the # syscalls.
 
   // Nope, there isn't.  Check to see if the file exists.
+  int FileDescriptor = -1;
   struct stat StatBuf;
-  //llvm::errs() << "STATING: " << Filename;
-  if (stat_cached(InterndFileName, &StatBuf) ||   // Error stat'ing.
-        S_ISDIR(StatBuf.st_mode)) {           // A directory?
-    // If this file doesn't exist, we leave a null in FileEntries for this path.
-    //llvm::errs() << ": Not existing\n";
+  if (getStatValue(InterndFileName, StatBuf, &FileDescriptor)) {
+    // There's no real file at the given path.
     return 0;
   }
-  //llvm::errs() << ": exists\n";
 
   // It exists.  See if we have already opened a file with the same inode.
   // This occurs when one dir is symlinked to another, for example.
-  FileEntry &UFE = UniqueFiles.getFile(InterndFileName, StatBuf);
+  FileEntry &UFE = UniqueRealFiles.getFile(InterndFileName, StatBuf);
 
   NamedFileEnt.setValue(&UFE);
-  if (UFE.getName())  // Already have an entry with this inode, return it.
+  if (UFE.getName()) { // Already have an entry with this inode, return it.
+    // If the stat process opened the file, close it to avoid a FD leak.
+    if (FileDescriptor != -1)
+      close(FileDescriptor);
+
     return &UFE;
+  }
 
   // Otherwise, we don't have this directory yet, add it.
-  // FIXME: Change the name to be a char* that points back to the 'FileEntries'
-  // key.
+  // FIXME: Change the name to be a char* that points back to the
+  // 'SeenFileEntries' key.
   UFE.Name    = InterndFileName;
   UFE.Size    = StatBuf.st_size;
   UFE.ModTime = StatBuf.st_mtime;
   UFE.Dir     = DirInfo;
   UFE.UID     = NextFileUID++;
+  UFE.FD      = FileDescriptor;
   return &UFE;
 }
 
 const FileEntry *
 FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
                             time_t ModificationTime) {
-  const char *NameStart = Filename.begin(), *NameEnd = Filename.end();
-
   ++NumFileLookups;
 
   // See if there is already an entry in the map.
   llvm::StringMapEntry<FileEntry *> &NamedFileEnt =
-    FileEntries.GetOrCreateValue(NameStart, NameEnd);
+    SeenFileEntries.GetOrCreateValue(Filename);
 
   // See if there is already an entry in the map.
-  if (NamedFileEnt.getValue())
-    return NamedFileEnt.getValue() == NON_EXISTENT_FILE
-                 ? 0 : NamedFileEnt.getValue();
+  if (NamedFileEnt.getValue() && NamedFileEnt.getValue() != NON_EXISTENT_FILE)
+    return NamedFileEnt.getValue();
 
   ++NumFileCacheMisses;
 
   // By default, initialize it to invalid.
   NamedFileEnt.setValue(NON_EXISTENT_FILE);
 
-  const DirectoryEntry *DirInfo
-    = getDirectoryFromFile(*this, NameStart, NameEnd);
-  if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
-    return 0;
+  addAncestorsAsVirtualDirs(Filename);
+  FileEntry *UFE = 0;
 
-  FileEntry *UFE = new FileEntry();
-  VirtualFileEntries.push_back(UFE);
-  NamedFileEnt.setValue(UFE);
+  // Now that all ancestors of Filename are in the cache, the
+  // following call is guaranteed to find the DirectoryEntry from the
+  // cache.
+  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
+  assert(DirInfo &&
+         "The directory of a virtual file should already be in the cache.");
 
-  UFE->Name    = NamedFileEnt.getKeyData();
+  // Check to see if the file exists. If so, drop the virtual file
+  int FileDescriptor = -1;
+  struct stat StatBuf;
+  const char *InterndFileName = NamedFileEnt.getKeyData();
+  if (getStatValue(InterndFileName, StatBuf, &FileDescriptor) == 0) {
+    // If the stat process opened the file, close it to avoid a FD leak.
+    if (FileDescriptor != -1)
+      close(FileDescriptor);
+
+    StatBuf.st_size = Size;
+    StatBuf.st_mtime = ModificationTime;
+    UFE = &UniqueRealFiles.getFile(InterndFileName, StatBuf);
+
+    NamedFileEnt.setValue(UFE);
+
+    // If we had already opened this file, close it now so we don't
+    // leak the descriptor. We're not going to use the file
+    // descriptor anyway, since this is a virtual file.
+    if (UFE->FD != -1) {
+      close(UFE->FD);
+      UFE->FD = -1;
+    }
+
+    // If we already have an entry with this inode, return it.
+    if (UFE->getName())
+      return UFE;
+  }
+
+  if (!UFE) {
+    UFE = new FileEntry();
+    VirtualFileEntries.push_back(UFE);
+    NamedFileEnt.setValue(UFE);
+  }
+
+  UFE->Name    = InterndFileName;
   UFE->Size    = Size;
   UFE->ModTime = ModificationTime;
   UFE->Dir     = DirInfo;
   UFE->UID     = NextFileUID++;
+  UFE->FD      = -1;
+  return UFE;
+}
+
+void FileManager::FixupRelativePath(llvm::sys::Path &path,
+                                    const FileSystemOptions &FSOpts) {
+  if (FSOpts.WorkingDir.empty() || llvm::sys::path::is_absolute(path.str()))
+    return;
+
+  llvm::SmallString<128> NewPath(FSOpts.WorkingDir);
+  llvm::sys::path::append(NewPath, path.str());
+  path = NewPath;
+}
+
+llvm::MemoryBuffer *FileManager::
+getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
+  llvm::OwningPtr<llvm::MemoryBuffer> Result;
+  llvm::error_code ec;
+  if (FileSystemOpts.WorkingDir.empty()) {
+    const char *Filename = Entry->getName();
+    // If the file is already open, use the open file descriptor.
+    if (Entry->FD != -1) {
+      ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result,
+                                           Entry->getSize());
+      if (ErrorStr)
+        *ErrorStr = ec.message();
+
+      close(Entry->FD);
+      Entry->FD = -1;
+      return Result.take();
+    }
+
+    // Otherwise, open the file.
+    ec = llvm::MemoryBuffer::getFile(Filename, Result, Entry->getSize());
+    if (ec && ErrorStr)
+      *ErrorStr = ec.message();
+    return Result.take();
+  }
   
-  // If this virtual file resolves to a file, also map that file to the 
-  // newly-created file entry.
-  const char *InterndFileName = NamedFileEnt.getKeyData();
-  struct stat StatBuf;
-  if (!stat_cached(InterndFileName, &StatBuf) &&
-      !S_ISDIR(StatBuf.st_mode)) {
-    llvm::sys::Path FilePath(InterndFileName);
-    FilePath.makeAbsolute();
-    FileEntries[FilePath.str()] = UFE;
+  llvm::sys::Path FilePath(Entry->getName());
+  FixupRelativePath(FilePath, FileSystemOpts);
+  ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result, Entry->getSize());
+  if (ec && ErrorStr)
+    *ErrorStr = ec.message();
+  return Result.take();
+}
+
+llvm::MemoryBuffer *FileManager::
+getBufferForFile(llvm::StringRef Filename, std::string *ErrorStr) {
+  llvm::OwningPtr<llvm::MemoryBuffer> Result;
+  llvm::error_code ec;
+  if (FileSystemOpts.WorkingDir.empty()) {
+    ec = llvm::MemoryBuffer::getFile(Filename, Result);
+    if (ec && ErrorStr)
+      *ErrorStr = ec.message();
+    return Result.take();
   }
+
+  llvm::sys::Path FilePath(Filename);
+  FixupRelativePath(FilePath, FileSystemOpts);
+  ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result);
+  if (ec && ErrorStr)
+    *ErrorStr = ec.message();
+  return Result.take();
+}
+
+/// getStatValue - Get the 'stat' information for the specified path,
+/// using the cache to accelerate it if possible.  This returns true
+/// if the path points to a virtual file or does not exist, or returns
+/// false if it's an existent real file.  If FileDescriptor is NULL,
+/// do directory look-up instead of file look-up.
+bool FileManager::getStatValue(const char *Path, struct stat &StatBuf,
+                               int *FileDescriptor) {
+  // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be
+  // absolute!
+  if (FileSystemOpts.WorkingDir.empty())
+    return FileSystemStatCache::get(Path, StatBuf, FileDescriptor,
+                                    StatCache.get());
+
+  llvm::sys::Path FilePath(Path);
+  FixupRelativePath(FilePath, FileSystemOpts);
+
+  return FileSystemStatCache::get(FilePath.c_str(), StatBuf, FileDescriptor,
+                                  StatCache.get());
+}
+
+void FileManager::GetUniqueIDMapping(
+                   llvm::SmallVectorImpl<const FileEntry *> &UIDToFiles) const {
+  UIDToFiles.clear();
+  UIDToFiles.resize(NextFileUID);
   
-  return UFE;
+  // Map file entries
+  for (llvm::StringMap<FileEntry*, llvm::BumpPtrAllocator>::const_iterator
+         FE = SeenFileEntries.begin(), FEEnd = SeenFileEntries.end();
+       FE != FEEnd; ++FE)
+    if (FE->getValue() && FE->getValue() != NON_EXISTENT_FILE)
+      UIDToFiles[FE->getValue()->getUID()] = FE->getValue();
+  
+  // Map virtual file entries
+  for (llvm::SmallVector<FileEntry*, 4>::const_iterator 
+         VFE = VirtualFileEntries.begin(), VFEEnd = VirtualFileEntries.end();
+       VFE != VFEEnd; ++VFE)
+    if (*VFE && *VFE != NON_EXISTENT_FILE)
+      UIDToFiles[(*VFE)->getUID()] = *VFE;
 }
 
+
 void FileManager::PrintStats() const {
   llvm::errs() << "\n*** File Manager Stats:\n";
-  llvm::errs() << UniqueFiles.size() << " files found, "
-               << UniqueDirs.size() << " dirs found.\n";
+  llvm::errs() << UniqueRealFiles.size() << " real files found, "
+               << UniqueRealDirs.size() << " real dirs found.\n";
+  llvm::errs() << VirtualFileEntries.size() << " virtual files found, "
+               << VirtualDirectoryEntries.size() << " virtual dirs found.\n";
   llvm::errs() << NumDirLookups << " dir lookups, "
                << NumDirCacheMisses << " dir cache misses.\n";
   llvm::errs() << NumFileLookups << " file lookups, "
@@ -395,20 +566,3 @@ void FileManager::PrintStats() const {
 
   //llvm::errs() << PagesMapped << BytesOfPagesMapped << FSLookups;
 }
-
-int MemorizeStatCalls::stat(const char *path, struct stat *buf) {
-  int result = StatSysCallCache::stat(path, buf);
-  
-  // Do not cache failed stats, it is easy to construct common inconsistent
-  // situations if we do, and they are not important for PCH performance (which
-  // currently only needs the stats to construct the initial FileManager
-  // entries).
-  if (result != 0)
-    return result;
-
-  // Cache file 'stat' results and directories with absolutely paths.
-  if (!S_ISDIR(buf->st_mode) || llvm::sys::Path(path).isAbsolute())
-    StatCalls[path] = StatResult(result, *buf);
-
-  return result;
-}
diff --git a/lib/Basic/FileSystemStatCache.cpp b/lib/Basic/FileSystemStatCache.cpp
new file mode 100644
index 000000000000..c8b07af295da
--- /dev/null
+++ b/lib/Basic/FileSystemStatCache.cpp
@@ -0,0 +1,120 @@
+//===--- FileSystemStatCache.cpp - Caching for 'stat' calls ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the FileSystemStatCache interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/FileSystemStatCache.h"
+#include "llvm/Support/Path.h"
+#include <fcntl.h>
+
+// FIXME: This is terrible, we need this for ::close.
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#include <sys/uio.h>
+#else
+#include <io.h>
+#endif
+using namespace clang;
+
+#if defined(_MSC_VER)
+#define S_ISDIR(s) ((_S_IFDIR & s) !=0)
+#endif
+
+/// FileSystemStatCache::get - Get the 'stat' information for the specified
+/// path, using the cache to accelerate it if possible.  This returns true if
+/// the path does not exist or false if it exists.
+///
+/// If FileDescriptor is non-null, then this lookup should only return success
+/// for files (not directories).  If it is null this lookup should only return
+/// success for directories (not files).  On a successful file lookup, the
+/// implementation can optionally fill in FileDescriptor with a valid
+/// descriptor and the client guarantees that it will close it.
+bool FileSystemStatCache::get(const char *Path, struct stat &StatBuf,
+                              int *FileDescriptor, FileSystemStatCache *Cache) {
+  LookupResult R;
+  bool isForDir = FileDescriptor == 0;
+
+  // If we have a cache, use it to resolve the stat query.
+  if (Cache)
+    R = Cache->getStat(Path, StatBuf, FileDescriptor);
+  else if (isForDir) {
+    // If this is a directory and we have no cache, just go to the file system.
+    R = ::stat(Path, &StatBuf) != 0 ? CacheMissing : CacheExists;
+  } else {
+    // Otherwise, we have to go to the filesystem.  We can always just use
+    // 'stat' here, but (for files) the client is asking whether the file exists
+    // because it wants to turn around and *open* it.  It is more efficient to
+    // do "open+fstat" on success than it is to do "stat+open".
+    //
+    // Because of this, check to see if the file exists with 'open'.  If the
+    // open succeeds, use fstat to get the stat info.
+    int OpenFlags = O_RDONLY;
+#ifdef O_BINARY
+    OpenFlags |= O_BINARY;  // Open input file in binary mode on win32.
+#endif
+    *FileDescriptor = ::open(Path, OpenFlags);
+    
+    if (*FileDescriptor == -1) {
+      // If the open fails, our "stat" fails.
+      R = CacheMissing;
+    } else {
+      // Otherwise, the open succeeded.  Do an fstat to get the information
+      // about the file.  We'll end up returning the open file descriptor to the
+      // client to do what they please with it.
+      if (::fstat(*FileDescriptor, &StatBuf) == 0)
+        R = CacheExists;
+      else {
+        // fstat rarely fails.  If it does, claim the initial open didn't
+        // succeed.
+        R = CacheMissing;
+        ::close(*FileDescriptor);
+        *FileDescriptor = -1;
+      }
+    }
+  }
+
+  // If the path doesn't exist, return failure.
+  if (R == CacheMissing) return true;
+  
+  // If the path exists, make sure that its "directoryness" matches the clients
+  // demands.
+  if (S_ISDIR(StatBuf.st_mode) != isForDir) {
+    // If not, close the file if opened.
+    if (FileDescriptor && *FileDescriptor != -1) {
+      ::close(*FileDescriptor);
+      *FileDescriptor = -1;
+    }
+    
+    return true;
+  }
+  
+  return false;
+}
+
+
+MemorizeStatCalls::LookupResult
+MemorizeStatCalls::getStat(const char *Path, struct stat &StatBuf,
+                           int *FileDescriptor) {
+  LookupResult Result = statChained(Path, StatBuf, FileDescriptor);
+  
+  // Do not cache failed stats, it is easy to construct common inconsistent
+  // situations if we do, and they are not important for PCH performance (which
+  // currently only needs the stats to construct the initial FileManager
+  // entries).
+  if (Result == CacheMissing)
+    return Result;
+  
+  // Cache file 'stat' results and directories with absolutely paths.
+  if (!S_ISDIR(StatBuf.st_mode) || llvm::sys::path::is_absolute(Path))
+    StatCalls[Path] = StatBuf;
+  
+  return Result;
+}
diff --git a/lib/Basic/IdentifierTable.cpp b/lib/Basic/IdentifierTable.cpp
index 6b673e39d365..ef11d658ed9e 100644
--- a/lib/Basic/IdentifierTable.cpp
+++ b/lib/Basic/IdentifierTable.cpp
@@ -44,8 +44,24 @@ IdentifierInfo::IdentifierInfo() {
 // IdentifierTable Implementation
 //===----------------------------------------------------------------------===//
 
+IdentifierIterator::~IdentifierIterator() { }
+
 IdentifierInfoLookup::~IdentifierInfoLookup() {}
 
+namespace {
+  /// \brief A simple identifier lookup iterator that represents an
+  /// empty sequence of identifiers.
+  class EmptyLookupIterator : public IdentifierIterator
+  {
+  public:
+    virtual llvm::StringRef Next() { return llvm::StringRef(); }
+  };
+}
+
+IdentifierIterator *IdentifierInfoLookup::getIdentifiers() const {
+  return new EmptyLookupIterator();
+}
+
 ExternalIdentifierLookup::~ExternalIdentifierLookup() {}
 
 IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
@@ -73,8 +89,9 @@ namespace {
     KEYMS = 32,
     BOOLSUPPORT = 64,
     KEYALTIVEC = 128,
-    KEYNOMS = 256,
-    KEYBORLAND = 512
+    KEYNOCXX = 256,
+    KEYBORLAND = 512,
+    KEYOPENCL = 1024
   };
 }
 
@@ -99,7 +116,8 @@ static void AddKeyword(llvm::StringRef Keyword,
   else if (LangOpts.Borland && (Flags & KEYBORLAND)) AddResult = 1;
   else if (LangOpts.Bool && (Flags & BOOLSUPPORT)) AddResult = 2;
   else if (LangOpts.AltiVec && (Flags & KEYALTIVEC)) AddResult = 2;
-  else if (!LangOpts.Microsoft && (Flags & KEYNOMS)) AddResult = 2;
+  else if (LangOpts.OpenCL && (Flags & KEYOPENCL)) AddResult = 2;
+  else if (!LangOpts.CPlusPlus && (Flags & KEYNOCXX)) AddResult = 2;
 
   // Don't add this keyword if disabled in this language.
   if (AddResult == 0) return;
@@ -308,6 +326,11 @@ IdentifierInfo *Selector::getIdentifierInfoForSlot(unsigned argIndex) const {
   return SI->getIdentifierInfoForSlot(argIndex);
 }
 
+llvm::StringRef Selector::getNameForSlot(unsigned int argIndex) const {
+  IdentifierInfo *II = getIdentifierInfoForSlot(argIndex);
+  return II? II->getName() : llvm::StringRef();
+}
+
 std::string MultiKeywordSelector::getName() const {
   llvm::SmallString<256> Str;
   llvm::raw_svector_ostream OS(Str);
@@ -374,7 +397,7 @@ Selector SelectorTable::getSelector(unsigned nKeys, IdentifierInfo **IIV) {
   unsigned Size = sizeof(MultiKeywordSelector) + nKeys*sizeof(IdentifierInfo *);
   MultiKeywordSelector *SI =
     (MultiKeywordSelector*)SelTabImpl.Allocator.Allocate(Size,
-                                         llvm::alignof<MultiKeywordSelector>());
+                                         llvm::alignOf<MultiKeywordSelector>());
   new (SI) MultiKeywordSelector(nKeys, IIV);
   SelTabImpl.Table.InsertNode(SI, InsertPos);
   return Selector(SI);
diff --git a/lib/Basic/Makefile b/lib/Basic/Makefile
index c15630459567..61accba0b5d2 100644
--- a/lib/Basic/Makefile
+++ b/lib/Basic/Makefile
@@ -16,10 +16,14 @@ LIBRARYNAME := clangBasic
 
 include $(CLANG_LEVEL)/Makefile
 
-SVN_REVISION := $(shell $(LLVM_SRC_ROOT)/utils/GetSourceVersion $(PROJ_SRC_DIR)/../..)
+SVN_REVISION := $(strip \
+	$(shell $(LLVM_SRC_ROOT)/utils/GetSourceVersion $(PROJ_SRC_DIR)/../..))
+
+SVN_REPOSITORY := $(strip \
+	$(shell $(LLVM_SRC_ROOT)/utils/GetRepositoryPath $(PROJ_SRC_DIR)/../..))
 
 CPP.Defines += -I$(PROJ_SRC_DIR)/../../include -I$(PROJ_OBJ_DIR)/../../include \
-         -DSVN_REVISION='"$(SVN_REVISION)"'
+         -DSVN_REVISION='"$(SVN_REVISION)"' -DSVN_REPOSITORY='"$(SVN_REPOSITORY)"'
 
 $(ObjDir)/.ver-svn .ver: $(ObjDir)/.dir
 	@if [ '$(SVN_REVISION)' != '$(shell cat $(ObjDir)/.ver-svn 2>/dev/null)' ]; then\
diff --git a/lib/Basic/SourceLocation.cpp b/lib/Basic/SourceLocation.cpp
index 7412b955606a..5062d43f58f2 100644
--- a/lib/Basic/SourceLocation.cpp
+++ b/lib/Basic/SourceLocation.cpp
@@ -43,6 +43,11 @@ void SourceLocation::print(llvm::raw_ostream &OS, const SourceManager &SM)const{
 
   if (isFileID()) {
     PresumedLoc PLoc = SM.getPresumedLoc(*this);
+    
+    if (PLoc.isInvalid()) {
+      OS << "<invalid>";
+      return;
+    }
     // The instantiation and spelling pos is identical for file locs.
     OS << PLoc.getFilename() << ':' << PLoc.getLine()
        << ':' << PLoc.getColumn();
@@ -105,6 +110,11 @@ bool FullSourceLoc::isInSystemHeader() const {
   return SrcMgr->isInSystemHeader(*this);
 }
 
+bool FullSourceLoc::isBeforeInTranslationUnitThan(SourceLocation Loc) const {
+  assert(isValid());
+  return SrcMgr->isBeforeInTranslationUnit(*this, Loc);
+}
+
 const char *FullSourceLoc::getCharacterData(bool *Invalid) const {
   assert(isValid());
   return SrcMgr->getCharacterData(*this, Invalid);
diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp
index 633d86c1eb8a..044c88dd2bf7 100644
--- a/lib/Basic/SourceManager.cpp
+++ b/lib/Basic/SourceManager.cpp
@@ -15,13 +15,16 @@
 #include "clang/Basic/SourceManagerInternals.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 #include <algorithm>
 #include <string>
 #include <cstring>
+#include <sys/stat.h>
 
 using namespace clang;
 using namespace SrcMgr;
@@ -66,100 +69,86 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
                                                   const SourceManager &SM,
                                                   SourceLocation Loc,
                                                   bool *Invalid) const {
-  if (Invalid)
-    *Invalid = false;
-      
-  // Lazily create the Buffer for ContentCaches that wrap files.
-  if (!Buffer.getPointer() && Entry) {
-    std::string ErrorStr;
-    struct stat FileInfo;
-    Buffer.setPointer(MemoryBuffer::getFile(Entry->getName(), &ErrorStr,
-                                            Entry->getSize(), &FileInfo));
+  // Lazily create the Buffer for ContentCaches that wrap files.  If we already
+  // computed it, jsut return what we have.
+  if (Buffer.getPointer() || Entry == 0) {
+    if (Invalid)
+      *Invalid = isBufferInvalid();
     
-    // If we were unable to open the file, then we are in an inconsistent
-    // situation where the content cache referenced a file which no longer
-    // exists. Most likely, we were using a stat cache with an invalid entry but
-    // the file could also have been removed during processing. Since we can't
-    // really deal with this situation, just create an empty buffer.
-    //
-    // FIXME: This is definitely not ideal, but our immediate clients can't
-    // currently handle returning a null entry here. Ideally we should detect
-    // that we are in an inconsistent situation and error out as quickly as
-    // possible.
-    if (!Buffer.getPointer()) {
-      const llvm::StringRef FillStr("<<<MISSING SOURCE FILE>>>\n");
-      Buffer.setPointer(MemoryBuffer::getNewMemBuffer(Entry->getSize(), 
-                                                      "<invalid>"));
-      char *Ptr = const_cast<char*>(Buffer.getPointer()->getBufferStart());
-      for (unsigned i = 0, e = Entry->getSize(); i != e; ++i)
-        Ptr[i] = FillStr[i % FillStr.size()];
-
-      if (Diag.isDiagnosticInFlight())
-        Diag.SetDelayedDiagnostic(diag::err_cannot_open_file, 
-                                  Entry->getName(), ErrorStr);
-      else 
-        Diag.Report(FullSourceLoc(Loc, SM), diag::err_cannot_open_file)
-          << Entry->getName() << ErrorStr;
-
-      Buffer.setInt(Buffer.getInt() | InvalidFlag);
-
-    // FIXME: This conditionalization is horrible, but we see spurious failures
-    // in the test suite due to this warning and no one has had time to hunt it
-    // down. So for now, we just don't emit this diagnostic on Win32, and hope
-    // nothing bad happens.
-    //
-    // PR6812.
-#if !defined(LLVM_ON_WIN32)
-    } else if (FileInfo.st_size != Entry->getSize() ||
-               FileInfo.st_mtime != Entry->getModificationTime()) {
-      // Check that the file's size and modification time are the same
-      // as in the file entry (which may have come from a stat cache).
-      if (Diag.isDiagnosticInFlight())
-        Diag.SetDelayedDiagnostic(diag::err_file_modified,
-                                  Entry->getName());
-      else
-        Diag.Report(FullSourceLoc(Loc, SM), diag::err_file_modified)
-          << Entry->getName();
-
-      Buffer.setInt(Buffer.getInt() | InvalidFlag);
-#endif
-    }
+    return Buffer.getPointer();
+  }    
+
+  std::string ErrorStr;
+  Buffer.setPointer(SM.getFileManager().getBufferForFile(Entry, &ErrorStr));
+
+  // If we were unable to open the file, then we are in an inconsistent
+  // situation where the content cache referenced a file which no longer
+  // exists. Most likely, we were using a stat cache with an invalid entry but
+  // the file could also have been removed during processing. Since we can't
+  // really deal with this situation, just create an empty buffer.
+  //
+  // FIXME: This is definitely not ideal, but our immediate clients can't
+  // currently handle returning a null entry here. Ideally we should detect
+  // that we are in an inconsistent situation and error out as quickly as
+  // possible.
+  if (!Buffer.getPointer()) {
+    const llvm::StringRef FillStr("<<<MISSING SOURCE FILE>>>\n");
+    Buffer.setPointer(MemoryBuffer::getNewMemBuffer(Entry->getSize(), 
+                                                    "<invalid>"));
+    char *Ptr = const_cast<char*>(Buffer.getPointer()->getBufferStart());
+    for (unsigned i = 0, e = Entry->getSize(); i != e; ++i)
+      Ptr[i] = FillStr[i % FillStr.size()];
+
+    if (Diag.isDiagnosticInFlight())
+      Diag.SetDelayedDiagnostic(diag::err_cannot_open_file, 
+                                Entry->getName(), ErrorStr);
+    else 
+      Diag.Report(Loc, diag::err_cannot_open_file)
+        << Entry->getName() << ErrorStr;
+
+    Buffer.setInt(Buffer.getInt() | InvalidFlag);
     
-    // If the buffer is valid, check to see if it has a UTF Byte Order Mark
-    // (BOM).  We only support UTF-8 without a BOM right now.  See
-    // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
-    if (!isBufferInvalid()) {
-      llvm::StringRef BufStr = Buffer.getPointer()->getBuffer();
-      const char *BOM = 0;
-      if (BufStr.startswith("\xFE\xBB\xBF"))
-        BOM = "UTF-8";
-      else if (BufStr.startswith("\xFE\xFF"))
-        BOM = "UTF-16 (BE)";
-      else if (BufStr.startswith("\xFF\xFE"))
-        BOM = "UTF-16 (LE)";
-      else if (BufStr.startswith(llvm::StringRef("\x00\x00\xFE\xFF", 4)))
-        BOM = "UTF-32 (BE)";
-      else if (BufStr.startswith(llvm::StringRef("\xFF\xFE\x00\x00", 4)))
-        BOM = "UTF-32 (LE)";
-      else if (BufStr.startswith("\x2B\x2F\x76"))
-        BOM = "UTF-7";
-      else if (BufStr.startswith("\xF7\x64\x4C"))
-        BOM = "UTF-1";
-      else if (BufStr.startswith("\xDD\x73\x66\x73"))
-        BOM = "UTF-EBCDIC";
-      else if (BufStr.startswith("\x0E\xFE\xFF"))
-        BOM = "SDSU";
-      else if (BufStr.startswith("\xFB\xEE\x28"))
-        BOM = "BOCU-1";
-      else if (BufStr.startswith("\x84\x31\x95\x33"))
-        BOM = "BOCU-1";
-      
-      if (BOM) {
-        Diag.Report(FullSourceLoc(Loc, SM), diag::err_unsupported_bom)
-          << BOM << Entry->getName();
-        Buffer.setInt(1);
-      }
-    }
+    if (Invalid) *Invalid = true;
+    return Buffer.getPointer();
+  }
+  
+  // Check that the file's size is the same as in the file entry (which may
+  // have come from a stat cache).
+  if (getRawBuffer()->getBufferSize() != (size_t)Entry->getSize()) {
+    if (Diag.isDiagnosticInFlight())
+      Diag.SetDelayedDiagnostic(diag::err_file_modified,
+                                Entry->getName());
+    else
+      Diag.Report(Loc, diag::err_file_modified)
+        << Entry->getName();
+
+    Buffer.setInt(Buffer.getInt() | InvalidFlag);
+    if (Invalid) *Invalid = true;
+    return Buffer.getPointer();
+  }
+  
+  // If the buffer is valid, check to see if it has a UTF Byte Order Mark
+  // (BOM).  We only support UTF-8 without a BOM right now.  See
+  // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
+  llvm::StringRef BufStr = Buffer.getPointer()->getBuffer();
+  const char *BOM = llvm::StringSwitch<const char *>(BufStr)
+    .StartsWith("\xEF\xBB\xBF", "UTF-8")
+    .StartsWith("\xFE\xFF", "UTF-16 (BE)")
+    .StartsWith("\xFF\xFE", "UTF-16 (LE)")
+    .StartsWith("\x00\x00\xFE\xFF", "UTF-32 (BE)")
+    .StartsWith("\xFF\xFE\x00\x00", "UTF-32 (LE)")
+    .StartsWith("\x2B\x2F\x76", "UTF-7")
+    .StartsWith("\xF7\x64\x4C", "UTF-1")
+    .StartsWith("\xDD\x73\x66\x73", "UTF-EBCDIC")
+    .StartsWith("\x0E\xFE\xFF", "SDSU")
+    .StartsWith("\xFB\xEE\x28", "BOCU-1")
+    .StartsWith("\x84\x31\x95\x33", "GB-18030")
+    .Default(0);
+
+  if (BOM) {
+    Diag.Report(Loc, diag::err_unsupported_bom)
+      << BOM << Entry->getName();
+    Buffer.setInt(Buffer.getInt() | InvalidFlag);
   }
   
   if (Invalid)
@@ -350,6 +339,14 @@ LineTableInfo &SourceManager::getLineTable() {
 // Private 'Create' methods.
 //===----------------------------------------------------------------------===//
 
+SourceManager::SourceManager(Diagnostic &Diag, FileManager &FileMgr)
+  : Diag(Diag), FileMgr(FileMgr),
+    ExternalSLocEntries(0), LineTable(0), NumLinearScans(0),
+    NumBinaryProbes(0) {
+  clearIDTables();
+  Diag.setSourceManager(this);
+}
+
 SourceManager::~SourceManager() {
   delete LineTable;
 
@@ -525,25 +522,32 @@ SourceManager::getMemoryBufferForFile(const FileEntry *File,
   return IR->getBuffer(Diag, *this, SourceLocation(), Invalid);
 }
 
-bool SourceManager::overrideFileContents(const FileEntry *SourceFile,
+void SourceManager::overrideFileContents(const FileEntry *SourceFile,
                                          const llvm::MemoryBuffer *Buffer,
                                          bool DoNotFree) {
   const SrcMgr::ContentCache *IR = getOrCreateContentCache(SourceFile);
-  if (IR == 0)
-    return true;
+  assert(IR && "getOrCreateContentCache() cannot return NULL");
 
   const_cast<SrcMgr::ContentCache *>(IR)->replaceBuffer(Buffer, DoNotFree);
-  return false;
 }
 
 llvm::StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const {
   bool MyInvalid = false;
-  const llvm::MemoryBuffer *Buf = getBuffer(FID, &MyInvalid);
+  const SLocEntry &SLoc = getSLocEntry(FID.ID);
+  if (!SLoc.isFile()) {
+    if (Invalid) 
+      *Invalid = true;
+    return "<<<<<INVALID SOURCE LOCATION>>>>>";
+  }
+  
+  const llvm::MemoryBuffer *Buf
+    = SLoc.getFile().getContentCache()->getBuffer(Diag, *this, SourceLocation(), 
+                                                  &MyInvalid);
   if (Invalid)
     *Invalid = MyInvalid;
 
   if (MyInvalid)
-    return "";
+    return "<<<<<INVALID SOURCE LOCATION>>>>>";
   
   return Buf->getBuffer();
 }
@@ -796,21 +800,30 @@ unsigned SourceManager::getColumnNumber(FileID FID, unsigned FilePos,
   return FilePos-LineStart+1;
 }
 
+// isInvalid - Return the result of calling loc.isInvalid(), and
+// if Invalid is not null, set its value to same.
+static bool isInvalid(SourceLocation Loc, bool *Invalid) {
+  bool MyInvalid = Loc.isInvalid();
+  if (Invalid)
+    *Invalid = MyInvalid;
+  return MyInvalid;
+}
+
 unsigned SourceManager::getSpellingColumnNumber(SourceLocation Loc,
                                                 bool *Invalid) const {
-  if (Loc.isInvalid()) return 0;
+  if (isInvalid(Loc, Invalid)) return 0;
   std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(Loc);
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
 unsigned SourceManager::getInstantiationColumnNumber(SourceLocation Loc,
                                                      bool *Invalid) const {
-  if (Loc.isInvalid()) return 0;
+  if (isInvalid(Loc, Invalid)) return 0;
   std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
-static DISABLE_INLINE void
+static LLVM_ATTRIBUTE_NOINLINE void
 ComputeLineNumbers(Diagnostic &Diag, ContentCache *FI,
                    llvm::BumpPtrAllocator &Alloc,
                    const SourceManager &SM, bool &Invalid);
@@ -974,13 +987,13 @@ unsigned SourceManager::getLineNumber(FileID FID, unsigned FilePos,
 
 unsigned SourceManager::getInstantiationLineNumber(SourceLocation Loc, 
                                                    bool *Invalid) const {
-  if (Loc.isInvalid()) return 0;
+  if (isInvalid(Loc, Invalid)) return 0;
   std::pair<FileID, unsigned> LocInfo = getDecomposedInstantiationLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
 unsigned SourceManager::getSpellingLineNumber(SourceLocation Loc, 
                                               bool *Invalid) const {
-  if (Loc.isInvalid()) return 0;
+  if (isInvalid(Loc, Invalid)) return 0;
   std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
@@ -1021,7 +1034,7 @@ SourceManager::getFileCharacteristic(SourceLocation Loc) const {
 /// for normal clients.
 const char *SourceManager::getBufferName(SourceLocation Loc, 
                                          bool *Invalid) const {
-  if (Loc.isInvalid()) return "<invalid loc>";
+  if (isInvalid(Loc, Invalid)) return "<invalid loc>";
 
   return getBuffer(getFileID(Loc), Invalid)->getBufferIdentifier();
 }
@@ -1051,8 +1064,14 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
     Filename = C->Entry->getName();
   else
     Filename = C->getBuffer(Diag, *this)->getBufferIdentifier();
-  unsigned LineNo = getLineNumber(LocInfo.first, LocInfo.second);
-  unsigned ColNo  = getColumnNumber(LocInfo.first, LocInfo.second);
+  bool Invalid = false;
+  unsigned LineNo = getLineNumber(LocInfo.first, LocInfo.second, &Invalid);
+  if (Invalid)
+    return PresumedLoc();
+  unsigned ColNo  = getColumnNumber(LocInfo.first, LocInfo.second, &Invalid);
+  if (Invalid)
+    return PresumedLoc();
+  
   SourceLocation IncludeLoc = FI.getIncludeLoc();
 
   // If we have #line directives in this file, update and overwrite the physical
@@ -1090,38 +1109,65 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
 // Other miscellaneous methods.
 //===----------------------------------------------------------------------===//
 
+/// \brief Retrieve the inode for the given file entry, if possible.
+///
+/// This routine involves a system call, and therefore should only be used
+/// in non-performance-critical code.
+static llvm::Optional<ino_t> getActualFileInode(const FileEntry *File) {
+  if (!File)
+    return llvm::Optional<ino_t>();
+  
+  struct stat StatBuf;
+  if (::stat(File->getName(), &StatBuf))
+    return llvm::Optional<ino_t>();
+    
+  return StatBuf.st_ino;
+}
+
 /// \brief Get the source location for the given file:line:col triplet.
 ///
 /// If the source file is included multiple times, the source location will
 /// be based upon the first inclusion.
 SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
-                                          unsigned Line, unsigned Col) const {
+                                          unsigned Line, unsigned Col) {
   assert(SourceFile && "Null source file!");
   assert(Line && Col && "Line and column should start from 1!");
 
-  fileinfo_iterator FI = FileInfos.find(SourceFile);
-  if (FI == FileInfos.end())
-    return SourceLocation();
-  ContentCache *Content = FI->second;
-
-  // If this is the first use of line information for this buffer, compute the
-  /// SourceLineCache for it on demand.
-  if (Content->SourceLineCache == 0) {
-    bool MyInvalid = false;
-    ComputeLineNumbers(Diag, Content, ContentCacheAlloc, *this, MyInvalid);
-    if (MyInvalid)
-      return SourceLocation();
-  }
-
   // Find the first file ID that corresponds to the given file.
   FileID FirstFID;
 
   // First, check the main file ID, since it is common to look for a
   // location in the main file.
+  llvm::Optional<ino_t> SourceFileInode;
+  llvm::Optional<llvm::StringRef> SourceFileName;
   if (!MainFileID.isInvalid()) {
     const SLocEntry &MainSLoc = getSLocEntry(MainFileID);
-    if (MainSLoc.isFile() && MainSLoc.getFile().getContentCache() == Content)
-      FirstFID = MainFileID;
+    if (MainSLoc.isFile()) {
+      const ContentCache *MainContentCache
+        = MainSLoc.getFile().getContentCache();
+      if (!MainContentCache) {
+        // Can't do anything
+      } else if (MainContentCache->Entry == SourceFile) {
+        FirstFID = MainFileID;
+      } else {
+        // Fall back: check whether we have the same base name and inode
+        // as the main file.
+        const FileEntry *MainFile = MainContentCache->Entry;
+        SourceFileName = llvm::sys::path::filename(SourceFile->getName());
+        if (*SourceFileName == llvm::sys::path::filename(MainFile->getName())) {
+          SourceFileInode = getActualFileInode(SourceFile);
+          if (SourceFileInode) {
+            if (llvm::Optional<ino_t> MainFileInode 
+                                               = getActualFileInode(MainFile)) {
+              if (*SourceFileInode == *MainFileInode) {
+                FirstFID = MainFileID;
+                SourceFile = MainFile;
+              }
+            }
+          }
+        }
+      }
+    }
   }
 
   if (FirstFID.isInvalid()) {
@@ -1129,16 +1175,63 @@ SourceLocation SourceManager::getLocation(const FileEntry *SourceFile,
     // through all of the source locations.
     for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) {
       const SLocEntry &SLoc = getSLocEntry(I);
-      if (SLoc.isFile() && SLoc.getFile().getContentCache() == Content) {
+      if (SLoc.isFile() && 
+          SLoc.getFile().getContentCache() &&
+          SLoc.getFile().getContentCache()->Entry == SourceFile) {
         FirstFID = FileID::get(I);
         break;
       }
     }
   }
+
+  // If we haven't found what we want yet, try again, but this time stat()
+  // each of the files in case the files have changed since we originally 
+  // parsed the file. 
+  if (FirstFID.isInvalid() &&
+      (SourceFileName || 
+       (SourceFileName = llvm::sys::path::filename(SourceFile->getName()))) &&
+      (SourceFileInode ||
+       (SourceFileInode = getActualFileInode(SourceFile)))) {
+    for (unsigned I = 0, N = sloc_entry_size(); I != N; ++I) {
+      const SLocEntry &SLoc = getSLocEntry(I);
+      if (SLoc.isFile()) { 
+        const ContentCache *FileContentCache 
+          = SLoc.getFile().getContentCache();
+        const FileEntry *Entry =FileContentCache? FileContentCache->Entry : 0;
+        if (Entry && 
+            *SourceFileName == llvm::sys::path::filename(Entry->getName())) {
+          if (llvm::Optional<ino_t> EntryInode = getActualFileInode(Entry)) {
+            if (*SourceFileInode == *EntryInode) {
+              FirstFID = FileID::get(I);
+              SourceFile = Entry;
+              break;
+            }
+          }
+        }
+      }
+    }      
+  }
     
   if (FirstFID.isInvalid())
     return SourceLocation();
 
+  if (Line == 1 && Col == 1)
+    return getLocForStartOfFile(FirstFID);
+
+  ContentCache *Content
+    = const_cast<ContentCache *>(getOrCreateContentCache(SourceFile));
+  if (!Content)
+    return SourceLocation();
+    
+  // If this is the first use of line information for this buffer, compute the
+  /// SourceLineCache for it on demand.
+  if (Content->SourceLineCache == 0) {
+    bool MyInvalid = false;
+    ComputeLineNumbers(Diag, Content, ContentCacheAlloc, *this, MyInvalid);
+    if (MyInvalid)
+      return SourceLocation();
+  }
+
   if (Line > Content->NumLines) {
     unsigned Size = Content->getBuffer(Diag, *this)->getBufferSize();
     if (Size > 0)
@@ -1190,6 +1283,13 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   if (LHS == RHS)
     return false;
 
+  // If both locations are macro instantiations, the order of their offsets
+  // reflect the order that the tokens, pointed to by these locations, were
+  // instantiated (during parsing each token that is instantiated by a macro,
+  // expands the SLocEntries).
+  if (LHS.isMacroID() && RHS.isMacroID())
+    return LHS.getOffset() < RHS.getOffset();
+
   std::pair<FileID, unsigned> LOffs = getDecomposedLoc(LHS);
   std::pair<FileID, unsigned> ROffs = getDecomposedLoc(RHS);
 
diff --git a/lib/Basic/TargetInfo.cpp b/lib/Basic/TargetInfo.cpp
index 6d42883cd13c..a9eeb8b4cc4e 100644
--- a/lib/Basic/TargetInfo.cpp
+++ b/lib/Basic/TargetInfo.cpp
@@ -15,6 +15,7 @@
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include <cctype>
 #include <cstdlib>
 using namespace clang;
 
@@ -25,6 +26,7 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   TLSSupported = true;
   NoAsmVariants = false;
   PointerWidth = PointerAlign = 32;
+  BoolWidth = BoolAlign = 8;
   IntWidth = IntAlign = 32;
   LongWidth = LongAlign = 32;
   LongLongWidth = LongLongAlign = 64;
@@ -54,6 +56,7 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   DescriptionString = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                       "i64:64:64-f32:32:32-f64:64:64-n32";
   UserLabelPrefix = "_";
+  MCountName = "mcount";
   HasAlignMac68kSupport = false;
 
   // Default to no types using fpret.
@@ -98,7 +101,7 @@ const char *TargetInfo::getTypeConstantSuffix(IntType T) {
   }
 }
 
-/// getTypeWidth - Return the width (in bits) of the specified integer type 
+/// getTypeWidth - Return the width (in bits) of the specified integer type
 /// enum. For example, SignedInt -> getIntWidth().
 unsigned TargetInfo::getTypeWidth(IntType T) const {
   switch (T) {
@@ -114,7 +117,7 @@ unsigned TargetInfo::getTypeWidth(IntType T) const {
   };
 }
 
-/// getTypeAlign - Return the alignment (in bits) of the specified integer type 
+/// getTypeAlign - Return the alignment (in bits) of the specified integer type
 /// enum. For example, SignedInt -> getIntAlign().
 unsigned TargetInfo::getTypeAlign(IntType T) const {
   switch (T) {
@@ -132,18 +135,18 @@ unsigned TargetInfo::getTypeAlign(IntType T) const {
 
 /// isTypeSigned - Return whether an integer types is signed. Returns true if
 /// the type is signed; false otherwise.
-bool TargetInfo::isTypeSigned(IntType T) const {
+bool TargetInfo::isTypeSigned(IntType T) {
   switch (T) {
   default: assert(0 && "not an integer!");
   case SignedShort:
   case SignedInt:
   case SignedLong:
-  case SignedLongLong:   
+  case SignedLongLong:
     return true;
   case UnsignedShort:
   case UnsignedInt:
   case UnsignedLong:
-  case UnsignedLongLong: 
+  case UnsignedLongLong:
     return false;
   };
 }
@@ -164,7 +167,7 @@ void TargetInfo::setForcedLangOptions(LangOptions &Opts) {
 static llvm::StringRef removeGCCRegisterPrefix(llvm::StringRef Name) {
   if (Name[0] == '%' || Name[0] == '#')
     Name = Name.substr(1);
-  
+
   return Name;
 }
 
@@ -174,7 +177,7 @@ static llvm::StringRef removeGCCRegisterPrefix(llvm::StringRef Name) {
 bool TargetInfo::isValidGCCRegisterName(llvm::StringRef Name) const {
   if (Name.empty())
     return false;
-  
+
   const char * const *Names;
   unsigned NumNames;
 
@@ -216,7 +219,7 @@ bool TargetInfo::isValidGCCRegisterName(llvm::StringRef Name) const {
   return false;
 }
 
-llvm::StringRef 
+llvm::StringRef
 TargetInfo::getNormalizedGCCRegisterName(llvm::StringRef Name) const {
   assert(isValidGCCRegisterName(Name) && "Invalid register passed in");
 
@@ -283,6 +286,10 @@ bool TargetInfo::validateOutputConstraint(ConstraintInfo &Info) const {
       Info.setAllowsRegister();
       break;
     case 'm': // memory operand.
+    case 'o': // offsetable memory operand.
+    case 'V': // non-offsetable memory operand.
+    case '<': // autodecrement memory operand.
+    case '>': // autoincrement memory operand.
       Info.setAllowsMemory();
       break;
     case 'g': // general register, memory operand or immediate integer.
@@ -291,13 +298,12 @@ bool TargetInfo::validateOutputConstraint(ConstraintInfo &Info) const {
       Info.setAllowsMemory();
       break;
     case ',': // multiple alternative constraint.  Pass it.
-      Name++;
       // Handle additional optional '=' or '+' modifiers.
-      if (*Name == '=' || *Name == '+')
+      if (Name[1] == '=' || Name[1] == '+')
         Name++;
       break;
     case '?': // Disparage slightly code.
-    case '!': // Disparage severly.
+    case '!': // Disparage severely.
       break;  // Pass them.
     }
 
@@ -347,6 +353,15 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
         if (i >= NumOutputs)
           return false;
 
+        // A number must refer to an output only operand.
+        if (OutputConstraints[i].isReadWrite())
+          return false;
+
+        // If the constraint is already tied, it must be tied to the 
+        // same operand referenced to by the number.
+        if (Info.hasTiedOperand() && Info.getTiedOperand() != i)
+          return false;
+
         // The constraint should have the same info as the respective
         // output constraint.
         Info.setTiedOperand(i, OutputConstraints[i]);
@@ -362,6 +377,11 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       if (!resolveSymbolicName(Name, OutputConstraints, NumOutputs, Index))
         return false;
 
+      // If the constraint is already tied, it must be tied to the 
+      // same operand referenced to by the number.
+      if (Info.hasTiedOperand() && Info.getTiedOperand() != Index)
+        return false;
+
       Info.setTiedOperand(Index, OutputConstraints[Index]);
       break;
     }
@@ -384,8 +404,10 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       Info.setAllowsRegister();
       break;
     case 'm': // memory operand.
-    case 'o': // offsettable memory operand
-    case 'V': // non-offsettable memory operand
+    case 'o': // offsettable memory operand.
+    case 'V': // non-offsettable memory operand.
+    case '<': // autodecrement memory operand.
+    case '>': // autoincrement memory operand.
       Info.setAllowsMemory();
       break;
     case 'g': // general register, memory operand or immediate integer.
@@ -393,6 +415,10 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       Info.setAllowsRegister();
       Info.setAllowsMemory();
       break;
+    case 'E': // immediate floating point.
+    case 'F': // immediate floating point.
+    case 'p': // address operand.
+      break;
     case ',': // multiple alternative constraint.  Ignore comma.
       break;
     case '?': // Disparage slightly code.
diff --git a/lib/Basic/Targets.cpp b/lib/Basic/Targets.cpp
index df20defa0fc0..a8198e4ae79e 100644
--- a/lib/Basic/Targets.cpp
+++ b/lib/Basic/Targets.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/Type.h"
 #include <algorithm>
 using namespace clang;
 
@@ -160,12 +161,12 @@ public:
     return llvm::MCSectionMachO::ParseSectionSpecifier(SR, Segment, Section,
                                                        TAA, StubSize);
   }
-  
+
   virtual const char *getStaticInitSectionSpecifier() const {
     // FIXME: We should return 0 when building kexts.
     return "__TEXT,__StaticInit,regular,pure_instructions";
   }
-  
+
 };
 
 
@@ -209,6 +210,25 @@ public:
   FreeBSDTargetInfo(const std::string &triple)
     : OSTargetInfo<Target>(triple) {
       this->UserLabelPrefix = "";
+
+      llvm::Triple Triple(triple);
+      switch (Triple.getArch()) {
+        default:
+        case llvm::Triple::x86:
+        case llvm::Triple::x86_64:
+          this->MCountName = ".mcount";
+          break;
+        case llvm::Triple::mips:
+        case llvm::Triple::mipsel:
+        case llvm::Triple::ppc:
+        case llvm::Triple::ppc64:
+          this->MCountName = "_mcount";
+          break;
+        case llvm::Triple::arm:
+          this->MCountName = "__mcount";
+          break;
+      }
+
     }
 };
 
@@ -256,6 +276,7 @@ public:
   LinuxTargetInfo(const std::string& triple)
     : OSTargetInfo<Target>(triple) {
     this->UserLabelPrefix = "";
+    this->WIntType = TargetInfo::UnsignedInt;
   }
 };
 
@@ -405,6 +426,54 @@ public:
     // FIXME: WIntType should be SignedLong
   }
 };
+
+// Windows target
+template<typename Target>
+class WindowsTargetInfo : public OSTargetInfo<Target> {
+protected:
+  virtual void getOSDefines(const LangOptions &Opts, const llvm::Triple &Triple,
+                            MacroBuilder &Builder) const {
+    Builder.defineMacro("_WIN32");
+  }
+  void getVisualStudioDefines(const LangOptions &Opts,
+                              MacroBuilder &Builder) const {
+    if (Opts.CPlusPlus) {
+      if (Opts.RTTI)
+        Builder.defineMacro("_CPPRTTI");
+
+      if (Opts.Exceptions)
+        Builder.defineMacro("_CPPUNWIND");
+    }
+
+    if (!Opts.CharIsSigned)
+      Builder.defineMacro("_CHAR_UNSIGNED");
+
+    // FIXME: POSIXThreads isn't exactly the option this should be defined for,
+    //        but it works for now.
+    if (Opts.POSIXThreads)
+      Builder.defineMacro("_MT");
+
+    if (Opts.MSCVersion != 0)
+      Builder.defineMacro("_MSC_VER", llvm::Twine(Opts.MSCVersion));
+
+    if (Opts.Microsoft) {
+      Builder.defineMacro("_MSC_EXTENSIONS");
+
+      if (Opts.CPlusPlus0x) {
+        Builder.defineMacro("_RVALUE_REFERENCES_V2_SUPPORTED");
+        Builder.defineMacro("_RVALUE_REFERENCES_SUPPORTED");
+        Builder.defineMacro("_NATIVE_NULLPTR_SUPPORTED");
+      }
+    }
+
+    Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
+  }
+
+public:
+  WindowsTargetInfo(const std::string &triple)
+    : OSTargetInfo<Target>(triple) {}
+};
+
 } // end anonymous namespace.
 
 //===----------------------------------------------------------------------===//
@@ -430,17 +499,6 @@ public:
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const;
 
-  virtual const char *getVAListDeclaration() const {
-    return "typedef char* __builtin_va_list;";
-    // This is the right definition for ABI/V4: System V.4/eabi.
-    /*return "typedef struct __va_list_tag {"
-           "  unsigned char gpr;"
-           "  unsigned char fpr;"
-           "  unsigned short reserved;"
-           "  void* overflow_arg_area;"
-           "  void* reg_save_area;"
-           "} __builtin_va_list[1];";*/
-  }
   virtual void getGCCRegNames(const char * const *&Names,
                               unsigned &NumNames) const;
   virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
@@ -458,16 +516,16 @@ public:
     // FIXME: The following are added to allow parsing.
     // I just took a guess at what the actions should be.
     // Also, is more specific checking needed?  I.e. specific registers?
-    case 'd': // Floating point register (containing 64-bit value) 
+    case 'd': // Floating point register (containing 64-bit value)
     case 'v': // Altivec vector register
       Info.setAllowsRegister();
       break;
     case 'w':
       switch (Name[1]) {
-        case 'd':// VSX vector register to hold vector double data 
-        case 'f':// VSX vector register to hold vector float data 
-        case 's':// VSX vector register to hold scalar float data 
-        case 'a':// Any VSX register 
+        case 'd':// VSX vector register to hold vector double data
+        case 'f':// VSX vector register to hold vector float data
+        case 's':// VSX vector register to hold scalar float data
+        case 'a':// Any VSX register
           break;
         default:
           return false;
@@ -475,27 +533,27 @@ public:
       Info.setAllowsRegister();
       Name++; // Skip over 'w'.
       break;
-    case 'h': // `MQ', `CTR', or `LINK' register 
-    case 'q': // `MQ' register 
-    case 'c': // `CTR' register 
-    case 'l': // `LINK' register 
-    case 'x': // `CR' register (condition register) number 0 
-    case 'y': // `CR' register (condition register) 
-    case 'z': // `XER[CA]' carry bit (part of the XER register) 
+    case 'h': // `MQ', `CTR', or `LINK' register
+    case 'q': // `MQ' register
+    case 'c': // `CTR' register
+    case 'l': // `LINK' register
+    case 'x': // `CR' register (condition register) number 0
+    case 'y': // `CR' register (condition register)
+    case 'z': // `XER[CA]' carry bit (part of the XER register)
       Info.setAllowsRegister();
       break;
-    case 'I': // Signed 16-bit constant 
+    case 'I': // Signed 16-bit constant
     case 'J': // Unsigned 16-bit constant shifted left 16 bits
-              //  (use `L' instead for SImode constants) 
-    case 'K': // Unsigned 16-bit constant 
-    case 'L': // Signed 16-bit constant shifted left 16 bits 
-    case 'M': // Constant larger than 31 
-    case 'N': // Exact power of 2 
-    case 'P': // Constant whose negation is a signed 16-bit constant 
+              //  (use `L' instead for SImode constants)
+    case 'K': // Unsigned 16-bit constant
+    case 'L': // Signed 16-bit constant shifted left 16 bits
+    case 'M': // Constant larger than 31
+    case 'N': // Exact power of 2
+    case 'P': // Constant whose negation is a signed 16-bit constant
     case 'G': // Floating point constant that can be loaded into a
-              // register with one instruction per word 
+              // register with one instruction per word
     case 'H': // Integer/Floating point constant that can be loaded
-              // into a register using three instructions 
+              // into a register using three instructions
       break;
     case 'm': // Memory operand. Note that on PowerPC targets, m can
               // include addresses that update the base register. It
@@ -503,13 +561,13 @@ public:
               // if that asm statement accesses the operand exactly once.
               // The asm statement must also use `%U<opno>' as a
               // placeholder for the "update" flag in the corresponding
-              // load or store instruction. For example: 
+              // load or store instruction. For example:
               // asm ("st%U0 %1,%0" : "=m" (mem) : "r" (val));
-              // is correct but: 
+              // is correct but:
               // asm ("st %1,%0" : "=m" (mem) : "r" (val));
               // is not. Use es rather than m if you don't want the base
-              // register to be updated. 
-    case 'e': 
+              // register to be updated.
+    case 'e':
       if (Name[1] != 's')
           return false;
               // es: A "stable" memory operand; that is, one which does not
@@ -521,23 +579,23 @@ public:
       Name++; // Skip over 'e'.
       break;
     case 'Q': // Memory operand that is an offset from a register (it is
-              // usually better to use `m' or `es' in asm statements) 
+              // usually better to use `m' or `es' in asm statements)
     case 'Z': // Memory operand that is an indexed or indirect from a
               // register (it is usually better to use `m' or `es' in
-              // asm statements) 
+              // asm statements)
       Info.setAllowsMemory();
       Info.setAllowsRegister();
       break;
-    case 'R': // AIX TOC entry 
+    case 'R': // AIX TOC entry
     case 'a': // Address operand that is an indexed or indirect from a
-              // register (`p' is preferable for asm statements) 
-    case 'S': // Constant suitable as a 64-bit mask operand 
-    case 'T': // Constant suitable as a 32-bit mask operand 
-    case 'U': // System V Release 4 small data area reference 
+              // register (`p' is preferable for asm statements)
+    case 'S': // Constant suitable as a 64-bit mask operand
+    case 'T': // Constant suitable as a 32-bit mask operand
+    case 'U': // System V Release 4 small data area reference
     case 't': // AND masks that can be performed by two rldic{l, r}
-              // instructions 
-    case 'W': // Vector constant that does not require memory 
-    case 'j': // Vector constant that is all zeros. 
+              // instructions
+    case 'W': // Vector constant that does not require memory
+    case 'j': // Vector constant that is all zeros.
       break;
     // End FIXME.
     }
@@ -549,8 +607,9 @@ public:
 };
 
 const Builtin::Info PPCTargetInfo::BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, false },
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER, false },
+#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, ALL_LANGUAGES, false },
+#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER,\
+                                              ALL_LANGUAGES, false },
 #include "clang/Basic/BuiltinsPPC.def"
 };
 
@@ -584,7 +643,7 @@ void PPCTargetInfo::getTargetDefines(const LangOptions &Opts,
 
   // FIXME: Should be controlled by command line option.
   Builder.defineMacro("__LONG_DOUBLE_128__");
-  
+
   if (Opts.AltiVec) {
     Builder.defineMacro("__VEC__", "10206");
     Builder.defineMacro("__ALTIVEC__");
@@ -704,7 +763,18 @@ public:
                         "i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32";
 
     if (getTriple().getOS() == llvm::Triple::FreeBSD)
-        this->SizeType = TargetInfo::UnsignedInt;
+        SizeType = UnsignedInt;
+  }
+
+  virtual const char *getVAListDeclaration() const {
+    // This is the ELF definition, and is overridden by the Darwin sub-target
+    return "typedef struct __va_list_tag {"
+           "  unsigned char gpr;"
+           "  unsigned char fpr;"
+           "  unsigned short reserved;"
+           "  void* overflow_arg_area;"
+           "  void* reg_save_area;"
+           "} __builtin_va_list[1];";
   }
 };
 } // end anonymous namespace.
@@ -720,17 +790,24 @@ public:
     DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64";
   }
+  virtual const char *getVAListDeclaration() const {
+    return "typedef char* __builtin_va_list;";
+  }
 };
 } // end anonymous namespace.
 
 
 namespace {
-class DarwinPPCTargetInfo :
-  public DarwinTargetInfo<PPCTargetInfo> {
+class DarwinPPC32TargetInfo :
+  public DarwinTargetInfo<PPC32TargetInfo> {
 public:
-  DarwinPPCTargetInfo(const std::string& triple)
-    : DarwinTargetInfo<PPCTargetInfo>(triple) {
+  DarwinPPC32TargetInfo(const std::string& triple)
+    : DarwinTargetInfo<PPC32TargetInfo>(triple) {
     HasAlignMac68kSupport = true;
+    BoolWidth = BoolAlign = 32; //XXX support -mone-byte-bool?
+  }
+  virtual const char *getVAListDeclaration() const {
+    return "typedef char* __builtin_va_list;";
   }
 };
 
@@ -752,8 +829,7 @@ class MBlazeTargetInfo : public TargetInfo {
 
 public:
   MBlazeTargetInfo(const std::string& triple) : TargetInfo(triple) {
-    DescriptionString = "E-p:32:32-i8:8:8-i16:16:16-i64:32:32-f64:32:32-"
-                        "v64:32:32-v128:32:32-n32";
+    DescriptionString = "E-p:32:32:32-i8:8:8-i16:16:16";
   }
 
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
@@ -875,8 +951,9 @@ void MBlazeTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
 namespace {
 // Namespace for x86 abstract base class
 const Builtin::Info BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, false },
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER, false },
+#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, ALL_LANGUAGES, false },
+#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER,\
+                                              ALL_LANGUAGES, false },
 #include "clang/Basic/BuiltinsX86.def"
 };
 
@@ -1010,6 +1087,9 @@ void X86TargetInfo::getDefaultFeatures(const std::string &CPU,
            CPU == "athlon-fx") {
     setFeatureEnabled(Features, "sse2", true);
     setFeatureEnabled(Features, "3dnowa", true);
+  } else if (CPU == "k8-sse3") {
+    setFeatureEnabled(Features, "sse3", true);
+    setFeatureEnabled(Features, "3dnowa", true);
   } else if (CPU == "c3-2")
     setFeatureEnabled(Features, "sse", true);
 }
@@ -1116,13 +1196,13 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
       .Case("mmx", MMX)
       .Default(NoMMXSSE);
     SSELevel = std::max(SSELevel, Level);
-    
-    AMD3DNowEnum ThreeDNowLevel = 
+
+    AMD3DNowEnum ThreeDNowLevel =
       llvm::StringSwitch<AMD3DNowEnum>(Features[i].substr(1))
         .Case("3dnowa", AMD3DNowAthlon)
         .Case("3dnow", AMD3DNow)
         .Default(NoAMD3DNow);
-    
+
     AMD3DNowLevel = std::max(AMD3DNowLevel, ThreeDNowLevel);
   }
 }
@@ -1184,7 +1264,24 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   case NoMMXSSE:
     break;
   }
-  
+
+  if (Opts.Microsoft && PointerWidth == 32) {
+    switch (SSELevel) {
+    case SSE42:
+    case SSE41:
+    case SSSE3:
+    case SSE3:
+    case SSE2:
+      Builder.defineMacro("_M_IX86_FP", llvm::Twine(2));
+      break;
+    case SSE1:
+      Builder.defineMacro("_M_IX86_FP", llvm::Twine(1));
+      break;
+    default:
+      Builder.defineMacro("_M_IX86_FP", llvm::Twine(0));
+    }
+  }
+
   // Each case falls through to the previous one here.
   switch (AMD3DNowLevel) {
   case AMD3DNowAthlon:
@@ -1241,6 +1338,7 @@ X86TargetInfo::validateAsmConstraint(const char *&Name,
   return false;
 }
 
+
 std::string
 X86TargetInfo::convertConstraint(const char Constraint) const {
   switch (Constraint) {
@@ -1250,6 +1348,8 @@ X86TargetInfo::convertConstraint(const char Constraint) const {
   case 'd': return std::string("{dx}");
   case 'S': return std::string("{si}");
   case 'D': return std::string("{di}");
+  case 'p': // address
+    return std::string("im");
   case 't': // top of floating point stack.
     return std::string("{st}");
   case 'u': // second from top of floating point stack.
@@ -1284,7 +1384,7 @@ public:
   virtual const char *getVAListDeclaration() const {
     return "typedef char* __builtin_va_list;";
   }
-  
+
   int getEHDataRegisterNumber(unsigned RegNo) const {
     if (RegNo == 0) return 0;
     if (RegNo == 1) return 2;
@@ -1325,10 +1425,10 @@ public:
 
 namespace {
 // x86-32 Windows target
-class WindowsX86_32TargetInfo : public X86_32TargetInfo {
+class WindowsX86_32TargetInfo : public WindowsTargetInfo<X86_32TargetInfo> {
 public:
   WindowsX86_32TargetInfo(const std::string& triple)
-    : X86_32TargetInfo(triple) {
+    : WindowsTargetInfo<X86_32TargetInfo>(triple) {
     TLSSupported = false;
     WCharType = UnsignedShort;
     DoubleAlign = LongLongAlign = 64;
@@ -1338,12 +1438,7 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    X86_32TargetInfo::getTargetDefines(Opts, Builder);
-    // This list is based off of the the list of things MingW defines
-    Builder.defineMacro("_WIN32");
-    DefineStd(Builder, "WIN32", Opts);
-    DefineStd(Builder, "WINNT", Opts);
-    Builder.defineMacro("_X86_");
+    WindowsTargetInfo<X86_32TargetInfo>::getTargetDefines(Opts, Builder);
   }
 };
 } // end anonymous namespace
@@ -1355,16 +1450,17 @@ class VisualStudioWindowsX86_32TargetInfo : public WindowsX86_32TargetInfo {
 public:
   VisualStudioWindowsX86_32TargetInfo(const std::string& triple)
     : WindowsX86_32TargetInfo(triple) {
+    LongDoubleWidth = 64;
+    LongDoubleFormat = &llvm::APFloat::IEEEdouble;
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     WindowsX86_32TargetInfo::getTargetDefines(Opts, Builder);
+    WindowsX86_32TargetInfo::getVisualStudioDefines(Opts, Builder);
     // The value of the following reflects processor type.
     // 300=386, 400=486, 500=Pentium, 600=Blend (default)
     // We lost the original triple, so we use the default.
     Builder.defineMacro("_M_IX86", "600");
-    Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
-    Builder.defineMacro("_STDCALL_SUPPORTED");
   }
 };
 } // end anonymous namespace
@@ -1379,6 +1475,9 @@ public:
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     WindowsX86_32TargetInfo::getTargetDefines(Opts, Builder);
+    DefineStd(Builder, "WIN32", Opts);
+    DefineStd(Builder, "WINNT", Opts);
+    Builder.defineMacro("_X86_");
     Builder.defineMacro("__MSVCRT__");
     Builder.defineMacro("__MINGW32__");
     Builder.defineMacro("__declspec", "__declspec");
@@ -1420,6 +1519,7 @@ public:
     SizeType = UnsignedLong;
     IntPtrType = SignedLong;
     PtrDiffType = SignedLong;
+    this->UserLabelPrefix = "";
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
@@ -1461,7 +1561,7 @@ public:
            "} __va_list_tag;"
            "typedef __va_list_tag __builtin_va_list[1];";
   }
-  
+
   int getEHDataRegisterNumber(unsigned RegNo) const {
     if (RegNo == 0) return 0;
     if (RegNo == 1) return 1;
@@ -1472,23 +1572,29 @@ public:
 
 namespace {
 // x86-64 Windows target
-class WindowsX86_64TargetInfo : public X86_64TargetInfo {
+class WindowsX86_64TargetInfo : public WindowsTargetInfo<X86_64TargetInfo> {
 public:
   WindowsX86_64TargetInfo(const std::string& triple)
-    : X86_64TargetInfo(triple) {
+    : WindowsTargetInfo<X86_64TargetInfo>(triple) {
     TLSSupported = false;
     WCharType = UnsignedShort;
     LongWidth = LongAlign = 32;
-    DoubleAlign = LongLongAlign = 64;      
+    DoubleAlign = LongLongAlign = 64;
     IntMaxType = SignedLongLong;
     UIntMaxType = UnsignedLongLong;
     Int64Type = SignedLongLong;
+    SizeType = UnsignedLongLong;
+    PtrDiffType = SignedLongLong;
+    IntPtrType = SignedLongLong;
+    this->UserLabelPrefix = "";
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    X86_64TargetInfo::getTargetDefines(Opts, Builder);
+    WindowsTargetInfo<X86_64TargetInfo>::getTargetDefines(Opts, Builder);
     Builder.defineMacro("_WIN64");
-    DefineStd(Builder, "WIN64", Opts);
+  }
+  virtual const char *getVAListDeclaration() const {
+    return "typedef char* __builtin_va_list;";
   }
 };
 } // end anonymous namespace
@@ -1503,11 +1609,9 @@ public:
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     WindowsX86_64TargetInfo::getTargetDefines(Opts, Builder);
+    WindowsX86_64TargetInfo::getVisualStudioDefines(Opts, Builder);
     Builder.defineMacro("_M_X64");
-    Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
-  }
-  virtual const char *getVAListDeclaration() const {
-    return "typedef char* __builtin_va_list;";
+    Builder.defineMacro("_M_AMD64");
   }
 };
 } // end anonymous namespace
@@ -1522,9 +1626,10 @@ public:
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     WindowsX86_64TargetInfo::getTargetDefines(Opts, Builder);
+    DefineStd(Builder, "WIN64", Opts);
     Builder.defineMacro("__MSVCRT__");
     Builder.defineMacro("__MINGW64__");
-    Builder.defineMacro("__declspec");
+    Builder.defineMacro("__declspec", "__declspec");
   }
 };
 } // end anonymous namespace
@@ -1590,7 +1695,7 @@ public:
     // {} in inline assembly are neon specifiers, not assembly variant
     // specifiers.
     NoAsmVariants = true;
-    
+
     // FIXME: Should we just treat this as a feature?
     IsThumb = getTriple().getArchName().startswith("thumb");
     if (IsThumb) {
@@ -1655,7 +1760,7 @@ public:
     else if (CPU == "cortex-a8" || CPU == "cortex-a9")
       Features["neon"] = true;
   }
-  
+
   virtual bool setFeatureEnabled(llvm::StringMap<bool> &Features,
                                  const std::string &Name,
                                  bool Enabled) const {
@@ -1715,6 +1820,7 @@ public:
       .Cases("arm1136jf-s", "mpcorenovfp", "mpcore", "6K")
       .Cases("arm1156t2-s", "arm1156t2f-s", "6T2")
       .Cases("cortex-a8", "cortex-a9", "7A")
+      .Case("cortex-m3", "7M")
       .Default(0);
   }
   virtual bool setCPU(const std::string &Name) {
@@ -1816,10 +1922,17 @@ const char * const ARMTargetInfo::GCCRegNames[] = {
   "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
   "s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15",
   "s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23",
-  "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31"
+  "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31",
 
-  // FIXME: Need double and NEON registers, but we need support for aliasing
-  // multiple registers for that.
+  // Double registers
+  "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
+  "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
+  "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
+  "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31",
+
+  // Quad registers
+  "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7",
+  "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15"
 };
 
 void ARMTargetInfo::getGCCRegNames(const char * const *&Names,
@@ -1845,6 +1958,8 @@ const TargetInfo::GCCRegAlias ARMTargetInfo::GCCRegAliases[] = {
   { { "r13" }, "sp" },
   { { "r14" }, "lr" },
   { { "r15" }, "pc" },
+  // The S, D and Q registers overlap, but aren't really aliases; we
+  // don't want to substitute one of these for a different-sized one.
 };
 
 void ARMTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
@@ -1854,8 +1969,9 @@ void ARMTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
 }
 
 const Builtin::Info ARMTargetInfo::BuiltinInfo[] = {
-#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, false },
-#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER, false },
+#define BUILTIN(ID, TYPE, ATTRS) { #ID, TYPE, ATTRS, 0, ALL_LANGUAGES, false },
+#define LIBBUILTIN(ID, TYPE, ATTRS, HEADER) { #ID, TYPE, ATTRS, HEADER,\
+                                              ALL_LANGUAGES, false },
 #include "clang/Basic/BuiltinsARM.def"
 };
 } // end anonymous namespace.
@@ -1882,17 +1998,37 @@ namespace {
 class SparcV8TargetInfo : public TargetInfo {
   static const TargetInfo::GCCRegAlias GCCRegAliases[];
   static const char * const GCCRegNames[];
+  bool SoftFloat;
 public:
   SparcV8TargetInfo(const std::string& triple) : TargetInfo(triple) {
     // FIXME: Support Sparc quad-precision long double?
     DescriptionString = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
                         "i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32";
   }
+  virtual bool setFeatureEnabled(llvm::StringMap<bool> &Features,
+                                 const std::string &Name,
+                                 bool Enabled) const {
+    if (Name == "soft-float")
+      Features[Name] = Enabled;
+    else
+      return false;
+
+    return true;
+  }
+  virtual void HandleTargetFeatures(std::vector<std::string> &Features) {
+    SoftFloat = false;
+    for (unsigned i = 0, e = Features.size(); i != e; ++i)
+      if (Features[i] == "+soft-float")
+        SoftFloat = true;
+  }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     DefineStd(Builder, "sparc", Opts);
     Builder.defineMacro("__sparcv8");
     Builder.defineMacro("__REGISTER_PREFIX__", "");
+
+    if (SoftFloat)
+      Builder.defineMacro("SOFT_FLOAT", "1");
   }
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
                                  unsigned &NumRecords) const {
@@ -1991,75 +2127,6 @@ public:
 } // end anonymous namespace.
 
 namespace {
-  class PIC16TargetInfo : public TargetInfo{
-  public:
-    PIC16TargetInfo(const std::string& triple) : TargetInfo(triple) {
-      TLSSupported = false;
-      IntWidth = 16;
-      LongWidth = LongLongWidth = 32;
-      PointerWidth = 16;
-      IntAlign = 8;
-      LongAlign = LongLongAlign = 8;
-      PointerAlign = 8;
-      SizeType = UnsignedInt;
-      IntMaxType = SignedLong;
-      UIntMaxType = UnsignedLong;
-      IntPtrType = SignedShort;
-      PtrDiffType = SignedInt;
-      SigAtomicType = SignedLong;
-      FloatWidth = 32;
-      FloatAlign = 32;
-      DoubleWidth = 32;
-      DoubleAlign = 32;
-      LongDoubleWidth = 32;
-      LongDoubleAlign = 32;
-      FloatFormat = &llvm::APFloat::IEEEsingle;
-      DoubleFormat = &llvm::APFloat::IEEEsingle;
-      LongDoubleFormat = &llvm::APFloat::IEEEsingle;
-      DescriptionString = "e-p:16:8:8-i8:8:8-i16:8:8-i32:8:8-f32:32:32-n8";
-
-    }
-    virtual uint64_t getPointerWidthV(unsigned AddrSpace) const { return 16; }
-    virtual uint64_t getPointerAlignV(unsigned AddrSpace) const { return 8; }
-    virtual void getTargetDefines(const LangOptions &Opts,
-                                MacroBuilder &Builder) const {
-      Builder.defineMacro("__pic16");
-      Builder.defineMacro("__PIC16");
-      Builder.defineMacro("rom", "__attribute__((address_space(1)))");
-      Builder.defineMacro("ram", "__attribute__((address_space(0)))");
-      Builder.defineMacro("__section(SectName)",
-             "__attribute__((section(SectName)))");
-      Builder.defineMacro("near",
-             "__attribute__((section(\"Address=NEAR\")))");
-      Builder.defineMacro("__address(Addr)",
-             "__attribute__((section(\"Address=\"#Addr)))");
-      Builder.defineMacro("__config(conf)", "asm(\"CONFIG \"#conf)");
-      Builder.defineMacro("__idlocs(value)", "asm(\"__IDLOCS \"#value)");
-      Builder.defineMacro("interrupt",
-             "__attribute__((section(\"interrupt=0x4\"))) \
-             __attribute__((used))");
-    }
-    virtual void getTargetBuiltins(const Builtin::Info *&Records,
-                                   unsigned &NumRecords) const {}
-    virtual const char *getVAListDeclaration() const {
-      return "typedef char* __builtin_va_list;";
-    }
-    virtual const char *getClobbers() const {
-      return "";
-    }
-    virtual void getGCCRegNames(const char * const *&Names,
-                                unsigned &NumNames) const {}
-    virtual bool validateAsmConstraint(const char *&Name,
-                                       TargetInfo::ConstraintInfo &info) const {
-      return true;
-    }
-    virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
-                                  unsigned &NumAliases) const {}
-    virtual bool useGlobalsForAutomaticVariables() const {return true;}
-  };
-}
-
-namespace {
   class MSP430TargetInfo : public TargetInfo {
     static const char * const GCCRegNames[];
   public:
@@ -2294,8 +2361,8 @@ namespace {
       LongDoubleFormat = &llvm::APFloat::IEEEsingle;
       DescriptionString = "E-p:32:32:32-i1:8:8-i8:8:32-"
                           "i16:16:32-i32:32:32-i64:32:32-"
-                          "f32:32:32-f64:64:64-v64:64:64-"
-                          "v128:128:128-a0:0:64-n32";
+                          "f32:32:32-f64:32:32-v64:32:32-"
+                          "v128:32:32-a0:0:32-n32";
     }
 
     virtual void getTargetDefines(const LangOptions &Opts,
@@ -2399,7 +2466,7 @@ public:
 };
 
 const char * const MipsTargetInfo::GCCRegNames[] = {
-  "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7", 
+  "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
   "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
   "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
   "$24",  "$25",  "$26",  "$27",  "$28",  "$sp",  "$fp",  "$31",
@@ -2526,12 +2593,9 @@ static TargetInfo *AllocateTarget(const std::string &T) {
       return new LinuxTargetInfo<MipselTargetInfo>(T);
     return new MipselTargetInfo(T);
 
-  case llvm::Triple::pic16:
-    return new PIC16TargetInfo(T);
-
   case llvm::Triple::ppc:
     if (os == llvm::Triple::Darwin)
-      return new DarwinPPCTargetInfo(T);
+      return new DarwinPPC32TargetInfo(T);
     else if (os == llvm::Triple::FreeBSD)
       return new FreeBSDTargetInfo<PPC32TargetInfo>(T);
     return new PPC32TargetInfo(T);
@@ -2615,10 +2679,13 @@ static TargetInfo *AllocateTarget(const std::string &T) {
       return new FreeBSDTargetInfo<X86_64TargetInfo>(T);
     case llvm::Triple::Solaris:
       return new SolarisTargetInfo<X86_64TargetInfo>(T);
-    case llvm::Triple::MinGW64:
+    case llvm::Triple::MinGW32:
       return new MinGWX86_64TargetInfo(T);
     case llvm::Triple::Win32:   // This is what Triple.h supports now.
-      return new VisualStudioWindowsX86_64TargetInfo(T);
+      if (Triple.getEnvironment() == llvm::Triple::MachO)
+        return new DarwinX86_64TargetInfo(T);
+      else
+        return new VisualStudioWindowsX86_64TargetInfo(T);
     default:
       return new X86_64TargetInfo(T);
     }
diff --git a/lib/Basic/Version.cpp b/lib/Basic/Version.cpp
index 28f29541aae6..6573eb96e3f3 100644
--- a/lib/Basic/Version.cpp
+++ b/lib/Basic/Version.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Basic/Version.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Config/config.h"
 #include <cstring>
 #include <cstdlib>
 
@@ -20,45 +21,52 @@ using namespace std;
 
 namespace clang {
   
-llvm::StringRef getClangRepositoryPath() {
-  static const char URL[] = "$URL: http://llvm.org/svn/llvm-project/cfe/tags/RELEASE_28/lib/Basic/Version.cpp $";
-  const char *URLEnd = URL + strlen(URL);
+std::string getClangRepositoryPath() {
+#ifdef SVN_REPOSITORY
+  llvm::StringRef URL(SVN_REPOSITORY);
+#else
+  llvm::StringRef URL("");
+#endif
 
-  const char *End = strstr(URL, "/lib/Basic");
-  if (End)
-    URLEnd = End;
+  // If the SVN_REPOSITORY is empty, try to use the SVN keyword. This helps us
+  // pick up a tag in an SVN export, for example.
+  static llvm::StringRef SVNRepository("$URL: http://llvm.org/svn/llvm-project/cfe/trunk/lib/Basic/Version.cpp $");
+  if (URL.empty()) {
+    URL = SVNRepository.slice(SVNRepository.find(':'),
+                              SVNRepository.find("/lib/Basic"));
+  }
 
   // Strip off version from a build from an integration branch.
-  End = strstr(URL, "/src/tools/clang");
-  if (End)
-    URLEnd = End;
+  URL = URL.slice(0, URL.find("/src/tools/clang"));
 
-  const char *Begin = strstr(URL, "cfe/");
-  if (Begin)
-    return llvm::StringRef(Begin + 4, URLEnd - Begin - 4);
+  // Trim path prefix off, assuming path came from standard cfe path.
+  size_t Start = URL.find("cfe/");
+  if (Start != llvm::StringRef::npos)
+    URL = URL.substr(Start + 4);
 
-  return llvm::StringRef(URL, URLEnd - URL);
+  return URL;
 }
 
 std::string getClangRevision() {
 #ifdef SVN_REVISION
-  if (SVN_REVISION[0] != '\0') {
-    std::string revision;
-    llvm::raw_string_ostream OS(revision);
-    OS << strtol(SVN_REVISION, 0, 10);
-    return OS.str();
-  }
-#endif
+  return SVN_REVISION;
+#else
   return "";
+#endif
 }
 
 std::string getClangFullRepositoryVersion() {
   std::string buf;
   llvm::raw_string_ostream OS(buf);
-  OS << getClangRepositoryPath();
-  const std::string &Revision = getClangRevision();
-  if (!Revision.empty())
-    OS << ' ' << Revision;
+  std::string Path = getClangRepositoryPath();
+  std::string Revision = getClangRevision();
+  if (!Path.empty())
+    OS << Path;
+  if (!Revision.empty()) {
+    if (!Path.empty())
+      OS << ' ';
+    OS << Revision;
+  }
   return OS.str();
 }
   
@@ -70,6 +78,12 @@ std::string getClangFullVersion() {
 #endif
   OS << "clang version " CLANG_VERSION_STRING " ("
      << getClangFullRepositoryVersion() << ')';
+
+  // If vendor supplied, include the base LLVM version as well.
+#ifdef CLANG_VENDOR
+  OS << " (based on LLVM " << PACKAGE_VERSION << ")";
+#endif
+
   return OS.str();
 }
 
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index bd5e342e028f..b4574344bc5f 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -12,4 +12,4 @@ add_subdirectory(Serialization)
 add_subdirectory(Frontend)
 add_subdirectory(FrontendTool)
 add_subdirectory(Index)
-add_subdirectory(Checker)
+add_subdirectory(StaticAnalyzer)
diff --git a/lib/Checker/BasicObjCFoundationChecks.cpp b/lib/Checker/BasicObjCFoundationChecks.cpp
deleted file mode 100644
index 3c1a6d1c8282..000000000000
--- a/lib/Checker/BasicObjCFoundationChecks.cpp
+++ /dev/null
@@ -1,585 +0,0 @@
-//== BasicObjCFoundationChecks.cpp - Simple Apple-Foundation checks -*- C++ -*--
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines BasicObjCFoundationChecks, a class that encapsulates
-//  a set of simple checks to run on Objective-C code using Apple's Foundation
-//  classes.
-//
-//===----------------------------------------------------------------------===//
-
-#include "BasicObjCFoundationChecks.h"
-
-#include "clang/Checker/PathSensitive/ExplodedGraph.h"
-#include "clang/Checker/PathSensitive/GRSimpleAPICheck.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/MemRegion.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/AST/DeclObjC.h"
-#include "clang/AST/Expr.h"
-#include "clang/AST/ExprObjC.h"
-#include "clang/AST/ASTContext.h"
-
-using namespace clang;
-
-static const ObjCInterfaceType* GetReceiverType(const ObjCMessageExpr* ME) {
-  QualType T;
-  switch (ME->getReceiverKind()) {
-  case ObjCMessageExpr::Instance:
-    T = ME->getInstanceReceiver()->getType();
-    break;
-
-  case ObjCMessageExpr::SuperInstance:
-    T = ME->getSuperType();
-    break;
-
-  case ObjCMessageExpr::Class:
-  case ObjCMessageExpr::SuperClass:
-    return 0;
-  }
-
-  if (const ObjCObjectPointerType *PT = T->getAs<ObjCObjectPointerType>())
-    return PT->getInterfaceType();
-
-  return NULL;
-}
-
-static const char* GetReceiverNameType(const ObjCMessageExpr* ME) {
-  if (const ObjCInterfaceType *ReceiverType = GetReceiverType(ME))
-    return ReceiverType->getDecl()->getIdentifier()->getNameStart();
-  return NULL;
-}
-
-namespace {
-
-class APIMisuse : public BugType {
-public:
-  APIMisuse(const char* name) : BugType(name, "API Misuse (Apple)") {}
-};
-
-class BasicObjCFoundationChecks : public GRSimpleAPICheck {
-  APIMisuse *BT;
-  BugReporter& BR;
-  ASTContext &Ctx;
-
-  bool isNSString(const ObjCInterfaceType *T, llvm::StringRef suffix);
-  bool AuditNSString(ExplodedNode* N, const ObjCMessageExpr* ME);
-
-  bool CheckNilArg(ExplodedNode* N, unsigned Arg);
-
-public:
-  BasicObjCFoundationChecks(ASTContext& ctx, BugReporter& br)
-    : BT(0), BR(br), Ctx(ctx) {}
-
-  bool Audit(ExplodedNode* N, GRStateManager&);
-
-private:
-  void WarnNilArg(ExplodedNode* N, const ObjCMessageExpr* ME, unsigned Arg) {
-    std::string sbuf;
-    llvm::raw_string_ostream os(sbuf);
-    os << "Argument to '" << GetReceiverNameType(ME) << "' method '"
-       << ME->getSelector().getAsString() << "' cannot be nil.";
-
-    // Lazily create the BugType object for NilArg.  This will be owned
-    // by the BugReporter object 'BR' once we call BR.EmitWarning.
-    if (!BT) BT = new APIMisuse("nil argument");
-
-    RangedBugReport *R = new RangedBugReport(*BT, os.str(), N);
-    R->addRange(ME->getArg(Arg)->getSourceRange());
-    BR.EmitReport(R);
-  }
-};
-
-} // end anonymous namespace
-
-
-GRSimpleAPICheck*
-clang::CreateBasicObjCFoundationChecks(ASTContext& Ctx, BugReporter& BR) {
-  return new BasicObjCFoundationChecks(Ctx, BR);
-}
-
-
-
-bool BasicObjCFoundationChecks::Audit(ExplodedNode* N,
-                                      GRStateManager&) {
-
-  const ObjCMessageExpr* ME =
-    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());
-
-  const ObjCInterfaceType *ReceiverType = GetReceiverType(ME);
-
-  if (!ReceiverType)
-    return false;
-
-  if (isNSString(ReceiverType,
-                 ReceiverType->getDecl()->getIdentifier()->getName()))
-    return AuditNSString(N, ME);
-
-  return false;
-}
-
-static inline bool isNil(SVal X) {
-  return isa<loc::ConcreteInt>(X);
-}
-
-//===----------------------------------------------------------------------===//
-// Error reporting.
-//===----------------------------------------------------------------------===//
-
-bool BasicObjCFoundationChecks::CheckNilArg(ExplodedNode* N, unsigned Arg) {
-  const ObjCMessageExpr* ME =
-    cast<ObjCMessageExpr>(cast<PostStmt>(N->getLocation()).getStmt());
-
-  const Expr * E = ME->getArg(Arg);
-
-  if (isNil(N->getState()->getSVal(E))) {
-    WarnNilArg(N, ME, Arg);
-    return true;
-  }
-
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// NSString checking.
-//===----------------------------------------------------------------------===//
-
-bool BasicObjCFoundationChecks::isNSString(const ObjCInterfaceType *T,
-                                           llvm::StringRef ClassName) {
-  return ClassName == "NSString" || ClassName == "NSMutableString";
-}
-
-bool BasicObjCFoundationChecks::AuditNSString(ExplodedNode* N,
-                                              const ObjCMessageExpr* ME) {
-
-  Selector S = ME->getSelector();
-
-  if (S.isUnarySelector())
-    return false;
-
-  // FIXME: This is going to be really slow doing these checks with
-  //  lexical comparisons.
-
-  std::string NameStr = S.getAsString();
-  llvm::StringRef Name(NameStr);
-  assert(!Name.empty());
-
-  // FIXME: Checking for initWithFormat: will not work in most cases
-  //  yet because [NSString alloc] returns id, not NSString*.  We will
-  //  need support for tracking expected-type information in the analyzer
-  //  to find these errors.
-  if (Name == "caseInsensitiveCompare:" ||
-      Name == "compare:" ||
-      Name == "compare:options:" ||
-      Name == "compare:options:range:" ||
-      Name == "compare:options:range:locale:" ||
-      Name == "componentsSeparatedByCharactersInSet:" ||
-      Name == "initWithFormat:")
-    return CheckNilArg(N, 0);
-
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// Error reporting.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class AuditCFNumberCreate : public GRSimpleAPICheck {
-  APIMisuse* BT;
-
-  // FIXME: Either this should be refactored into GRSimpleAPICheck, or
-  //   it should always be passed with a call to Audit.  The latter
-  //   approach makes this class more stateless.
-  ASTContext& Ctx;
-  IdentifierInfo* II;
-  BugReporter& BR;
-
-public:
-  AuditCFNumberCreate(ASTContext& ctx, BugReporter& br)
-  : BT(0), Ctx(ctx), II(&Ctx.Idents.get("CFNumberCreate")), BR(br){}
-
-  ~AuditCFNumberCreate() {}
-
-  bool Audit(ExplodedNode* N, GRStateManager&);
-
-private:
-  void AddError(const TypedRegion* R, const Expr* Ex, ExplodedNode *N,
-                uint64_t SourceSize, uint64_t TargetSize, uint64_t NumberKind);
-};
-} // end anonymous namespace
-
-enum CFNumberType {
-  kCFNumberSInt8Type = 1,
-  kCFNumberSInt16Type = 2,
-  kCFNumberSInt32Type = 3,
-  kCFNumberSInt64Type = 4,
-  kCFNumberFloat32Type = 5,
-  kCFNumberFloat64Type = 6,
-  kCFNumberCharType = 7,
-  kCFNumberShortType = 8,
-  kCFNumberIntType = 9,
-  kCFNumberLongType = 10,
-  kCFNumberLongLongType = 11,
-  kCFNumberFloatType = 12,
-  kCFNumberDoubleType = 13,
-  kCFNumberCFIndexType = 14,
-  kCFNumberNSIntegerType = 15,
-  kCFNumberCGFloatType = 16
-};
-
-namespace {
-  template<typename T>
-  class Optional {
-    bool IsKnown;
-    T Val;
-  public:
-    Optional() : IsKnown(false), Val(0) {}
-    Optional(const T& val) : IsKnown(true), Val(val) {}
-
-    bool isKnown() const { return IsKnown; }
-
-    const T& getValue() const {
-      assert (isKnown());
-      return Val;
-    }
-
-    operator const T&() const {
-      return getValue();
-    }
-  };
-}
-
-static Optional<uint64_t> GetCFNumberSize(ASTContext& Ctx, uint64_t i) {
-  static const unsigned char FixedSize[] = { 8, 16, 32, 64, 32, 64 };
-
-  if (i < kCFNumberCharType)
-    return FixedSize[i-1];
-
-  QualType T;
-
-  switch (i) {
-    case kCFNumberCharType:     T = Ctx.CharTy;     break;
-    case kCFNumberShortType:    T = Ctx.ShortTy;    break;
-    case kCFNumberIntType:      T = Ctx.IntTy;      break;
-    case kCFNumberLongType:     T = Ctx.LongTy;     break;
-    case kCFNumberLongLongType: T = Ctx.LongLongTy; break;
-    case kCFNumberFloatType:    T = Ctx.FloatTy;    break;
-    case kCFNumberDoubleType:   T = Ctx.DoubleTy;   break;
-    case kCFNumberCFIndexType:
-    case kCFNumberNSIntegerType:
-    case kCFNumberCGFloatType:
-      // FIXME: We need a way to map from names to Type*.
-    default:
-      return Optional<uint64_t>();
-  }
-
-  return Ctx.getTypeSize(T);
-}
-
-#if 0
-static const char* GetCFNumberTypeStr(uint64_t i) {
-  static const char* Names[] = {
-    "kCFNumberSInt8Type",
-    "kCFNumberSInt16Type",
-    "kCFNumberSInt32Type",
-    "kCFNumberSInt64Type",
-    "kCFNumberFloat32Type",
-    "kCFNumberFloat64Type",
-    "kCFNumberCharType",
-    "kCFNumberShortType",
-    "kCFNumberIntType",
-    "kCFNumberLongType",
-    "kCFNumberLongLongType",
-    "kCFNumberFloatType",
-    "kCFNumberDoubleType",
-    "kCFNumberCFIndexType",
-    "kCFNumberNSIntegerType",
-    "kCFNumberCGFloatType"
-  };
-
-  return i <= kCFNumberCGFloatType ? Names[i-1] : "Invalid CFNumberType";
-}
-#endif
-
-bool AuditCFNumberCreate::Audit(ExplodedNode* N,GRStateManager&){
-  const CallExpr* CE =
-    cast<CallExpr>(cast<PostStmt>(N->getLocation()).getStmt());
-  const Expr* Callee = CE->getCallee();
-  SVal CallV = N->getState()->getSVal(Callee);
-  const FunctionDecl* FD = CallV.getAsFunctionDecl();
-
-  if (!FD || FD->getIdentifier() != II || CE->getNumArgs()!=3)
-    return false;
-
-  // Get the value of the "theType" argument.
-  SVal TheTypeVal = N->getState()->getSVal(CE->getArg(1));
-
-    // FIXME: We really should allow ranges of valid theType values, and
-    //   bifurcate the state appropriately.
-  nonloc::ConcreteInt* V = dyn_cast<nonloc::ConcreteInt>(&TheTypeVal);
-
-  if (!V)
-    return false;
-
-  uint64_t NumberKind = V->getValue().getLimitedValue();
-  Optional<uint64_t> TargetSize = GetCFNumberSize(Ctx, NumberKind);
-
-  // FIXME: In some cases we can emit an error.
-  if (!TargetSize.isKnown())
-    return false;
-
-  // Look at the value of the integer being passed by reference.  Essentially
-  // we want to catch cases where the value passed in is not equal to the
-  // size of the type being created.
-  SVal TheValueExpr = N->getState()->getSVal(CE->getArg(2));
-
-  // FIXME: Eventually we should handle arbitrary locations.  We can do this
-  //  by having an enhanced memory model that does low-level typing.
-  loc::MemRegionVal* LV = dyn_cast<loc::MemRegionVal>(&TheValueExpr);
-
-  if (!LV)
-    return false;
-
-  const TypedRegion* R = dyn_cast<TypedRegion>(LV->StripCasts());
-
-  if (!R)
-    return false;
-
-  QualType T = Ctx.getCanonicalType(R->getValueType());
-
-  // FIXME: If the pointee isn't an integer type, should we flag a warning?
-  //  People can do weird stuff with pointers.
-
-  if (!T->isIntegerType())
-    return false;
-
-  uint64_t SourceSize = Ctx.getTypeSize(T);
-
-  // CHECK: is SourceSize == TargetSize
-
-  if (SourceSize == TargetSize)
-    return false;
-
-  AddError(R, CE->getArg(2), N, SourceSize, TargetSize, NumberKind);
-
-  // FIXME: We can actually create an abstract "CFNumber" object that has
-  //  the bits initialized to the provided values.
-  return SourceSize < TargetSize;
-}
-
-void AuditCFNumberCreate::AddError(const TypedRegion* R, const Expr* Ex,
-                                   ExplodedNode *N,
-                                   uint64_t SourceSize, uint64_t TargetSize,
-                                   uint64_t NumberKind) {
-
-  std::string sbuf;
-  llvm::raw_string_ostream os(sbuf);
-
-  os << (SourceSize == 8 ? "An " : "A ")
-     << SourceSize << " bit integer is used to initialize a CFNumber "
-        "object that represents "
-     << (TargetSize == 8 ? "an " : "a ")
-     << TargetSize << " bit integer. ";
-
-  if (SourceSize < TargetSize)
-    os << (TargetSize - SourceSize)
-       << " bits of the CFNumber value will be garbage." ;
-  else
-    os << (SourceSize - TargetSize)
-       << " bits of the input integer will be lost.";
-
-  // Lazily create the BugType object.  This will be owned
-  // by the BugReporter object 'BR' once we call BR.EmitWarning.
-  if (!BT) BT = new APIMisuse("Bad use of CFNumberCreate");
-  RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
-  report->addRange(Ex->getSourceRange());
-  BR.EmitReport(report);
-}
-
-GRSimpleAPICheck*
-clang::CreateAuditCFNumberCreate(ASTContext& Ctx, BugReporter& BR) {
-  return new AuditCFNumberCreate(Ctx, BR);
-}
-
-//===----------------------------------------------------------------------===//
-// CFRetain/CFRelease checking for null arguments.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class CFRetainReleaseChecker : public CheckerVisitor<CFRetainReleaseChecker> {
-  APIMisuse *BT;
-  IdentifierInfo *Retain, *Release;
-
-public:
-  CFRetainReleaseChecker(ASTContext& Ctx): BT(NULL),
-    Retain(&Ctx.Idents.get("CFRetain")), Release(&Ctx.Idents.get("CFRelease"))
-    {}
-
-  static void *getTag() { static int x = 0; return &x; }
-
-  void PreVisitCallExpr(CheckerContext& C, const CallExpr* CE);
-};
-} // end anonymous namespace
-
-
-void CFRetainReleaseChecker::PreVisitCallExpr(CheckerContext& C,
-                                              const CallExpr* CE) {
-  // If the CallExpr doesn't have exactly 1 argument just give up checking.
-  if (CE->getNumArgs() != 1)
-    return;
-
-  // Get the function declaration of the callee.
-  const GRState* state = C.getState();
-  SVal X = state->getSVal(CE->getCallee());
-  const FunctionDecl* FD = X.getAsFunctionDecl();
-
-  if (!FD)
-    return;
-
-  // Check if we called CFRetain/CFRelease.
-  const IdentifierInfo *FuncII = FD->getIdentifier();
-  if (!(FuncII == Retain || FuncII == Release))
-    return;
-
-  // FIXME: The rest of this just checks that the argument is non-null.
-  // It should probably be refactored and combined with AttrNonNullChecker.
-
-  // Get the argument's value.
-  const Expr *Arg = CE->getArg(0);
-  SVal ArgVal = state->getSVal(Arg);
-  DefinedSVal *DefArgVal = dyn_cast<DefinedSVal>(&ArgVal);
-  if (!DefArgVal)
-    return;
-
-  // Get a NULL value.
-  ValueManager &ValMgr = C.getValueManager();
-  DefinedSVal Zero = cast<DefinedSVal>(ValMgr.makeZeroVal(Arg->getType()));
-
-  // Make an expression asserting that they're equal.
-  SValuator &SVator = ValMgr.getSValuator();
-  DefinedOrUnknownSVal ArgIsNull = SVator.EvalEQ(state, Zero, *DefArgVal);
-
-  // Are they equal?
-  const GRState *stateTrue, *stateFalse;
-  llvm::tie(stateTrue, stateFalse) = state->Assume(ArgIsNull);
-
-  if (stateTrue && !stateFalse) {
-    ExplodedNode *N = C.GenerateSink(stateTrue);
-    if (!N)
-      return;
-
-    if (!BT)
-      BT = new APIMisuse("null passed to CFRetain/CFRelease");
-
-    const char *description = (FuncII == Retain)
-                            ? "Null pointer argument in call to CFRetain"
-                            : "Null pointer argument in call to CFRelease";
-
-    EnhancedBugReport *report = new EnhancedBugReport(*BT, description, N);
-    report->addRange(Arg->getSourceRange());
-    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Arg);
-
-    C.EmitReport(report);
-    return;
-  }
-
-  // From here on, we know the argument is non-null.
-  C.addTransition(stateFalse);
-}
-
-//===----------------------------------------------------------------------===//
-// Check for sending 'retain', 'release', or 'autorelease' directly to a Class.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class ClassReleaseChecker :
-    public CheckerVisitor<ClassReleaseChecker> {
-  Selector releaseS;
-  Selector retainS;
-  Selector autoreleaseS;
-  Selector drainS;
-  BugType *BT;
-public:
-  ClassReleaseChecker(ASTContext &Ctx)
-    : releaseS(GetNullarySelector("release", Ctx)),
-      retainS(GetNullarySelector("retain", Ctx)),
-      autoreleaseS(GetNullarySelector("autorelease", Ctx)),
-      drainS(GetNullarySelector("drain", Ctx)),
-      BT(0) {}
-
-  static void *getTag() { static int x = 0; return &x; }
-      
-  void PreVisitObjCMessageExpr(CheckerContext &C, const ObjCMessageExpr *ME);    
-};
-}
-
-void ClassReleaseChecker::PreVisitObjCMessageExpr(CheckerContext &C,
-                                                  const ObjCMessageExpr *ME) {
-  ObjCInterfaceDecl *Class = 0;
-  switch (ME->getReceiverKind()) {
-  case ObjCMessageExpr::Class:
-    Class = ME->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
-    break;
-
-  case ObjCMessageExpr::SuperClass:
-    Class = ME->getSuperType()->getAs<ObjCObjectType>()->getInterface();
-    break;
-
-  case ObjCMessageExpr::Instance:
-  case ObjCMessageExpr::SuperInstance:
-    return;
-  }
-
-  Selector S = ME->getSelector();
-  if (!(S == releaseS || S == retainS || S == autoreleaseS || S == drainS))
-    return;
-  
-  if (!BT)
-    BT = new APIMisuse("message incorrectly sent to class instead of class "
-                       "instance");
-  
-  ExplodedNode *N = C.GenerateNode();
-
-  if (!N)
-    return;
-  
-  llvm::SmallString<200> buf;
-  llvm::raw_svector_ostream os(buf);
-
-  os << "The '" << S.getAsString() << "' message should be sent to instances "
-        "of class '" << Class->getName()
-     << "' and not the class directly";
-  
-  RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
-  report->addRange(ME->getSourceRange());
-  C.EmitReport(report);
-}
-
-//===----------------------------------------------------------------------===//
-// Check registration.
-//===----------------------------------------------------------------------===//
-
-void clang::RegisterAppleChecks(GRExprEngine& Eng, const Decl &D) {
-  ASTContext& Ctx = Eng.getContext();
-  BugReporter &BR = Eng.getBugReporter();
-
-  Eng.AddCheck(CreateBasicObjCFoundationChecks(Ctx, BR),
-               Stmt::ObjCMessageExprClass);
-  Eng.AddCheck(CreateAuditCFNumberCreate(Ctx, BR), Stmt::CallExprClass);
-
-  RegisterNSErrorChecks(BR, Eng, D);
-  RegisterNSAutoreleasePoolChecks(Eng);
-
-  Eng.registerCheck(new CFRetainReleaseChecker(Ctx));
-  Eng.registerCheck(new ClassReleaseChecker(Ctx));
-}
diff --git a/lib/Checker/GRCXXExprEngine.cpp b/lib/Checker/GRCXXExprEngine.cpp
deleted file mode 100644
index a49989b5fda5..000000000000
--- a/lib/Checker/GRCXXExprEngine.cpp
+++ /dev/null
@@ -1,240 +0,0 @@
-//===- GRCXXExprEngine.cpp - C++ expr evaluation engine ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the C++ expression evaluation engine.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Checker/PathSensitive/AnalysisManager.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/AST/DeclCXX.h"
-
-using namespace clang;
-
-void GRExprEngine::EvalArguments(ConstExprIterator AI, ConstExprIterator AE,
-                                 const FunctionProtoType *FnType, 
-                                 ExplodedNode *Pred, ExplodedNodeSet &Dst) {
-  llvm::SmallVector<CallExprWLItem, 20> WorkList;
-  WorkList.reserve(AE - AI);
-  WorkList.push_back(CallExprWLItem(AI, Pred));
-
-  while (!WorkList.empty()) {
-    CallExprWLItem Item = WorkList.back();
-    WorkList.pop_back();
-
-    if (Item.I == AE) {
-      Dst.insert(Item.N);
-      continue;
-    }
-
-    ExplodedNodeSet Tmp;
-    const unsigned ParamIdx = Item.I - AI;
-    bool VisitAsLvalue = FnType? FnType->getArgType(ParamIdx)->isReferenceType()
-                               : false;
-    if (VisitAsLvalue)
-      VisitLValue(*Item.I, Item.N, Tmp);
-    else
-      Visit(*Item.I, Item.N, Tmp);
-
-    ++(Item.I);
-    for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI != NE; ++NI)
-      WorkList.push_back(CallExprWLItem(Item.I, *NI));
-  }
-}
-
-const CXXThisRegion *GRExprEngine::getCXXThisRegion(const CXXMethodDecl *D,
-                                                 const StackFrameContext *SFC) {
-  Type *T = D->getParent()->getTypeForDecl();
-  QualType PT = getContext().getPointerType(QualType(T,0));
-  return ValMgr.getRegionManager().getCXXThisRegion(PT, SFC);
-}
-
-void GRExprEngine::CreateCXXTemporaryObject(const Expr *Ex, ExplodedNode *Pred,
-                                            ExplodedNodeSet &Dst) {
-  ExplodedNodeSet Tmp;
-  Visit(Ex, Pred, Tmp);
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E; ++I) {
-    const GRState *state = GetState(*I);
-
-    // Bind the temporary object to the value of the expression. Then bind
-    // the expression to the location of the object.
-    SVal V = state->getSVal(Ex);
-
-    const MemRegion *R =
-      ValMgr.getRegionManager().getCXXObjectRegion(Ex,
-                                                   Pred->getLocationContext());
-
-    state = state->bindLoc(loc::MemRegionVal(R), V);
-    MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, loc::MemRegionVal(R)));
-  }
-}
-
-void GRExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E, SVal Dest,
-                                         ExplodedNode *Pred,
-                                         ExplodedNodeSet &Dst) {
-  if (E->isElidable()) {
-    VisitAggExpr(E->getArg(0), Dest, Pred, Dst);
-    return;
-  }
-
-  const CXXConstructorDecl *CD = E->getConstructor();
-  assert(CD);
-
-  if (!(CD->isThisDeclarationADefinition() && AMgr.shouldInlineCall()))
-    // FIXME: invalidate the object.
-    return;
-
-  
-  // Evaluate other arguments.
-  ExplodedNodeSet ArgsEvaluated;
-  const FunctionProtoType *FnType = CD->getType()->getAs<FunctionProtoType>();
-  EvalArguments(E->arg_begin(), E->arg_end(), FnType, Pred, ArgsEvaluated);
-  // The callee stack frame context used to create the 'this' parameter region.
-  const StackFrameContext *SFC = AMgr.getStackFrame(CD, 
-                                                    Pred->getLocationContext(),
-                                   E, Builder->getBlock(), Builder->getIndex());
-
-  const CXXThisRegion *ThisR = getCXXThisRegion(E->getConstructor(), SFC);
-
-  CallEnter Loc(E, SFC->getAnalysisContext(), Pred->getLocationContext());
-  for (ExplodedNodeSet::iterator NI = ArgsEvaluated.begin(),
-                                 NE = ArgsEvaluated.end(); NI != NE; ++NI) {
-    const GRState *state = GetState(*NI);
-    // Setup 'this' region, so that the ctor is evaluated on the object pointed
-    // by 'Dest'.
-    state = state->bindLoc(loc::MemRegionVal(ThisR), Dest);
-    ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
-    if (N)
-      Dst.Add(N);
-  }
-}
-
-void GRExprEngine::VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE, 
-                                          ExplodedNode *Pred, 
-                                          ExplodedNodeSet &Dst) {
-  // Get the method type.
-  const FunctionProtoType *FnType = 
-                       MCE->getCallee()->getType()->getAs<FunctionProtoType>();
-  assert(FnType && "Method type not available");
-
-  // Evaluate explicit arguments with a worklist.
-  ExplodedNodeSet ArgsEvaluated;
-  EvalArguments(MCE->arg_begin(), MCE->arg_end(), FnType, Pred, ArgsEvaluated);
- 
-  // Evaluate the implicit object argument.
-  ExplodedNodeSet AllArgsEvaluated;
-  const MemberExpr *ME = dyn_cast<MemberExpr>(MCE->getCallee()->IgnoreParens());
-  if (!ME)
-    return;
-  Expr *ObjArgExpr = ME->getBase();
-  for (ExplodedNodeSet::iterator I = ArgsEvaluated.begin(), 
-                                 E = ArgsEvaluated.end(); I != E; ++I) {
-    if (ME->isArrow())
-      Visit(ObjArgExpr, *I, AllArgsEvaluated);
-    else
-      VisitLValue(ObjArgExpr, *I, AllArgsEvaluated);
-  }
-
-  const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
-  assert(MD && "not a CXXMethodDecl?");
-
-  if (!(MD->isThisDeclarationADefinition() && AMgr.shouldInlineCall()))
-    // FIXME: conservative method call evaluation.
-    return;
-
-  const StackFrameContext *SFC = AMgr.getStackFrame(MD, 
-                                                    Pred->getLocationContext(),
-                                                    MCE, 
-                                                    Builder->getBlock(), 
-                                                    Builder->getIndex());
-  const CXXThisRegion *ThisR = getCXXThisRegion(MD, SFC);
-  CallEnter Loc(MCE, SFC->getAnalysisContext(), Pred->getLocationContext());
-  for (ExplodedNodeSet::iterator I = AllArgsEvaluated.begin(),
-         E = AllArgsEvaluated.end(); I != E; ++I) {
-    // Set up 'this' region.
-    const GRState *state = GetState(*I);
-    state = state->bindLoc(loc::MemRegionVal(ThisR),state->getSVal(ObjArgExpr));
-    ExplodedNode *N = Builder->generateNode(Loc, state, *I);
-    if (N)
-      Dst.Add(N);
-  }
-}
-
-void GRExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
-                                   ExplodedNodeSet &Dst) {
-  if (CNE->isArray()) {
-    // FIXME: allocating an array has not been handled.
-    return;
-  }
-
-  unsigned Count = Builder->getCurrentBlockCount();
-  DefinedOrUnknownSVal SymVal = getValueManager().getConjuredSymbolVal(NULL,CNE,
-                                                         CNE->getType(), Count);
-  const MemRegion *NewReg = cast<loc::MemRegionVal>(SymVal).getRegion();
-
-  QualType ObjTy = CNE->getType()->getAs<PointerType>()->getPointeeType();
-
-  const ElementRegion *EleReg = 
-                         getStoreManager().GetElementZeroRegion(NewReg, ObjTy);
-
-  // Evaluate constructor arguments.
-  const FunctionProtoType *FnType = NULL;
-  const CXXConstructorDecl *CD = CNE->getConstructor();
-  if (CD)
-    FnType = CD->getType()->getAs<FunctionProtoType>();
-  ExplodedNodeSet ArgsEvaluated;
-  EvalArguments(CNE->constructor_arg_begin(), CNE->constructor_arg_end(),
-                FnType, Pred, ArgsEvaluated);
-
-  // Initialize the object region and bind the 'new' expression.
-  for (ExplodedNodeSet::iterator I = ArgsEvaluated.begin(), 
-                                 E = ArgsEvaluated.end(); I != E; ++I) {
-    const GRState *state = GetState(*I);
-
-    if (ObjTy->isRecordType()) {
-      state = state->InvalidateRegion(EleReg, CNE, Count);
-    } else {
-      if (CNE->hasInitializer()) {
-        SVal V = state->getSVal(*CNE->constructor_arg_begin());
-        state = state->bindLoc(loc::MemRegionVal(EleReg), V);
-      } else {
-        // Explicitly set to undefined, because currently we retrieve symbolic
-        // value from symbolic region.
-        state = state->bindLoc(loc::MemRegionVal(EleReg), UndefinedVal());
-      }
-    }
-    state = state->BindExpr(CNE, loc::MemRegionVal(EleReg));
-    MakeNode(Dst, CNE, *I, state);
-  }
-}
-
-void GRExprEngine::VisitCXXDeleteExpr(const CXXDeleteExpr *CDE, 
-                                      ExplodedNode *Pred,ExplodedNodeSet &Dst) {
-  // Should do more checking.
-  ExplodedNodeSet ArgEvaluated;
-  Visit(CDE->getArgument(), Pred, ArgEvaluated);
-  for (ExplodedNodeSet::iterator I = ArgEvaluated.begin(), 
-                                 E = ArgEvaluated.end(); I != E; ++I) {
-    const GRState *state = GetState(*I);
-    MakeNode(Dst, CDE, *I, state);
-  }
-}
-
-void GRExprEngine::VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred,
-                                    ExplodedNodeSet &Dst) {
-  // Get the this object region from StoreManager.
-  const MemRegion *R =
-    ValMgr.getRegionManager().getCXXThisRegion(
-                                  getContext().getCanonicalType(TE->getType()),
-                                               Pred->getLocationContext());
-
-  const GRState *state = GetState(Pred);
-  SVal V = state->getSVal(loc::MemRegionVal(R));
-  MakeNode(Dst, TE, Pred, state->BindExpr(TE, V));
-}
diff --git a/lib/Checker/GRExprEngineExperimentalChecks.h b/lib/Checker/GRExprEngineExperimentalChecks.h
deleted file mode 100644
index 7b5b0ed7ba00..000000000000
--- a/lib/Checker/GRExprEngineExperimentalChecks.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//=-- GRExprEngineExperimentalChecks.h ------------------------------*- C++ -*-=
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines functions to instantiate and register experimental
-//  checks in GRExprEngine.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_GREXPRENGINE_EXPERIMENTAL_CHECKS
-#define LLVM_CLANG_GREXPRENGINE_EXPERIMENTAL_CHECKS
-
-namespace clang {
-
-class GRExprEngine;
-
-void RegisterCStringChecker(GRExprEngine &Eng);
-void RegisterIdempotentOperationChecker(GRExprEngine &Eng);
-void RegisterMallocChecker(GRExprEngine &Eng);
-void RegisterPthreadLockChecker(GRExprEngine &Eng);
-void RegisterStreamChecker(GRExprEngine &Eng);
-void RegisterUnreachableCodeChecker(GRExprEngine &Eng);
-
-} // end clang namespace
-#endif
diff --git a/lib/Checker/GRExprEngineInternalChecks.h b/lib/Checker/GRExprEngineInternalChecks.h
deleted file mode 100644
index f91a759b3299..000000000000
--- a/lib/Checker/GRExprEngineInternalChecks.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//=-- GRExprEngineInternalChecks.h- Builtin GRExprEngine Checks -----*- C++ -*-=
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines functions to instantiate and register the "built-in"
-//  checks in GRExprEngine.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_GREXPRENGINE_INTERNAL_CHECKS
-#define LLVM_CLANG_GREXPRENGINE_INTERNAL_CHECKS
-
-namespace clang {
-
-class GRExprEngine;
-
-// Foundational checks that handle basic semantics.
-void RegisterAdjustedReturnValueChecker(GRExprEngine &Eng);
-void RegisterArrayBoundChecker(GRExprEngine &Eng);
-void RegisterAttrNonNullChecker(GRExprEngine &Eng);
-void RegisterBuiltinFunctionChecker(GRExprEngine &Eng);
-void RegisterCallAndMessageChecker(GRExprEngine &Eng);
-void RegisterCastToStructChecker(GRExprEngine &Eng);
-void RegisterCastSizeChecker(GRExprEngine &Eng);
-void RegisterDereferenceChecker(GRExprEngine &Eng);
-void RegisterDivZeroChecker(GRExprEngine &Eng);
-void RegisterFixedAddressChecker(GRExprEngine &Eng);
-void RegisterNoReturnFunctionChecker(GRExprEngine &Eng);
-void RegisterPointerArithChecker(GRExprEngine &Eng);
-void RegisterPointerSubChecker(GRExprEngine &Eng);
-void RegisterReturnPointerRangeChecker(GRExprEngine &Eng);
-void RegisterReturnUndefChecker(GRExprEngine &Eng);
-void RegisterStackAddrLeakChecker(GRExprEngine &Eng);
-void RegisterUndefBranchChecker(GRExprEngine &Eng);
-void RegisterUndefCapturedBlockVarChecker(GRExprEngine &Eng);
-void RegisterUndefResultChecker(GRExprEngine &Eng);
-void RegisterUndefinedArraySubscriptChecker(GRExprEngine &Eng);
-void RegisterUndefinedAssignmentChecker(GRExprEngine &Eng);
-void RegisterVLASizeChecker(GRExprEngine &Eng);
-
-// API checks.
-void RegisterMacOSXAPIChecker(GRExprEngine &Eng);
-void RegisterOSAtomicChecker(GRExprEngine &Eng);
-void RegisterUnixAPIChecker(GRExprEngine &Eng);
-
-} // end clang namespace
-#endif
diff --git a/lib/Checker/ManagerRegistry.cpp b/lib/Checker/ManagerRegistry.cpp
deleted file mode 100644
index d11a997cc0f6..000000000000
--- a/lib/Checker/ManagerRegistry.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===- ManagerRegistry.cpp - Pluggble Analyzer module creators --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the pluggable analyzer module creators.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Checker/ManagerRegistry.h"
-
-using namespace clang;
-
-StoreManagerCreator ManagerRegistry::StoreMgrCreator = 0;
-
-ConstraintManagerCreator ManagerRegistry::ConstraintMgrCreator = 0;
diff --git a/lib/Checker/SValuator.cpp b/lib/Checker/SValuator.cpp
deleted file mode 100644
index 273e5742a8e2..000000000000
--- a/lib/Checker/SValuator.cpp
+++ /dev/null
@@ -1,157 +0,0 @@
-// SValuator.cpp - Basic class for all SValuator implementations --*- C++ -*--//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines SValuator, the base class for all (complete) SValuator
-//  implementations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Checker/PathSensitive/SValuator.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-
-using namespace clang;
-
-
-SVal SValuator::EvalBinOp(const GRState *ST, BinaryOperator::Opcode Op,
-                          SVal L, SVal R, QualType T) {
-
-  if (L.isUndef() || R.isUndef())
-    return UndefinedVal();
-
-  if (L.isUnknown() || R.isUnknown())
-    return UnknownVal();
-
-  if (isa<Loc>(L)) {
-    if (isa<Loc>(R))
-      return EvalBinOpLL(ST, Op, cast<Loc>(L), cast<Loc>(R), T);
-
-    return EvalBinOpLN(ST, Op, cast<Loc>(L), cast<NonLoc>(R), T);
-  }
-
-  if (isa<Loc>(R)) {
-    // Support pointer arithmetic where the addend is on the left
-    // and the pointer on the right.
-    assert(Op == BO_Add);
-
-    // Commute the operands.
-    return EvalBinOpLN(ST, Op, cast<Loc>(R), cast<NonLoc>(L), T);
-  }
-
-  return EvalBinOpNN(ST, Op, cast<NonLoc>(L), cast<NonLoc>(R), T);
-}
-
-DefinedOrUnknownSVal SValuator::EvalEQ(const GRState *ST,
-                                       DefinedOrUnknownSVal L,
-                                       DefinedOrUnknownSVal R) {
-  return cast<DefinedOrUnknownSVal>(EvalBinOp(ST, BO_EQ, L, R,
-                                              ValMgr.getContext().IntTy));
-}
-
-SVal SValuator::EvalCast(SVal val, QualType castTy, QualType originalTy) {
-  if (val.isUnknownOrUndef() || castTy == originalTy)
-    return val;
-
-  ASTContext &C = ValMgr.getContext();
-
-  // For const casts, just propagate the value.
-  if (!castTy->isVariableArrayType() && !originalTy->isVariableArrayType())
-    if (C.hasSameUnqualifiedType(castTy, originalTy))
-      return val;
-
-  // Check for casts to real or complex numbers.  We don't handle these at all
-  // right now.
-  if (castTy->isFloatingType() || castTy->isAnyComplexType())
-    return UnknownVal();
-  
-  // Check for casts from integers to integers.
-  if (castTy->isIntegerType() && originalTy->isIntegerType())
-    return EvalCastNL(cast<NonLoc>(val), castTy);
-
-  // Check for casts from pointers to integers.
-  if (castTy->isIntegerType() && Loc::IsLocType(originalTy))
-    return EvalCastL(cast<Loc>(val), castTy);
-
-  // Check for casts from integers to pointers.
-  if (Loc::IsLocType(castTy) && originalTy->isIntegerType()) {
-    if (nonloc::LocAsInteger *LV = dyn_cast<nonloc::LocAsInteger>(&val)) {
-      if (const MemRegion *R = LV->getLoc().getAsRegion()) {
-        StoreManager &storeMgr = ValMgr.getStateManager().getStoreManager();
-        R = storeMgr.CastRegion(R, castTy);
-        return R ? SVal(loc::MemRegionVal(R)) : UnknownVal();
-      }
-      return LV->getLoc();
-    }
-    goto DispatchCast;
-  }
-
-  // Just pass through function and block pointers.
-  if (originalTy->isBlockPointerType() || originalTy->isFunctionPointerType()) {
-    assert(Loc::IsLocType(castTy));
-    return val;
-  }
-
-  // Check for casts from array type to another type.
-  if (originalTy->isArrayType()) {
-    // We will always decay to a pointer.
-    val = ValMgr.getStateManager().ArrayToPointer(cast<Loc>(val));
-
-    // Are we casting from an array to a pointer?  If so just pass on
-    // the decayed value.
-    if (castTy->isPointerType())
-      return val;
-
-    // Are we casting from an array to an integer?  If so, cast the decayed
-    // pointer value to an integer.
-    assert(castTy->isIntegerType());
-
-    // FIXME: Keep these here for now in case we decide soon that we
-    // need the original decayed type.
-    //    QualType elemTy = cast<ArrayType>(originalTy)->getElementType();
-    //    QualType pointerTy = C.getPointerType(elemTy);
-    return EvalCastL(cast<Loc>(val), castTy);
-  }
-
-  // Check for casts from a region to a specific type.
-  if (const MemRegion *R = val.getAsRegion()) {
-    // FIXME: We should handle the case where we strip off view layers to get
-    //  to a desugared type.
-
-    assert(Loc::IsLocType(castTy));
-    // We get a symbolic function pointer for a dereference of a function
-    // pointer, but it is of function type. Example:
-
-    //  struct FPRec {
-    //    void (*my_func)(int * x);
-    //  };
-    //
-    //  int bar(int x);
-    //
-    //  int f1_a(struct FPRec* foo) {
-    //    int x;
-    //    (*foo->my_func)(&x);
-    //    return bar(x)+1; // no-warning
-    //  }
-
-    assert(Loc::IsLocType(originalTy) || originalTy->isFunctionType() ||
-           originalTy->isBlockPointerType());
-
-    StoreManager &storeMgr = ValMgr.getStateManager().getStoreManager();
-
-    // Delegate to store manager to get the result of casting a region to a
-    // different type.  If the MemRegion* returned is NULL, this expression
-    // evaluates to UnknownVal.
-    R = storeMgr.CastRegion(R, castTy);
-    return R ? SVal(loc::MemRegionVal(R)) : UnknownVal();
-  }
-
-DispatchCast:
-  // All other cases.
-  return isa<Loc>(val) ? EvalCastL(cast<Loc>(val), castTy)
-                       : EvalCastNL(cast<NonLoc>(val), castTy);
-}
diff --git a/lib/Checker/ValueManager.cpp b/lib/Checker/ValueManager.cpp
deleted file mode 100644
index 8b7cd7bbdb68..000000000000
--- a/lib/Checker/ValueManager.cpp
+++ /dev/null
@@ -1,162 +0,0 @@
-//== ValueManager.cpp - Aggregate manager of symbols and SVals --*- C++ -*--==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines ValueManager, a class that manages symbolic values
-//  and SVals created for use by GRExprEngine and related classes.  It
-//  wraps and owns SymbolManager, MemRegionManager, and BasicValueFactory.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Checker/PathSensitive/ValueManager.h"
-#include "clang/Analysis/AnalysisContext.h"
-
-using namespace clang;
-using namespace llvm;
-
-//===----------------------------------------------------------------------===//
-// Utility methods for constructing SVals.
-//===----------------------------------------------------------------------===//
-
-DefinedOrUnknownSVal ValueManager::makeZeroVal(QualType T) {
-  if (Loc::IsLocType(T))
-    return makeNull();
-
-  if (T->isIntegerType())
-    return makeIntVal(0, T);
-
-  // FIXME: Handle floats.
-  // FIXME: Handle structs.
-  return UnknownVal();
-}
-
-//===----------------------------------------------------------------------===//
-// Utility methods for constructing Non-Locs.
-//===----------------------------------------------------------------------===//
-
-NonLoc ValueManager::makeNonLoc(const SymExpr *lhs, BinaryOperator::Opcode op,
-                                const APSInt& v, QualType T) {
-  // The Environment ensures we always get a persistent APSInt in
-  // BasicValueFactory, so we don't need to get the APSInt from
-  // BasicValueFactory again.
-  assert(!Loc::IsLocType(T));
-  return nonloc::SymExprVal(SymMgr.getSymIntExpr(lhs, op, v, T));
-}
-
-NonLoc ValueManager::makeNonLoc(const SymExpr *lhs, BinaryOperator::Opcode op,
-                                const SymExpr *rhs, QualType T) {
-  assert(SymMgr.getType(lhs) == SymMgr.getType(rhs));
-  assert(!Loc::IsLocType(T));
-  return nonloc::SymExprVal(SymMgr.getSymSymExpr(lhs, op, rhs, T));
-}
-
-
-SVal ValueManager::convertToArrayIndex(SVal V) {
-  if (V.isUnknownOrUndef())
-    return V;
-
-  // Common case: we have an appropriately sized integer.
-  if (nonloc::ConcreteInt* CI = dyn_cast<nonloc::ConcreteInt>(&V)) {
-    const llvm::APSInt& I = CI->getValue();
-    if (I.getBitWidth() == ArrayIndexWidth && I.isSigned())
-      return V;
-  }
-
-  return SVator->EvalCastNL(cast<NonLoc>(V), ArrayIndexTy);
-}
-
-DefinedOrUnknownSVal 
-ValueManager::getRegionValueSymbolVal(const TypedRegion* R) {
-  QualType T = R->getValueType();
-
-  if (!SymbolManager::canSymbolicate(T))
-    return UnknownVal();
-
-  SymbolRef sym = SymMgr.getRegionValueSymbol(R);
-
-  if (Loc::IsLocType(T))
-    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
-
-  return nonloc::SymbolVal(sym);
-}
-
-DefinedOrUnknownSVal ValueManager::getConjuredSymbolVal(const void *SymbolTag,
-                                                        const Expr *E,
-                                                        unsigned Count) {
-  QualType T = E->getType();
-
-  if (!SymbolManager::canSymbolicate(T))
-    return UnknownVal();
-
-  SymbolRef sym = SymMgr.getConjuredSymbol(E, Count, SymbolTag);
-
-  if (Loc::IsLocType(T))
-    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
-
-  return nonloc::SymbolVal(sym);
-}
-
-DefinedOrUnknownSVal ValueManager::getConjuredSymbolVal(const void *SymbolTag,
-                                                        const Expr *E,
-                                                        QualType T,
-                                                        unsigned Count) {
-  
-  if (!SymbolManager::canSymbolicate(T))
-    return UnknownVal();
-
-  SymbolRef sym = SymMgr.getConjuredSymbol(E, T, Count, SymbolTag);
-
-  if (Loc::IsLocType(T))
-    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
-
-  return nonloc::SymbolVal(sym);
-}
-
-DefinedSVal ValueManager::getMetadataSymbolVal(const void *SymbolTag,
-                                               const MemRegion *MR,
-                                               const Expr *E, QualType T,
-                                               unsigned Count) {
-  assert(SymbolManager::canSymbolicate(T) && "Invalid metadata symbol type");
-
-  SymbolRef sym = SymMgr.getMetadataSymbol(MR, E, T, Count, SymbolTag);
-
-  if (Loc::IsLocType(T))
-    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
-
-  return nonloc::SymbolVal(sym);
-}
-
-DefinedOrUnknownSVal
-ValueManager::getDerivedRegionValueSymbolVal(SymbolRef parentSymbol,
-                                             const TypedRegion *R) {
-  QualType T = R->getValueType();
-
-  if (!SymbolManager::canSymbolicate(T))
-    return UnknownVal();
-
-  SymbolRef sym = SymMgr.getDerivedSymbol(parentSymbol, R);
-
-  if (Loc::IsLocType(T))
-    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
-
-  return nonloc::SymbolVal(sym);
-}
-
-DefinedSVal ValueManager::getFunctionPointer(const FunctionDecl* FD) {
-  return loc::MemRegionVal(MemMgr.getFunctionTextRegion(FD));
-}
-
-DefinedSVal ValueManager::getBlockPointer(const BlockDecl *D,
-                                          CanQualType locTy,
-                                          const LocationContext *LC) {
-  const BlockTextRegion *BC =
-    MemMgr.getBlockTextRegion(D, locTy, LC->getAnalysisContext());
-  const BlockDataRegion *BD = MemMgr.getBlockDataRegion(BC, LC);
-  return loc::MemRegionVal(BD);
-}
-
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h
index 91b7742557f5..ce1039849b7c 100644
--- a/lib/CodeGen/ABIInfo.h
+++ b/lib/CodeGen/ABIInfo.h
@@ -70,11 +70,12 @@ namespace clang {
     Kind TheKind;
     llvm::PATypeHolder TypeData;
     unsigned UIntData;
-    bool BoolData;
+    bool BoolData0;
+    bool BoolData1;
 
     ABIArgInfo(Kind K, const llvm::Type *TD=0,
-               unsigned UI=0, bool B = false) 
-      : TheKind(K), TypeData(TD), UIntData(UI), BoolData(B) {}
+               unsigned UI=0, bool B0 = false, bool B1 = false)
+      : TheKind(K), TypeData(TD), UIntData(UI), BoolData0(B0), BoolData1(B1) {}
 
   public:
     ABIArgInfo() : TheKind(Direct), TypeData(0), UIntData(0) {}
@@ -88,8 +89,9 @@ namespace clang {
     static ABIArgInfo getIgnore() {
       return ABIArgInfo(Ignore);
     }
-    static ABIArgInfo getIndirect(unsigned Alignment, bool ByVal = true) {
-      return ABIArgInfo(Indirect, 0, Alignment, ByVal);
+    static ABIArgInfo getIndirect(unsigned Alignment, bool ByVal = true
+                                  , bool Realign = false) {
+      return ABIArgInfo(Indirect, 0, Alignment, ByVal, Realign);
     }
     static ABIArgInfo getExpand() {
       return ABIArgInfo(Expand);
@@ -105,7 +107,7 @@ namespace clang {
     bool canHaveCoerceToType() const {
       return TheKind == Direct || TheKind == Extend;
     }
-    
+
     // Direct/Extend accessors
     unsigned getDirectOffset() const {
       assert((isDirect() || isExtend()) && "Not a direct or extend kind");
@@ -115,12 +117,12 @@ namespace clang {
       assert(canHaveCoerceToType() && "Invalid kind!");
       return TypeData;
     }
-    
+
     void setCoerceToType(const llvm::Type *T) {
       assert(canHaveCoerceToType() && "Invalid kind!");
       TypeData = T;
     }
-    
+
     // Indirect accessors
     unsigned getIndirectAlign() const {
       assert(TheKind == Indirect && "Invalid kind!");
@@ -129,9 +131,14 @@ namespace clang {
 
     bool getIndirectByVal() const {
       assert(TheKind == Indirect && "Invalid kind!");
-      return BoolData;
+      return BoolData0;
     }
-    
+
+    bool getIndirectRealign() const {
+      assert(TheKind == Indirect && "Invalid kind!");
+      return BoolData1;
+    }
+
     void dump() const;
   };
 
@@ -140,10 +147,10 @@ namespace clang {
   class ABIInfo {
   public:
     CodeGen::CodeGenTypes &CGT;
-    
+
     ABIInfo(CodeGen::CodeGenTypes &cgt) : CGT(cgt) {}
     virtual ~ABIInfo();
-    
+
     ASTContext &getContext() const;
     llvm::LLVMContext &getVMContext() const;
     const llvm::TargetData &getTargetData() const;
diff --git a/lib/CodeGen/BackendUtil.cpp b/lib/CodeGen/BackendUtil.cpp
index 69efe438cc9a..9897b1b1a028 100644
--- a/lib/CodeGen/BackendUtil.cpp
+++ b/lib/CodeGen/BackendUtil.cpp
@@ -26,6 +26,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/SubtargetFeature.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetRegistry.h"
@@ -42,14 +43,14 @@ class EmitAssemblyHelper {
 
   Timer CodeGenerationTime;
 
-  mutable FunctionPassManager *CodeGenPasses;
+  mutable PassManager *CodeGenPasses;
   mutable PassManager *PerModulePasses;
   mutable FunctionPassManager *PerFunctionPasses;
 
 private:
-  FunctionPassManager *getCodeGenPasses() const {
+  PassManager *getCodeGenPasses() const {
     if (!CodeGenPasses) {
-      CodeGenPasses = new FunctionPassManager(TheModule);
+      CodeGenPasses = new PassManager();
       CodeGenPasses->add(new TargetData(TheModule));
     }
     return CodeGenPasses;
@@ -107,14 +108,22 @@ void EmitAssemblyHelper::CreatePasses() {
     OptLevel = 0;
     Inlining = CodeGenOpts.NoInlining;
   }
+  
+  FunctionPassManager *FPM = getPerFunctionPasses();
+  
+  TargetLibraryInfo *TLI =
+    new TargetLibraryInfo(Triple(TheModule->getTargetTriple()));
+  if (!CodeGenOpts.SimplifyLibCalls)
+    TLI->disableAllFunctions();
+  FPM->add(TLI);
 
   // In -O0 if checking is disabled, we don't even have per-function passes.
   if (CodeGenOpts.VerifyModule)
-    getPerFunctionPasses()->add(createVerifierPass());
+    FPM->add(createVerifierPass());
 
   // Assume that standard function passes aren't run for -O0.
   if (OptLevel > 0)
-    llvm::createStandardFunctionPasses(getPerFunctionPasses(), OptLevel);
+    llvm::createStandardFunctionPasses(FPM, OptLevel);
 
   llvm::Pass *InliningPass = 0;
   switch (Inlining) {
@@ -136,8 +145,15 @@ void EmitAssemblyHelper::CreatePasses() {
     break;
   }
 
+  PassManager *MPM = getPerModulePasses();
+  
+  TLI = new TargetLibraryInfo(Triple(TheModule->getTargetTriple()));
+  if (!CodeGenOpts.SimplifyLibCalls)
+    TLI->disableAllFunctions();
+  MPM->add(TLI);
+
   // For now we always create per module passes.
-  llvm::createStandardModulePasses(getPerModulePasses(), OptLevel,
+  llvm::createStandardModulePasses(MPM, OptLevel,
                                    CodeGenOpts.OptimizeSize,
                                    CodeGenOpts.UnitAtATime,
                                    CodeGenOpts.UnrollLoops,
@@ -183,7 +199,11 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
     llvm::FloatABIType = llvm::FloatABI::Default;
   }
 
+  llvm::LessPreciseFPMADOption = CodeGenOpts.LessPreciseFPMAD;
+  llvm::NoInfsFPMath = CodeGenOpts.NoInfsFPMath;
+  llvm::NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath;
   NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS;
+  llvm::UnsafeFPMath = CodeGenOpts.UnsafeFPMath;
   llvm::UseSoftFloat = CodeGenOpts.SoftFloat;
   UnwindTablesMandatory = CodeGenOpts.UnwindTables;
 
@@ -248,7 +268,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
     TM->setMCRelaxAll(true);
 
   // Create the code generator passes.
-  FunctionPassManager *PM = getCodeGenPasses();
+  PassManager *PM = getCodeGenPasses();
   CodeGenOpt::Level OptLevel = CodeGenOpt::Default;
 
   switch (CodeGenOpts.OptimizationLevel) {
@@ -320,13 +340,7 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) {
 
   if (CodeGenPasses) {
     PrettyStackTraceString CrashInfo("Code generation");
-
-    CodeGenPasses->doInitialization();
-    for (Module::iterator I = TheModule->begin(),
-           E = TheModule->end(); I != E; ++I)
-      if (!I->isDeclaration())
-        CodeGenPasses->run(*I);
-    CodeGenPasses->doFinalization();
+    CodeGenPasses->run(*TheModule);
   }
 }
 
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 04f1ef24b297..a35648d23d51 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenFunction.h"
 #include "CGObjCRuntime.h"
 #include "CodeGenModule.h"
+#include "CGBlocks.h"
 #include "clang/AST/DeclObjC.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/SmallSet.h"
@@ -24,397 +25,631 @@
 using namespace clang;
 using namespace CodeGen;
 
-CGBlockInfo::CGBlockInfo(const char *N)
-  : Name(N), CXXThisRef(0), NeedsObjCSelf(false) {
+CGBlockInfo::CGBlockInfo(const BlockExpr *blockExpr, const char *N)
+  : Name(N), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
+    HasCXXObject(false), StructureType(0), Block(blockExpr) {
     
   // Skip asm prefix, if any.
   if (Name && Name[0] == '\01')
     ++Name;
 }
 
+/// Build the given block as a global block.
+static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
+                                        const CGBlockInfo &blockInfo,
+                                        llvm::Constant *blockFn);
 
-llvm::Constant *CodeGenFunction::
-BuildDescriptorBlockDecl(const BlockExpr *BE, const CGBlockInfo &Info,
-                         const llvm::StructType* Ty,
-                         llvm::Constant *BlockVarLayout,
-                         std::vector<HelperInfo> *NoteForHelper) {
-  bool BlockHasCopyDispose = Info.BlockHasCopyDispose;
-  CharUnits Size = Info.BlockSize;
-  const llvm::Type *UnsignedLongTy
-    = CGM.getTypes().ConvertType(getContext().UnsignedLongTy);
-  llvm::Constant *C;
-  std::vector<llvm::Constant*> Elts;
+/// Build the helper function to copy a block.
+static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
+                                       const CGBlockInfo &blockInfo) {
+  return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
+}
+
+/// Build the helper function to dipose of a block.
+static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
+                                          const CGBlockInfo &blockInfo) {
+  return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
+}
+
+/// Build the block descriptor constant for a block.
+static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
+                                            const CGBlockInfo &blockInfo) {
+  ASTContext &C = CGM.getContext();
+
+  const llvm::Type *ulong = CGM.getTypes().ConvertType(C.UnsignedLongTy);
+  const llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy);
+
+  llvm::SmallVector<llvm::Constant*, 6> elements;
 
   // reserved
-  C = llvm::ConstantInt::get(UnsignedLongTy, 0);
-  Elts.push_back(C);
+  elements.push_back(llvm::ConstantInt::get(ulong, 0));
 
   // Size
   // FIXME: What is the right way to say this doesn't fit?  We should give
   // a user diagnostic in that case.  Better fix would be to change the
   // API to size_t.
-  C = llvm::ConstantInt::get(UnsignedLongTy, Size.getQuantity());
-  Elts.push_back(C);
+  elements.push_back(llvm::ConstantInt::get(ulong,
+                                            blockInfo.BlockSize.getQuantity()));
 
-  // optional copy/dispose helpers
-  if (BlockHasCopyDispose) {
+  // Optional copy/dispose helpers.
+  if (blockInfo.NeedsCopyDispose) {
     // copy_func_helper_decl
-    Elts.push_back(BuildCopyHelper(Ty, NoteForHelper));
+    elements.push_back(buildCopyHelper(CGM, blockInfo));
 
     // destroy_func_decl
-    Elts.push_back(BuildDestroyHelper(Ty, NoteForHelper));
+    elements.push_back(buildDisposeHelper(CGM, blockInfo));
   }
 
-  // Signature.  non-optional ObjC-style method descriptor @encode sequence
-  std::string BlockTypeEncoding;
-  CGM.getContext().getObjCEncodingForBlock(BE, BlockTypeEncoding);
-
-  Elts.push_back(llvm::ConstantExpr::getBitCast(
-          CGM.GetAddrOfConstantCString(BlockTypeEncoding), PtrToInt8Ty));
+  // Signature.  Mandatory ObjC-style method descriptor @encode sequence.
+  std::string typeAtEncoding =
+    CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr());
+  elements.push_back(llvm::ConstantExpr::getBitCast(
+                          CGM.GetAddrOfConstantCString(typeAtEncoding), i8p));
   
-  // Layout.
-  C = BlockVarLayout;
-    
-  Elts.push_back(C);
+  // GC layout.
+  if (C.getLangOptions().ObjC1)
+    elements.push_back(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
+  else
+    elements.push_back(llvm::Constant::getNullValue(i8p));
+
+  llvm::Constant *init =
+    llvm::ConstantStruct::get(CGM.getLLVMContext(), elements.data(),
+                              elements.size(), false);
 
-  C = llvm::ConstantStruct::get(VMContext, Elts, false);
+  llvm::GlobalVariable *global =
+    new llvm::GlobalVariable(CGM.getModule(), init->getType(), true,
+                             llvm::GlobalValue::InternalLinkage,
+                             init, "__block_descriptor_tmp");
 
-  C = new llvm::GlobalVariable(CGM.getModule(), C->getType(), true,
-                               llvm::GlobalValue::InternalLinkage,
-                               C, "__block_descriptor_tmp");
-  return C;
+  return llvm::ConstantExpr::getBitCast(global, CGM.getBlockDescriptorType());
 }
 
-static void CollectBlockDeclRefInfo(const Stmt *S, CGBlockInfo &Info) {
-  for (Stmt::const_child_iterator I = S->child_begin(), E = S->child_end();
-       I != E; ++I)
-    if (*I)
-      CollectBlockDeclRefInfo(*I, Info);
-
-  // We want to ensure we walk down into block literals so we can find
-  // all nested BlockDeclRefExprs.
-  if (const BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
-    Info.InnerBlocks.insert(BE->getBlockDecl());
-    CollectBlockDeclRefInfo(BE->getBody(), Info);
-  }
+static BlockFlags computeBlockFlag(CodeGenModule &CGM,
+                                   const BlockExpr *BE,
+                                   BlockFlags flags) {
+  const FunctionType *ftype = BE->getFunctionType();
+  
+  // This is a bit overboard.
+  CallArgList args;
+  const CGFunctionInfo &fnInfo =
+    CGM.getTypes().getFunctionInfo(ftype->getResultType(), args,
+                                   ftype->getExtInfo());
+
+  if (CGM.ReturnTypeUsesSRet(fnInfo))
+    flags |= BLOCK_USE_STRET;
 
-  else if (const BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(S)) {
-    const ValueDecl *D = BDRE->getDecl();
-    // FIXME: Handle enums.
-    if (isa<FunctionDecl>(D))
-      return;
-
-    if (isa<ImplicitParamDecl>(D) &&
-        isa<ObjCMethodDecl>(D->getDeclContext()) &&
-        cast<ObjCMethodDecl>(D->getDeclContext())->getSelfDecl() == D) {
-      Info.NeedsObjCSelf = true;
-      return;
+  return flags;
+}
+
+/*
+  Purely notional variadic template describing the layout of a block.
+
+  template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
+  struct Block_literal {
+    /// Initialized to one of:
+    ///   extern void *_NSConcreteStackBlock[];
+    ///   extern void *_NSConcreteGlobalBlock[];
+    ///
+    /// In theory, we could start one off malloc'ed by setting
+    /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
+    /// this isa:
+    ///   extern void *_NSConcreteMallocBlock[];
+    struct objc_class *isa;
+
+    /// These are the flags (with corresponding bit number) that the
+    /// compiler is actually supposed to know about.
+    ///  25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
+    ///   descriptor provides copy and dispose helper functions
+    ///  26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
+    ///   object with a nontrivial destructor or copy constructor
+    ///  28. BLOCK_IS_GLOBAL - indicates that the block is allocated
+    ///   as global memory
+    ///  29. BLOCK_USE_STRET - indicates that the block function
+    ///   uses stret, which objc_msgSend needs to know about
+    ///  30. BLOCK_HAS_SIGNATURE - indicates that the block has an
+    ///   @encoded signature string
+    /// And we're not supposed to manipulate these:
+    ///  24. BLOCK_NEEDS_FREE - indicates that the block has been moved
+    ///   to malloc'ed memory
+    ///  27. BLOCK_IS_GC - indicates that the block has been moved to
+    ///   to GC-allocated memory
+    /// Additionally, the bottom 16 bits are a reference count which
+    /// should be zero on the stack.
+    int flags;
+
+    /// Reserved;  should be zero-initialized.
+    int reserved;
+
+    /// Function pointer generated from block literal.
+    _ResultType (*invoke)(Block_literal *, _ParamTypes...);
+
+    /// Block description metadata generated from block literal.
+    struct Block_descriptor *block_descriptor;
+
+    /// Captured values follow.
+    _CapturesTypes captures...;
+  };
+ */
+
+/// The number of fields in a block header.
+const unsigned BlockHeaderSize = 5;
+
+namespace {
+  /// A chunk of data that we actually have to capture in the block.
+  struct BlockLayoutChunk {
+    CharUnits Alignment;
+    CharUnits Size;
+    const BlockDecl::Capture *Capture; // null for 'this'
+    const llvm::Type *Type;
+
+    BlockLayoutChunk(CharUnits align, CharUnits size,
+                     const BlockDecl::Capture *capture,
+                     const llvm::Type *type)
+      : Alignment(align), Size(size), Capture(capture), Type(type) {}
+
+    /// Tell the block info that this chunk has the given field index.
+    void setIndex(CGBlockInfo &info, unsigned index) {
+      if (!Capture)
+        info.CXXThisIndex = index;
+      else
+        info.Captures[Capture->getVariable()]
+          = CGBlockInfo::Capture::makeIndex(index);
     }
+  };
 
-    // Only Decls that escape are added.
-    if (!Info.InnerBlocks.count(D->getDeclContext()))
-      Info.DeclRefs.push_back(BDRE);
+  /// Order by descending alignment.
+  bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
+    return left.Alignment > right.Alignment;
   }
+}
 
-  // Make sure to capture implicit 'self' references due to super calls.
-  else if (const ObjCMessageExpr *E = dyn_cast<ObjCMessageExpr>(S)) {
-    if (E->getReceiverKind() == ObjCMessageExpr::SuperClass || 
-        E->getReceiverKind() == ObjCMessageExpr::SuperInstance)
-      Info.NeedsObjCSelf = true;
+/// Determines if the given record type has a mutable field.
+static bool hasMutableField(const CXXRecordDecl *record) {
+  for (CXXRecordDecl::field_iterator
+         i = record->field_begin(), e = record->field_end(); i != e; ++i)
+    if ((*i)->isMutable())
+      return true;
+
+  for (CXXRecordDecl::base_class_const_iterator
+         i = record->bases_begin(), e = record->bases_end(); i != e; ++i) {
+    const RecordType *record = i->getType()->castAs<RecordType>();
+    if (hasMutableField(cast<CXXRecordDecl>(record->getDecl())))
+      return true;
   }
 
-  // Getter/setter uses may also cause implicit super references,
-  // which we can check for with:
-  else if (isa<ObjCSuperExpr>(S))
-    Info.NeedsObjCSelf = true;
+  return false;
+}
+
+/// Determines if the given type is safe for constant capture in C++.
+static bool isSafeForCXXConstantCapture(QualType type) {
+  const RecordType *recordType =
+    type->getBaseElementTypeUnsafe()->getAs<RecordType>();
 
-  else if (isa<CXXThisExpr>(S))
-    Info.CXXThisRef = cast<CXXThisExpr>(S);
+  // Only records can be unsafe.
+  if (!recordType) return true;
+
+  const CXXRecordDecl *record = cast<CXXRecordDecl>(recordType->getDecl());
+
+  // Maintain semantics for classes with non-trivial dtors or copy ctors.
+  if (!record->hasTrivialDestructor()) return false;
+  if (!record->hasTrivialCopyConstructor()) return false;
+
+  // Otherwise, we just have to make sure there aren't any mutable
+  // fields that might have changed since initialization.
+  return !hasMutableField(record);
 }
 
-/// CanBlockBeGlobal - Given a CGBlockInfo struct, determines if a block can be
-/// declared as a global variable instead of on the stack.
-static bool CanBlockBeGlobal(const CGBlockInfo &Info) {
-  return Info.DeclRefs.empty();
+/// It is illegal to modify a const object after initialization.
+/// Therefore, if a const object has a constant initializer, we don't
+/// actually need to keep storage for it in the block; we'll just
+/// rematerialize it at the start of the block function.  This is
+/// acceptable because we make no promises about address stability of
+/// captured variables.
+static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
+                                            const VarDecl *var) {
+  QualType type = var->getType();
+
+  // We can only do this if the variable is const.
+  if (!type.isConstQualified()) return 0;
+
+  // Furthermore, in C++ we have to worry about mutable fields:
+  // C++ [dcl.type.cv]p4:
+  //   Except that any class member declared mutable can be
+  //   modified, any attempt to modify a const object during its
+  //   lifetime results in undefined behavior.
+  if (CGM.getLangOptions().CPlusPlus && !isSafeForCXXConstantCapture(type))
+    return 0;
+
+  // If the variable doesn't have any initializer (shouldn't this be
+  // invalid?), it's not clear what we should do.  Maybe capture as
+  // zero?
+  const Expr *init = var->getInit();
+  if (!init) return 0;
+
+  return CGM.EmitConstantExpr(init, var->getType());
 }
 
-/// AllocateAllBlockDeclRefs - Preallocate all nested BlockDeclRefExprs to
-/// ensure we can generate the debug information for the parameter for the block
-/// invoke function.
-static void AllocateAllBlockDeclRefs(CodeGenFunction &CGF, CGBlockInfo &Info) {
-  if (Info.CXXThisRef)
-    CGF.AllocateBlockCXXThisPointer(Info.CXXThisRef);
-
-  for (size_t i = 0; i < Info.DeclRefs.size(); ++i)
-    CGF.AllocateBlockDecl(Info.DeclRefs[i]);
-
-  if (Info.NeedsObjCSelf) {
-    ValueDecl *Self = cast<ObjCMethodDecl>(CGF.CurFuncDecl)->getSelfDecl();
-    BlockDeclRefExpr *BDRE =
-      new (CGF.getContext()) BlockDeclRefExpr(Self, Self->getType(),
-                                              SourceLocation(), false);
-    Info.DeclRefs.push_back(BDRE);
-    CGF.AllocateBlockDecl(BDRE);
-  }
+/// Get the low bit of a nonzero character count.  This is the
+/// alignment of the nth byte if the 0th byte is universally aligned.
+static CharUnits getLowBit(CharUnits v) {
+  return CharUnits::fromQuantity(v.getQuantity() & (~v.getQuantity() + 1));
 }
 
-static unsigned computeBlockFlag(CodeGenModule &CGM,
-                                 const BlockExpr *BE, unsigned flags) {
-  QualType BPT = BE->getType();
-  const FunctionType *ftype = BPT->getPointeeType()->getAs<FunctionType>();
-  QualType ResultType = ftype->getResultType();
-  
-  CallArgList Args;
-  CodeGenTypes &Types = CGM.getTypes();
-  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, Args,
-                                                       FunctionType::ExtInfo());
-  if (CGM.ReturnTypeUsesSRet(FnInfo))
-    flags |= CodeGenFunction::BLOCK_USE_STRET;
-  return flags;
+static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
+                                std::vector<const llvm::Type*> &elementTypes) {
+  ASTContext &C = CGM.getContext();
+
+  // The header is basically a 'struct { void *; int; int; void *; void *; }'.
+  CharUnits ptrSize, ptrAlign, intSize, intAlign;
+  llvm::tie(ptrSize, ptrAlign) = C.getTypeInfoInChars(C.VoidPtrTy);
+  llvm::tie(intSize, intAlign) = C.getTypeInfoInChars(C.IntTy);
+
+  // Are there crazy embedded platforms where this isn't true?
+  assert(intSize <= ptrSize && "layout assumptions horribly violated");
+
+  CharUnits headerSize = ptrSize;
+  if (2 * intSize < ptrAlign) headerSize += ptrSize;
+  else headerSize += 2 * intSize;
+  headerSize += 2 * ptrSize;
+
+  info.BlockAlign = ptrAlign;
+  info.BlockSize = headerSize;
+
+  assert(elementTypes.empty());
+  const llvm::Type *i8p = CGM.getTypes().ConvertType(C.VoidPtrTy);
+  const llvm::Type *intTy = CGM.getTypes().ConvertType(C.IntTy);
+  elementTypes.push_back(i8p);
+  elementTypes.push_back(intTy);
+  elementTypes.push_back(intTy);
+  elementTypes.push_back(i8p);
+  elementTypes.push_back(CGM.getBlockDescriptorType());
+
+  assert(elementTypes.size() == BlockHeaderSize);
 }
 
-// FIXME: Push most into CGM, passing down a few bits, like current function
-// name.
-llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
-  std::string Name = CurFn->getName();
-  CGBlockInfo Info(Name.c_str());
-  Info.InnerBlocks.insert(BE->getBlockDecl());
-  CollectBlockDeclRefInfo(BE->getBody(), Info);
+/// Compute the layout of the given block.  Attempts to lay the block
+/// out with minimal space requirements.
+static void computeBlockInfo(CodeGenModule &CGM, CGBlockInfo &info) {
+  ASTContext &C = CGM.getContext();
+  const BlockDecl *block = info.getBlockDecl();
 
-  // Check if the block can be global.
-  // FIXME: This test doesn't work for nested blocks yet.  Longer term, I'd like
-  // to just have one code path.  We should move this function into CGM and pass
-  // CGF, then we can just check to see if CGF is 0.
-  if (0 && CanBlockBeGlobal(Info))
-    return CGM.GetAddrOfGlobalBlock(BE, Name.c_str());
+  std::vector<const llvm::Type*> elementTypes;
+  initializeForBlockHeader(CGM, info, elementTypes);
 
-  size_t BlockFields = 5;
+  if (!block->hasCaptures()) {
+    info.StructureType =
+      llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
+    info.CanBeGlobal = true;
+    return;
+  }
 
-  std::vector<llvm::Constant*> Elts(BlockFields);
+  // Collect the layout chunks.
+  llvm::SmallVector<BlockLayoutChunk, 16> layout;
+  layout.reserve(block->capturesCXXThis() +
+                 (block->capture_end() - block->capture_begin()));
 
-  llvm::Constant *C;
-  llvm::Value *V;
+  CharUnits maxFieldAlign;
 
-  {
-    llvm::Constant *BlockVarLayout;
-    // C = BuildBlockStructInitlist();
-    unsigned int flags = BLOCK_HAS_SIGNATURE;
+  // First, 'this'.
+  if (block->capturesCXXThis()) {
+    const DeclContext *DC = block->getDeclContext();
+    for (; isa<BlockDecl>(DC); DC = cast<BlockDecl>(DC)->getDeclContext())
+      ;
+    QualType thisType = cast<CXXMethodDecl>(DC)->getThisType(C);
 
-    // We run this first so that we set BlockHasCopyDispose from the entire
-    // block literal.
-    // __invoke
-    llvm::Function *Fn
-      = CodeGenFunction(CGM).GenerateBlockFunction(CurGD, BE, Info, CurFuncDecl,
-                                                   BlockVarLayout,
-                                                   LocalDeclMap);
-    BlockHasCopyDispose |= Info.BlockHasCopyDispose;
-    Elts[3] = Fn;
-
-    // FIXME: Don't use BlockHasCopyDispose, it is set more often then
-    // necessary, for example: { ^{ __block int i; ^{ i = 1; }(); }(); }
-    if (Info.BlockHasCopyDispose)
-      flags |= BLOCK_HAS_COPY_DISPOSE;
-
-    // __isa
-    C = CGM.getNSConcreteStackBlock();
-    C = llvm::ConstantExpr::getBitCast(C, PtrToInt8Ty);
-    Elts[0] = C;
-
-    // __flags
-    flags = computeBlockFlag(CGM, BE, flags);
-    const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(
-      CGM.getTypes().ConvertType(CGM.getContext().IntTy));
-    C = llvm::ConstantInt::get(IntTy, flags);
-    Elts[1] = C;
-
-    // __reserved
-    C = llvm::ConstantInt::get(IntTy, 0);
-    Elts[2] = C;
-
-    if (Info.BlockLayout.empty()) {
-      // __descriptor
-      C = llvm::Constant::getNullValue(PtrToInt8Ty);
-      Elts[4] = BuildDescriptorBlockDecl(BE, Info, 0, C, 0);
-
-      // Optimize to being a global block.
-      Elts[0] = CGM.getNSConcreteGlobalBlock();
-      
-      Elts[1] = llvm::ConstantInt::get(IntTy, flags|BLOCK_IS_GLOBAL);
-
-      C = llvm::ConstantStruct::get(VMContext, Elts, false);
-
-      C = new llvm::GlobalVariable(CGM.getModule(), C->getType(), true,
-                                   llvm::GlobalValue::InternalLinkage, C,
-                                   "__block_holder_tmp_" +
-                                   llvm::Twine(CGM.getGlobalUniqueCount()));
-      QualType BPT = BE->getType();
-      C = llvm::ConstantExpr::getBitCast(C, ConvertType(BPT));
-      return C;
+    const llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType);
+    std::pair<CharUnits,CharUnits> tinfo
+      = CGM.getContext().getTypeInfoInChars(thisType);
+    maxFieldAlign = std::max(maxFieldAlign, tinfo.second);
+
+    layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first, 0, llvmType));
+  }
+
+  // Next, all the block captures.
+  for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
+         ce = block->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+
+    if (ci->isByRef()) {
+      // We have to copy/dispose of the __block reference.
+      info.NeedsCopyDispose = true;
+
+      // Just use void* instead of a pointer to the byref type.
+      QualType byRefPtrTy = C.VoidPtrTy;
+
+      const llvm::Type *llvmType = CGM.getTypes().ConvertType(byRefPtrTy);
+      std::pair<CharUnits,CharUnits> tinfo
+        = CGM.getContext().getTypeInfoInChars(byRefPtrTy);
+      maxFieldAlign = std::max(maxFieldAlign, tinfo.second);
+
+      layout.push_back(BlockLayoutChunk(tinfo.second, tinfo.first,
+                                        &*ci, llvmType));
+      continue;
+    }
+
+    // Otherwise, build a layout chunk with the size and alignment of
+    // the declaration.
+    if (llvm::Constant *constant = tryCaptureAsConstant(CGM, variable)) {
+      info.Captures[variable] = CGBlockInfo::Capture::makeConstant(constant);
+      continue;
     }
 
-    std::vector<const llvm::Type *> Types(BlockFields+Info.BlockLayout.size());
-    for (int i=0; i<4; ++i)
-      Types[i] = Elts[i]->getType();
-    Types[4] = PtrToInt8Ty;
-
-    for (unsigned i = 0, n = Info.BlockLayout.size(); i != n; ++i) {
-      const Expr *E = Info.BlockLayout[i];
-      const BlockDeclRefExpr *BDRE = dyn_cast<BlockDeclRefExpr>(E);
-      QualType Ty = E->getType();
-      if (BDRE && BDRE->isByRef()) {
-        Types[i+BlockFields] = 
-          llvm::PointerType::get(BuildByRefType(BDRE->getDecl()), 0);
-      } else if (BDRE && BDRE->getDecl()->getType()->isReferenceType()) {
-         Types[i+BlockFields] = llvm::PointerType::get(ConvertType(Ty), 0);
-      } else 
-        Types[i+BlockFields] = ConvertType(Ty);
+    // Block pointers require copy/dispose.
+    if (variable->getType()->isBlockPointerType()) {
+      info.NeedsCopyDispose = true;
+
+    // So do Objective-C pointers.
+    } else if (variable->getType()->isObjCObjectPointerType() ||
+               C.isObjCNSObjectType(variable->getType())) {
+      info.NeedsCopyDispose = true;
+
+    // So do types that require non-trivial copy construction.
+    } else if (ci->hasCopyExpr()) {
+      info.NeedsCopyDispose = true;
+      info.HasCXXObject = true;
+
+    // And so do types with destructors.
+    } else if (CGM.getLangOptions().CPlusPlus) {
+      if (const CXXRecordDecl *record =
+            variable->getType()->getAsCXXRecordDecl()) {
+        if (!record->hasTrivialDestructor()) {
+          info.HasCXXObject = true;
+          info.NeedsCopyDispose = true;
+        }
+      }
     }
 
-    llvm::StructType *Ty = llvm::StructType::get(VMContext, Types, true);
+    CharUnits size = C.getTypeSizeInChars(variable->getType());
+    CharUnits align = C.getDeclAlign(variable);
+    maxFieldAlign = std::max(maxFieldAlign, align);
 
-    llvm::AllocaInst *A = CreateTempAlloca(Ty);
-    A->setAlignment(Info.BlockAlign.getQuantity());
-    V = A;
+    const llvm::Type *llvmType =
+      CGM.getTypes().ConvertTypeForMem(variable->getType());
+
+    layout.push_back(BlockLayoutChunk(align, size, &*ci, llvmType));
+  }
+
+  // If that was everything, we're done here.
+  if (layout.empty()) {
+    info.StructureType =
+      llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
+    info.CanBeGlobal = true;
+    return;
+  }
 
-    // Build layout / cleanup information for all the data entries in the
-    // layout, and write the enclosing fields into the type.
-    std::vector<HelperInfo> NoteForHelper(Info.BlockLayout.size());
-    unsigned NumHelpers = 0;
+  // Sort the layout by alignment.  We have to use a stable sort here
+  // to get reproducible results.  There should probably be an
+  // llvm::array_pod_stable_sort.
+  std::stable_sort(layout.begin(), layout.end());
+
+  CharUnits &blockSize = info.BlockSize;
+  info.BlockAlign = std::max(maxFieldAlign, info.BlockAlign);
+
+  // Assuming that the first byte in the header is maximally aligned,
+  // get the alignment of the first byte following the header.
+  CharUnits endAlign = getLowBit(blockSize);
+
+  // If the end of the header isn't satisfactorily aligned for the
+  // maximum thing, look for things that are okay with the header-end
+  // alignment, and keep appending them until we get something that's
+  // aligned right.  This algorithm is only guaranteed optimal if
+  // that condition is satisfied at some point; otherwise we can get
+  // things like:
+  //   header                 // next byte has alignment 4
+  //   something_with_size_5; // next byte has alignment 1
+  //   something_with_alignment_8;
+  // which has 7 bytes of padding, as opposed to the naive solution
+  // which might have less (?).
+  if (endAlign < maxFieldAlign) {
+    llvm::SmallVectorImpl<BlockLayoutChunk>::iterator
+      li = layout.begin() + 1, le = layout.end();
+
+    // Look for something that the header end is already
+    // satisfactorily aligned for.
+    for (; li != le && endAlign < li->Alignment; ++li)
+      ;
+
+    // If we found something that's naturally aligned for the end of
+    // the header, keep adding things...
+    if (li != le) {
+      llvm::SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
+      for (; li != le; ++li) {
+        assert(endAlign >= li->Alignment);
+
+        li->setIndex(info, elementTypes.size());
+        elementTypes.push_back(li->Type);
+        blockSize += li->Size;
+        endAlign = getLowBit(blockSize);
+
+        // ...until we get to the alignment of the maximum field.
+        if (endAlign >= maxFieldAlign)
+          break;
+      }
 
-    for (unsigned i=0; i<4; ++i)
-      Builder.CreateStore(Elts[i], Builder.CreateStructGEP(V, i, "block.tmp"));
+      // Don't re-append everything we just appended.
+      layout.erase(first, li);
+    }
+  }
 
-    for (unsigned i=0; i < Info.BlockLayout.size(); ++i) {
-      const Expr *E = Info.BlockLayout[i];
+  // At this point, we just have to add padding if the end align still
+  // isn't aligned right.
+  if (endAlign < maxFieldAlign) {
+    CharUnits padding = maxFieldAlign - endAlign;
 
-      // Skip padding.
-      if (isa<DeclRefExpr>(E)) continue;
+    elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,
+                                                padding.getQuantity()));
+    blockSize += padding;
 
-      llvm::Value* Addr = Builder.CreateStructGEP(V, i+BlockFields, "tmp");
-      HelperInfo &Note = NoteForHelper[NumHelpers++];
+    endAlign = getLowBit(blockSize);
+    assert(endAlign >= maxFieldAlign);
+  }
 
-      Note.index = i+5;
+  // Slam everything else on now.  This works because they have
+  // strictly decreasing alignment and we expect that size is always a
+  // multiple of alignment.
+  for (llvm::SmallVectorImpl<BlockLayoutChunk>::iterator
+         li = layout.begin(), le = layout.end(); li != le; ++li) {
+    assert(endAlign >= li->Alignment);
+    li->setIndex(info, elementTypes.size());
+    elementTypes.push_back(li->Type);
+    blockSize += li->Size;
+    endAlign = getLowBit(blockSize);
+  }
 
-      if (isa<CXXThisExpr>(E)) {
-        Note.RequiresCopying = false;
-        Note.flag = BLOCK_FIELD_IS_OBJECT;
+  info.StructureType =
+    llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
+}
 
-        Builder.CreateStore(LoadCXXThis(), Addr);
-        continue;
-      }
+/// Emit a block literal expression in the current function.
+llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
+  std::string Name = CurFn->getName();
+  CGBlockInfo blockInfo(blockExpr, Name.c_str());
+
+  // Compute information about the layout, etc., of this block.
+  computeBlockInfo(CGM, blockInfo);
+
+  // Using that metadata, generate the actual block function.
+  llvm::Constant *blockFn
+    = CodeGenFunction(CGM).GenerateBlockFunction(CurGD, blockInfo,
+                                                 CurFuncDecl, LocalDeclMap);
+  blockFn = llvm::ConstantExpr::getBitCast(blockFn, VoidPtrTy);
+
+  // If there is nothing to capture, we can emit this as a global block.
+  if (blockInfo.CanBeGlobal)
+    return buildGlobalBlock(CGM, blockInfo, blockFn);
+
+  // Otherwise, we have to emit this as a local block.
+
+  llvm::Constant *isa = CGM.getNSConcreteStackBlock();
+  isa = llvm::ConstantExpr::getBitCast(isa, VoidPtrTy);
+
+  // Build the block descriptor.
+  llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo);
+
+  const llvm::Type *intTy = ConvertType(getContext().IntTy);
+
+  llvm::AllocaInst *blockAddr =
+    CreateTempAlloca(blockInfo.StructureType, "block");
+  blockAddr->setAlignment(blockInfo.BlockAlign.getQuantity());
+
+  // Compute the initial on-stack block flags.
+  BlockFlags flags = BLOCK_HAS_SIGNATURE;
+  if (blockInfo.NeedsCopyDispose) flags |= BLOCK_HAS_COPY_DISPOSE;
+  if (blockInfo.HasCXXObject) flags |= BLOCK_HAS_CXX_OBJ;
+  flags = computeBlockFlag(CGM, blockInfo.getBlockExpr(), flags);
+
+  // Initialize the block literal.
+  Builder.CreateStore(isa, Builder.CreateStructGEP(blockAddr, 0, "block.isa"));
+  Builder.CreateStore(llvm::ConstantInt::get(intTy, flags.getBitMask()),
+                      Builder.CreateStructGEP(blockAddr, 1, "block.flags"));
+  Builder.CreateStore(llvm::ConstantInt::get(intTy, 0),
+                      Builder.CreateStructGEP(blockAddr, 2, "block.reserved"));
+  Builder.CreateStore(blockFn, Builder.CreateStructGEP(blockAddr, 3,
+                                                       "block.invoke"));
+  Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockAddr, 4,
+                                                          "block.descriptor"));
+
+  // Finally, capture all the values into the block.
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
+
+  // First, 'this'.
+  if (blockDecl->capturesCXXThis()) {
+    llvm::Value *addr = Builder.CreateStructGEP(blockAddr,
+                                                blockInfo.CXXThisIndex,
+                                                "block.captured-this.addr");
+    Builder.CreateStore(LoadCXXThis(), addr);
+  }
 
-      const BlockDeclRefExpr *BDRE = cast<BlockDeclRefExpr>(E);
-      const ValueDecl *VD = BDRE->getDecl();
-      QualType T = VD->getType();
+  // Next, captured variables.
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+         ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
 
-      Note.RequiresCopying = BlockRequiresCopying(T);
+    // Ignore constant captures.
+    if (capture.isConstant()) continue;
 
-      if (BDRE->isByRef()) {
-        Note.flag = BLOCK_FIELD_IS_BYREF;
-        if (T.isObjCGCWeak())
-          Note.flag |= BLOCK_FIELD_IS_WEAK;
-      } else if (T->isBlockPointerType()) {
-        Note.flag = BLOCK_FIELD_IS_BLOCK;
-      } else {
-        Note.flag = BLOCK_FIELD_IS_OBJECT;
-      }
+    QualType type = variable->getType();
 
-      if (LocalDeclMap[VD]) {
-        if (BDRE->isByRef()) {
-          llvm::Value *Loc = LocalDeclMap[VD];
-          Loc = Builder.CreateStructGEP(Loc, 1, "forwarding");
-          Loc = Builder.CreateLoad(Loc);
-          Builder.CreateStore(Loc, Addr);
-          continue;
-        } else {
-          if (BDRE->getCopyConstructorExpr()) {
-            E = BDRE->getCopyConstructorExpr();
-            PushDestructorCleanup(E->getType(), Addr);
-          }
-            else {
-              E = new (getContext()) DeclRefExpr(const_cast<ValueDecl*>(VD),
-                                            VD->getType().getNonReferenceType(),
-                                            SourceLocation());
-              if (VD->getType()->isReferenceType()) {
-                E = new (getContext())
-                    UnaryOperator(const_cast<Expr*>(E), UO_AddrOf,
-                                getContext().getPointerType(E->getType()),
-                                SourceLocation());
-              } 
-            }
-          }
-        }
+    // This will be a [[type]]*, except that a byref entry will just be
+    // an i8**.
+    llvm::Value *blockField =
+      Builder.CreateStructGEP(blockAddr, capture.getIndex(),
+                              "block.captured");
 
-      if (BDRE->isByRef()) {
-        E = new (getContext())
-          UnaryOperator(const_cast<Expr*>(E), UO_AddrOf,
-                        getContext().getPointerType(E->getType()),
-                        SourceLocation());
-      }
+    // Compute the address of the thing we're going to move into the
+    // block literal.
+    llvm::Value *src;
+    if (ci->isNested()) {
+      // We need to use the capture from the enclosing block.
+      const CGBlockInfo::Capture &enclosingCapture =
+        BlockInfo->getCapture(variable);
+
+      // This is a [[type]]*, except that a byref entry wil just be an i8**.
+      src = Builder.CreateStructGEP(LoadBlockStruct(),
+                                    enclosingCapture.getIndex(),
+                                    "block.capture.addr");
+    } else {
+      // This is a [[type]]*.
+      src = LocalDeclMap[variable];
+    }
 
-      RValue r = EmitAnyExpr(E, Addr, false);
-      if (r.isScalar()) {
-        llvm::Value *Loc = r.getScalarVal();
-        const llvm::Type *Ty = Types[i+BlockFields];
-        if  (BDRE->isByRef()) {
-          // E is now the address of the value field, instead, we want the
-          // address of the actual ByRef struct.  We optimize this slightly
-          // compared to gcc by not grabbing the forwarding slot as this must
-          // be done during Block_copy for us, and we can postpone the work
-          // until then.
-          CharUnits offset = BlockDecls[BDRE->getDecl()];
-
-          llvm::Value *BlockLiteral = LoadBlockStruct();
-
-          Loc = Builder.CreateGEP(BlockLiteral,
-                     llvm::ConstantInt::get(Int64Ty, offset.getQuantity()),
-                                  "block.literal");
-          Ty = llvm::PointerType::get(Ty, 0);
-          Loc = Builder.CreateBitCast(Loc, Ty);
-          Loc = Builder.CreateLoad(Loc);
-          // Loc = Builder.CreateBitCast(Loc, Ty);
-        }
-        Builder.CreateStore(Loc, Addr);
-      } else if (r.isComplex())
-        // FIXME: implement
-        ErrorUnsupported(BE, "complex in block literal");
-      else if (r.isAggregate())
-        ; // Already created into the destination
+    // For byrefs, we just write the pointer to the byref struct into
+    // the block field.  There's no need to chase the forwarding
+    // pointer at this point, since we're building something that will
+    // live a shorter life than the stack byref anyway.
+    if (ci->isByRef()) {
+      // Get a void* that points to the byref struct.
+      if (ci->isNested())
+        src = Builder.CreateLoad(src, "byref.capture");
       else
-        assert (0 && "bad block variable");
-      // FIXME: Ensure that the offset created by the backend for
-      // the struct matches the previously computed offset in BlockDecls.
+        src = Builder.CreateBitCast(src, VoidPtrTy);
+
+      // Write that void* into the capture field.
+      Builder.CreateStore(src, blockField);
+
+    // If we have a copy constructor, evaluate that into the block field.
+    } else if (const Expr *copyExpr = ci->getCopyExpr()) {
+      EmitSynthesizedCXXCopyCtor(blockField, src, copyExpr);
+
+    // If it's a reference variable, copy the reference into the block field.
+    } else if (type->isReferenceType()) {
+      Builder.CreateStore(Builder.CreateLoad(src, "ref.val"), blockField);
+
+    // Otherwise, fake up a POD copy into the block field.
+    } else {
+      // We use one of these or the other depending on whether the
+      // reference is nested.
+      DeclRefExpr notNested(const_cast<VarDecl*>(variable), type, VK_LValue,
+                            SourceLocation());
+      BlockDeclRefExpr nested(const_cast<VarDecl*>(variable), type,
+                              VK_LValue, SourceLocation(), /*byref*/ false);
+
+      Expr *declRef = 
+        (ci->isNested() ? static_cast<Expr*>(&nested) : &notNested);
+
+      ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
+                           declRef, VK_RValue);
+      EmitAnyExprToMem(&l2r, blockField, /*volatile*/ false, /*init*/ true);
     }
-    NoteForHelper.resize(NumHelpers);
-
-    // __descriptor
-    llvm::Value *Descriptor = BuildDescriptorBlockDecl(BE, Info, Ty,
-                                                       BlockVarLayout,
-                                                       &NoteForHelper);
-    Descriptor = Builder.CreateBitCast(Descriptor, PtrToInt8Ty);
-    Builder.CreateStore(Descriptor, Builder.CreateStructGEP(V, 4, "block.tmp"));
-  }
 
-  QualType BPT = BE->getType();
-  V = Builder.CreateBitCast(V, ConvertType(BPT));
-  // See if this is a __weak block variable and the must call objc_read_weak
-  // on it.
-  const FunctionType *ftype = BPT->getPointeeType()->getAs<FunctionType>();
-  QualType RES = ftype->getResultType();
-  if (RES.isObjCGCWeak()) {
-    // Must cast argument to id*
-    const llvm::Type *ObjectPtrTy = 
-      ConvertType(CGM.getContext().getObjCIdType());
-    const llvm::Type *PtrObjectPtrTy = 
-      llvm::PointerType::getUnqual(ObjectPtrTy);
-    V = Builder.CreateBitCast(V, PtrObjectPtrTy);
-    V =  CGM.getObjCRuntime().EmitObjCWeakRead(*this, V);
+    // Push a destructor if necessary.  The semantics for when this
+    // actually gets run are really obscure.
+    if (!ci->isByRef() && CGM.getLangOptions().CPlusPlus)
+      PushDestructorCleanup(type, blockField);
   }
-  return V;
+
+  // Cast to the converted block-pointer type, which happens (somewhat
+  // unfortunately) to be a pointer to function type.
+  llvm::Value *result =
+    Builder.CreateBitCast(blockAddr,
+                          ConvertType(blockInfo.getBlockExpr()->getType()));
+
+  return result;
 }
 
 
-const llvm::Type *BlockModule::getBlockDescriptorType() {
+const llvm::Type *CodeGenModule::getBlockDescriptorType() {
   if (BlockDescriptorType)
     return BlockDescriptorType;
 
@@ -443,18 +678,16 @@ const llvm::Type *BlockModule::getBlockDescriptorType() {
   getModule().addTypeName("struct.__block_descriptor",
                           BlockDescriptorType);
 
+  // Now form a pointer to that.
+  BlockDescriptorType = llvm::PointerType::getUnqual(BlockDescriptorType);
   return BlockDescriptorType;
 }
 
-const llvm::Type *BlockModule::getGenericBlockLiteralType() {
+const llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
   if (GenericBlockLiteralType)
     return GenericBlockLiteralType;
 
-  const llvm::Type *BlockDescPtrTy =
-    llvm::PointerType::getUnqual(getBlockDescriptorType());
-
-  const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(
-    getTypes().ConvertType(getContext().IntTy));
+  const llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
 
   // struct __block_literal_generic {
   //   void *__isa;
@@ -463,11 +696,11 @@ const llvm::Type *BlockModule::getGenericBlockLiteralType() {
   //   void (*__invoke)(void *);
   //   struct __block_descriptor *__descriptor;
   // };
-  GenericBlockLiteralType = llvm::StructType::get(IntTy->getContext(),
-                                                  PtrToInt8Ty,
+  GenericBlockLiteralType = llvm::StructType::get(getLLVMContext(),
+                                                  VoidPtrTy,
                                                   IntTy,
                                                   IntTy,
-                                                  PtrToInt8Ty,
+                                                  VoidPtrTy,
                                                   BlockDescPtrTy,
                                                   NULL);
 
@@ -496,10 +729,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   // Get the function pointer from the literal.
   llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3, "tmp");
 
-  BlockLiteral =
-    Builder.CreateBitCast(BlockLiteral,
-                          llvm::Type::getInt8PtrTy(VMContext),
-                          "tmp");
+  BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy, "tmp");
 
   // Add the block literal.
   QualType VoidPtrTy = getContext().getPointerType(getContext().VoidTy);
@@ -533,318 +763,222 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr* E,
   return EmitCall(FnInfo, Func, ReturnValue, Args);
 }
 
-void CodeGenFunction::AllocateBlockCXXThisPointer(const CXXThisExpr *E) {
-  assert(BlockCXXThisOffset.isZero() && "already computed 'this' pointer");
-
-  // Figure out what the offset is.
-  QualType T = E->getType();
-  std::pair<CharUnits,CharUnits> TypeInfo = getContext().getTypeInfoInChars(T);
-  CharUnits Offset = getBlockOffset(TypeInfo.first, TypeInfo.second);
+llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable,
+                                                 bool isByRef) {
+  assert(BlockInfo && "evaluating block ref without block information?");
+  const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable);
 
-  BlockCXXThisOffset = Offset;
-  BlockLayout.push_back(E);
-}
-
-void CodeGenFunction::AllocateBlockDecl(const BlockDeclRefExpr *E) {
-  const ValueDecl *VD = E->getDecl();
-  CharUnits &Offset = BlockDecls[VD];
+  // Handle constant captures.
+  if (capture.isConstant()) return LocalDeclMap[variable];
 
-  // See if we have already allocated an offset for this variable.
-  if (!Offset.isZero())
-    return;
+  llvm::Value *addr =
+    Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(),
+                            "block.capture.addr");
 
-  // Don't run the expensive check, unless we have to.
-  if (!BlockHasCopyDispose)
-    if (E->isByRef()
-        || BlockRequiresCopying(E->getType()))
-      BlockHasCopyDispose = true;
+  if (isByRef) {
+    // addr should be a void** right now.  Load, then cast the result
+    // to byref*.
 
-  const ValueDecl *D = cast<ValueDecl>(E->getDecl());
+    addr = Builder.CreateLoad(addr);
+    const llvm::PointerType *byrefPointerType
+      = llvm::PointerType::get(BuildByRefType(variable), 0);
+    addr = Builder.CreateBitCast(addr, byrefPointerType,
+                                 "byref.addr");
 
-  CharUnits Size;
-  CharUnits Align;
+    // Follow the forwarding pointer.
+    addr = Builder.CreateStructGEP(addr, 1, "byref.forwarding");
+    addr = Builder.CreateLoad(addr, "byref.addr.forwarded");
 
-  if (E->isByRef()) {
-    llvm::tie(Size,Align) =
-      getContext().getTypeInfoInChars(getContext().VoidPtrTy);
-  } else {
-    Size = getContext().getTypeSizeInChars(D->getType());
-    Align = getContext().getDeclAlign(D);
+    // Cast back to byref* and GEP over to the actual object.
+    addr = Builder.CreateBitCast(addr, byrefPointerType);
+    addr = Builder.CreateStructGEP(addr, getByRefValueLLVMField(variable), 
+                                   variable->getNameAsString());
   }
 
-  Offset = getBlockOffset(Size, Align);
-  BlockLayout.push_back(E);
-}
+  if (variable->getType()->isReferenceType())
+    addr = Builder.CreateLoad(addr, "ref.tmp");
 
-llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const ValueDecl *VD,
-                                                 bool IsByRef) {
-  
-  CharUnits offset = BlockDecls[VD];
-  assert(!offset.isZero() && "getting address of unallocated decl");
-
-  llvm::Value *BlockLiteral = LoadBlockStruct();
-  llvm::Value *V = Builder.CreateGEP(BlockLiteral,
-                       llvm::ConstantInt::get(Int64Ty, offset.getQuantity()),
-                                     "block.literal");
-  if (IsByRef) {
-    const llvm::Type *PtrStructTy
-      = llvm::PointerType::get(BuildByRefType(VD), 0);
-    // The block literal will need a copy/destroy helper.
-    BlockHasCopyDispose = true;
-    
-    const llvm::Type *Ty = PtrStructTy;
-    Ty = llvm::PointerType::get(Ty, 0);
-    V = Builder.CreateBitCast(V, Ty);
-    V = Builder.CreateLoad(V);
-    V = Builder.CreateStructGEP(V, 1, "forwarding");
-    V = Builder.CreateLoad(V);
-    V = Builder.CreateBitCast(V, PtrStructTy);
-    V = Builder.CreateStructGEP(V, getByRefValueLLVMField(VD), 
-                                VD->getNameAsString());
-    if (VD->getType()->isReferenceType())
-      V = Builder.CreateLoad(V);
-  } else {
-    const llvm::Type *Ty = CGM.getTypes().ConvertType(VD->getType());
-    Ty = llvm::PointerType::get(Ty, 0);
-    V = Builder.CreateBitCast(V, Ty);
-    if (VD->getType()->isReferenceType())
-      V = Builder.CreateLoad(V, "ref.tmp");
-  }
-  return V;
+  return addr;
 }
 
 llvm::Constant *
-BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
-  // Generate the block descriptor.
-  const llvm::Type *UnsignedLongTy = Types.ConvertType(Context.UnsignedLongTy);
-  const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(
-    getTypes().ConvertType(getContext().IntTy));
-
-  llvm::Constant *DescriptorFields[4];
-
-  // Reserved
-  DescriptorFields[0] = llvm::Constant::getNullValue(UnsignedLongTy);
-
-  // Block literal size. For global blocks we just use the size of the generic
-  // block literal struct.
-  CharUnits BlockLiteralSize = 
-    CGM.GetTargetTypeStoreSize(getGenericBlockLiteralType());
-  DescriptorFields[1] =
-    llvm::ConstantInt::get(UnsignedLongTy,BlockLiteralSize.getQuantity());
-  
-  // signature.  non-optional ObjC-style method descriptor @encode sequence
-  std::string BlockTypeEncoding;
-  CGM.getContext().getObjCEncodingForBlock(BE, BlockTypeEncoding);
-
-  DescriptorFields[2] = llvm::ConstantExpr::getBitCast(
-          CGM.GetAddrOfConstantCString(BlockTypeEncoding), PtrToInt8Ty);
-  
-  // layout
-  DescriptorFields[3] =
-    llvm::ConstantInt::get(UnsignedLongTy,0);
+CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *blockExpr,
+                                    const char *name) {
+  CGBlockInfo blockInfo(blockExpr, name);
 
-  // build the structure from the 4 elements
-  llvm::Constant *DescriptorStruct =
-    llvm::ConstantStruct::get(VMContext, &DescriptorFields[0], 4, false);
+  // Compute information about the layout, etc., of this block.
+  computeBlockInfo(*this, blockInfo);
 
-  llvm::GlobalVariable *Descriptor =
-    new llvm::GlobalVariable(getModule(), DescriptorStruct->getType(), true,
-                             llvm::GlobalVariable::InternalLinkage,
-                             DescriptorStruct, "__block_descriptor_global");
+  // Using that metadata, generate the actual block function.
+  llvm::Constant *blockFn;
+  {
+    llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
+    blockFn = CodeGenFunction(*this).GenerateBlockFunction(GlobalDecl(),
+                                                           blockInfo,
+                                                           0, LocalDeclMap);
+  }
+  blockFn = llvm::ConstantExpr::getBitCast(blockFn, VoidPtrTy);
 
-  int FieldCount = 5;
-  // Generate the constants for the block literal.
+  return buildGlobalBlock(*this, blockInfo, blockFn);
+}
 
-  std::vector<llvm::Constant*> LiteralFields(FieldCount);
+static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
+                                        const CGBlockInfo &blockInfo,
+                                        llvm::Constant *blockFn) {
+  assert(blockInfo.CanBeGlobal);
 
-  CGBlockInfo Info(n);
-  llvm::Constant *BlockVarLayout;
-  llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
-  llvm::Function *Fn
-    = CodeGenFunction(CGM).GenerateBlockFunction(GlobalDecl(), BE, 
-                                                 Info, 0, BlockVarLayout,
-                                                 LocalDeclMap);
-  assert(Info.BlockSize == BlockLiteralSize
-         && "no imports allowed for global block");
+  // Generate the constants for the block literal initializer.
+  llvm::Constant *fields[BlockHeaderSize];
 
   // isa
-  LiteralFields[0] = CGM.getNSConcreteGlobalBlock();
+  fields[0] = CGM.getNSConcreteGlobalBlock();
 
   // __flags
-  unsigned flags = computeBlockFlag(CGM, BE,
-                                    (BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE));
-  LiteralFields[1] =
-    llvm::ConstantInt::get(IntTy, flags);
+  BlockFlags flags = computeBlockFlag(CGM, blockInfo.getBlockExpr(),
+                                      BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE);
+  fields[1] = llvm::ConstantInt::get(CGM.IntTy, flags.getBitMask());
 
   // Reserved
-  LiteralFields[2] = llvm::Constant::getNullValue(IntTy);
+  fields[2] = llvm::Constant::getNullValue(CGM.IntTy);
 
   // Function
-  LiteralFields[3] = Fn;
+  fields[3] = blockFn;
 
   // Descriptor
-  LiteralFields[4] = Descriptor;
-  
-  llvm::Constant *BlockLiteralStruct =
-    llvm::ConstantStruct::get(VMContext, LiteralFields, false);
-
-  llvm::GlobalVariable *BlockLiteral =
-    new llvm::GlobalVariable(getModule(), BlockLiteralStruct->getType(), true,
-                             llvm::GlobalVariable::InternalLinkage,
-                             BlockLiteralStruct, "__block_literal_global");
+  fields[4] = buildBlockDescriptor(CGM, blockInfo);
 
-  return BlockLiteral;
-}
+  llvm::Constant *init =
+    llvm::ConstantStruct::get(CGM.getLLVMContext(), fields, BlockHeaderSize,
+                              /*packed*/ false);
 
-llvm::Value *CodeGenFunction::LoadBlockStruct() {
-  llvm::Value *V = Builder.CreateLoad(LocalDeclMap[getBlockStructDecl()],
-                                      "self");
-  // For now, we codegen based upon byte offsets.
-  return Builder.CreateBitCast(V, PtrToInt8Ty);
+  llvm::GlobalVariable *literal =
+    new llvm::GlobalVariable(CGM.getModule(),
+                             init->getType(),
+                             /*constant*/ true,
+                             llvm::GlobalVariable::InternalLinkage,
+                             init,
+                             "__block_literal_global");
+  literal->setAlignment(blockInfo.BlockAlign.getQuantity());
+
+  // Return a constant of the appropriately-casted type.
+  const llvm::Type *requiredType =
+    CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType());
+  return llvm::ConstantExpr::getBitCast(literal, requiredType);
 }
 
 llvm::Function *
-CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
-                                       CGBlockInfo &Info,
-                                       const Decl *OuterFuncDecl,
-                                       llvm::Constant *& BlockVarLayout,
-                                  llvm::DenseMap<const Decl*, llvm::Value*> ldm) {
+CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
+                                       const CGBlockInfo &blockInfo,
+                                       const Decl *outerFnDecl,
+                                       const DeclMapTy &ldm) {
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
 
-  // Check if we should generate debug info for this block.
-  if (CGM.getDebugInfo())
-    DebugInfo = CGM.getDebugInfo();
+  DebugInfo = CGM.getDebugInfo();
+  BlockInfo = &blockInfo;
 
   // Arrange for local static and local extern declarations to appear
-  // to be local to this function as well, as they are directly referenced
-  // in a block.
-  for (llvm::DenseMap<const Decl *, llvm::Value*>::iterator i = ldm.begin();
-       i != ldm.end();
-       ++i) {
-    const VarDecl *VD = dyn_cast<VarDecl>(i->first);
-
-    if (VD->getStorageClass() == SC_Static || VD->hasExternalStorage())
-      LocalDeclMap[VD] = i->second;
-  }
-
-  BlockOffset = 
-      CGM.GetTargetTypeStoreSize(CGM.getGenericBlockLiteralType());
-  BlockAlign = getContext().getTypeAlignInChars(getContext().VoidPtrTy);
-
-  const FunctionType *BlockFunctionType = BExpr->getFunctionType();
-  QualType ResultType;
-  FunctionType::ExtInfo EInfo = getFunctionExtInfo(*BlockFunctionType);
-  bool IsVariadic;
-  if (const FunctionProtoType *FTy =
-      dyn_cast<FunctionProtoType>(BlockFunctionType)) {
-    ResultType = FTy->getResultType();
-    IsVariadic = FTy->isVariadic();
-  } else {
-    // K&R style block.
-    ResultType = BlockFunctionType->getResultType();
-    IsVariadic = false;
+  // to be local to this function as well, in case they're directly
+  // referenced in a block.
+  for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) {
+    const VarDecl *var = dyn_cast<VarDecl>(i->first);
+    if (var && !var->hasLocalStorage())
+      LocalDeclMap[var] = i->second;
   }
 
-  FunctionArgList Args;
-
-  CurFuncDecl = OuterFuncDecl;
+  // Begin building the function declaration.
 
-  const BlockDecl *BD = BExpr->getBlockDecl();
+  // Build the argument list.
+  FunctionArgList args;
 
+  // The first argument is the block pointer.  Just take it as a void*
+  // and cast it later.
+  QualType selfTy = getContext().VoidPtrTy;
   IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor");
 
-  // Build the block struct now.
-  AllocateAllBlockDeclRefs(*this, Info);
+  // FIXME: this leaks, and we only need it very temporarily.
+  ImplicitParamDecl *selfDecl =
+    ImplicitParamDecl::Create(getContext(),
+                              const_cast<BlockDecl*>(blockDecl),
+                              SourceLocation(), II, selfTy);
+  args.push_back(std::make_pair(selfDecl, selfTy));
+
+  // Now add the rest of the parameters.
+  for (BlockDecl::param_const_iterator i = blockDecl->param_begin(),
+       e = blockDecl->param_end(); i != e; ++i)
+    args.push_back(std::make_pair(*i, (*i)->getType()));
+
+  // Create the function declaration.
+  const FunctionProtoType *fnType =
+    cast<FunctionProtoType>(blockInfo.getBlockExpr()->getFunctionType());
+  const CGFunctionInfo &fnInfo =
+    CGM.getTypes().getFunctionInfo(fnType->getResultType(), args,
+                                   fnType->getExtInfo());
+  const llvm::FunctionType *fnLLVMType =
+    CGM.getTypes().GetFunctionType(fnInfo, fnType->isVariadic());
+
+  MangleBuffer name;
+  CGM.getBlockMangledName(GD, name, blockDecl);
+  llvm::Function *fn =
+    llvm::Function::Create(fnLLVMType, llvm::GlobalValue::InternalLinkage, 
+                           name.getString(), &CGM.getModule());
+  CGM.SetInternalFunctionAttributes(blockDecl, fn, fnInfo);
+
+  // Begin generating the function.
+  StartFunction(blockDecl, fnType->getResultType(), fn, args,
+                blockInfo.getBlockExpr()->getBody()->getLocEnd());
+  CurFuncDecl = outerFnDecl; // StartFunction sets this to blockDecl
+
+  // Okay.  Undo some of what StartFunction did.  We really don't need
+  // an alloca for the block address;  in theory we could remove it,
+  // but that might do unpleasant things to debug info.
+  llvm::AllocaInst *blockAddrAlloca
+    = cast<llvm::AllocaInst>(LocalDeclMap[selfDecl]);
+  llvm::Value *blockAddr = Builder.CreateLoad(blockAddrAlloca);
+  BlockPointer = Builder.CreateBitCast(blockAddr,
+                                       blockInfo.StructureType->getPointerTo(),
+                                       "block");
 
-  // Capture block layout info. here.
-  if (CGM.getContext().getLangOptions().ObjC1)
-    BlockVarLayout = CGM.getObjCRuntime().GCBlockLayout(*this, Info.DeclRefs);
-  else
-    BlockVarLayout = llvm::Constant::getNullValue(PtrToInt8Ty);
-  
-  QualType ParmTy = getContext().getBlockParmType(BlockHasCopyDispose,
-                                                  BlockLayout);
-
-  // FIXME: This leaks
-  ImplicitParamDecl *SelfDecl =
-    ImplicitParamDecl::Create(getContext(), const_cast<BlockDecl*>(BD),
-                              SourceLocation(), II,
-                              ParmTy);
-
-  Args.push_back(std::make_pair(SelfDecl, SelfDecl->getType()));
-  BlockStructDecl = SelfDecl;
+  // If we have a C++ 'this' reference, go ahead and force it into
+  // existence now.
+  if (blockDecl->capturesCXXThis()) {
+    llvm::Value *addr = Builder.CreateStructGEP(BlockPointer,
+                                                blockInfo.CXXThisIndex,
+                                                "block.captured-this");
+    CXXThisValue = Builder.CreateLoad(addr, "this");
+  }
 
-  for (BlockDecl::param_const_iterator i = BD->param_begin(),
-       e = BD->param_end(); i != e; ++i)
-    Args.push_back(std::make_pair(*i, (*i)->getType()));
+  // LoadObjCSelf() expects there to be an entry for 'self' in LocalDeclMap;
+  // appease it.
+  if (const ObjCMethodDecl *method
+        = dyn_cast_or_null<ObjCMethodDecl>(CurFuncDecl)) {
+    const VarDecl *self = method->getSelfDecl();
+
+    // There might not be a capture for 'self', but if there is...
+    if (blockInfo.Captures.count(self)) {
+      const CGBlockInfo::Capture &capture = blockInfo.getCapture(self);
+      llvm::Value *selfAddr = Builder.CreateStructGEP(BlockPointer,
+                                                      capture.getIndex(),
+                                                      "block.captured-self");
+      LocalDeclMap[self] = selfAddr;
+    }
+  }
 
-  const CGFunctionInfo &FI =
-    CGM.getTypes().getFunctionInfo(ResultType, Args, EInfo);
+  // Also force all the constant captures.
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+         ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+    if (!capture.isConstant()) continue;
 
-  CodeGenTypes &Types = CGM.getTypes();
-  const llvm::FunctionType *LTy = Types.GetFunctionType(FI, IsVariadic);
+    unsigned align = getContext().getDeclAlign(variable).getQuantity();
 
-  MangleBuffer Name;
-  CGM.getMangledName(GD, Name, BD);
-  llvm::Function *Fn =
-    llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage, 
-                           Name.getString(), &CGM.getModule());
+    llvm::AllocaInst *alloca =
+      CreateMemTemp(variable->getType(), "block.captured-const");
+    alloca->setAlignment(align);
 
-  CGM.SetInternalFunctionAttributes(BD, Fn, FI);
-  StartFunction(BD, ResultType, Fn, Args,
-                BExpr->getBody()->getLocEnd());
-  
-  CurFuncDecl = OuterFuncDecl;
-  
-  QualType FnType(BlockFunctionType, 0);
-  bool HasPrototype = isa<FunctionProtoType>(BlockFunctionType);
-  
-  IdentifierInfo *ID = &getContext().Idents.get(Name.getString());
-  CurCodeDecl = FunctionDecl::Create(getContext(),
-                                     getContext().getTranslationUnitDecl(),
-                                     SourceLocation(), ID, FnType, 
-                                     0,
-                                     SC_Static,
-                                     SC_None,
-                                     false, HasPrototype);
-  if (FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FnType)) {
-    const FunctionDecl *CFD = dyn_cast<FunctionDecl>(CurCodeDecl);
-    FunctionDecl *FD = const_cast<FunctionDecl *>(CFD);
-    llvm::SmallVector<ParmVarDecl*, 16> Params;
-    for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i)
-      Params.push_back(ParmVarDecl::Create(getContext(), FD, 
-                                           SourceLocation(), 0,
-                                           FT->getArgType(i), /*TInfo=*/0,
-                                           SC_None, SC_None, 0));
-    FD->setParams(Params.data(), Params.size());
-  }
-  
-  
-  // If we have a C++ 'this' reference, go ahead and force it into
-  // existence now.
-  if (Info.CXXThisRef) {
-    assert(!BlockCXXThisOffset.isZero() &&
-           "haven't yet allocated 'this' reference");
-
-    // TODO: I have a dream that one day this will be typed.
-    llvm::Value *BlockLiteral = LoadBlockStruct();
-    llvm::Value *ThisPtrRaw =
-      Builder.CreateConstInBoundsGEP1_64(BlockLiteral,
-                                         BlockCXXThisOffset.getQuantity(),
-                                         "this.ptr.raw");
-
-    const llvm::Type *Ty =
-      CGM.getTypes().ConvertType(Info.CXXThisRef->getType());
-    Ty = llvm::PointerType::get(Ty, 0);  
-    llvm::Value *ThisPtr = Builder.CreateBitCast(ThisPtrRaw, Ty, "this.ptr");
-
-    CXXThisValue = Builder.CreateLoad(ThisPtr, "this");
-  }
+    Builder.CreateStore(capture.getConstant(), alloca, align);
 
-  // If we have an Objective C 'self' reference, go ahead and force it
-  // into existence now.
-  if (Info.NeedsObjCSelf) {
-    ValueDecl *Self = cast<ObjCMethodDecl>(CurFuncDecl)->getSelfDecl();
-    LocalDeclMap[Self] = GetAddrOfBlockDecl(Self, false);
+    LocalDeclMap[variable] = alloca;
   }
 
   // Save a spot to insert the debug information for all the BlockDeclRefDecls.
@@ -852,7 +986,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
   llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
   --entry_ptr;
 
-  EmitStmt(BExpr->getBody());
+  EmitStmt(blockDecl->getBody());
 
   // Remember where we were...
   llvm::BasicBlock *resume = Builder.GetInsertBlock();
@@ -861,96 +995,78 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
   ++entry_ptr;
   Builder.SetInsertPoint(entry, entry_ptr);
 
+  // Emit debug information for all the BlockDeclRefDecls.
+  // FIXME: also for 'this'
   if (CGDebugInfo *DI = getDebugInfo()) {
-    // Emit debug information for all the BlockDeclRefDecls.
-    // FIXME: also for 'this'
-    for (unsigned i = 0, e = BlockLayout.size(); i != e; ++i) {
-      if (const BlockDeclRefExpr *BDRE =
-            dyn_cast<BlockDeclRefExpr>(BlockLayout[i])) {
-        const ValueDecl *D = BDRE->getDecl();
-        DI->setLocation(D->getLocation());
-        DI->EmitDeclareOfBlockDeclRefVariable(BDRE,
-                                             LocalDeclMap[getBlockStructDecl()],
-                                              Builder, this);
+    for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+           ce = blockDecl->capture_end(); ci != ce; ++ci) {
+      const VarDecl *variable = ci->getVariable();
+      DI->setLocation(variable->getLocation());
+
+      const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+      if (capture.isConstant()) {
+        DI->EmitDeclareOfAutoVariable(variable, LocalDeclMap[variable],
+                                      Builder);
+        continue;
       }
+
+      DI->EmitDeclareOfBlockDeclRefVariable(variable, blockAddrAlloca,
+                                            Builder, blockInfo);
     }
   }
+
   // And resume where we left off.
   if (resume == 0)
     Builder.ClearInsertionPoint();
   else
     Builder.SetInsertPoint(resume);
 
-  FinishFunction(cast<CompoundStmt>(BExpr->getBody())->getRBracLoc());
+  FinishFunction(cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc());
 
-  // The runtime needs a minimum alignment of a void *.
-  CharUnits MinAlign = getContext().getTypeAlignInChars(getContext().VoidPtrTy);
-  BlockOffset = CharUnits::fromQuantity(
-      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), 
-                               MinAlign.getQuantity()));
-
-  Info.BlockSize = BlockOffset;
-  Info.BlockAlign = BlockAlign;
-  Info.BlockLayout = BlockLayout;
-  Info.BlockHasCopyDispose = BlockHasCopyDispose;
-  return Fn;
+  return fn;
 }
 
-CharUnits BlockFunction::getBlockOffset(CharUnits Size, CharUnits Align) {
-  assert((Align.isPositive()) && "alignment must be 1 byte or more");
-
-  CharUnits OldOffset = BlockOffset;
-
-  // Ensure proper alignment, even if it means we have to have a gap
-  BlockOffset = CharUnits::fromQuantity(
-      llvm::RoundUpToAlignment(BlockOffset.getQuantity(), Align.getQuantity()));
-  BlockAlign = std::max(Align, BlockAlign);
-
-  CharUnits Pad = BlockOffset - OldOffset;
-  if (Pad.isPositive()) {
-    QualType PadTy = getContext().getConstantArrayType(getContext().CharTy,
-                                                       llvm::APInt(32, 
-                                                         Pad.getQuantity()),
-                                                       ArrayType::Normal, 0);
-    ValueDecl *PadDecl = VarDecl::Create(getContext(), 
-                                         getContext().getTranslationUnitDecl(),
-                                         SourceLocation(),
-                                         0, QualType(PadTy), 0,
-                                         SC_None, SC_None);
-    Expr *E = new (getContext()) DeclRefExpr(PadDecl, PadDecl->getType(),
-                                             SourceLocation());
-    BlockLayout.push_back(E);
-  }
+/*
+    notes.push_back(HelperInfo());
+    HelperInfo &note = notes.back();
+    note.index = capture.getIndex();
+    note.RequiresCopying = (ci->hasCopyExpr() || BlockRequiresCopying(type));
+    note.cxxbar_import = ci->getCopyExpr();
+
+    if (ci->isByRef()) {
+      note.flag = BLOCK_FIELD_IS_BYREF;
+      if (type.isObjCGCWeak())
+        note.flag |= BLOCK_FIELD_IS_WEAK;
+    } else if (type->isBlockPointerType()) {
+      note.flag = BLOCK_FIELD_IS_BLOCK;
+    } else {
+      note.flag = BLOCK_FIELD_IS_OBJECT;
+    }
+ */
 
-  BlockOffset += Size;
-  return BlockOffset - Size;
-}
 
-llvm::Constant *BlockFunction::
-GenerateCopyHelperFunction(bool BlockHasCopyDispose, const llvm::StructType *T,
-                           std::vector<HelperInfo> *NoteForHelperp) {
-  QualType R = getContext().VoidTy;
 
-  FunctionArgList Args;
+
+
+llvm::Constant *
+CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
+  ASTContext &C = getContext();
+
+  FunctionArgList args;
   // FIXME: This leaks
-  ImplicitParamDecl *Dst =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
-  Args.push_back(std::make_pair(Dst, Dst->getType()));
-  ImplicitParamDecl *Src =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
-  Args.push_back(std::make_pair(Src, Src->getType()));
+  ImplicitParamDecl *dstDecl =
+    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(std::make_pair(dstDecl, dstDecl->getType()));
+  ImplicitParamDecl *srcDecl =
+    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(std::make_pair(srcDecl, srcDecl->getType()));
 
   const CGFunctionInfo &FI =
-      CGM.getTypes().getFunctionInfo(R, Args, FunctionType::ExtInfo());
+      CGM.getTypes().getFunctionInfo(C.VoidTy, args, FunctionType::ExtInfo());
 
-  // FIXME: We'd like to put these into a mergable by content, with
-  // internal linkage.
-  CodeGenTypes &Types = CGM.getTypes();
-  const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
+  // FIXME: it would be nice if these were mergeable with things with
+  // identical semantics.
+  const llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
 
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
@@ -959,80 +1075,89 @@ GenerateCopyHelperFunction(bool BlockHasCopyDispose, const llvm::StructType *T,
   IdentifierInfo *II
     = &CGM.getContext().Idents.get("__copy_helper_block_");
 
-  FunctionDecl *FD = FunctionDecl::Create(getContext(),
-                                          getContext().getTranslationUnitDecl(),
-                                          SourceLocation(), II, R, 0,
+  FunctionDecl *FD = FunctionDecl::Create(C,
+                                          C.getTranslationUnitDecl(),
+                                          SourceLocation(), II, C.VoidTy, 0,
                                           SC_Static,
                                           SC_None,
                                           false,
                                           true);
-  CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
-
-  llvm::Value *SrcObj = CGF.GetAddrOfLocalVar(Src);
-  llvm::Type *PtrPtrT;
-
-  if (NoteForHelperp) {
-    std::vector<HelperInfo> &NoteForHelper = *NoteForHelperp;
-
-    PtrPtrT = llvm::PointerType::get(llvm::PointerType::get(T, 0), 0);
-    SrcObj = Builder.CreateBitCast(SrcObj, PtrPtrT);
-    SrcObj = Builder.CreateLoad(SrcObj);
-
-    llvm::Value *DstObj = CGF.GetAddrOfLocalVar(Dst);
-    llvm::Type *PtrPtrT;
-    PtrPtrT = llvm::PointerType::get(llvm::PointerType::get(T, 0), 0);
-    DstObj = Builder.CreateBitCast(DstObj, PtrPtrT);
-    DstObj = Builder.CreateLoad(DstObj);
-
-    for (unsigned i=0; i < NoteForHelper.size(); ++i) {
-      int flag = NoteForHelper[i].flag;
-      int index = NoteForHelper[i].index;
-
-      if ((NoteForHelper[i].flag & BLOCK_FIELD_IS_BYREF)
-          || NoteForHelper[i].RequiresCopying) {
-        llvm::Value *Srcv = SrcObj;
-        Srcv = Builder.CreateStructGEP(Srcv, index);
-        Srcv = Builder.CreateBitCast(Srcv,
-                                     llvm::PointerType::get(PtrToInt8Ty, 0));
-        Srcv = Builder.CreateLoad(Srcv);
-
-        llvm::Value *Dstv = Builder.CreateStructGEP(DstObj, index);
-        Dstv = Builder.CreateBitCast(Dstv, PtrToInt8Ty);
-
-        llvm::Value *N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
-        llvm::Value *F = CGM.getBlockObjectAssign();
-        Builder.CreateCall3(F, Dstv, Srcv, N);
-      }
+  StartFunction(FD, C.VoidTy, Fn, args, SourceLocation());
+
+  const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
+
+  llvm::Value *src = GetAddrOfLocalVar(srcDecl);
+  src = Builder.CreateLoad(src);
+  src = Builder.CreateBitCast(src, structPtrTy, "block.source");
+
+  llvm::Value *dst = GetAddrOfLocalVar(dstDecl);
+  dst = Builder.CreateLoad(dst);
+  dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
+
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
+
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+         ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    QualType type = variable->getType();
+
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+    if (capture.isConstant()) continue;
+
+    const Expr *copyExpr = ci->getCopyExpr();
+    unsigned flags = 0;
+
+    if (copyExpr) {
+      assert(!ci->isByRef());
+      // don't bother computing flags
+    } else if (ci->isByRef()) {
+      flags = BLOCK_FIELD_IS_BYREF;
+      if (type.isObjCGCWeak()) flags |= BLOCK_FIELD_IS_WEAK;
+    } else if (type->isBlockPointerType()) {
+      flags = BLOCK_FIELD_IS_BLOCK;
+    } else if (type->isObjCObjectPointerType() || C.isObjCNSObjectType(type)) {
+      flags = BLOCK_FIELD_IS_OBJECT;
+    }
+
+    if (!copyExpr && !flags) continue;
+
+    unsigned index = capture.getIndex();
+    llvm::Value *srcField = Builder.CreateStructGEP(src, index);
+    llvm::Value *dstField = Builder.CreateStructGEP(dst, index);
+
+    // If there's an explicit copy expression, we do that.
+    if (copyExpr) {
+      EmitSynthesizedCXXCopyCtor(dstField, srcField, copyExpr);
+    } else {
+      llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
+      srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy);
+      llvm::Value *dstAddr = Builder.CreateBitCast(dstField, VoidPtrTy);
+      Builder.CreateCall3(CGM.getBlockObjectAssign(), dstAddr, srcValue,
+                          llvm::ConstantInt::get(Int32Ty, flags));
     }
   }
 
-  CGF.FinishFunction();
+  FinishFunction();
 
-  return llvm::ConstantExpr::getBitCast(Fn, PtrToInt8Ty);
+  return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
 }
 
-llvm::Constant *BlockFunction::
-GenerateDestroyHelperFunction(bool BlockHasCopyDispose,
-                              const llvm::StructType* T,
-                              std::vector<HelperInfo> *NoteForHelperp) {
-  QualType R = getContext().VoidTy;
+llvm::Constant *
+CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
+  ASTContext &C = getContext();
 
-  FunctionArgList Args;
+  FunctionArgList args;
   // FIXME: This leaks
-  ImplicitParamDecl *Src =
-    ImplicitParamDecl::Create(getContext(), 0,
-                              SourceLocation(), 0,
-                              getContext().getPointerType(getContext().VoidTy));
-
-  Args.push_back(std::make_pair(Src, Src->getType()));
+  ImplicitParamDecl *srcDecl =
+    ImplicitParamDecl::Create(C, 0, SourceLocation(), 0, C.VoidPtrTy);
+  args.push_back(std::make_pair(srcDecl, srcDecl->getType()));
 
   const CGFunctionInfo &FI =
-      CGM.getTypes().getFunctionInfo(R, Args, FunctionType::ExtInfo());
+      CGM.getTypes().getFunctionInfo(C.VoidTy, args, FunctionType::ExtInfo());
 
   // FIXME: We'd like to put these into a mergable by content, with
   // internal linkage.
-  CodeGenTypes &Types = CGM.getTypes();
-  const llvm::FunctionType *LTy = Types.GetFunctionType(FI, false);
+  const llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI, false);
 
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
@@ -1041,59 +1166,76 @@ GenerateDestroyHelperFunction(bool BlockHasCopyDispose,
   IdentifierInfo *II
     = &CGM.getContext().Idents.get("__destroy_helper_block_");
 
-  FunctionDecl *FD = FunctionDecl::Create(getContext(),
-                                          getContext().getTranslationUnitDecl(),
-                                          SourceLocation(), II, R, 0,
+  FunctionDecl *FD = FunctionDecl::Create(C, C.getTranslationUnitDecl(),
+                                          SourceLocation(), II, C.VoidTy, 0,
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
-
-  if (NoteForHelperp) {
-    std::vector<HelperInfo> &NoteForHelper = *NoteForHelperp;
-
-    llvm::Value *SrcObj = CGF.GetAddrOfLocalVar(Src);
-    llvm::Type *PtrPtrT;
-    PtrPtrT = llvm::PointerType::get(llvm::PointerType::get(T, 0), 0);
-    SrcObj = Builder.CreateBitCast(SrcObj, PtrPtrT);
-    SrcObj = Builder.CreateLoad(SrcObj);
-
-    for (unsigned i=0; i < NoteForHelper.size(); ++i) {
-      int flag = NoteForHelper[i].flag;
-      int index = NoteForHelper[i].index;
-
-      if ((NoteForHelper[i].flag & BLOCK_FIELD_IS_BYREF)
-          || NoteForHelper[i].RequiresCopying) {
-        llvm::Value *Srcv = SrcObj;
-        Srcv = Builder.CreateStructGEP(Srcv, index);
-        Srcv = Builder.CreateBitCast(Srcv,
-                                     llvm::PointerType::get(PtrToInt8Ty, 0));
-        Srcv = Builder.CreateLoad(Srcv);
-
-        BuildBlockRelease(Srcv, flag);
-      }
+  StartFunction(FD, C.VoidTy, Fn, args, SourceLocation());
+
+  const llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
+
+  llvm::Value *src = GetAddrOfLocalVar(srcDecl);
+  src = Builder.CreateLoad(src);
+  src = Builder.CreateBitCast(src, structPtrTy, "block");
+
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
+
+  CodeGenFunction::RunCleanupsScope cleanups(*this);
+
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+         ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    QualType type = variable->getType();
+
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+    if (capture.isConstant()) continue;
+
+    BlockFieldFlags flags;
+    const CXXDestructorDecl *dtor = 0;
+
+    if (ci->isByRef()) {
+      flags = BLOCK_FIELD_IS_BYREF;
+      if (type.isObjCGCWeak()) flags |= BLOCK_FIELD_IS_WEAK;
+    } else if (type->isBlockPointerType()) {
+      flags = BLOCK_FIELD_IS_BLOCK;
+    } else if (type->isObjCObjectPointerType() || C.isObjCNSObjectType(type)) {
+      flags = BLOCK_FIELD_IS_OBJECT;
+    } else if (C.getLangOptions().CPlusPlus) {
+      if (const CXXRecordDecl *record = type->getAsCXXRecordDecl())
+        if (!record->hasTrivialDestructor())
+          dtor = record->getDestructor();
     }
-  }
 
-  CGF.FinishFunction();
+    if (!dtor && flags.empty()) continue;
 
-  return llvm::ConstantExpr::getBitCast(Fn, PtrToInt8Ty);
-}
+    unsigned index = capture.getIndex();
+    llvm::Value *srcField = Builder.CreateStructGEP(src, index);
 
-llvm::Constant *BlockFunction::BuildCopyHelper(const llvm::StructType *T,
-                                       std::vector<HelperInfo> *NoteForHelper) {
-  return CodeGenFunction(CGM).GenerateCopyHelperFunction(BlockHasCopyDispose,
-                                                         T, NoteForHelper);
-}
+    // If there's an explicit copy expression, we do that.
+    if (dtor) {
+      PushDestructorCleanup(dtor, srcField);
+
+    // Otherwise we call _Block_object_dispose.  It wouldn't be too
+    // hard to just emit this as a cleanup if we wanted to make sure
+    // that things were done in reverse.
+    } else {
+      llvm::Value *value = Builder.CreateLoad(srcField);
+      value = Builder.CreateBitCast(value, VoidPtrTy);
+      BuildBlockRelease(value, flags);
+    }
+  }
+
+  cleanups.ForceCleanup();
 
-llvm::Constant *BlockFunction::BuildDestroyHelper(const llvm::StructType *T,
-                                      std::vector<HelperInfo> *NoteForHelperp) {
-  return CodeGenFunction(CGM).GenerateDestroyHelperFunction(BlockHasCopyDispose,
-                                                            T, NoteForHelperp);
+  FinishFunction();
+
+  return llvm::ConstantExpr::getBitCast(Fn, VoidPtrTy);
 }
 
-llvm::Constant *BlockFunction::
-GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
+llvm::Constant *CodeGenFunction::
+GeneratebyrefCopyHelperFunction(const llvm::Type *T, BlockFieldFlags flags,
+                                const VarDecl *variable) {
   QualType R = getContext().VoidTy;
 
   FunctionArgList Args;
@@ -1121,10 +1263,10 @@ GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
   // internal linkage.
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
-                           "__Block_byref_id_object_copy_", &CGM.getModule());
+                           "__Block_byref_object_copy_", &CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__Block_byref_id_object_copy_");
+    = &CGM.getContext().Idents.get("__Block_byref_object_copy_");
 
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
@@ -1132,37 +1274,43 @@ GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
+  StartFunction(FD, R, Fn, Args, SourceLocation());
 
   // dst->x
-  llvm::Value *V = CGF.GetAddrOfLocalVar(Dst);
+  llvm::Value *V = GetAddrOfLocalVar(Dst);
   V = Builder.CreateBitCast(V, llvm::PointerType::get(T, 0));
   V = Builder.CreateLoad(V);
   V = Builder.CreateStructGEP(V, 6, "x");
-  llvm::Value *DstObj = Builder.CreateBitCast(V, PtrToInt8Ty);
+  llvm::Value *DstObj = V;
 
   // src->x
-  V = CGF.GetAddrOfLocalVar(Src);
+  V = GetAddrOfLocalVar(Src);
   V = Builder.CreateLoad(V);
   V = Builder.CreateBitCast(V, T);
   V = Builder.CreateStructGEP(V, 6, "x");
-  V = Builder.CreateBitCast(V, llvm::PointerType::get(PtrToInt8Ty, 0));
-  llvm::Value *SrcObj = Builder.CreateLoad(V);
-
-  flag |= BLOCK_BYREF_CALLER;
-
-  llvm::Value *N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
-  llvm::Value *F = CGM.getBlockObjectAssign();
-  Builder.CreateCall3(F, DstObj, SrcObj, N);
-
-  CGF.FinishFunction();
+  
+  if (Expr *copyExpr = getContext().getBlockVarCopyInits(variable)) {
+    llvm::Value *SrcObj = V;
+    EmitSynthesizedCXXCopyCtor(DstObj, SrcObj, copyExpr);
+  } else {
+    DstObj = Builder.CreateBitCast(DstObj, VoidPtrTy);
+    V = Builder.CreateBitCast(V, VoidPtrPtrTy);
+    llvm::Value *SrcObj = Builder.CreateLoad(V);
+    flags |= BLOCK_BYREF_CALLER;
+    llvm::Value *N = llvm::ConstantInt::get(Int32Ty, flags.getBitMask());
+    llvm::Value *F = CGM.getBlockObjectAssign();
+    Builder.CreateCall3(F, DstObj, SrcObj, N);
+  }
+  
+  FinishFunction();
 
-  return llvm::ConstantExpr::getBitCast(Fn, PtrToInt8Ty);
+  return llvm::ConstantExpr::getBitCast(Fn, Int8PtrTy);
 }
 
 llvm::Constant *
-BlockFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
-                                                  int flag) {
+CodeGenFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
+                                                    BlockFieldFlags flags,
+                                                    const VarDecl *variable) {
   QualType R = getContext().VoidTy;
 
   FunctionArgList Args;
@@ -1184,11 +1332,11 @@ BlockFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
   // internal linkage.
   llvm::Function *Fn =
     llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
-                           "__Block_byref_id_object_dispose_",
+                           "__Block_byref_object_dispose_",
                            &CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__Block_byref_id_object_dispose_");
+    = &CGM.getContext().Idents.get("__Block_byref_object_dispose_");
 
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
@@ -1196,72 +1344,78 @@ BlockFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
                                           SC_Static,
                                           SC_None,
                                           false, true);
-  CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
+  StartFunction(FD, R, Fn, Args, SourceLocation());
 
-  llvm::Value *V = CGF.GetAddrOfLocalVar(Src);
+  llvm::Value *V = GetAddrOfLocalVar(Src);
   V = Builder.CreateBitCast(V, llvm::PointerType::get(T, 0));
   V = Builder.CreateLoad(V);
   V = Builder.CreateStructGEP(V, 6, "x");
-  V = Builder.CreateBitCast(V, llvm::PointerType::get(PtrToInt8Ty, 0));
-  V = Builder.CreateLoad(V);
 
-  flag |= BLOCK_BYREF_CALLER;
-  BuildBlockRelease(V, flag);
-  CGF.FinishFunction();
+  // If it's not any kind of special object, it must have a destructor
+  // or something.
+  if (!flags.isSpecialPointer()) {
+    EHScopeStack::stable_iterator CleanupDepth = EHStack.stable_begin();
+    PushDestructorCleanup(variable->getType(), V);
+    PopCleanupBlocks(CleanupDepth);
 
-  return llvm::ConstantExpr::getBitCast(Fn, PtrToInt8Ty);
+  // Otherwise, call _Block_object_dispose.
+  } else {
+    V = Builder.CreateBitCast(V, llvm::PointerType::get(Int8PtrTy, 0));
+    V = Builder.CreateLoad(V);
+
+    flags |= BLOCK_BYREF_CALLER;
+    BuildBlockRelease(V, flags);
+  }
+
+  FinishFunction();
+
+  return llvm::ConstantExpr::getBitCast(Fn, Int8PtrTy);
 }
 
-llvm::Constant *BlockFunction::BuildbyrefCopyHelper(const llvm::Type *T,
-                                                    int Flag, unsigned Align) {
+llvm::Constant *CodeGenModule::BuildbyrefCopyHelper(const llvm::Type *T,
+                                                    BlockFieldFlags flags,
+                                                    unsigned align,
+                                                    const VarDecl *var) {
   // All alignments below that of pointer alignment collapse down to just
   // pointer alignment, as we always have at least that much alignment to begin
   // with.
-  Align /= unsigned(CGF.Target.getPointerAlign(0)/8);
+  align /= unsigned(getTarget().getPointerAlign(0) / 8);
   
   // As an optimization, we only generate a single function of each kind we
   // might need.  We need a different one for each alignment and for each
   // setting of flags.  We mix Align and flag to get the kind.
-  uint64_t Kind = (uint64_t)Align*BLOCK_BYREF_CURRENT_MAX + Flag;
-  llvm::Constant *&Entry = CGM.AssignCache[Kind];
-  if (Entry)
-    return Entry;
-  return Entry = CodeGenFunction(CGM).GeneratebyrefCopyHelperFunction(T, Flag);
+  uint64_t Kind = (uint64_t)align*BLOCK_BYREF_CURRENT_MAX + flags.getBitMask();
+  llvm::Constant *&Entry = AssignCache[Kind];
+  if (!Entry)
+    Entry = CodeGenFunction(*this).
+                GeneratebyrefCopyHelperFunction(T, flags, var);
+  return Entry;
 }
 
-llvm::Constant *BlockFunction::BuildbyrefDestroyHelper(const llvm::Type *T,
-                                                       int Flag,
-                                                       unsigned Align) {
+llvm::Constant *CodeGenModule::BuildbyrefDestroyHelper(const llvm::Type *T,
+                                                       BlockFieldFlags flags,
+                                                       unsigned align,
+                                                       const VarDecl *var) {
   // All alignments below that of pointer alignment collpase down to just
   // pointer alignment, as we always have at least that much alignment to begin
   // with.
-  Align /= unsigned(CGF.Target.getPointerAlign(0)/8);
+  align /= unsigned(getTarget().getPointerAlign(0) / 8);
   
   // As an optimization, we only generate a single function of each kind we
   // might need.  We need a different one for each alignment and for each
   // setting of flags.  We mix Align and flag to get the kind.
-  uint64_t Kind = (uint64_t)Align*BLOCK_BYREF_CURRENT_MAX + Flag;
-  llvm::Constant *&Entry = CGM.DestroyCache[Kind];
-  if (Entry)
-    return Entry;
-  return Entry=CodeGenFunction(CGM).GeneratebyrefDestroyHelperFunction(T, Flag);
+  uint64_t Kind = (uint64_t)align*BLOCK_BYREF_CURRENT_MAX + flags.getBitMask();
+  llvm::Constant *&Entry = DestroyCache[Kind];
+  if (!Entry)
+    Entry = CodeGenFunction(*this).
+                GeneratebyrefDestroyHelperFunction(T, flags, var);
+  return Entry;
 }
 
-void BlockFunction::BuildBlockRelease(llvm::Value *V, int flag) {
+void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags) {
   llvm::Value *F = CGM.getBlockObjectDispose();
   llvm::Value *N;
-  V = Builder.CreateBitCast(V, PtrToInt8Ty);
-  N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
+  V = Builder.CreateBitCast(V, Int8PtrTy);
+  N = llvm::ConstantInt::get(Int32Ty, flags.getBitMask());
   Builder.CreateCall2(F, V, N);
 }
-
-ASTContext &BlockFunction::getContext() const { return CGM.getContext(); }
-
-BlockFunction::BlockFunction(CodeGenModule &cgm, CodeGenFunction &cgf,
-                             CGBuilderTy &B)
-  : CGM(cgm), VMContext(cgm.getLLVMContext()), CGF(cgf), Builder(B) {
-  PtrToInt8Ty = llvm::PointerType::getUnqual(
-            llvm::Type::getInt8Ty(VMContext));
-
-  BlockHasCopyDispose = false;
-}
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index 743e3c83be33..bee729dccee9 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -24,9 +24,6 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 
-#include <vector>
-#include <map>
-
 #include "CGBuilder.h"
 #include "CGCall.h"
 #include "CGValue.h"
@@ -46,157 +43,153 @@ namespace llvm {
 namespace clang {
 
 namespace CodeGen {
+
 class CodeGenModule;
+class CGBlockInfo;
+
+enum BlockFlag_t {
+  BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
+  BLOCK_HAS_CXX_OBJ =       (1 << 26),
+  BLOCK_IS_GLOBAL =         (1 << 28),
+  BLOCK_USE_STRET =         (1 << 29),
+  BLOCK_HAS_SIGNATURE  =    (1 << 30)
+};
+class BlockFlags {
+  uint32_t flags;
 
-class BlockBase {
+  BlockFlags(uint32_t flags) : flags(flags) {}
 public:
-    enum {
-        BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
-        BLOCK_HAS_CXX_OBJ =       (1 << 26),
-        BLOCK_IS_GLOBAL =         (1 << 28),
-        BLOCK_USE_STRET =         (1 << 29),
-        BLOCK_HAS_SIGNATURE  =    (1 << 30)
-    };
-};
+  BlockFlags() : flags(0) {}
+  BlockFlags(BlockFlag_t flag) : flags(flag) {}
 
+  uint32_t getBitMask() const { return flags; }
+  bool empty() const { return flags == 0; }
 
-class BlockModule : public BlockBase {
-  ASTContext &Context;
-  llvm::Module &TheModule;
-  const llvm::TargetData &TheTargetData;
-  CodeGenTypes &Types;
-  CodeGenModule &CGM;
-  llvm::LLVMContext &VMContext;
+  friend BlockFlags operator|(BlockFlags l, BlockFlags r) {
+    return BlockFlags(l.flags | r.flags);
+  }
+  friend BlockFlags &operator|=(BlockFlags &l, BlockFlags r) {
+    l.flags |= r.flags;
+    return l;
+  }
+  friend bool operator&(BlockFlags l, BlockFlags r) {
+    return (l.flags & r.flags);
+  }
+};
+inline BlockFlags operator|(BlockFlag_t l, BlockFlag_t r) {
+  return BlockFlags(l) | BlockFlags(r);
+}
 
-  ASTContext &getContext() const { return Context; }
-  llvm::Module &getModule() const { return TheModule; }
-  CodeGenTypes &getTypes() { return Types; }
-  const llvm::TargetData &getTargetData() const { return TheTargetData; }
-public:
-  int getGlobalUniqueCount() { return ++Block.GlobalUniqueCount; }
-  const llvm::Type *getBlockDescriptorType();
+enum BlockFieldFlag_t {
+  BLOCK_FIELD_IS_OBJECT   = 0x03,  /* id, NSObject, __attribute__((NSObject)),
+                                    block, ... */
+  BLOCK_FIELD_IS_BLOCK    = 0x07,  /* a block variable */
 
-  const llvm::Type *getGenericBlockLiteralType();
+  BLOCK_FIELD_IS_BYREF    = 0x08,  /* the on stack structure holding the __block
+                                    variable */
+  BLOCK_FIELD_IS_WEAK     = 0x10,  /* declared __weak, only used in byref copy
+                                    helpers */
 
-  llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *);
+  BLOCK_BYREF_CALLER      = 128,   /* called from __block (byref) copy/dispose
+                                      support routines */
+  BLOCK_BYREF_CURRENT_MAX = 256
+};
 
-  const llvm::Type *BlockDescriptorType;
-  const llvm::Type *GenericBlockLiteralType;
+class BlockFieldFlags {
+  uint32_t flags;
 
-  struct {
-    int GlobalUniqueCount;
-  } Block;
+  BlockFieldFlags(uint32_t flags) : flags(flags) {}
+public:
+  BlockFieldFlags() : flags(0) {}
+  BlockFieldFlags(BlockFieldFlag_t flag) : flags(flag) {}
 
-  const llvm::PointerType *PtrToInt8Ty;
+  uint32_t getBitMask() const { return flags; }
+  bool empty() const { return flags == 0; }
 
-  std::map<uint64_t, llvm::Constant *> AssignCache;
-  std::map<uint64_t, llvm::Constant *> DestroyCache;
+  /// Answers whether the flags indicate that this field is an object
+  /// or block pointer that requires _Block_object_assign/dispose.
+  bool isSpecialPointer() const { return flags & BLOCK_FIELD_IS_OBJECT; }
 
-  BlockModule(ASTContext &C, llvm::Module &M, const llvm::TargetData &TD,
-              CodeGenTypes &T, CodeGenModule &CodeGen)
-    : Context(C), TheModule(M), TheTargetData(TD), Types(T),
-      CGM(CodeGen), VMContext(M.getContext()),
-      BlockDescriptorType(0), GenericBlockLiteralType(0) {
-    Block.GlobalUniqueCount = 0;
-    PtrToInt8Ty = llvm::Type::getInt8PtrTy(M.getContext());
+  friend BlockFieldFlags operator|(BlockFieldFlags l, BlockFieldFlags r) {
+    return BlockFieldFlags(l.flags | r.flags);
+  }
+  friend BlockFieldFlags &operator|=(BlockFieldFlags &l, BlockFieldFlags r) {
+    l.flags |= r.flags;
+    return l;
+  }
+  friend bool operator&(BlockFieldFlags l, BlockFieldFlags r) {
+    return (l.flags & r.flags);
   }
-
-  bool BlockRequiresCopying(QualType Ty)
-    { return getContext().BlockRequiresCopying(Ty); }
 };
+inline BlockFieldFlags operator|(BlockFieldFlag_t l, BlockFieldFlag_t r) {
+  return BlockFieldFlags(l) | BlockFieldFlags(r);
+}
 
-class BlockFunction : public BlockBase {
-  CodeGenModule &CGM;
-  ASTContext &getContext() const;
-
-protected:
-  llvm::LLVMContext &VMContext;
-
+/// CGBlockInfo - Information to generate a block literal.
+class CGBlockInfo {
 public:
-  CodeGenFunction &CGF;
-
-  const llvm::PointerType *PtrToInt8Ty;
-  struct HelperInfo {
-    int index;
-    int flag;
-    bool RequiresCopying;
+  /// Name - The name of the block, kindof.
+  const char *Name;
+
+  /// The field index of 'this' within the block, if there is one.
+  unsigned CXXThisIndex;
+
+  class Capture {
+    uintptr_t Data;
+
+  public:
+    bool isIndex() const { return (Data & 1) != 0; }
+    bool isConstant() const { return !isIndex(); }
+    unsigned getIndex() const { assert(isIndex()); return Data >> 1; }
+    llvm::Value *getConstant() const {
+      assert(isConstant());
+      return reinterpret_cast<llvm::Value*>(Data);
+    }
+
+    static Capture makeIndex(unsigned index) {
+      Capture v;
+      v.Data = (index << 1) | 1;
+      return v;
+    }
+
+    static Capture makeConstant(llvm::Value *value) {
+      Capture v;
+      v.Data = reinterpret_cast<uintptr_t>(value);
+      return v;
+    }    
   };
 
-  enum {
-    BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
-                                      block, ... */
-    BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
-    BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the __block
-                                      variable */
-    BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
-                                      helpers */
-    BLOCK_BYREF_CALLER      = 128,  /* called from __block (byref) copy/dispose
-                                      support routines */
-    BLOCK_BYREF_CURRENT_MAX = 256
-  };
+  /// The mapping of allocated indexes within the block.
+  llvm::DenseMap<const VarDecl*, Capture> Captures;  
+
+  /// CanBeGlobal - True if the block can be global, i.e. it has
+  /// no non-constant captures.
+  bool CanBeGlobal : 1;
 
-  CGBuilderTy &Builder;
+  /// True if the block needs a custom copy or dispose function.
+  bool NeedsCopyDispose : 1;
 
-  BlockFunction(CodeGenModule &cgm, CodeGenFunction &cgf, CGBuilderTy &B);
+  /// HasCXXObject - True if the block's custom copy/dispose functions
+  /// need to be run even in GC mode.
+  bool HasCXXObject : 1;
 
-  /// BlockOffset - The offset in bytes for the next allocation of an
-  /// imported block variable.
-  CharUnits BlockOffset;
-  /// BlockAlign - Maximal alignment needed for the Block expressed in 
-  /// characters.
+  const llvm::StructType *StructureType;
+  const BlockExpr *Block;
+  CharUnits BlockSize;
   CharUnits BlockAlign;
+  llvm::SmallVector<const Expr*, 8> BlockLayout;
+
+  const Capture &getCapture(const VarDecl *var) const {
+    llvm::DenseMap<const VarDecl*, Capture>::const_iterator
+      it = Captures.find(var);
+    assert(it != Captures.end() && "no entry for variable!");
+    return it->second;
+  }
+
+  const BlockDecl *getBlockDecl() const { return Block->getBlockDecl(); }
+  const BlockExpr *getBlockExpr() const { return Block; }
 
-  /// getBlockOffset - Allocate a location within the block's storage
-  /// for a value with the given size and alignment requirements.
-  CharUnits getBlockOffset(CharUnits Size, CharUnits Align);
-
-  /// BlockHasCopyDispose - True iff the block uses copy/dispose.
-  bool BlockHasCopyDispose;
-
-  /// BlockLayout - The layout of the block's storage, represented as
-  /// a sequence of expressions which require such storage.  The
-  /// expressions can be:
-  /// - a BlockDeclRefExpr, indicating that the given declaration
-  ///   from an enclosing scope is needed by the block;
-  /// - a DeclRefExpr, which always wraps an anonymous VarDecl with
-  ///   array type, used to insert padding into the block; or
-  /// - a CXXThisExpr, indicating that the C++ 'this' value should
-  ///   propagate from the parent to the block.
-  /// This is a really silly representation.
-  llvm::SmallVector<const Expr *, 8> BlockLayout;
-
-  /// BlockDecls - Offsets for all Decls in BlockDeclRefExprs.
-  llvm::DenseMap<const Decl*, CharUnits> BlockDecls;
-  
-  /// BlockCXXThisOffset - The offset of the C++ 'this' value within
-  /// the block structure.
-  CharUnits BlockCXXThisOffset;
-
-  ImplicitParamDecl *BlockStructDecl;
-  ImplicitParamDecl *getBlockStructDecl() { return BlockStructDecl; }
-
-  llvm::Constant *GenerateCopyHelperFunction(bool, const llvm::StructType *,
-                                             std::vector<HelperInfo> *);
-  llvm::Constant *GenerateDestroyHelperFunction(bool, const llvm::StructType *,
-                                                std::vector<HelperInfo> *);
-
-  llvm::Constant *BuildCopyHelper(const llvm::StructType *,
-                                  std::vector<HelperInfo> *);
-  llvm::Constant *BuildDestroyHelper(const llvm::StructType *,
-                                     std::vector<HelperInfo> *);
-
-  llvm::Constant *GeneratebyrefCopyHelperFunction(const llvm::Type *, int flag);
-  llvm::Constant *GeneratebyrefDestroyHelperFunction(const llvm::Type *T, int);
-
-  llvm::Constant *BuildbyrefCopyHelper(const llvm::Type *T, int flag,
-                                       unsigned Align);
-  llvm::Constant *BuildbyrefDestroyHelper(const llvm::Type *T, int flag,
-                                          unsigned Align);
-
-  void BuildBlockRelease(llvm::Value *DeclPtr, int flag = BLOCK_FIELD_IS_BYREF);
-
-  bool BlockRequiresCopying(QualType Ty)
-    { return getContext().BlockRequiresCopying(Ty); }
+  CGBlockInfo(const BlockExpr *blockExpr, const char *Name);
 };
 
 }  // end namespace CodeGen
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 986f621f64f5..5eeb605f18ee 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -30,40 +30,39 @@ static void EmitMemoryBarrier(CodeGenFunction &CGF,
                               bool LoadLoad, bool LoadStore,
                               bool StoreLoad, bool StoreStore,
                               bool Device) {
-  Value *True = llvm::ConstantInt::getTrue(CGF.getLLVMContext());
-  Value *False = llvm::ConstantInt::getFalse(CGF.getLLVMContext());
+  Value *True = CGF.Builder.getTrue();
+  Value *False = CGF.Builder.getFalse();
   Value *C[5] = { LoadLoad ? True : False,
                   LoadStore ? True : False,
                   StoreLoad ? True : False,
-                  StoreStore  ? True : False,
+                  StoreStore ? True : False,
                   Device ? True : False };
   CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(Intrinsic::memory_barrier),
                          C, C + 5);
 }
 
-static Value *EmitCastToInt(CodeGenFunction &CGF,
-                            const llvm::Type *ToType, Value *Val) {
-  if (Val->getType()->isPointerTy()) {
-    return CGF.Builder.CreatePtrToInt(Val, ToType);
-  }
-  assert(Val->getType()->isIntegerTy() &&
-         "Used a non-integer and non-pointer type with atomic builtin");
-  assert(Val->getType()->getScalarSizeInBits() <=
-         ToType->getScalarSizeInBits() && "Integer type too small");
-  return CGF.Builder.CreateSExtOrBitCast(Val, ToType);
+/// Emit the conversions required to turn the given value into an
+/// integer of the given size.
+static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
+                        QualType T, const llvm::IntegerType *IntType) {
+  V = CGF.EmitToMemory(V, T);
+
+  if (V->getType()->isPointerTy())
+    return CGF.Builder.CreatePtrToInt(V, IntType);
+
+  assert(V->getType() == IntType);
+  return V;
 }
 
-static Value *EmitCastFromInt(CodeGenFunction &CGF, QualType ToQualType,
-                              Value *Val) {
-  const llvm::Type *ToType = CGF.ConvertType(ToQualType);
-  if (ToType->isPointerTy()) {
-    return CGF.Builder.CreateIntToPtr(Val, ToType);
-  }
-  assert(Val->getType()->isIntegerTy() &&
-         "Used a non-integer and non-pointer type with atomic builtin");
-  assert(Val->getType()->getScalarSizeInBits() >=
-         ToType->getScalarSizeInBits() && "Integer type too small");
-  return CGF.Builder.CreateTruncOrBitCast(Val, ToType);
+static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
+                          QualType T, const llvm::Type *ResultType) {
+  V = CGF.EmitFromMemory(V, T);
+
+  if (ResultType->isPointerTy())
+    return CGF.Builder.CreateIntToPtr(V, ResultType);
+
+  assert(V->getType() == ResultType);
+  return V;
 }
 
 // The atomic builtins are also full memory barriers. This is a utility for
@@ -85,43 +84,69 @@ static Value *EmitCallWithBarrier(CodeGenFunction &CGF, Value *Fn,
 /// and the expression node.
 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
                                Intrinsic::ID Id, const CallExpr *E) {
-  const llvm::Type *ValueType =
+  QualType T = E->getType();
+  assert(E->getArg(0)->getType()->isPointerType());
+  assert(CGF.getContext().hasSameUnqualifiedType(T,
+                                  E->getArg(0)->getType()->getPointeeType()));
+  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
+
+  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
+  unsigned AddrSpace =
+    cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+
+  const llvm::IntegerType *IntType =
     llvm::IntegerType::get(CGF.getLLVMContext(),
-                           CGF.getContext().getTypeSize(E->getType()));
-  const llvm::Type *PtrType = ValueType->getPointerTo();
-  const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
-  Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
-
-  Value *Args[2] = { CGF.Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
-                                               PtrType),
-                     EmitCastToInt(CGF, ValueType,
-                                   CGF.EmitScalarExpr(E->getArg(1))) };
-  return RValue::get(EmitCastFromInt(CGF, E->getType(),
-                                     EmitCallWithBarrier(CGF, AtomF, Args,
-                                                         Args + 2)));
+                           CGF.getContext().getTypeSize(T));
+  const llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
+
+  const llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
+  llvm::Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
+
+  llvm::Value *Args[2];
+  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
+  Args[1] = CGF.EmitScalarExpr(E->getArg(1));
+  const llvm::Type *ValueType = Args[1]->getType();
+  Args[1] = EmitToInt(CGF, Args[1], T, IntType);
+
+  llvm::Value *Result = EmitCallWithBarrier(CGF, AtomF, Args, Args + 2);
+  Result = EmitFromInt(CGF, Result, T, ValueType);
+  return RValue::get(Result);
 }
 
 /// Utility to insert an atomic instruction based Instrinsic::ID and
-// the expression node, where the return value is the result of the
-// operation.
+/// the expression node, where the return value is the result of the
+/// operation.
 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
                                    Intrinsic::ID Id, const CallExpr *E,
                                    Instruction::BinaryOps Op) {
-  const llvm::Type *ValueType =
+  QualType T = E->getType();
+  assert(E->getArg(0)->getType()->isPointerType());
+  assert(CGF.getContext().hasSameUnqualifiedType(T,
+                                  E->getArg(0)->getType()->getPointeeType()));
+  assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
+
+  llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
+  unsigned AddrSpace =
+    cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+
+  const llvm::IntegerType *IntType =
     llvm::IntegerType::get(CGF.getLLVMContext(),
-                           CGF.getContext().getTypeSize(E->getType()));
-  const llvm::Type *PtrType = ValueType->getPointerTo();
-  const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
-  Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
-
-  Value *Args[2] = { CGF.Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
-                                               PtrType),
-                     EmitCastToInt(CGF, ValueType,
-                                   CGF.EmitScalarExpr(E->getArg(1))) };
-  Value *Result = EmitCallWithBarrier(CGF, AtomF, Args, Args + 2);
-  return RValue::get(EmitCastFromInt(CGF, E->getType(),
-                                     CGF.Builder.CreateBinOp(Op, Result,
-                                                             Args[1])));
+                           CGF.getContext().getTypeSize(T));
+  const llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
+
+  const llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
+  llvm::Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
+
+  llvm::Value *Args[2];
+  Args[1] = CGF.EmitScalarExpr(E->getArg(1));
+  const llvm::Type *ValueType = Args[1]->getType();
+  Args[1] = EmitToInt(CGF, Args[1], T, IntType);
+  Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
+
+  llvm::Value *Result = EmitCallWithBarrier(CGF, AtomF, Args, Args + 2);
+  Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
+  Result = EmitFromInt(CGF, Result, T, ValueType);
+  return RValue::get(Result);
 }
 
 /// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy,
@@ -153,10 +178,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   Expr::EvalResult Result;
   if (E->Evaluate(Result, CGM.getContext())) {
     if (Result.Val.isInt())
-      return RValue::get(llvm::ConstantInt::get(VMContext,
+      return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
                                                 Result.Val.getInt()));
-    else if (Result.Val.isFloat())
-      return RValue::get(ConstantFP::get(VMContext, Result.Val.getFloat()));
+    if (Result.Val.isFloat())
+      return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
+                                               Result.Val.getFloat()));
   }
 
   switch (BuiltinID) {
@@ -168,7 +194,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__builtin_va_start:
   case Builtin::BI__builtin_va_end: {
     Value *ArgValue = EmitVAListRef(E->getArg(0));
-    const llvm::Type *DestType = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::Type *DestType = Int8PtrTy;
     if (ArgValue->getType() != DestType)
       ArgValue = Builder.CreateBitCast(ArgValue, DestType,
                                        ArgValue->getName().data());
@@ -181,7 +207,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *DstPtr = EmitVAListRef(E->getArg(0));
     Value *SrcPtr = EmitVAListRef(E->getArg(1));
 
-    const llvm::Type *Type = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::Type *Type = Int8PtrTy;
 
     DstPtr = Builder.CreateBitCast(DstPtr, Type);
     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
@@ -310,7 +336,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
     assert(CI);
     uint64_t val = CI->getZExtValue();
-    CI = ConstantInt::get(llvm::Type::getInt1Ty(VMContext), (val & 0x2) >> 1);    
+    CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);    
     
     Value *F = CGM.getIntrinsic(Intrinsic::objectsize, ResType, 1);
     return RValue::get(Builder.CreateCall2(F,
@@ -332,11 +358,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     return RValue::get(Builder.CreateCall(F));
   }
   case Builtin::BI__builtin_unreachable: {
-    if (CatchUndefined && HaveInsertPoint())
+    if (CatchUndefined)
       EmitBranch(getTrapBB());
-    Value *V = Builder.CreateUnreachable();
-    Builder.ClearInsertionPoint();
-    return RValue::get(V);
+    else
+      Builder.CreateUnreachable();
+
+    // We do need to preserve an insertion point.
+    EmitBlock(createBasicBlock("unreachable.cont"));
+
+    return RValue::get(0);
   }
       
   case Builtin::BI__builtin_powi:
@@ -495,18 +525,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BIalloca:
   case Builtin::BI__builtin_alloca: {
     Value *Size = EmitScalarExpr(E->getArg(0));
-    return RValue::get(Builder.CreateAlloca(llvm::Type::getInt8Ty(VMContext), Size, "tmp"));
+    return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size, "tmp"));
   }
   case Builtin::BIbzero:
   case Builtin::BI__builtin_bzero: {
     Value *Address = EmitScalarExpr(E->getArg(0));
     Value *SizeVal = EmitScalarExpr(E->getArg(1));
-    Builder.CreateCall5(CGM.getMemSetFn(Address->getType(), SizeVal->getType()),
-                   Address,
-                   llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext), 0),
-                   SizeVal,
-                   llvm::ConstantInt::get(Int32Ty, 1),
-                   llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), 0));
+    Builder.CreateMemSet(Address, Builder.getInt8(0), SizeVal, 1, false);
     return RValue::get(Address);
   }
   case Builtin::BImemcpy:
@@ -514,11 +539,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Address = EmitScalarExpr(E->getArg(0));
     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    Builder.CreateCall5(CGM.getMemCpyFn(Address->getType(), SrcAddr->getType(),
-                                        SizeVal->getType()),
-                  Address, SrcAddr, SizeVal, 
-                  llvm::ConstantInt::get(Int32Ty, 1),
-                  llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), 0));
+    Builder.CreateMemCpy(Address, SrcAddr, SizeVal, 1, false);
     return RValue::get(Address);
   }
       
@@ -536,24 +557,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *Address = EmitScalarExpr(E->getArg(0));
     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    Builder.CreateCall5(CGM.getMemMoveFn(Address->getType(), SrcAddr->getType(),
-                                         SizeVal->getType()),
-                  Address, SrcAddr, SizeVal, 
-                  llvm::ConstantInt::get(Int32Ty, 1),
-                  llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), 0));
+    Builder.CreateMemMove(Address, SrcAddr, SizeVal, 1, false);
     return RValue::get(Address);
   }
   case Builtin::BImemset:
   case Builtin::BI__builtin_memset: {
     Value *Address = EmitScalarExpr(E->getArg(0));
+    Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
+                                         Builder.getInt8Ty());
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    Builder.CreateCall5(CGM.getMemSetFn(Address->getType(), SizeVal->getType()),
-                  Address,
-                  Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
-                                      llvm::Type::getInt8Ty(VMContext)),
-                  SizeVal,
-                  llvm::ConstantInt::get(Int32Ty, 1),
-                  llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), 0));
+    Builder.CreateMemSet(Address, ByteVal, SizeVal, 1, false);
     return RValue::get(Address);
   }
   case Builtin::BI__builtin_dwarf_cfa: {
@@ -621,9 +634,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
                                   : Intrinsic::eh_return_i64,
                                 0, 0);
     Builder.CreateCall2(F, Int, Ptr);
-    Value *V = Builder.CreateUnreachable();
-    Builder.ClearInsertionPoint();
-    return RValue::get(V);
+    Builder.CreateUnreachable();
+
+    // We do need to preserve an insertion point.
+    EmitBlock(createBasicBlock("builtin_eh_return.cont"));
+
+    return RValue::get(0);
   }
   case Builtin::BI__builtin_unwind_init: {
     Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init, 0, 0);
@@ -640,11 +656,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     //
     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
 
-    LLVMContext &C = CGM.getLLVMContext();
-
     // Cast the pointer to intptr_t.
     Value *Ptr = EmitScalarExpr(E->getArg(0));
-    const llvm::IntegerType *IntPtrTy = CGM.getTargetData().getIntPtrType(C);
     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
 
     // If that's 64 bits, we're done.
@@ -676,20 +689,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
     // Call LLVM's EH setjmp, which is lightweight.
     Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
-    Buf = Builder.CreateBitCast(Buf, llvm::Type::getInt8PtrTy(VMContext));
+    Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
     return RValue::get(Builder.CreateCall(F, Buf));
   }
   case Builtin::BI__builtin_longjmp: {
     Value *Buf = EmitScalarExpr(E->getArg(0));
-    Buf = Builder.CreateBitCast(Buf, llvm::Type::getInt8PtrTy(VMContext));
+    Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
 
     // Call LLVM's EH longjmp, which is lightweight.
     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
 
-    // longjmp doesn't return; mark this as unreachable
-    Value *V = Builder.CreateUnreachable();
-    Builder.ClearInsertionPoint();
-    return RValue::get(V);
+    // longjmp doesn't return; mark this as unreachable.
+    Builder.CreateUnreachable();
+
+    // We do need to preserve an insertion point.
+    EmitBlock(createBasicBlock("longjmp.cont"));
+
+    return RValue::get(0);
   }
   case Builtin::BI__sync_fetch_and_add:
   case Builtin::BI__sync_fetch_and_sub:
@@ -788,23 +804,29 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_val_compare_and_swap_4:
   case Builtin::BI__sync_val_compare_and_swap_8:
   case Builtin::BI__sync_val_compare_and_swap_16: {
-    const llvm::Type *ValueType =
-      llvm::IntegerType::get(CGF.getLLVMContext(),
-                             CGF.getContext().getTypeSize(E->getType()));
-    const llvm::Type *PtrType = ValueType->getPointerTo();
-    const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+    QualType T = E->getType();
+    llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
+    unsigned AddrSpace =
+      cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+    
+    const llvm::IntegerType *IntType =
+      llvm::IntegerType::get(getLLVMContext(),
+                             getContext().getTypeSize(T));
+    const llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
+    const llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
     Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
                                     IntrinsicTypes, 2);
 
-    Value *Args[3] = { Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
-                                             PtrType),
-                       EmitCastToInt(CGF, ValueType,
-                                     CGF.EmitScalarExpr(E->getArg(1))),
-                       EmitCastToInt(CGF, ValueType,
-                                     CGF.EmitScalarExpr(E->getArg(2))) };
-    return RValue::get(EmitCastFromInt(CGF, E->getType(),
-                                       EmitCallWithBarrier(CGF, AtomF, Args,
-                                                           Args + 3)));
+    Value *Args[3];
+    Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
+    Args[1] = EmitScalarExpr(E->getArg(1));
+    const llvm::Type *ValueType = Args[1]->getType();
+    Args[1] = EmitToInt(*this, Args[1], T, IntType);
+    Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
+
+    Value *Result = EmitCallWithBarrier(*this, AtomF, Args, Args + 3);
+    Result = EmitFromInt(*this, Result, T, ValueType);
+    return RValue::get(Result);
   }
 
   case Builtin::BI__sync_bool_compare_and_swap_1:
@@ -812,26 +834,30 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_bool_compare_and_swap_4:
   case Builtin::BI__sync_bool_compare_and_swap_8:
   case Builtin::BI__sync_bool_compare_and_swap_16: {
-    const llvm::Type *ValueType =
-      llvm::IntegerType::get(
-        CGF.getLLVMContext(),
-        CGF.getContext().getTypeSize(E->getArg(1)->getType()));
-    const llvm::Type *PtrType = ValueType->getPointerTo();
-    const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+    QualType T = E->getArg(1)->getType();
+    llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
+    unsigned AddrSpace =
+      cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+    
+    const llvm::IntegerType *IntType =
+      llvm::IntegerType::get(getLLVMContext(),
+                             getContext().getTypeSize(T));
+    const llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
+    const llvm::Type *IntrinsicTypes[2] = { IntType, IntPtrType };
     Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
                                     IntrinsicTypes, 2);
 
-    Value *Args[3] = { Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
-                                             PtrType),
-                       EmitCastToInt(CGF, ValueType,
-                                     CGF.EmitScalarExpr(E->getArg(1))),
-                       EmitCastToInt(CGF, ValueType,
-                                     CGF.EmitScalarExpr(E->getArg(2))) };
+    Value *Args[3];
+    Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
+    Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType);
+    Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
+
     Value *OldVal = Args[1];
     Value *PrevVal = EmitCallWithBarrier(*this, AtomF, Args, Args + 3);
     Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal);
     // zext bool to int.
-    return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
+    Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
+    return RValue::get(Result);
   }
 
   case Builtin::BI__sync_lock_test_and_set_1:
@@ -935,11 +961,30 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   if (IntrinsicID != Intrinsic::not_intrinsic) {
     SmallVector<Value*, 16> Args;
 
+    // Find out if any arguments are required to be integer constant
+    // expressions.
+    unsigned ICEArguments = 0;
+    ASTContext::GetBuiltinTypeError Error;
+    getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
+    assert(Error == ASTContext::GE_None && "Should not codegen an error");
+
     Function *F = CGM.getIntrinsic(IntrinsicID);
     const llvm::FunctionType *FTy = F->getFunctionType();
 
     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
-      Value *ArgValue = EmitScalarExpr(E->getArg(i));
+      Value *ArgValue;
+      // If this is a normal argument, just emit it as a scalar.
+      if ((ICEArguments & (1 << i)) == 0) {
+        ArgValue = EmitScalarExpr(E->getArg(i));
+      } else {
+        // If this is required to be a constant, constant fold it so that we 
+        // know that the generated intrinsic gets a ConstantInt.
+        llvm::APSInt Result;
+        bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
+        assert(IsConst && "Constant arg isn't actually constant?");
+        (void)IsConst;
+        ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
+      }
 
       // If the intrinsic arg type is different from the builtin arg type
       // we need to do a bit cast.
@@ -956,7 +1001,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *V = Builder.CreateCall(F, Args.data(), Args.data() + Args.size());
     QualType BuiltinRetType = E->getType();
 
-    const llvm::Type *RetTy = llvm::Type::getVoidTy(VMContext);
+    const llvm::Type *RetTy = llvm::Type::getVoidTy(getLLVMContext());
     if (!BuiltinRetType->isVoidType()) RetTy = ConvertType(BuiltinRetType);
 
     if (RetTy != V->getType()) {
@@ -997,7 +1042,8 @@ Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
   }
 }
 
-const llvm::VectorType *GetNeonType(LLVMContext &C, unsigned type, bool q) {
+static const llvm::VectorType *GetNeonType(LLVMContext &C, unsigned type,
+                                           bool q) {
   switch (type) {
     default: break;
     case 0: 
@@ -1012,17 +1058,15 @@ const llvm::VectorType *GetNeonType(LLVMContext &C, unsigned type, bool q) {
   return 0;
 }
 
-Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C, bool widen) {
+Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
   unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
-  if (widen)
-    nElts <<= 1;
   SmallVector<Constant*, 16> Indices(nElts, C);
-  Value* SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+  Value* SV = llvm::ConstantVector::get(Indices);
   return Builder.CreateShuffleVector(V, V, SV, "lane");
 }
 
 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
-                                     const char *name, bool splat,
+                                     const char *name,
                                      unsigned shift, bool rightshift) {
   unsigned j = 0;
   for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
@@ -1032,10 +1076,6 @@ Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
     else
       Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
 
-  if (splat) {
-    Ops[j-1] = EmitNeonSplat(Ops[j-1], cast<Constant>(Ops[j]));
-    Ops.resize(j);
-  }
   return Builder.CreateCall(F, Ops.begin(), Ops.end(), name);
 }
 
@@ -1047,7 +1087,7 @@ Value *CodeGenFunction::EmitNeonShiftVector(Value *V, const llvm::Type *Ty,
   const llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
   llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
   SmallVector<llvm::Constant*, 16> CV(VTy->getNumElements(), C);
-  return llvm::ConstantVector::get(CV.begin(), CV.size());
+  return llvm::ConstantVector::get(CV);
 }
 
 /// GetPointeeAlignment - Given an expression with a pointer type, find the
@@ -1099,9 +1139,9 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     // Determine the overloaded type of this builtin.
     const llvm::Type *Ty;
     if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
-      Ty = llvm::Type::getFloatTy(VMContext);
+      Ty = llvm::Type::getFloatTy(getLLVMContext());
     else
-      Ty = llvm::Type::getDoubleTy(VMContext);
+      Ty = llvm::Type::getDoubleTy(getLLVMContext());
     
     // Determine whether this is an unsigned conversion or not.
     bool usgn = Result.getZExtValue() == 1;
@@ -1117,9 +1157,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   bool usgn = type & 0x08;
   bool quad = type & 0x10;
   bool poly = (type & 0x7) == 5 || (type & 0x7) == 6;
-  bool splat = false;
+  (void)poly;  // Only used in assert()s.
+  bool rightShift = false;
 
-  const llvm::VectorType *VTy = GetNeonType(VMContext, type & 0x7, quad);
+  const llvm::VectorType *VTy = GetNeonType(getLLVMContext(), type & 0x7, quad);
   const llvm::Type *Ty = VTy;
   if (!Ty)
     return 0;
@@ -1127,37 +1168,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   unsigned Int;
   switch (BuiltinID) {
   default: return 0;
-  case ARM::BI__builtin_neon_vaba_v:
-  case ARM::BI__builtin_neon_vabaq_v: {
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    SmallVector<Value*, 2> Args;
-    Args.push_back(Ops[1]);
-    Args.push_back(Ops[2]);
-    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
-    Ops[1] = EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Args, "vaba");
-    return Builder.CreateAdd(Ops[0], Ops[1], "vaba");
-  }
-  case ARM::BI__builtin_neon_vabal_v: {
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    SmallVector<Value*, 2> Args;
-    Args.push_back(Ops[1]);
-    Args.push_back(Ops[2]);
-    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[1] = EmitNeonCall(CGM.getIntrinsic(Int, &DTy, 1), Args, "vabal");
-    Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    return Builder.CreateAdd(Ops[0], Ops[1], "vabal");
-  }
   case ARM::BI__builtin_neon_vabd_v:
   case ARM::BI__builtin_neon_vabdq_v:
     Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vabd");
-  case ARM::BI__builtin_neon_vabdl_v: {
-    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, &DTy, 1), Ops, "vabdl");
-    return Builder.CreateZExt(Ops[0], Ty, "vabdl");
-  }
   case ARM::BI__builtin_neon_vabs_v:
   case ARM::BI__builtin_neon_vabsq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vabs, &Ty, 1),
@@ -1165,51 +1179,28 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vaddhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vaddhn, &Ty, 1),
                         Ops, "vaddhn");
-  case ARM::BI__builtin_neon_vaddl_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    if (usgn) {
-      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    } else {
-      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-    }
-    return Builder.CreateAdd(Ops[0], Ops[1], "vaddl");
-  }
-  case ARM::BI__builtin_neon_vaddw_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    if (usgn)
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    else
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-    return Builder.CreateAdd(Ops[0], Ops[1], "vaddw");
-  }
   case ARM::BI__builtin_neon_vcale_v:
     std::swap(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vcage_v: {
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacged, &Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacged);
     return EmitNeonCall(F, Ops, "vcage");
   }
   case ARM::BI__builtin_neon_vcaleq_v:
     std::swap(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vcageq_v: {
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgeq, &Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgeq);
     return EmitNeonCall(F, Ops, "vcage");
   }
   case ARM::BI__builtin_neon_vcalt_v:
     std::swap(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vcagt_v: {
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtd, &Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtd);
     return EmitNeonCall(F, Ops, "vcagt");
   }
   case ARM::BI__builtin_neon_vcaltq_v:
     std::swap(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vcagtq_v: {
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtq, &Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vacgtq);
     return EmitNeonCall(F, Ops, "vcagt");
   }
   case ARM::BI__builtin_neon_vcls_v:
@@ -1227,11 +1218,20 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcnt, &Ty, 1);
     return EmitNeonCall(F, Ops, "vcnt");
   }
-  // FIXME: intrinsics for f16<->f32 convert missing from ARM target.
+  case ARM::BI__builtin_neon_vcvt_f16_v: {
+    assert((type & 0x7) == 7 && !quad && "unexpected vcvt_f16_v builtin");
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvtfp2hf);
+    return EmitNeonCall(F, Ops, "vcvt");
+  }
+  case ARM::BI__builtin_neon_vcvt_f32_f16: {
+    assert((type & 0x7) == 7 && !quad && "unexpected vcvt_f32_f16 builtin");
+    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vcvthf2fp);
+    return EmitNeonCall(F, Ops, "vcvt");
+  }
   case ARM::BI__builtin_neon_vcvt_f32_v:
   case ARM::BI__builtin_neon_vcvtq_f32_v: {
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ty = GetNeonType(VMContext, 4, quad);
+    Ty = GetNeonType(getLLVMContext(), 4, quad);
     return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   }
@@ -1239,13 +1239,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vcvt_u32_v:
   case ARM::BI__builtin_neon_vcvtq_s32_v:
   case ARM::BI__builtin_neon_vcvtq_u32_v: {
-    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(VMContext, 4, quad));
+    Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(getLLVMContext(), 4, quad));
     return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 
                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
   }
   case ARM::BI__builtin_neon_vcvt_n_f32_v:
   case ARM::BI__builtin_neon_vcvtq_n_f32_v: {
-    const llvm::Type *Tys[2] = { GetNeonType(VMContext, 4, quad), Ty };
+    const llvm::Type *Tys[2] = { GetNeonType(getLLVMContext(), 4, quad), Ty };
     Int = usgn ? Intrinsic::arm_neon_vcvtfxu2fp : Intrinsic::arm_neon_vcvtfxs2fp;
     Function *F = CGM.getIntrinsic(Int, Tys, 2);
     return EmitNeonCall(F, Ops, "vcvt_n");
@@ -1254,28 +1254,21 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vcvt_n_u32_v:
   case ARM::BI__builtin_neon_vcvtq_n_s32_v:
   case ARM::BI__builtin_neon_vcvtq_n_u32_v: {
-    const llvm::Type *Tys[2] = { Ty, GetNeonType(VMContext, 4, quad) };
+    const llvm::Type *Tys[2] = { Ty, GetNeonType(getLLVMContext(), 4, quad) };
     Int = usgn ? Intrinsic::arm_neon_vcvtfp2fxu : Intrinsic::arm_neon_vcvtfp2fxs;
     Function *F = CGM.getIntrinsic(Int, Tys, 2);
     return EmitNeonCall(F, Ops, "vcvt_n");
   }
-  case ARM::BI__builtin_neon_vdup_lane_v: 
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    return EmitNeonSplat(Ops[0], cast<Constant>(Ops[1]));
-  case ARM::BI__builtin_neon_vdupq_lane_v:
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    return EmitNeonSplat(Ops[0], cast<Constant>(Ops[1]), true);
   case ARM::BI__builtin_neon_vext_v:
   case ARM::BI__builtin_neon_vextq_v: {
-    ConstantInt *C = dyn_cast<ConstantInt>(Ops[2]);
-    int CV = C->getSExtValue();
+    int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
     SmallVector<Constant*, 16> Indices;
     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
       Indices.push_back(ConstantInt::get(Int32Ty, i+CV));
     
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
-    Value* SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+    Value *SV = llvm::ConstantVector::get(Indices);
     return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
   }
   case ARM::BI__builtin_neon_vget_lane_i8:
@@ -1386,6 +1379,27 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld2_dup_v:
   case ARM::BI__builtin_neon_vld3_dup_v:
   case ARM::BI__builtin_neon_vld4_dup_v: {
+    // Handle 64-bit elements as a special-case.  There is no "dup" needed.
+    if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
+      switch (BuiltinID) {
+      case ARM::BI__builtin_neon_vld2_dup_v: 
+        Int = Intrinsic::arm_neon_vld2; 
+        break;
+      case ARM::BI__builtin_neon_vld3_dup_v:
+        Int = Intrinsic::arm_neon_vld2; 
+        break;
+      case ARM::BI__builtin_neon_vld4_dup_v:
+        Int = Intrinsic::arm_neon_vld2; 
+        break;
+      default: assert(0 && "unknown vld_dup intrinsic?");
+      }
+      Function *F = CGM.getIntrinsic(Int, &Ty, 1);
+      Value *Align = GetPointeeAlignment(*this, E->getArg(1));
+      Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
+      Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
+      Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+      return Builder.CreateStore(Ops[1], Ops[0]);
+    }
     switch (BuiltinID) {
     case ARM::BI__builtin_neon_vld2_dup_v: 
       Int = Intrinsic::arm_neon_vld2lane; 
@@ -1430,46 +1444,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vminq_v:
     Int = usgn ? Intrinsic::arm_neon_vminu : Intrinsic::arm_neon_vmins;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmin");
-  case ARM::BI__builtin_neon_vmlal_lane_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
-    Ops[2] = EmitNeonSplat(Ops[2], cast<Constant>(Ops[3]));
-  }
-  case ARM::BI__builtin_neon_vmlal_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
-    if (usgn) {
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-      Ops[2] = Builder.CreateZExt(Ops[2], Ty);
-    } else {
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-      Ops[2] = Builder.CreateSExt(Ops[2], Ty);
-    }
-    Ops[1] = Builder.CreateMul(Ops[1], Ops[2]);
-    return Builder.CreateAdd(Ops[0], Ops[1], "vmlal");
-  }
-  case ARM::BI__builtin_neon_vmlsl_lane_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
-    Ops[2] = EmitNeonSplat(Ops[2], cast<Constant>(Ops[3]));
-  }
-  case ARM::BI__builtin_neon_vmlsl_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
-    if (usgn) {
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-      Ops[2] = Builder.CreateZExt(Ops[2], Ty);
-    } else {
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-      Ops[2] = Builder.CreateSExt(Ops[2], Ty);
-    }
-    Ops[1] = Builder.CreateMul(Ops[1], Ops[2]);
-    return Builder.CreateSub(Ops[0], Ops[1], "vmlsl");
-  }
   case ARM::BI__builtin_neon_vmovl_v: {
     const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
     Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
@@ -1482,38 +1456,41 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
     return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
   }
-  case ARM::BI__builtin_neon_vmull_lane_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    Ops[1] = EmitNeonSplat(Ops[1], cast<Constant>(Ops[2]));
-  }
-  case ARM::BI__builtin_neon_vmull_v: {
-    if (poly)
-      return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmullp, &Ty, 1),
-                          Ops, "vmull");
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    if (usgn) {
-      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    } else {
-      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-    }
-    return Builder.CreateMul(Ops[0], Ops[1], "vmull");
-  }
+  case ARM::BI__builtin_neon_vmul_v:
+  case ARM::BI__builtin_neon_vmulq_v:
+    assert(poly && "vmul builtin only supported for polynomial types");
+    return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmulp, &Ty, 1),
+                        Ops, "vmul");
+  case ARM::BI__builtin_neon_vmull_v:
+    assert(poly && "vmull builtin only supported for polynomial types");
+    return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmullp, &Ty, 1),
+                        Ops, "vmull");
   case ARM::BI__builtin_neon_vpadal_v:
-  case ARM::BI__builtin_neon_vpadalq_v:
+  case ARM::BI__builtin_neon_vpadalq_v: {
     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vpadal");
+    // The source operand type has twice as many elements of half the size.
+    unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
+    const llvm::Type *EltTy =
+      llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
+    const llvm::Type *NarrowTy =
+      llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
+    const llvm::Type *Tys[2] = { Ty, NarrowTy };
+    return EmitNeonCall(CGM.getIntrinsic(Int, Tys, 2), Ops, "vpadal");
+  }
   case ARM::BI__builtin_neon_vpadd_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vpadd, &Ty, 1),
                         Ops, "vpadd");
   case ARM::BI__builtin_neon_vpaddl_v:
-  case ARM::BI__builtin_neon_vpaddlq_v:
+  case ARM::BI__builtin_neon_vpaddlq_v: {
     Int = usgn ? Intrinsic::arm_neon_vpaddlu : Intrinsic::arm_neon_vpaddls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vpaddl");
+    // The source operand type has twice as many elements of half the size.
+    unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
+    const llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
+    const llvm::Type *NarrowTy =
+      llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
+    const llvm::Type *Tys[2] = { Ty, NarrowTy };
+    return EmitNeonCall(CGM.getIntrinsic(Int, Tys, 2), Ops, "vpaddl");
+  }
   case ARM::BI__builtin_neon_vpmax_v:
     Int = usgn ? Intrinsic::arm_neon_vpmaxu : Intrinsic::arm_neon_vpmaxs;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vpmax");
@@ -1528,28 +1505,19 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vqaddq_v:
     Int = usgn ? Intrinsic::arm_neon_vqaddu : Intrinsic::arm_neon_vqadds;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqadd");
-  case ARM::BI__builtin_neon_vqdmlal_lane_v:
-    splat = true;
   case ARM::BI__builtin_neon_vqdmlal_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmlal, &Ty, 1),
-                        Ops, "vqdmlal", splat);
-  case ARM::BI__builtin_neon_vqdmlsl_lane_v:
-    splat = true;
+                        Ops, "vqdmlal");
   case ARM::BI__builtin_neon_vqdmlsl_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmlsl, &Ty, 1),
-                        Ops, "vqdmlsl", splat);
-  case ARM::BI__builtin_neon_vqdmulh_lane_v:
-  case ARM::BI__builtin_neon_vqdmulhq_lane_v:
-    splat = true;
+                        Ops, "vqdmlsl");
   case ARM::BI__builtin_neon_vqdmulh_v:
   case ARM::BI__builtin_neon_vqdmulhq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmulh, &Ty, 1),
-                        Ops, "vqdmulh", splat);
-  case ARM::BI__builtin_neon_vqdmull_lane_v:
-    splat = true;
+                        Ops, "vqdmulh");
   case ARM::BI__builtin_neon_vqdmull_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqdmull, &Ty, 1),
-                        Ops, "vqdmull", splat);
+                        Ops, "vqdmull");
   case ARM::BI__builtin_neon_vqmovn_v:
     Int = usgn ? Intrinsic::arm_neon_vqmovnu : Intrinsic::arm_neon_vqmovns;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqmovn");
@@ -1557,26 +1525,24 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqmovnsu, &Ty, 1),
                         Ops, "vqdmull");
   case ARM::BI__builtin_neon_vqneg_v:
+  case ARM::BI__builtin_neon_vqnegq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqneg, &Ty, 1),
                         Ops, "vqneg");
-  case ARM::BI__builtin_neon_vqrdmulh_lane_v:
-  case ARM::BI__builtin_neon_vqrdmulhq_lane_v:
-    splat = true;
   case ARM::BI__builtin_neon_vqrdmulh_v:
   case ARM::BI__builtin_neon_vqrdmulhq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrdmulh, &Ty, 1),
-                        Ops, "vqrdmulh", splat);
+                        Ops, "vqrdmulh");
   case ARM::BI__builtin_neon_vqrshl_v:
   case ARM::BI__builtin_neon_vqrshlq_v:
     Int = usgn ? Intrinsic::arm_neon_vqrshiftu : Intrinsic::arm_neon_vqrshifts;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqrshl");
   case ARM::BI__builtin_neon_vqrshrn_n_v:
     Int = usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqrshrn_n", false,
+    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqrshrn_n",
                         1, true);
   case ARM::BI__builtin_neon_vqrshrun_n_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, &Ty, 1),
-                        Ops, "vqrshrun_n", false, 1, true);
+                        Ops, "vqrshrun_n", 1, true);
   case ARM::BI__builtin_neon_vqshl_v:
   case ARM::BI__builtin_neon_vqshlq_v:
     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
@@ -1584,7 +1550,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vqshl_n_v:
   case ARM::BI__builtin_neon_vqshlq_n_v:
     Int = usgn ? Intrinsic::arm_neon_vqshiftu : Intrinsic::arm_neon_vqshifts;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqshl_n", false,
+    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqshl_n",
                         1, false);
   case ARM::BI__builtin_neon_vqshlu_n_v:
   case ARM::BI__builtin_neon_vqshluq_n_v:
@@ -1592,11 +1558,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
                         Ops, "vqshlu", 1, false);
   case ARM::BI__builtin_neon_vqshrn_n_v:
     Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqshrn_n", false,
+    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vqshrn_n",
                         1, true);
   case ARM::BI__builtin_neon_vqshrun_n_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, &Ty, 1),
-                        Ops, "vqshrun_n", false, 1, true);
+                        Ops, "vqshrun_n", 1, true);
   case ARM::BI__builtin_neon_vqsub_v:
   case ARM::BI__builtin_neon_vqsubq_v:
     Int = usgn ? Intrinsic::arm_neon_vqsubu : Intrinsic::arm_neon_vqsubs;
@@ -1622,12 +1588,11 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vrshl");
   case ARM::BI__builtin_neon_vrshrn_n_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, &Ty, 1),
-                        Ops, "vrshrn_n", false, 1, true);
+                        Ops, "vrshrn_n", 1, true);
   case ARM::BI__builtin_neon_vrshr_n_v:
   case ARM::BI__builtin_neon_vrshrq_n_v:
     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vrshr_n", false,
-                        1, true);
+    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vrshr_n", 1, true);
   case ARM::BI__builtin_neon_vrsqrte_v:
   case ARM::BI__builtin_neon_vrsqrteq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsqrte, &Ty, 1),
@@ -1665,14 +1630,14 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vshl");
   case ARM::BI__builtin_neon_vshll_n_v:
     Int = usgn ? Intrinsic::arm_neon_vshiftlu : Intrinsic::arm_neon_vshiftls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vshll", false, 1);
+    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vshll", 1);
   case ARM::BI__builtin_neon_vshl_n_v:
   case ARM::BI__builtin_neon_vshlq_n_v:
     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
     return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], "vshl_n");
   case ARM::BI__builtin_neon_vshrn_n_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftn, &Ty, 1),
-                        Ops, "vshrn_n", false, 1, true);
+                        Ops, "vshrn_n", 1, true);
   case ARM::BI__builtin_neon_vshr_n_v:
   case ARM::BI__builtin_neon_vshrq_n_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1683,10 +1648,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       return Builder.CreateAShr(Ops[0], Ops[1], "vshr_n");
   case ARM::BI__builtin_neon_vsri_n_v:
   case ARM::BI__builtin_neon_vsriq_n_v:
-    poly = true;
+    rightShift = true;
   case ARM::BI__builtin_neon_vsli_n_v:
   case ARM::BI__builtin_neon_vsliq_n_v:
-    Ops[2] = EmitNeonShiftVector(Ops[2], Ty, poly);
+    Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, &Ty, 1),
                         Ops, "vsli_n");
   case ARM::BI__builtin_neon_vsra_n_v:
@@ -1743,29 +1708,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vsubhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vsubhn, &Ty, 1),
                         Ops, "vsubhn");
-  case ARM::BI__builtin_neon_vsubl_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    if (usgn) {
-      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    } else {
-      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-    }
-    return Builder.CreateSub(Ops[0], Ops[1], "vsubl");
-  }
-  case ARM::BI__builtin_neon_vsubw_v: {
-    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
-    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
-    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
-    if (usgn)
-      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
-    else
-      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
-    return Builder.CreateSub(Ops[0], Ops[1], "vsubw");
-  }
   case ARM::BI__builtin_neon_vtbl1_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
                         Ops, "vtbl1");
@@ -1804,7 +1746,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
-    Value *SV;
+    Value *SV = 0;
 
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
@@ -1813,7 +1755,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi));
       }
       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
-      SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+      SV = llvm::ConstantVector::get(Indices);
       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
       SV = Builder.CreateStore(SV, Addr);
     }
@@ -1824,7 +1766,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
-    Value *SV;
+    Value *SV = 0;
     
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
@@ -1832,7 +1774,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
 
       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
-      SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+      SV = llvm::ConstantVector::get(Indices);
       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
       SV = Builder.CreateStore(SV, Addr);
     }
@@ -1843,7 +1785,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
-    Value *SV;
+    Value *SV = 0;
     
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
@@ -1852,7 +1794,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
       }
       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
-      SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+      SV = llvm::ConstantVector::get(Indices);
       SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
       SV = Builder.CreateStore(SV, Addr);
     }
@@ -1861,13 +1803,57 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   }
 }
 
+llvm::Value *CodeGenFunction::
+BuildVector(const llvm::SmallVectorImpl<llvm::Value*> &Ops) {
+  assert((Ops.size() & (Ops.size() - 1)) == 0 &&
+         "Not a power-of-two sized vector!");
+  bool AllConstants = true;
+  for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
+    AllConstants &= isa<Constant>(Ops[i]);
+
+  // If this is a constant vector, create a ConstantVector.
+  if (AllConstants) {
+    std::vector<llvm::Constant*> CstOps;
+    for (unsigned i = 0, e = Ops.size(); i != e; ++i)
+      CstOps.push_back(cast<Constant>(Ops[i]));
+    return llvm::ConstantVector::get(CstOps);
+  }
+
+  // Otherwise, insertelement the values to build the vector.
+  Value *Result =
+    llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
+
+  for (unsigned i = 0, e = Ops.size(); i != e; ++i)
+    Result = Builder.CreateInsertElement(Result, Ops[i],
+               llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), i));
+
+  return Result;
+}
+
 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
-
   llvm::SmallVector<Value*, 4> Ops;
 
-  for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
-    Ops.push_back(EmitScalarExpr(E->getArg(i)));
+  // Find out if any arguments are required to be integer constant expressions.
+  unsigned ICEArguments = 0;
+  ASTContext::GetBuiltinTypeError Error;
+  getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
+  assert(Error == ASTContext::GE_None && "Should not codegen an error");
+
+  for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
+    // If this is a normal argument, just emit it as a scalar.
+    if ((ICEArguments & (1 << i)) == 0) {
+      Ops.push_back(EmitScalarExpr(E->getArg(i)));
+      continue;
+    }
+
+    // If this is required to be a constant, constant fold it so that we know
+    // that the generated intrinsic gets a ConstantInt.
+    llvm::APSInt Result;
+    bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
+    assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
+    Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
+  }
 
   switch (BuiltinID) {
   default: return 0;
@@ -1926,6 +1912,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     llvm::Function *F = CGM.getIntrinsic(ID);
     return Builder.CreateCall(F, &Ops[0], &Ops[0] + Ops.size(), name);
   }
+  case X86::BI__builtin_ia32_vec_init_v8qi:
+  case X86::BI__builtin_ia32_vec_init_v4hi:
+  case X86::BI__builtin_ia32_vec_init_v2si:
+    return Builder.CreateBitCast(BuildVector(Ops),
+                                 llvm::Type::getX86_MMXTy(getLLVMContext()));
+  case X86::BI__builtin_ia32_vec_ext_v2si:
+    return Builder.CreateExtractElement(Ops[0],
+                                  llvm::ConstantInt::get(Ops[1]->getType(), 0));
   case X86::BI__builtin_ia32_pslldi:
   case X86::BI__builtin_ia32_psllqi:
   case X86::BI__builtin_ia32_psllwi:
@@ -1987,7 +1981,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     return Builder.CreateCall(F, &Ops[0], &Ops[0] + Ops.size(), "cmpss");
   }
   case X86::BI__builtin_ia32_ldmxcsr: {
-    const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::Type *PtrTy = Int8PtrTy;
     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
     Value *Tmp = Builder.CreateAlloca(Int32Ty, One, "tmp");
     Builder.CreateStore(Ops[0], Tmp);
@@ -1995,7 +1989,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
                               Builder.CreateBitCast(Tmp, PtrTy));
   }
   case X86::BI__builtin_ia32_stmxcsr: {
-    const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::Type *PtrTy = Int8PtrTy;
     Value *One = llvm::ConstantInt::get(Int32Ty, 1);
     Value *Tmp = Builder.CreateAlloca(Int32Ty, One, "tmp");
     One = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
@@ -2037,7 +2031,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
       for (unsigned i = 0; i != 8; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
       
-      Value* SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+      Value* SV = llvm::ConstantVector::get(Indices);
       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
     }
     
@@ -2068,7 +2062,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
       for (unsigned i = 0; i != 16; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
       
-      Value* SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
+      Value* SV = llvm::ConstantVector::get(Indices);
       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
     }
     
@@ -2112,7 +2106,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   case PPC::BI__builtin_altivec_lvsl:
   case PPC::BI__builtin_altivec_lvsr:
   {
-    Ops[1] = Builder.CreateBitCast(Ops[1], llvm::Type::getInt8PtrTy(VMContext));
+    Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
 
     Ops[0] = Builder.CreateGEP(Ops[1], Ops[0], "tmp");
     Ops.pop_back();
@@ -2152,7 +2146,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
   case PPC::BI__builtin_altivec_stvehx:
   case PPC::BI__builtin_altivec_stvewx:
   {
-    Ops[2] = Builder.CreateBitCast(Ops[2], llvm::Type::getInt8PtrTy(VMContext));
+    Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
     Ops[1] = Builder.CreateGEP(Ops[2], Ops[1], "tmp");
     Ops.pop_back();
 
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index 179716f631a8..7ffc6e732554 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -1,4 +1,4 @@
-//===--- CGDecl.cpp - Emit LLVM Code for declarations ---------------------===//
+//===--- CGCXX.cpp - Emit LLVM Code for declarations ----------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -16,12 +16,12 @@
 #include "CGCXXABI.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
-#include "Mangle.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/Mangle.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/ADT/StringExtras.h"
@@ -60,13 +60,12 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
     return true;
   }
 
-  // If any fields have a non-trivial destructor, we have to emit it
-  // separately.
+  // If any field has a non-trivial destructor, we have to emit the
+  // destructor separately.
   for (CXXRecordDecl::field_iterator I = Class->field_begin(),
          E = Class->field_end(); I != E; ++I)
-    if (const RecordType *RT = (*I)->getType()->getAs<RecordType>())
-      if (!cast<CXXRecordDecl>(RT->getDecl())->hasTrivialDestructor())
-        return true;
+    if ((*I)->getType().isDestructedType())
+      return true;
 
   // Try to find a unique base class with a non-trivial destructor.
   const CXXRecordDecl *UniqueBase = 0;
@@ -103,7 +102,7 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
 
   // If the base is at a non-zero offset, give up.
   const ASTRecordLayout &ClassLayout = Context.getASTRecordLayout(Class);
-  if (ClassLayout.getBaseClassOffset(UniqueBase) != 0)
+  if (ClassLayout.getBaseClassOffsetInBits(UniqueBase) != 0)
     return true;
 
   return TryEmitDefinitionAsAlias(GlobalDecl(D, Dtor_Base),
@@ -180,7 +179,7 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl,
   }
 
   // Finally, set up the alias with its proper name and attributes.
-  SetCommonAttributes(AliasDecl.getDecl(), Alias);
+  SetCommonAttributes(cast<NamedDecl>(AliasDecl.getDecl()), Alias);
 
   return false;
 }
@@ -227,7 +226,8 @@ CodeGenModule::GetAddrOfCXXConstructor(const CXXConstructorDecl *D,
   const llvm::FunctionType *FTy =
     getTypes().GetFunctionType(getTypes().getFunctionInfo(D, Type), 
                                FPT->isVariadic());
-  return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FTy, GD));
+  return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FTy, GD,
+                                                      /*ForVTable=*/false));
 }
 
 void CodeGenModule::EmitCXXDestructors(const CXXDestructorDecl *D) {
@@ -284,16 +284,15 @@ CodeGenModule::GetAddrOfCXXDestructor(const CXXDestructorDecl *D,
   const llvm::FunctionType *FTy =
     getTypes().GetFunctionType(getTypes().getFunctionInfo(D, Type), false);
 
-  return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FTy, GD));
+  return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FTy, GD,
+                                                      /*ForVTable=*/false));
 }
 
 static llvm::Value *BuildVirtualCall(CodeGenFunction &CGF, uint64_t VTableIndex, 
                                      llvm::Value *This, const llvm::Type *Ty) {
-  Ty = Ty->getPointerTo()->getPointerTo()->getPointerTo();
-  
-  llvm::Value *VTable = CGF.Builder.CreateBitCast(This, Ty);
-  VTable = CGF.Builder.CreateLoad(VTable);
+  Ty = Ty->getPointerTo()->getPointerTo();
   
+  llvm::Value *VTable = CGF.GetVTablePtr(This, Ty);
   llvm::Value *VFuncPtr = 
     CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn");
   return CGF.Builder.CreateLoad(VFuncPtr);
@@ -308,164 +307,84 @@ CodeGenFunction::BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
   return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
 }
 
+/// BuildVirtualCall - This routine is to support gcc's kext ABI making
+/// indirect call to virtual functions. It makes the call through indexing
+/// into the vtable.
 llvm::Value *
-CodeGenFunction::BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
-                                  llvm::Value *&This, const llvm::Type *Ty) {
-  DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
-  uint64_t VTableIndex = 
-    CGM.getVTables().getMethodVTableIndex(GlobalDecl(DD, Type));
-
-  return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
-}
-
-/// Implementation for CGCXXABI.  Possibly this should be moved into
-/// the incomplete ABI implementations?
-
-CGCXXABI::~CGCXXABI() {}
-
-static void ErrorUnsupportedABI(CodeGenFunction &CGF,
-                                llvm::StringRef S) {
-  Diagnostic &Diags = CGF.CGM.getDiags();
-  unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
-                                          "cannot yet compile %1 in this ABI");
-  Diags.Report(CGF.getContext().getFullLoc(CGF.CurCodeDecl->getLocation()),
-               DiagID)
-    << S;
-}
-
-static llvm::Constant *GetBogusMemberPointer(CodeGenModule &CGM,
-                                             QualType T) {
-  return llvm::Constant::getNullValue(CGM.getTypes().ConvertType(T));
-}
-
-const llvm::Type *
-CGCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
-  return CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
-}
-
-llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
-                                                       llvm::Value *&This,
-                                                       llvm::Value *MemPtr,
-                                                 const MemberPointerType *MPT) {
-  ErrorUnsupportedABI(CGF, "calls through member pointers");
-
-  const FunctionProtoType *FPT = 
-    MPT->getPointeeType()->getAs<FunctionProtoType>();
-  const CXXRecordDecl *RD = 
-    cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
-  const llvm::FunctionType *FTy = 
-    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
-                                   FPT->isVariadic());
-  return llvm::Constant::getNullValue(FTy->getPointerTo());
-}
-
-llvm::Value *CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
-                                                    llvm::Value *Base,
-                                                    llvm::Value *MemPtr,
-                                              const MemberPointerType *MPT) {
-  ErrorUnsupportedABI(CGF, "loads of member pointers");
-  const llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
-  return llvm::Constant::getNullValue(Ty);
-}
-
-llvm::Value *CGCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
-                                                   const CastExpr *E,
-                                                   llvm::Value *Src) {
-  ErrorUnsupportedABI(CGF, "member function pointer conversions");
-  return GetBogusMemberPointer(CGM, E->getType());
+CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD, 
+                                  NestedNameSpecifier *Qual,
+                                  const llvm::Type *Ty) {
+  llvm::Value *VTable = 0;
+  assert((Qual->getKind() == NestedNameSpecifier::TypeSpec) &&
+         "BuildAppleKextVirtualCall - bad Qual kind");
+  
+  const Type *QTy = Qual->getAsType();
+  QualType T = QualType(QTy, 0);
+  const RecordType *RT = T->getAs<RecordType>();
+  assert(RT && "BuildAppleKextVirtualCall - Qual type must be record");
+  const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+  
+  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(MD))
+    return BuildAppleKextVirtualDestructorCall(DD, Dtor_Complete, RD);
+  
+  VTable = CGM.getVTables().GetAddrOfVTable(RD);
+  Ty = Ty->getPointerTo()->getPointerTo();
+  VTable = Builder.CreateBitCast(VTable, Ty);
+  assert(VTable && "BuildVirtualCall = kext vtbl pointer is null");
+  MD = MD->getCanonicalDecl();
+  uint64_t VTableIndex = CGM.getVTables().getMethodVTableIndex(MD);
+  uint64_t AddressPoint = 
+    CGM.getVTables().getAddressPoint(BaseSubobject(RD, 0), RD);
+  VTableIndex += AddressPoint;
+  llvm::Value *VFuncPtr = 
+    Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt");
+  return Builder.CreateLoad(VFuncPtr);
 }
 
+/// BuildVirtualCall - This routine makes indirect vtable call for
+/// call to virtual destructors. It returns 0 if it could not do it.
 llvm::Value *
-CGCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
-                                      llvm::Value *L,
-                                      llvm::Value *R,
-                                      const MemberPointerType *MPT,
-                                      bool Inequality) {
-  ErrorUnsupportedABI(CGF, "member function pointer comparison");
-  return CGF.Builder.getFalse();
+CodeGenFunction::BuildAppleKextVirtualDestructorCall(
+                                            const CXXDestructorDecl *DD,
+                                            CXXDtorType Type,
+                                            const CXXRecordDecl *RD) {
+  llvm::Value * Callee = 0;
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(DD);
+  // FIXME. Dtor_Base dtor is always direct!!
+  // It need be somehow inline expanded into the caller.
+  // -O does that. But need to support -O0 as well.
+  if (MD->isVirtual() && Type != Dtor_Base) {
+    // Compute the function type we're calling.
+    const CGFunctionInfo *FInfo = 
+    &CGM.getTypes().getFunctionInfo(cast<CXXDestructorDecl>(MD),
+                                    Dtor_Complete);
+    const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+    const llvm::Type *Ty
+      = CGM.getTypes().GetFunctionType(*FInfo, FPT->isVariadic());
+
+    llvm::Value *VTable = CGM.getVTables().GetAddrOfVTable(RD);
+    Ty = Ty->getPointerTo()->getPointerTo();
+    VTable = Builder.CreateBitCast(VTable, Ty);
+    DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
+    uint64_t VTableIndex = 
+      CGM.getVTables().getMethodVTableIndex(GlobalDecl(DD, Type));
+    uint64_t AddressPoint =
+      CGM.getVTables().getAddressPoint(BaseSubobject(RD, 0), RD);
+    VTableIndex += AddressPoint;
+    llvm::Value *VFuncPtr =
+      Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfnkxt");
+    Callee = Builder.CreateLoad(VFuncPtr);
+  }
+  return Callee;
 }
 
 llvm::Value *
-CGCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
-                                     llvm::Value *MemPtr,
-                                     const MemberPointerType *MPT) {
-  ErrorUnsupportedABI(CGF, "member function pointer null testing");
-  return CGF.Builder.getFalse();
-}
-
-llvm::Constant *
-CGCXXABI::EmitMemberPointerConversion(llvm::Constant *C, const CastExpr *E) {
-  return GetBogusMemberPointer(CGM, E->getType());
-}
-
-llvm::Constant *
-CGCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
-  return GetBogusMemberPointer(CGM, QualType(MPT, 0));
-}
-
-llvm::Constant *CGCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
-  return GetBogusMemberPointer(CGM,
-                         CGM.getContext().getMemberPointerType(MD->getType(),
-                                         MD->getParent()->getTypeForDecl()));
-}
-
-llvm::Constant *CGCXXABI::EmitMemberPointer(const FieldDecl *FD) {
-  return GetBogusMemberPointer(CGM,
-                         CGM.getContext().getMemberPointerType(FD->getType(),
-                                         FD->getParent()->getTypeForDecl()));
-}
-
-bool CGCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
-  // Fake answer.
-  return true;
-}
-
-void CGCXXABI::BuildThisParam(CodeGenFunction &CGF, FunctionArgList &Params) {
-  const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
-
-  // FIXME: I'm not entirely sure I like using a fake decl just for code
-  // generation. Maybe we can come up with a better way?
-  ImplicitParamDecl *ThisDecl
-    = ImplicitParamDecl::Create(CGM.getContext(), 0, MD->getLocation(),
-                                &CGM.getContext().Idents.get("this"),
-                                MD->getThisType(CGM.getContext()));
-  Params.push_back(std::make_pair(ThisDecl, ThisDecl->getType()));
-  getThisDecl(CGF) = ThisDecl;
-}
-
-void CGCXXABI::EmitThisParam(CodeGenFunction &CGF) {
-  /// Initialize the 'this' slot.
-  assert(getThisDecl(CGF) && "no 'this' variable for function");
-  getThisValue(CGF)
-    = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getThisDecl(CGF)),
-                             "this");
-}
-
-void CGCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
-                                   RValue RV, QualType ResultType) {
-  CGF.EmitReturnOfRValue(RV, ResultType);
-}
-
-CharUnits CGCXXABI::GetArrayCookieSize(QualType ElementType) {
-  return CharUnits::Zero();
-}
+CodeGenFunction::BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
+                                  llvm::Value *This, const llvm::Type *Ty) {
+  DD = cast<CXXDestructorDecl>(DD->getCanonicalDecl());
+  uint64_t VTableIndex = 
+    CGM.getVTables().getMethodVTableIndex(GlobalDecl(DD, Type));
 
-llvm::Value *CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
-                                             llvm::Value *NewPtr,
-                                             llvm::Value *NumElements,
-                                             QualType ElementType) {
-  // Should never be called.
-  ErrorUnsupportedABI(CGF, "array cookie initialization");
-  return 0;
+  return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
 }
 
-void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
-                               QualType ElementType, llvm::Value *&NumElements,
-                               llvm::Value *&AllocPtr, CharUnits &CookieSize) {
-  ErrorUnsupportedABI(CGF, "array cookie reading");
-
-  // This should be enough to avoid assertions.
-  NumElements = 0;
-  AllocPtr = llvm::Constant::getNullValue(CGF.Builder.getInt8PtrTy());
-  CookieSize = CharUnits::Zero();
-}
diff --git a/lib/CodeGen/CGCXX.h b/lib/CodeGen/CGCXX.h
deleted file mode 100644
index 1e6adb05a0d9..000000000000
--- a/lib/CodeGen/CGCXX.h
+++ /dev/null
@@ -1,36 +0,0 @@
-//===----- CGCXX.h - C++ related code CodeGen declarations ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// These classes wrap the information about a call or function
-// definition used to handle ABI compliancy.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef CLANG_CODEGEN_CGCXX_H
-#define CLANG_CODEGEN_CGCXX_H
-
-namespace clang {
-
-/// CXXCtorType - C++ constructor types
-enum CXXCtorType {
-    Ctor_Complete,          // Complete object ctor
-    Ctor_Base,              // Base object ctor
-    Ctor_CompleteAllocating // Complete object allocating ctor
-};
-
-/// CXXDtorType - C++ destructor types
-enum CXXDtorType {
-    Dtor_Deleting, // Deleting dtor
-    Dtor_Complete, // Complete object dtor
-    Dtor_Base      // Base object dtor
-};
-
-} // end namespace clang
-
-#endif // CLANG_CODEGEN_CGCXX_H
diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp
new file mode 100644
index 000000000000..8373b660b2f4
--- /dev/null
+++ b/lib/CodeGen/CGCXXABI.cpp
@@ -0,0 +1,174 @@
+//===----- CGCXXABI.cpp - Interface to C++ ABIs -----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for C++ code generation. Concrete subclasses
+// of this implement code generation for specific C++ ABIs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CGCXXABI.h"
+
+using namespace clang;
+using namespace CodeGen;
+
+CGCXXABI::~CGCXXABI() { }
+
+static void ErrorUnsupportedABI(CodeGenFunction &CGF,
+                                llvm::StringRef S) {
+  Diagnostic &Diags = CGF.CGM.getDiags();
+  unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+                                          "cannot yet compile %1 in this ABI");
+  Diags.Report(CGF.getContext().getFullLoc(CGF.CurCodeDecl->getLocation()),
+               DiagID)
+    << S;
+}
+
+static llvm::Constant *GetBogusMemberPointer(CodeGenModule &CGM,
+                                             QualType T) {
+  return llvm::Constant::getNullValue(CGM.getTypes().ConvertType(T));
+}
+
+const llvm::Type *
+CGCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
+  return CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
+}
+
+llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
+                                                       llvm::Value *&This,
+                                                       llvm::Value *MemPtr,
+                                                 const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "calls through member pointers");
+
+  const FunctionProtoType *FPT = 
+    MPT->getPointeeType()->getAs<FunctionProtoType>();
+  const CXXRecordDecl *RD = 
+    cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
+  const llvm::FunctionType *FTy = 
+    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
+                                   FPT->isVariadic());
+  return llvm::Constant::getNullValue(FTy->getPointerTo());
+}
+
+llvm::Value *CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
+                                                    llvm::Value *Base,
+                                                    llvm::Value *MemPtr,
+                                              const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "loads of member pointers");
+  const llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
+  return llvm::Constant::getNullValue(Ty);
+}
+
+llvm::Value *CGCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
+                                                   const CastExpr *E,
+                                                   llvm::Value *Src) {
+  ErrorUnsupportedABI(CGF, "member function pointer conversions");
+  return GetBogusMemberPointer(CGM, E->getType());
+}
+
+llvm::Value *
+CGCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
+                                      llvm::Value *L,
+                                      llvm::Value *R,
+                                      const MemberPointerType *MPT,
+                                      bool Inequality) {
+  ErrorUnsupportedABI(CGF, "member function pointer comparison");
+  return CGF.Builder.getFalse();
+}
+
+llvm::Value *
+CGCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
+                                     llvm::Value *MemPtr,
+                                     const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "member function pointer null testing");
+  return CGF.Builder.getFalse();
+}
+
+llvm::Constant *
+CGCXXABI::EmitMemberPointerConversion(llvm::Constant *C, const CastExpr *E) {
+  return GetBogusMemberPointer(CGM, E->getType());
+}
+
+llvm::Constant *
+CGCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
+  return GetBogusMemberPointer(CGM, QualType(MPT, 0));
+}
+
+llvm::Constant *CGCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
+  return GetBogusMemberPointer(CGM,
+                         CGM.getContext().getMemberPointerType(MD->getType(),
+                                         MD->getParent()->getTypeForDecl()));
+}
+
+llvm::Constant *CGCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
+                                                CharUnits offset) {
+  return GetBogusMemberPointer(CGM, QualType(MPT, 0));
+}
+
+bool CGCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
+  // Fake answer.
+  return true;
+}
+
+void CGCXXABI::BuildThisParam(CodeGenFunction &CGF, FunctionArgList &Params) {
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
+
+  // FIXME: I'm not entirely sure I like using a fake decl just for code
+  // generation. Maybe we can come up with a better way?
+  ImplicitParamDecl *ThisDecl
+    = ImplicitParamDecl::Create(CGM.getContext(), 0, MD->getLocation(),
+                                &CGM.getContext().Idents.get("this"),
+                                MD->getThisType(CGM.getContext()));
+  Params.push_back(std::make_pair(ThisDecl, ThisDecl->getType()));
+  getThisDecl(CGF) = ThisDecl;
+}
+
+void CGCXXABI::EmitThisParam(CodeGenFunction &CGF) {
+  /// Initialize the 'this' slot.
+  assert(getThisDecl(CGF) && "no 'this' variable for function");
+  getThisValue(CGF)
+    = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getThisDecl(CGF)),
+                             "this");
+}
+
+void CGCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
+                                   RValue RV, QualType ResultType) {
+  CGF.EmitReturnOfRValue(RV, ResultType);
+}
+
+CharUnits CGCXXABI::GetArrayCookieSize(const CXXNewExpr *expr) {
+  return CharUnits::Zero();
+}
+
+llvm::Value *CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+                                             llvm::Value *NewPtr,
+                                             llvm::Value *NumElements,
+                                             const CXXNewExpr *expr,
+                                             QualType ElementType) {
+  // Should never be called.
+  ErrorUnsupportedABI(CGF, "array cookie initialization");
+  return 0;
+}
+
+void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                               const CXXDeleteExpr *expr, QualType ElementType,
+                               llvm::Value *&NumElements,
+                               llvm::Value *&AllocPtr, CharUnits &CookieSize) {
+  ErrorUnsupportedABI(CGF, "array cookie reading");
+
+  // This should be enough to avoid assertions.
+  NumElements = 0;
+  AllocPtr = llvm::Constant::getNullValue(CGF.Builder.getInt8PtrTy());
+  CookieSize = CharUnits::Zero();
+}
+
+void CGCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
+                               const VarDecl &D,
+                               llvm::GlobalVariable *GV) {
+  ErrorUnsupportedABI(CGF, "static local variable initialization");
+}
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index 367e345918a2..de4df3dcbe84 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -32,18 +32,20 @@ namespace clang {
   class CXXMethodDecl;
   class CXXRecordDecl;
   class FieldDecl;
+  class MangleContext;
 
 namespace CodeGen {
   class CodeGenFunction;
   class CodeGenModule;
-  class MangleContext;
 
 /// Implements C++ ABI-specific code generation functions.
 class CGCXXABI {
 protected:
   CodeGenModule &CGM;
+  llvm::OwningPtr<MangleContext> MangleCtx;
 
-  CGCXXABI(CodeGenModule &CGM) : CGM(CGM) {}
+  CGCXXABI(CodeGenModule &CGM)
+    : CGM(CGM), MangleCtx(CGM.getContext().createMangleContext()) {}
 
 protected:
   ImplicitParamDecl *&getThisDecl(CodeGenFunction &CGF) {
@@ -74,7 +76,9 @@ public:
   virtual ~CGCXXABI();
 
   /// Gets the mangle context.
-  virtual MangleContext &getMangleContext() = 0;
+  MangleContext &getMangleContext() {
+    return *MangleCtx;
+  }
 
   /// Find the LLVM type used to represent the given member pointer
   /// type.
@@ -118,7 +122,8 @@ public:
   virtual llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD);
 
   /// Create a member pointer for the given field.
-  virtual llvm::Constant *EmitMemberPointer(const FieldDecl *FD);
+  virtual llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
+                                                CharUnits offset);
 
   /// Emit a comparison between two member pointers.  Returns an i1.
   virtual llvm::Value *
@@ -185,7 +190,7 @@ public:
   ///
   /// \param ElementType - the allocated type of the expression,
   ///   i.e. the pointee type of the expression result type
-  virtual CharUnits GetArrayCookieSize(QualType ElementType);
+  virtual CharUnits GetArrayCookieSize(const CXXNewExpr *expr);
 
   /// Initialize the array cookie for the given allocation.
   ///
@@ -198,6 +203,7 @@ public:
   virtual llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
                                              llvm::Value *NewPtr,
                                              llvm::Value *NumElements,
+                                             const CXXNewExpr *expr,
                                              QualType ElementType);
 
   /// Reads the array cookie associated with the given pointer,
@@ -214,9 +220,21 @@ public:
   /// \param CookieSize - an out parameter which will be initialized
   ///   with the size of the cookie, or zero if there is no cookie
   virtual void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                               const CXXDeleteExpr *expr,
                                QualType ElementType, llvm::Value *&NumElements,
                                llvm::Value *&AllocPtr, CharUnits &CookieSize);
 
+  /*************************** Static local guards ****************************/
+
+  /// Emits the guarded initializer and destructor setup for the given
+  /// variable, given that it couldn't be emitted as a constant.
+  ///
+  /// The variable may be:
+  ///   - a static local variable
+  ///   - a static data member of a class template instantiation
+  virtual void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
+                               llvm::GlobalVariable *DeclPtr);
+
 };
 
 /// Creates an instance of a C++ ABI class.
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 475dfa5c102a..ae84b6196dfa 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -131,7 +131,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
   return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
 }
 
-const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D, 
+const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
                                                     CXXCtorType Type) {
   llvm::SmallVector<CanQualType, 16> ArgTys;
   ArgTys.push_back(GetThisType(Context, D->getParent()));
@@ -170,7 +170,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
   CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
   assert(isa<FunctionType>(FTy));
   if (isa<FunctionNoProtoType>(FTy))
-    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());  
+    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
   assert(isa<FunctionProtoType>(FTy));
   return getFunctionInfo(FTy.getAs<FunctionProtoType>());
 }
@@ -195,13 +195,13 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
   // FIXME: Do we need to handle ObjCMethodDecl?
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
-                                              
+
   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
     return getFunctionInfo(CD, GD.getCtorType());
 
   if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
     return getFunctionInfo(DD, GD.getDtorType());
-  
+
   return getFunctionInfo(FD);
 }
 
@@ -256,14 +256,14 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
 
   // Compute ABI information.
   getABIInfo().computeInfo(*FI);
-  
+
   // Loop over all of the computed argument and return value info.  If any of
   // them are direct or extend without a specified coerce type, specify the
   // default now.
   ABIArgInfo &RetInfo = FI->getReturnInfo();
   if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
     RetInfo.setCoerceToType(ConvertTypeRecursive(FI->getReturnType()));
-  
+
   for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
        I != E; ++I)
     if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
@@ -274,7 +274,7 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
   // we *just* filled in the FunctionInfo for.
   if (!IsRecursive && !PointersToResolve.empty())
     HandleLateResolvedPointers();
-  
+
   return *FI;
 }
 
@@ -288,7 +288,7 @@ CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
     NoReturn(_NoReturn), RegParm(_RegParm)
 {
   NumArgs = NumArgTys;
-  
+
   // FIXME: Coallocate with the CGFunctionInfo object.
   Args = new ArgInfo[1 + NumArgTys];
   Args[0].type = ResTy;
@@ -386,20 +386,20 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
                                    uint64_t DstSize, CodeGenFunction &CGF) {
   // We can't dive into a zero-element struct.
   if (SrcSTy->getNumElements() == 0) return SrcPtr;
-  
+
   const llvm::Type *FirstElt = SrcSTy->getElementType(0);
-  
+
   // If the first elt is at least as large as what we're looking for, or if the
   // first element is the same size as the whole struct, we can enter it.
-  uint64_t FirstEltSize = 
+  uint64_t FirstEltSize =
     CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
-  if (FirstEltSize < DstSize && 
+  if (FirstEltSize < DstSize &&
       FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
     return SrcPtr;
-  
+
   // GEP into the first element.
   SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
-  
+
   // If the first element is a struct, recurse.
   const llvm::Type *SrcTy =
     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
@@ -417,23 +417,23 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
                                              CodeGenFunction &CGF) {
   if (Val->getType() == Ty)
     return Val;
-  
+
   if (isa<llvm::PointerType>(Val->getType())) {
     // If this is Pointer->Pointer avoid conversion to and from int.
     if (isa<llvm::PointerType>(Ty))
       return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
-  
+
     // Convert the pointer to an integer so we can play with its width.
     Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
   }
-  
+
   const llvm::Type *DestIntTy = Ty;
   if (isa<llvm::PointerType>(DestIntTy))
     DestIntTy = CGF.IntPtrTy;
-  
+
   if (Val->getType() != DestIntTy)
     Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
-  
+
   if (isa<llvm::PointerType>(Ty))
     Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
   return Val;
@@ -452,18 +452,18 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
                                       CodeGenFunction &CGF) {
   const llvm::Type *SrcTy =
     cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
-  
+
   // If SrcTy and Ty are the same, just do a load.
   if (SrcTy == Ty)
     return CGF.Builder.CreateLoad(SrcPtr);
-  
+
   uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
-  
+
   if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
     SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
     SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
   }
-  
+
   uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
 
   // If the source and destination are integer or pointer types, just do an
@@ -473,7 +473,7 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
     llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
     return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
   }
-  
+
   // If load is legal, just bitcast the src pointer.
   if (SrcSize >= DstSize) {
     // Generally SrcSize is never greater than DstSize, since this means we are
@@ -489,7 +489,7 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
     Load->setAlignment(1);
     return Load;
   }
-  
+
   // Otherwise do coercion through memory. This is stupid, but
   // simple.
   llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
@@ -518,14 +518,14 @@ static void CreateCoercedStore(llvm::Value *Src,
     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
     return;
   }
-  
+
   uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
-  
+
   if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
     DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
     DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
   }
-  
+
   // If the source and destination are integer or pointer types, just do an
   // extension or truncation to the desired type.
   if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
@@ -534,7 +534,7 @@ static void CreateCoercedStore(llvm::Value *Src,
     CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
     return;
   }
-  
+
   uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
 
   // If store is legal, just bitcast the src pointer.
@@ -590,7 +590,7 @@ bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
 
 const llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
   const CGFunctionInfo &FI = getFunctionInfo(GD);
-  
+
   // For definition purposes, don't consider a K&R function variadic.
   bool Variadic = false;
   if (const FunctionProtoType *FPT =
@@ -673,7 +673,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
 const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
-  
+
   if (!VerifyFuncTypeComplete(FPT)) {
     const CGFunctionInfo *Info;
     if (isa<CXXDestructorDecl>(MD))
@@ -688,7 +688,7 @@ const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
 
 void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
                                            const Decl *TargetDecl,
-                                           AttributeListType &PAL, 
+                                           AttributeListType &PAL,
                                            unsigned &CallingConv) {
   unsigned FuncAttrs = 0;
   unsigned RetAttrs = 0;
@@ -755,10 +755,10 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   if (RetAttrs)
     PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
 
-  // FIXME: we need to honor command line settings also.
-  // FIXME: RegParm should be reduced in case of nested functions and/or global
-  // register variable.
+  // FIXME: RegParm should be reduced in case of global register variable.
   signed RegParm = FI.getRegParm();
+  if (!RegParm)
+    RegParm = CodeGenOpts.NumRegisterParameters;
 
   unsigned PointerWidth = getContext().Target.getPointerWidth(0);
   for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
@@ -769,7 +769,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
 
     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
     // have the corresponding parameter variable.  It doesn't make
-    // sense to do it here because parameters are so fucked up.
+    // sense to do it here because parameters are so messed up.
     switch (AI.getKind()) {
     case ABIArgInfo::Extend:
       if (ParamType->isSignedIntegerType())
@@ -786,7 +786,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
           Attributes |= llvm::Attribute::InReg;
       }
       // FIXME: handle sseregparm someday...
-        
+
       if (const llvm::StructType *STy =
             dyn_cast<llvm::StructType>(AI.getCoerceToType()))
         Index += STy->getNumElements()-1;  // 1 will be added below.
@@ -866,13 +866,32 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     switch (ArgI.getKind()) {
     case ABIArgInfo::Indirect: {
       llvm::Value *V = AI;
+
       if (hasAggregateLLVMType(Ty)) {
-        // Do nothing, aggregates and complex variables are accessed by
-        // reference.
+        // Aggregates and complex variables are accessed by reference.  All we
+        // need to do is realign the value, if requested
+        if (ArgI.getIndirectRealign()) {
+          llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
+
+          // Copy from the incoming argument pointer to the temporary with the
+          // appropriate alignment.
+          //
+          // FIXME: We should have a common utility for generating an aggregate
+          // copy.
+          const llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
+          CharUnits Size = getContext().getTypeSizeInChars(Ty);
+          Builder.CreateMemCpy(Builder.CreateBitCast(AlignedTemp, I8PtrTy),
+                               Builder.CreateBitCast(V, I8PtrTy),
+                               llvm::ConstantInt::get(IntPtrTy, 
+                                                      Size.getQuantity()),
+                               ArgI.getIndirectAlign(),
+                               false);
+          V = AlignedTemp;
+        }
       } else {
         // Load scalar value from indirect argument.
-        unsigned Alignment = getContext().getTypeAlignInChars(Ty).getQuantity();
-        V = EmitLoadOfScalar(V, false, Alignment, Ty);
+        CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
+        V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty);
         if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
           // This must be a promotion, for something like
           // "void a(x) short x; {..."
@@ -891,7 +910,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
           ArgI.getDirectOffset() == 0) {
         assert(AI != Fn->arg_end() && "Argument mismatch!");
         llvm::Value *V = AI;
-        
+
         if (Arg->getType().isRestrictQualified())
           AI->addAttr(llvm::Attribute::NoAlias);
 
@@ -905,33 +924,33 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       }
 
       llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
-      
+
       // The alignment we need to use is the max of the requested alignment for
       // the argument plus the alignment required by our access code below.
-      unsigned AlignmentToUse = 
-        CGF.CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
+      unsigned AlignmentToUse =
+        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
       AlignmentToUse = std::max(AlignmentToUse,
                         (unsigned)getContext().getDeclAlign(Arg).getQuantity());
-      
+
       Alloca->setAlignment(AlignmentToUse);
       llvm::Value *V = Alloca;
       llvm::Value *Ptr = V;    // Pointer to store into.
-      
+
       // If the value is offset in memory, apply the offset now.
       if (unsigned Offs = ArgI.getDirectOffset()) {
         Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
         Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
-        Ptr = Builder.CreateBitCast(Ptr, 
+        Ptr = Builder.CreateBitCast(Ptr,
                           llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
       }
-      
+
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
       if (const llvm::StructType *STy =
             dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
         Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
-        
+
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           assert(AI != Fn->arg_end() && "Argument mismatch!");
           AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
@@ -944,8 +963,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         AI->setName(Arg->getName() + ".coerce");
         CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
       }
-      
-      
+
+
       // Match to what EmitParmDecl is expecting for this type.
       if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
         V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
@@ -1024,13 +1043,13 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
         RetAI.getDirectOffset() == 0) {
       // The internal return value temp always will have pointer-to-return-type
       // type, just do a load.
-        
+
       // If the instruction right before the insertion point is a store to the
       // return value, we can elide the load, zap the store, and usually zap the
       // alloca.
       llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
       llvm::StoreInst *SI = 0;
-      if (InsertBB->empty() || 
+      if (InsertBB->empty() ||
           !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
           SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
         RV = Builder.CreateLoad(ReturnValue);
@@ -1039,7 +1058,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
         RetDbgLoc = SI->getDebugLoc();
         RV = SI->getValueOperand();
         SI->eraseFromParent();
-        
+
         // If that was the only use of the return value, nuke it as well now.
         if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
           cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
@@ -1052,10 +1071,10 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
       if (unsigned Offs = RetAI.getDirectOffset()) {
         V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
         V = Builder.CreateConstGEP1_32(V, Offs);
-        V = Builder.CreateBitCast(V, 
+        V = Builder.CreateBitCast(V,
                          llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
       }
-      
+
       RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
     }
     break;
@@ -1079,7 +1098,7 @@ RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
   llvm::Value *Local = GetAddrOfLocalVar(Param);
 
   QualType ArgType = Param->getType();
- 
+
   // For the most part, we just need to load the alloca, except:
   // 1) aggregate r-values are actually pointers to temporaries, and
   // 2) references to aggregates are pointers directly to the aggregate.
@@ -1180,7 +1199,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
     case ABIArgInfo::Ignore:
       break;
-        
+
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
       if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
@@ -1204,16 +1223,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
       } else
         SrcPtr = RV.getAggregateAddr();
-      
+
       // If the value is offset in memory, apply the offset now.
       if (unsigned Offs = ArgInfo.getDirectOffset()) {
         SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
         SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
-        SrcPtr = Builder.CreateBitCast(SrcPtr, 
+        SrcPtr = Builder.CreateBitCast(SrcPtr,
                        llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
 
       }
-      
+
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
@@ -1233,7 +1252,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
                                          *this));
       }
-      
+
       break;
     }
 
@@ -1332,7 +1351,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     // If we are ignoring an argument that had a result, make sure to
     // construct the appropriate return value for our caller.
     return GetUndefRValue(RetTy);
-    
+
   case ABIArgInfo::Extend:
   case ABIArgInfo::Direct: {
     if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
@@ -1355,25 +1374,25 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       }
       return RValue::get(CI);
     }
-      
+
     llvm::Value *DestPtr = ReturnValue.getValue();
     bool DestIsVolatile = ReturnValue.isVolatile();
-    
+
     if (!DestPtr) {
       DestPtr = CreateMemTemp(RetTy, "coerce");
       DestIsVolatile = false;
     }
-    
+
     // If the value is offset in memory, apply the offset now.
     llvm::Value *StorePtr = DestPtr;
     if (unsigned Offs = RetAI.getDirectOffset()) {
       StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
       StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
-      StorePtr = Builder.CreateBitCast(StorePtr, 
+      StorePtr = Builder.CreateBitCast(StorePtr,
                          llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
     }
     CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
-    
+
     unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
     if (RetTy->isAnyComplexType())
       return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index bf26799b83f3..8e88beb3cf0d 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -14,6 +14,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/EvaluatedExprVisitor.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtCXX.h"
 
@@ -40,7 +41,7 @@ ComputeNonVirtualBaseClassOffset(ASTContext &Context,
       cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
     
     // Add the offset.
-    Offset += Layout.getBaseClassOffset(BaseDecl);
+    Offset += Layout.getBaseClassOffsetInBits(BaseDecl);
     
     RD = BaseDecl;
   }
@@ -86,9 +87,9 @@ CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(llvm::Value *This,
   uint64_t Offset;
   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
   if (BaseIsVirtual)
-    Offset = Layout.getVBaseClassOffset(Base);
+    Offset = Layout.getVBaseClassOffsetInBits(Base);
   else
-    Offset = Layout.getBaseClassOffset(Base);
+    Offset = Layout.getBaseClassOffsetInBits(Base);
 
   // Shift and cast down to the base type.
   // TODO: for complete types, this should be possible with a GEP.
@@ -179,8 +180,17 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
 
   llvm::Value *VirtualOffset = 0;
 
-  if (VBase)
-    VirtualOffset = GetVirtualBaseClassOffset(Value, Derived, VBase);
+  if (VBase) {
+    if (Derived->hasAttr<FinalAttr>()) {
+      VirtualOffset = 0;
+
+      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
+
+      uint64_t VBaseOffset = Layout.getVBaseClassOffsetInBits(VBase);
+      NonVirtualOffset += VBaseOffset / 8;
+    } else
+      VirtualOffset = GetVirtualBaseClassOffset(Value, Derived, VBase);
+  }
 
   // Apply the offsets.
   Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset, 
@@ -294,7 +304,8 @@ static llvm::Value *GetVTTParameter(CodeGenFunction &CGF, GlobalDecl GD,
     const ASTRecordLayout &Layout = 
       CGF.getContext().getASTRecordLayout(RD);
     uint64_t BaseOffset = ForVirtualBase ? 
-      Layout.getVBaseClassOffset(Base) : Layout.getBaseClassOffset(Base);
+      Layout.getVBaseClassOffsetInBits(Base) : 
+      Layout.getBaseClassOffsetInBits(Base);
 
     SubVTTIndex = 
       CGF.CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
@@ -307,7 +318,7 @@ static llvm::Value *GetVTTParameter(CodeGenFunction &CGF, GlobalDecl GD,
     VTT = CGF.Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
   } else {
     // We're the complete constructor, so get the VTT by name.
-    VTT = CGF.CGM.getVTables().getVTT(RD);
+    VTT = CGF.CGM.getVTables().GetAddrOfVTT(RD);
     VTT = CGF.Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
   }
 
@@ -334,11 +345,34 @@ namespace {
       CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual, Addr);
     }
   };
+
+  /// A visitor which checks whether an initializer uses 'this' in a
+  /// way which requires the vtable to be properly set.
+  struct DynamicThisUseChecker : EvaluatedExprVisitor<DynamicThisUseChecker> {
+    typedef EvaluatedExprVisitor<DynamicThisUseChecker> super;
+
+    bool UsesThis;
+
+    DynamicThisUseChecker(ASTContext &C) : super(C), UsesThis(false) {}
+
+    // Black-list all explicit and implicit references to 'this'.
+    //
+    // Do we need to worry about external references to 'this' derived
+    // from arbitrary code?  If so, then anything which runs arbitrary
+    // external code might potentially access the vtable.
+    void VisitCXXThisExpr(CXXThisExpr *E) { UsesThis = true; }
+  };
+}
+
+static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
+  DynamicThisUseChecker Checker(C);
+  Checker.Visit(const_cast<Expr*>(Init));
+  return Checker.UsesThis;
 }
 
 static void EmitBaseInitializer(CodeGenFunction &CGF, 
                                 const CXXRecordDecl *ClassDecl,
-                                CXXBaseOrMemberInitializer *BaseInit,
+                                CXXCtorInitializer *BaseInit,
                                 CXXCtorType CtorType) {
   assert(BaseInit->isBaseInitializer() &&
          "Must have base initializer!");
@@ -355,6 +389,12 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
   if (CtorType == Ctor_Base && isBaseVirtual)
     return;
 
+  // If the initializer for the base (other than the constructor
+  // itself) accesses 'this' in any way, we need to initialize the
+  // vtables.
+  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
+    CGF.InitializeVTablePointers(ClassDecl);
+
   // We can pretend to be a complete class because it only matters for
   // virtual bases, and we only do virtual bases for complete ctors.
   llvm::Value *V = 
@@ -362,9 +402,12 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
                                               BaseClassDecl,
                                               isBaseVirtual);
 
-  CGF.EmitAggExpr(BaseInit->getInit(), V, false, false, true);
+  AggValueSlot AggSlot = AggValueSlot::forAddr(V, false, /*Lifetime*/ true);
+
+  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
   
-  if (CGF.Exceptions && !BaseClassDecl->hasTrivialDestructor())
+  if (CGF.CGM.getLangOptions().areExceptionsEnabled() && 
+      !BaseClassDecl->hasTrivialDestructor())
     CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
                                           isBaseVirtual);
 }
@@ -372,7 +415,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
 static void EmitAggMemberInitializer(CodeGenFunction &CGF,
                                      LValue LHS,
                                      llvm::Value *ArrayIndexVar,
-                                     CXXBaseOrMemberInitializer *MemberInit,
+                                     CXXCtorInitializer *MemberInit,
                                      QualType T,
                                      unsigned Index) {
   if (Index == MemberInit->getNumArrayIndices()) {
@@ -388,11 +431,11 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
       Next = CGF.Builder.CreateAdd(ArrayIndex, Next, "inc");
       CGF.Builder.CreateStore(Next, ArrayIndexVar);      
     }
+
+    AggValueSlot Slot = AggValueSlot::forAddr(Dest, LHS.isVolatileQualified(),
+                                              /*Lifetime*/ true);
     
-    CGF.EmitAggExpr(MemberInit->getInit(), Dest, 
-                    LHS.isVolatileQualified(),
-                    /*IgnoreResult*/ false,
-                    /*IsInitializer*/ true);
+    CGF.EmitAggExpr(MemberInit->getInit(), Slot);
     
     return;
   }
@@ -476,29 +519,29 @@ namespace {
   
 static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   const CXXRecordDecl *ClassDecl,
-                                  CXXBaseOrMemberInitializer *MemberInit,
+                                  CXXCtorInitializer *MemberInit,
                                   const CXXConstructorDecl *Constructor,
                                   FunctionArgList &Args) {
-  assert(MemberInit->isMemberInitializer() &&
+  assert(MemberInit->isAnyMemberInitializer() &&
          "Must have member initializer!");
   
   // non-static data member initializers.
-  FieldDecl *Field = MemberInit->getMember();
+  FieldDecl *Field = MemberInit->getAnyMember();
   QualType FieldType = CGF.getContext().getCanonicalType(Field->getType());
 
   llvm::Value *ThisPtr = CGF.LoadCXXThis();
   LValue LHS;
   
   // If we are initializing an anonymous union field, drill down to the field.
-  if (MemberInit->getAnonUnionMember()) {
-    Field = MemberInit->getAnonUnionMember();
-    LHS = CGF.EmitLValueForAnonRecordField(ThisPtr, Field, 0);
-    FieldType = Field->getType();
+  if (MemberInit->isIndirectMemberInitializer()) {
+    LHS = CGF.EmitLValueForAnonRecordField(ThisPtr,
+                                           MemberInit->getIndirectMember(), 0);
+    FieldType = MemberInit->getIndirectMember()->getAnonField()->getType();
   } else {
     LHS = CGF.EmitLValueForFieldInitialization(ThisPtr, Field, 0);
   }
 
-  // FIXME: If there's no initializer and the CXXBaseOrMemberInitializer
+  // FIXME: If there's no initializer and the CXXCtorInitializer
   // was implicitly generated, we shouldn't be zeroing memory.
   RValue RHS;
   if (FieldType->isReferenceType()) {
@@ -557,12 +600,12 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
       
       // Emit the block variables for the array indices, if any.
       for (unsigned I = 0, N = MemberInit->getNumArrayIndices(); I != N; ++I)
-        CGF.EmitLocalBlockVarDecl(*MemberInit->getArrayIndex(I));
+        CGF.EmitAutoVarDecl(*MemberInit->getArrayIndex(I));
     }
     
     EmitAggMemberInitializer(CGF, LHS, ArrayIndexVar, MemberInit, FieldType, 0);
     
-    if (!CGF.Exceptions)
+    if (!CGF.CGM.getLangOptions().areExceptionsEnabled())
       return;
 
     // FIXME: If we have an array of classes w/ non-trivial destructors, 
@@ -674,12 +717,12 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
                                        FunctionArgList &Args) {
   const CXXRecordDecl *ClassDecl = CD->getParent();
 
-  llvm::SmallVector<CXXBaseOrMemberInitializer *, 8> MemberInitializers;
+  llvm::SmallVector<CXXCtorInitializer *, 8> MemberInitializers;
   
   for (CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
        E = CD->init_end();
        B != E; ++B) {
-    CXXBaseOrMemberInitializer *Member = (*B);
+    CXXCtorInitializer *Member = (*B);
     
     if (Member->isBaseInitializer())
       EmitBaseInitializer(*this, ClassDecl, Member, CtorType);
@@ -754,6 +797,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
       assert(Dtor->isImplicit() && "bodyless dtor not implicit");
       // nothing to do besides what's in the epilogue
     }
+    // -fapple-kext must inline any call to this dtor into
+    // the caller's body.
+    if (getContext().getLangOptions().AppleKext)
+      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
     break;
   }
 
@@ -1118,6 +1165,64 @@ CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
 }
 
 void
+CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
+                                        llvm::Value *This, llvm::Value *Src,
+                                        CallExpr::const_arg_iterator ArgBeg,
+                                        CallExpr::const_arg_iterator ArgEnd) {
+  if (D->isTrivial()) {
+    assert(ArgBeg + 1 == ArgEnd && "unexpected argcount for trivial ctor");
+    assert(D->isCopyConstructor() && "trivial 1-arg ctor not a copy ctor");
+    EmitAggregateCopy(This, Src, (*ArgBeg)->getType());
+    return;
+  }
+  llvm::Value *Callee = CGM.GetAddrOfCXXConstructor(D, 
+                                                    clang::Ctor_Complete);
+  assert(D->isInstance() &&
+         "Trying to emit a member call expr on a static method!");
+  
+  const FunctionProtoType *FPT = D->getType()->getAs<FunctionProtoType>();
+  
+  CallArgList Args;
+  
+  // Push the this ptr.
+  Args.push_back(std::make_pair(RValue::get(This),
+                                D->getThisType(getContext())));
+  
+  
+  // Push the src ptr.
+  QualType QT = *(FPT->arg_type_begin());
+  const llvm::Type *t = CGM.getTypes().ConvertType(QT);
+  Src = Builder.CreateBitCast(Src, t);
+  Args.push_back(std::make_pair(RValue::get(Src), QT));
+  
+  // Skip over first argument (Src).
+  ++ArgBeg;
+  CallExpr::const_arg_iterator Arg = ArgBeg;
+  for (FunctionProtoType::arg_type_iterator I = FPT->arg_type_begin()+1,
+       E = FPT->arg_type_end(); I != E; ++I, ++Arg) {
+    assert(Arg != ArgEnd && "Running over edge of argument list!");
+    QualType ArgType = *I;
+    Args.push_back(std::make_pair(EmitCallArg(*Arg, ArgType),
+                                  ArgType));
+  }
+  // Either we've emitted all the call args, or we have a call to a
+  // variadic function.
+  assert((Arg == ArgEnd || FPT->isVariadic()) &&
+         "Extra arguments in non-variadic function!");
+  // If we still have any arguments, emit them using the type of the argument.
+  for (; Arg != ArgEnd; ++Arg) {
+    QualType ArgType = Arg->getType();
+    Args.push_back(std::make_pair(EmitCallArg(*Arg, ArgType),
+                                  ArgType));
+  }
+  
+  QualType ResultType = FPT->getResultType();
+  EmitCall(CGM.getTypes().getFunctionInfo(ResultType, Args,
+                                          FPT->getExtInfo()),
+                  Callee, ReturnValueSlot(), Args, D);
+}
+
+void
 CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
                                                 CXXCtorType CtorType,
                                                 const FunctionArgList &Args) {
@@ -1164,7 +1269,13 @@ void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
                                             llvm::Value *This) {
   llvm::Value *VTT = GetVTTParameter(*this, GlobalDecl(DD, Type), 
                                      ForVirtualBase);
-  llvm::Value *Callee = CGM.GetAddrOfCXXDestructor(DD, Type);
+  llvm::Value *Callee = 0;
+  if (getContext().getLangOptions().AppleKext)
+    Callee = BuildAppleKextVirtualDestructorCall(DD, Type, 
+                                                 DD->getParent());
+    
+  if (!Callee)
+    Callee = CGM.GetAddrOfCXXDestructor(DD, Type);
   
   EmitCXXMemberCall(DD, Callee, ReturnValueSlot(), This, VTT, 0, 0);
 }
@@ -1202,13 +1313,7 @@ llvm::Value *
 CodeGenFunction::GetVirtualBaseClassOffset(llvm::Value *This,
                                            const CXXRecordDecl *ClassDecl,
                                            const CXXRecordDecl *BaseClassDecl) {
-  const llvm::Type *Int8PtrTy = 
-    llvm::Type::getInt8Ty(VMContext)->getPointerTo();
-
-  llvm::Value *VTablePtr = Builder.CreateBitCast(This, 
-                                                 Int8PtrTy->getPointerTo());
-  VTablePtr = Builder.CreateLoad(VTablePtr, "vtable");
-
+  llvm::Value *VTablePtr = GetVTablePtr(This, Int8PtrTy);
   int64_t VBaseOffsetOffset = 
     CGM.getVTables().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl);
   
@@ -1326,15 +1431,16 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base,
       const ASTRecordLayout &Layout = 
         getContext().getASTRecordLayout(VTableClass);
 
-      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
+      BaseOffset = Layout.getVBaseClassOffsetInBits(BaseDecl);
       BaseOffsetFromNearestVBase = 0;
       BaseDeclIsNonVirtualPrimaryBase = false;
     } else {
       const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
 
-      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
+      BaseOffset = 
+        Base.getBaseOffset() + Layout.getBaseClassOffsetInBits(BaseDecl);
       BaseOffsetFromNearestVBase = 
-        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
+        OffsetFromNearestVBase + Layout.getBaseClassOffsetInBits(BaseDecl);
       BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
     }
     
@@ -1361,3 +1467,9 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
                            /*BaseIsNonVirtualPrimaryBase=*/false, 
                            VTable, RD, VBases);
 }
+
+llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This,
+                                           const llvm::Type *Ty) {
+  llvm::Value *VTablePtrSrc = Builder.CreateBitCast(This, Ty->getPointerTo());
+  return Builder.CreateLoad(VTablePtrSrc, "vtable");
+}
diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp
new file mode 100644
index 000000000000..374ede880d20
--- /dev/null
+++ b/lib/CodeGen/CGCleanup.cpp
@@ -0,0 +1,1144 @@
+//===--- CGCleanup.cpp - Bookkeeping and code emission for cleanups -------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code dealing with the IR generation for cleanups
+// and related information.
+//
+// A "cleanup" is a piece of code which needs to be executed whenever
+// control transfers out of a particular scope.  This can be
+// conditionalized to occur only on exceptional control flow, only on
+// normal control flow, or both.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenFunction.h"
+#include "CGCleanup.h"
+
+using namespace clang;
+using namespace CodeGen;
+
+bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) {
+  if (rv.isScalar())
+    return DominatingLLVMValue::needsSaving(rv.getScalarVal());
+  if (rv.isAggregate())
+    return DominatingLLVMValue::needsSaving(rv.getAggregateAddr());
+  return true;
+}
+
+DominatingValue<RValue>::saved_type
+DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
+  if (rv.isScalar()) {
+    llvm::Value *V = rv.getScalarVal();
+
+    // These automatically dominate and don't need to be saved.
+    if (!DominatingLLVMValue::needsSaving(V))
+      return saved_type(V, ScalarLiteral);
+
+    // Everything else needs an alloca.
+    llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
+    CGF.Builder.CreateStore(V, addr);
+    return saved_type(addr, ScalarAddress);
+  }
+
+  if (rv.isComplex()) {
+    CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
+    const llvm::Type *ComplexTy =
+      llvm::StructType::get(CGF.getLLVMContext(),
+                            V.first->getType(), V.second->getType(),
+                            (void*) 0);
+    llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex");
+    CGF.StoreComplexToAddr(V, addr, /*volatile*/ false);
+    return saved_type(addr, ComplexAddress);
+  }
+
+  assert(rv.isAggregate());
+  llvm::Value *V = rv.getAggregateAddr(); // TODO: volatile?
+  if (!DominatingLLVMValue::needsSaving(V))
+    return saved_type(V, AggregateLiteral);
+
+  llvm::Value *addr = CGF.CreateTempAlloca(V->getType(), "saved-rvalue");
+  CGF.Builder.CreateStore(V, addr);
+  return saved_type(addr, AggregateAddress);  
+}
+
+/// Given a saved r-value produced by SaveRValue, perform the code
+/// necessary to restore it to usability at the current insertion
+/// point.
+RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
+  switch (K) {
+  case ScalarLiteral:
+    return RValue::get(Value);
+  case ScalarAddress:
+    return RValue::get(CGF.Builder.CreateLoad(Value));
+  case AggregateLiteral:
+    return RValue::getAggregate(Value);
+  case AggregateAddress:
+    return RValue::getAggregate(CGF.Builder.CreateLoad(Value));
+  case ComplexAddress:
+    return RValue::getComplex(CGF.LoadComplexFromAddr(Value, false));
+  }
+
+  llvm_unreachable("bad saved r-value kind");
+  return RValue();
+}
+
+/// Push an entry of the given size onto this protected-scope stack.
+char *EHScopeStack::allocate(size_t Size) {
+  if (!StartOfBuffer) {
+    unsigned Capacity = 1024;
+    while (Capacity < Size) Capacity *= 2;
+    StartOfBuffer = new char[Capacity];
+    StartOfData = EndOfBuffer = StartOfBuffer + Capacity;
+  } else if (static_cast<size_t>(StartOfData - StartOfBuffer) < Size) {
+    unsigned CurrentCapacity = EndOfBuffer - StartOfBuffer;
+    unsigned UsedCapacity = CurrentCapacity - (StartOfData - StartOfBuffer);
+
+    unsigned NewCapacity = CurrentCapacity;
+    do {
+      NewCapacity *= 2;
+    } while (NewCapacity < UsedCapacity + Size);
+
+    char *NewStartOfBuffer = new char[NewCapacity];
+    char *NewEndOfBuffer = NewStartOfBuffer + NewCapacity;
+    char *NewStartOfData = NewEndOfBuffer - UsedCapacity;
+    memcpy(NewStartOfData, StartOfData, UsedCapacity);
+    delete [] StartOfBuffer;
+    StartOfBuffer = NewStartOfBuffer;
+    EndOfBuffer = NewEndOfBuffer;
+    StartOfData = NewStartOfData;
+  }
+
+  assert(StartOfBuffer + Size <= StartOfData);
+  StartOfData -= Size;
+  return StartOfData;
+}
+
+EHScopeStack::stable_iterator
+EHScopeStack::getEnclosingEHCleanup(iterator it) const {
+  assert(it != end());
+  do {
+    if (isa<EHCleanupScope>(*it)) {
+      if (cast<EHCleanupScope>(*it).isEHCleanup())
+        return stabilize(it);
+      return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
+    }
+    ++it;
+  } while (it != end());
+  return stable_end();
+}
+
+
+void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
+  assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned");
+  char *Buffer = allocate(EHCleanupScope::getSizeForCleanupSize(Size));
+  bool IsNormalCleanup = Kind & NormalCleanup;
+  bool IsEHCleanup = Kind & EHCleanup;
+  bool IsActive = !(Kind & InactiveCleanup);
+  EHCleanupScope *Scope =
+    new (Buffer) EHCleanupScope(IsNormalCleanup,
+                                IsEHCleanup,
+                                IsActive,
+                                Size,
+                                BranchFixups.size(),
+                                InnermostNormalCleanup,
+                                InnermostEHCleanup);
+  if (IsNormalCleanup)
+    InnermostNormalCleanup = stable_begin();
+  if (IsEHCleanup)
+    InnermostEHCleanup = stable_begin();
+
+  return Scope->getCleanupBuffer();
+}
+
+void EHScopeStack::popCleanup() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  assert(isa<EHCleanupScope>(*begin()));
+  EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
+  InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
+  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+  StartOfData += Cleanup.getAllocatedSize();
+
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
+  // Destroy the cleanup.
+  Cleanup.~EHCleanupScope();
+
+  // Check whether we can shrink the branch-fixups stack.
+  if (!BranchFixups.empty()) {
+    // If we no longer have any normal cleanups, all the fixups are
+    // complete.
+    if (!hasNormalCleanups())
+      BranchFixups.clear();
+
+    // Otherwise we can still trim out unnecessary nulls.
+    else
+      popNullFixups();
+  }
+}
+
+EHFilterScope *EHScopeStack::pushFilter(unsigned NumFilters) {
+  char *Buffer = allocate(EHFilterScope::getSizeForNumFilters(NumFilters));
+  CatchDepth++;
+  return new (Buffer) EHFilterScope(NumFilters);
+}
+
+void EHScopeStack::popFilter() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  EHFilterScope &Filter = cast<EHFilterScope>(*begin());
+  StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
+
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
+  assert(CatchDepth > 0 && "mismatched filter push/pop");
+  CatchDepth--;
+}
+
+EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
+  char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
+  CatchDepth++;
+  EHCatchScope *Scope = new (Buffer) EHCatchScope(NumHandlers);
+  for (unsigned I = 0; I != NumHandlers; ++I)
+    Scope->getHandlers()[I].Index = getNextEHDestIndex();
+  return Scope;
+}
+
+void EHScopeStack::pushTerminate() {
+  char *Buffer = allocate(EHTerminateScope::getSize());
+  CatchDepth++;
+  new (Buffer) EHTerminateScope(getNextEHDestIndex());
+}
+
+/// Remove any 'null' fixups on the stack.  However, we can't pop more
+/// fixups than the fixup depth on the innermost normal cleanup, or
+/// else fixups that we try to add to that cleanup will end up in the
+/// wrong place.  We *could* try to shrink fixup depths, but that's
+/// actually a lot of work for little benefit.
+void EHScopeStack::popNullFixups() {
+  // We expect this to only be called when there's still an innermost
+  // normal cleanup;  otherwise there really shouldn't be any fixups.
+  assert(hasNormalCleanups());
+
+  EHScopeStack::iterator it = find(InnermostNormalCleanup);
+  unsigned MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
+  assert(BranchFixups.size() >= MinSize && "fixup stack out of order");
+
+  while (BranchFixups.size() > MinSize &&
+         BranchFixups.back().Destination == 0)
+    BranchFixups.pop_back();
+}
+
+void CodeGenFunction::initFullExprCleanup() {
+  // Create a variable to decide whether the cleanup needs to be run.
+  llvm::AllocaInst *active
+    = CreateTempAlloca(Builder.getInt1Ty(), "cleanup.cond");
+
+  // Initialize it to false at a site that's guaranteed to be run
+  // before each evaluation.
+  llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
+  new llvm::StoreInst(Builder.getFalse(), active, &block->back());
+
+  // Initialize it to true at the current location.
+  Builder.CreateStore(Builder.getTrue(), active);
+
+  // Set that as the active flag in the cleanup.
+  EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin());
+  assert(cleanup.getActiveFlag() == 0 && "cleanup already has active flag?");
+  cleanup.setActiveFlag(active);
+
+  if (cleanup.isNormalCleanup()) cleanup.setTestFlagInNormalCleanup();
+  if (cleanup.isEHCleanup()) cleanup.setTestFlagInEHCleanup();
+}
+
+EHScopeStack::Cleanup::~Cleanup() {
+  llvm_unreachable("Cleanup is indestructable");
+}
+
+/// All the branch fixups on the EH stack have propagated out past the
+/// outermost normal cleanup; resolve them all by adding cases to the
+/// given switch instruction.
+static void ResolveAllBranchFixups(CodeGenFunction &CGF,
+                                   llvm::SwitchInst *Switch,
+                                   llvm::BasicBlock *CleanupEntry) {
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> CasesAdded;
+
+  for (unsigned I = 0, E = CGF.EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination isn't set.
+    BranchFixup &Fixup = CGF.EHStack.getBranchFixup(I);
+    if (Fixup.Destination == 0) continue;
+
+    // If there isn't an OptimisticBranchBlock, then InitialBranch is
+    // still pointing directly to its destination; forward it to the
+    // appropriate cleanup entry.  This is required in the specific
+    // case of
+    //   { std::string s; goto lbl; }
+    //   lbl:
+    // i.e. where there's an unresolved fixup inside a single cleanup
+    // entry which we're currently popping.
+    if (Fixup.OptimisticBranchBlock == 0) {
+      new llvm::StoreInst(CGF.Builder.getInt32(Fixup.DestinationIndex),
+                          CGF.getNormalCleanupDestSlot(),
+                          Fixup.InitialBranch);
+      Fixup.InitialBranch->setSuccessor(0, CleanupEntry);
+    }
+
+    // Don't add this case to the switch statement twice.
+    if (!CasesAdded.insert(Fixup.Destination)) continue;
+
+    Switch->addCase(CGF.Builder.getInt32(Fixup.DestinationIndex),
+                    Fixup.Destination);
+  }
+
+  CGF.EHStack.clearFixups();
+}
+
+/// Transitions the terminator of the given exit-block of a cleanup to
+/// be a cleanup switch.
+static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
+                                                   llvm::BasicBlock *Block) {
+  // If it's a branch, turn it into a switch whose default
+  // destination is its original target.
+  llvm::TerminatorInst *Term = Block->getTerminator();
+  assert(Term && "can't transition block without terminator");
+
+  if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
+    assert(Br->isUnconditional());
+    llvm::LoadInst *Load =
+      new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term);
+    llvm::SwitchInst *Switch =
+      llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
+    Br->eraseFromParent();
+    return Switch;
+  } else {
+    return cast<llvm::SwitchInst>(Term);
+  }
+}
+
+void CodeGenFunction::ResolveBranchFixups(llvm::BasicBlock *Block) {
+  assert(Block && "resolving a null target block");
+  if (!EHStack.getNumBranchFixups()) return;
+
+  assert(EHStack.hasNormalCleanups() &&
+         "branch fixups exist with no normal cleanups on stack");
+
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> ModifiedOptimisticBlocks;
+  bool ResolvedAny = false;
+
+  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination doesn't match.
+    BranchFixup &Fixup = EHStack.getBranchFixup(I);
+    if (Fixup.Destination != Block) continue;
+
+    Fixup.Destination = 0;
+    ResolvedAny = true;
+
+    // If it doesn't have an optimistic branch block, LatestBranch is
+    // already pointing to the right place.
+    llvm::BasicBlock *BranchBB = Fixup.OptimisticBranchBlock;
+    if (!BranchBB)
+      continue;
+
+    // Don't process the same optimistic branch block twice.
+    if (!ModifiedOptimisticBlocks.insert(BranchBB))
+      continue;
+
+    llvm::SwitchInst *Switch = TransitionToCleanupSwitch(*this, BranchBB);
+
+    // Add a case to the switch.
+    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex), Block);
+  }
+
+  if (ResolvedAny)
+    EHStack.popNullFixups();
+}
+
+/// Pops cleanup blocks until the given savepoint is reached.
+void CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old) {
+  assert(Old.isValid());
+
+  while (EHStack.stable_begin() != Old) {
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
+
+    // As long as Old strictly encloses the scope's enclosing normal
+    // cleanup, we're going to emit another normal cleanup which
+    // fallthrough can propagate through.
+    bool FallThroughIsBranchThrough =
+      Old.strictlyEncloses(Scope.getEnclosingNormalCleanup());
+
+    PopCleanupBlock(FallThroughIsBranchThrough);
+  }
+}
+
+static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
+                                           EHCleanupScope &Scope) {
+  assert(Scope.isNormalCleanup());
+  llvm::BasicBlock *Entry = Scope.getNormalBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("cleanup");
+    Scope.setNormalBlock(Entry);
+  }
+  return Entry;
+}
+
+static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
+                                       EHCleanupScope &Scope) {
+  assert(Scope.isEHCleanup());
+  llvm::BasicBlock *Entry = Scope.getEHBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("eh.cleanup");
+    Scope.setEHBlock(Entry);
+  }
+  return Entry;
+}
+
+/// Attempts to reduce a cleanup's entry block to a fallthrough.  This
+/// is basically llvm::MergeBlockIntoPredecessor, except
+/// simplified/optimized for the tighter constraints on cleanup blocks.
+///
+/// Returns the new block, whatever it is.
+static llvm::BasicBlock *SimplifyCleanupEntry(CodeGenFunction &CGF,
+                                              llvm::BasicBlock *Entry) {
+  llvm::BasicBlock *Pred = Entry->getSinglePredecessor();
+  if (!Pred) return Entry;
+
+  llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Pred->getTerminator());
+  if (!Br || Br->isConditional()) return Entry;
+  assert(Br->getSuccessor(0) == Entry);
+
+  // If we were previously inserting at the end of the cleanup entry
+  // block, we'll need to continue inserting at the end of the
+  // predecessor.
+  bool WasInsertBlock = CGF.Builder.GetInsertBlock() == Entry;
+  assert(!WasInsertBlock || CGF.Builder.GetInsertPoint() == Entry->end());
+
+  // Kill the branch.
+  Br->eraseFromParent();
+
+  // Merge the blocks.
+  Pred->getInstList().splice(Pred->end(), Entry->getInstList());
+
+  // Replace all uses of the entry with the predecessor, in case there
+  // are phis in the cleanup.
+  Entry->replaceAllUsesWith(Pred);
+
+  // Kill the entry block.
+  Entry->eraseFromParent();
+
+  if (WasInsertBlock)
+    CGF.Builder.SetInsertPoint(Pred);
+
+  return Pred;
+}
+
+static void EmitCleanup(CodeGenFunction &CGF,
+                        EHScopeStack::Cleanup *Fn,
+                        bool ForEH,
+                        llvm::Value *ActiveFlag) {
+  // EH cleanups always occur within a terminate scope.
+  if (ForEH) CGF.EHStack.pushTerminate();
+
+  // If there's an active flag, load it and skip the cleanup if it's
+  // false.
+  llvm::BasicBlock *ContBB = 0;
+  if (ActiveFlag) {
+    ContBB = CGF.createBasicBlock("cleanup.done");
+    llvm::BasicBlock *CleanupBB = CGF.createBasicBlock("cleanup.action");
+    llvm::Value *IsActive
+      = CGF.Builder.CreateLoad(ActiveFlag, "cleanup.is_active");
+    CGF.Builder.CreateCondBr(IsActive, CleanupBB, ContBB);
+    CGF.EmitBlock(CleanupBB);
+  }
+
+  // Ask the cleanup to emit itself.
+  Fn->Emit(CGF, ForEH);
+  assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
+
+  // Emit the continuation block if there was an active flag.
+  if (ActiveFlag)
+    CGF.EmitBlock(ContBB);
+
+  // Leave the terminate scope.
+  if (ForEH) CGF.EHStack.popTerminate();
+}
+
+static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit,
+                                          llvm::BasicBlock *From,
+                                          llvm::BasicBlock *To) {
+  // Exit is the exit block of a cleanup, so it always terminates in
+  // an unconditional branch or a switch.
+  llvm::TerminatorInst *Term = Exit->getTerminator();
+
+  if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
+    assert(Br->isUnconditional() && Br->getSuccessor(0) == From);
+    Br->setSuccessor(0, To);
+  } else {
+    llvm::SwitchInst *Switch = cast<llvm::SwitchInst>(Term);
+    for (unsigned I = 0, E = Switch->getNumSuccessors(); I != E; ++I)
+      if (Switch->getSuccessor(I) == From)
+        Switch->setSuccessor(I, To);
+  }
+}
+
+/// Pops a cleanup block.  If the block includes a normal cleanup, the
+/// current insertion point is threaded through the cleanup, as are
+/// any branch fixups on the cleanup.
+void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
+  assert(!EHStack.empty() && "cleanup stack is empty!");
+  assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
+  assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
+
+  // Remember activation information.
+  bool IsActive = Scope.isActive();
+  llvm::Value *NormalActiveFlag =
+    Scope.shouldTestFlagInNormalCleanup() ? Scope.getActiveFlag() : 0;
+  llvm::Value *EHActiveFlag = 
+    Scope.shouldTestFlagInEHCleanup() ? Scope.getActiveFlag() : 0;
+
+  // Check whether we need an EH cleanup.  This is only true if we've
+  // generated a lazy EH cleanup block.
+  bool RequiresEHCleanup = Scope.hasEHBranches();
+
+  // Check the three conditions which might require a normal cleanup:
+
+  // - whether there are branch fix-ups through this cleanup
+  unsigned FixupDepth = Scope.getFixupDepth();
+  bool HasFixups = EHStack.getNumBranchFixups() != FixupDepth;
+
+  // - whether there are branch-throughs or branch-afters
+  bool HasExistingBranches = Scope.hasBranches();
+
+  // - whether there's a fallthrough
+  llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock();
+  bool HasFallthrough = (FallthroughSource != 0 && IsActive);
+
+  // Branch-through fall-throughs leave the insertion point set to the
+  // end of the last cleanup, which points to the current scope.  The
+  // rest of IR gen doesn't need to worry about this; it only happens
+  // during the execution of PopCleanupBlocks().
+  bool HasPrebranchedFallthrough =
+    (FallthroughSource && FallthroughSource->getTerminator());
+
+  // If this is a normal cleanup, then having a prebranched
+  // fallthrough implies that the fallthrough source unconditionally
+  // jumps here.
+  assert(!Scope.isNormalCleanup() || !HasPrebranchedFallthrough ||
+         (Scope.getNormalBlock() &&
+          FallthroughSource->getTerminator()->getSuccessor(0)
+            == Scope.getNormalBlock()));
+
+  bool RequiresNormalCleanup = false;
+  if (Scope.isNormalCleanup() &&
+      (HasFixups || HasExistingBranches || HasFallthrough)) {
+    RequiresNormalCleanup = true;
+  }
+
+  // Even if we don't need the normal cleanup, we might still have
+  // prebranched fallthrough to worry about.
+  if (Scope.isNormalCleanup() && !RequiresNormalCleanup &&
+      HasPrebranchedFallthrough) {
+    assert(!IsActive);
+
+    llvm::BasicBlock *NormalEntry = Scope.getNormalBlock();
+
+    // If we're branching through this cleanup, just forward the
+    // prebranched fallthrough to the next cleanup, leaving the insert
+    // point in the old block.
+    if (FallthroughIsBranchThrough) {
+      EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
+      llvm::BasicBlock *EnclosingEntry = 
+        CreateNormalEntry(*this, cast<EHCleanupScope>(S));
+
+      ForwardPrebranchedFallthrough(FallthroughSource,
+                                    NormalEntry, EnclosingEntry);
+      assert(NormalEntry->use_empty() &&
+             "uses of entry remain after forwarding?");
+      delete NormalEntry;
+
+    // Otherwise, we're branching out;  just emit the next block.
+    } else {
+      EmitBlock(NormalEntry);
+      SimplifyCleanupEntry(*this, NormalEntry);
+    }
+  }
+
+  // If we don't need the cleanup at all, we're done.
+  if (!RequiresNormalCleanup && !RequiresEHCleanup) {
+    EHStack.popCleanup(); // safe because there are no fixups
+    assert(EHStack.getNumBranchFixups() == 0 ||
+           EHStack.hasNormalCleanups());
+    return;
+  }
+
+  // Copy the cleanup emission data out.  Note that SmallVector
+  // guarantees maximal alignment for its buffer regardless of its
+  // type parameter.
+  llvm::SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
+  CleanupBuffer.reserve(Scope.getCleanupSize());
+  memcpy(CleanupBuffer.data(),
+         Scope.getCleanupBuffer(), Scope.getCleanupSize());
+  CleanupBuffer.set_size(Scope.getCleanupSize());
+  EHScopeStack::Cleanup *Fn =
+    reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
+
+  // We want to emit the EH cleanup after the normal cleanup, but go
+  // ahead and do the setup for the EH cleanup while the scope is still
+  // alive.
+  llvm::BasicBlock *EHEntry = 0;
+  llvm::SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
+  if (RequiresEHCleanup) {
+    EHEntry = CreateEHEntry(*this, Scope);
+
+    // Figure out the branch-through dest if necessary.
+    llvm::BasicBlock *EHBranchThroughDest = 0;
+    if (Scope.hasEHBranchThroughs()) {
+      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
+      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
+      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
+    }
+
+    // If we have exactly one branch-after and no branch-throughs, we
+    // can dispatch it without a switch.
+    if (!Scope.hasEHBranchThroughs() &&
+        Scope.getNumEHBranchAfters() == 1) {
+      assert(!EHBranchThroughDest);
+
+      // TODO: remove the spurious eh.cleanup.dest stores if this edge
+      // never went through any switches.
+      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
+    
+    // Otherwise, if we have any branch-afters, we need a switch.
+    } else if (Scope.getNumEHBranchAfters()) {
+      // The default of the switch belongs to the branch-throughs if
+      // they exist.
+      llvm::BasicBlock *Default =
+        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
+
+      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
+
+      llvm::LoadInst *Load =
+        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
+      llvm::SwitchInst *Switch =
+        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+      EHInstsToAppend.push_back(Load);
+      EHInstsToAppend.push_back(Switch);
+
+      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
+        Switch->addCase(Scope.getEHBranchAfterIndex(I),
+                        Scope.getEHBranchAfterBlock(I));
+
+    // Otherwise, we have only branch-throughs; jump to the next EH
+    // cleanup.
+    } else {
+      assert(EHBranchThroughDest);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
+    }
+  }
+
+  if (!RequiresNormalCleanup) {
+    EHStack.popCleanup();
+  } else {
+    // If we have a fallthrough and no other need for the cleanup,
+    // emit it directly.
+    if (HasFallthrough && !HasPrebranchedFallthrough &&
+        !HasFixups && !HasExistingBranches) {
+
+      // Fixups can cause us to optimistically create a normal block,
+      // only to later have no real uses for it.  Just delete it in
+      // this case.
+      // TODO: we can potentially simplify all the uses after this.
+      if (Scope.getNormalBlock()) {
+        Scope.getNormalBlock()->replaceAllUsesWith(getUnreachableBlock());
+        delete Scope.getNormalBlock();
+      }
+
+      EHStack.popCleanup();
+
+      EmitCleanup(*this, Fn, /*ForEH*/ false, NormalActiveFlag);
+
+    // Otherwise, the best approach is to thread everything through
+    // the cleanup block and then try to clean up after ourselves.
+    } else {
+      // Force the entry block to exist.
+      llvm::BasicBlock *NormalEntry = CreateNormalEntry(*this, Scope);
+
+      // I.  Set up the fallthrough edge in.
+
+      // If there's a fallthrough, we need to store the cleanup
+      // destination index.  For fall-throughs this is always zero.
+      if (HasFallthrough) {
+        if (!HasPrebranchedFallthrough)
+          Builder.CreateStore(Builder.getInt32(0), getNormalCleanupDestSlot());
+
+      // Otherwise, clear the IP if we don't have fallthrough because
+      // the cleanup is inactive.  We don't need to save it because
+      // it's still just FallthroughSource.
+      } else if (FallthroughSource) {
+        assert(!IsActive && "source without fallthrough for active cleanup");
+        Builder.ClearInsertionPoint();
+      }
+
+      // II.  Emit the entry block.  This implicitly branches to it if
+      // we have fallthrough.  All the fixups and existing branches
+      // should already be branched to it.
+      EmitBlock(NormalEntry);
+
+      // III.  Figure out where we're going and build the cleanup
+      // epilogue.
+
+      bool HasEnclosingCleanups =
+        (Scope.getEnclosingNormalCleanup() != EHStack.stable_end());
+
+      // Compute the branch-through dest if we need it:
+      //   - if there are branch-throughs threaded through the scope
+      //   - if fall-through is a branch-through
+      //   - if there are fixups that will be optimistically forwarded
+      //     to the enclosing cleanup
+      llvm::BasicBlock *BranchThroughDest = 0;
+      if (Scope.hasBranchThroughs() ||
+          (FallthroughSource && FallthroughIsBranchThrough) ||
+          (HasFixups && HasEnclosingCleanups)) {
+        assert(HasEnclosingCleanups);
+        EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
+        BranchThroughDest = CreateNormalEntry(*this, cast<EHCleanupScope>(S));
+      }
+
+      llvm::BasicBlock *FallthroughDest = 0;
+      llvm::SmallVector<llvm::Instruction*, 2> InstsToAppend;
+
+      // If there's exactly one branch-after and no other threads,
+      // we can route it without a switch.
+      if (!Scope.hasBranchThroughs() && !HasFixups && !HasFallthrough &&
+          Scope.getNumBranchAfters() == 1) {
+        assert(!BranchThroughDest || !IsActive);
+
+        // TODO: clean up the possibly dead stores to the cleanup dest slot.
+        llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter));
+
+      // Build a switch-out if we need it:
+      //   - if there are branch-afters threaded through the scope
+      //   - if fall-through is a branch-after
+      //   - if there are fixups that have nowhere left to go and
+      //     so must be immediately resolved
+      } else if (Scope.getNumBranchAfters() ||
+                 (HasFallthrough && !FallthroughIsBranchThrough) ||
+                 (HasFixups && !HasEnclosingCleanups)) {
+
+        llvm::BasicBlock *Default =
+          (BranchThroughDest ? BranchThroughDest : getUnreachableBlock());
+
+        // TODO: base this on the number of branch-afters and fixups
+        const unsigned SwitchCapacity = 10;
+
+        llvm::LoadInst *Load =
+          new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest");
+        llvm::SwitchInst *Switch =
+          llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+        InstsToAppend.push_back(Load);
+        InstsToAppend.push_back(Switch);
+
+        // Branch-after fallthrough.
+        if (FallthroughSource && !FallthroughIsBranchThrough) {
+          FallthroughDest = createBasicBlock("cleanup.cont");
+          if (HasFallthrough)
+            Switch->addCase(Builder.getInt32(0), FallthroughDest);
+        }
+
+        for (unsigned I = 0, E = Scope.getNumBranchAfters(); I != E; ++I) {
+          Switch->addCase(Scope.getBranchAfterIndex(I),
+                          Scope.getBranchAfterBlock(I));
+        }
+
+        // If there aren't any enclosing cleanups, we can resolve all
+        // the fixups now.
+        if (HasFixups && !HasEnclosingCleanups)
+          ResolveAllBranchFixups(*this, Switch, NormalEntry);
+      } else {
+        // We should always have a branch-through destination in this case.
+        assert(BranchThroughDest);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchThroughDest));
+      }
+
+      // IV.  Pop the cleanup and emit it.
+      EHStack.popCleanup();
+      assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups);
+
+      EmitCleanup(*this, Fn, /*ForEH*/ false, NormalActiveFlag);
+
+      // Append the prepared cleanup prologue from above.
+      llvm::BasicBlock *NormalExit = Builder.GetInsertBlock();
+      for (unsigned I = 0, E = InstsToAppend.size(); I != E; ++I)
+        NormalExit->getInstList().push_back(InstsToAppend[I]);
+
+      // Optimistically hope that any fixups will continue falling through.
+      for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+           I < E; ++I) {
+        BranchFixup &Fixup = EHStack.getBranchFixup(I);
+        if (!Fixup.Destination) continue;
+        if (!Fixup.OptimisticBranchBlock) {
+          new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex),
+                              getNormalCleanupDestSlot(),
+                              Fixup.InitialBranch);
+          Fixup.InitialBranch->setSuccessor(0, NormalEntry);
+        }
+        Fixup.OptimisticBranchBlock = NormalExit;
+      }
+
+      // V.  Set up the fallthrough edge out.
+      
+      // Case 1: a fallthrough source exists but shouldn't branch to
+      // the cleanup because the cleanup is inactive.
+      if (!HasFallthrough && FallthroughSource) {
+        assert(!IsActive);
+
+        // If we have a prebranched fallthrough, that needs to be
+        // forwarded to the right block.
+        if (HasPrebranchedFallthrough) {
+          llvm::BasicBlock *Next;
+          if (FallthroughIsBranchThrough) {
+            Next = BranchThroughDest;
+            assert(!FallthroughDest);
+          } else {
+            Next = FallthroughDest;
+          }
+
+          ForwardPrebranchedFallthrough(FallthroughSource, NormalEntry, Next);
+        }
+        Builder.SetInsertPoint(FallthroughSource);
+
+      // Case 2: a fallthrough source exists and should branch to the
+      // cleanup, but we're not supposed to branch through to the next
+      // cleanup.
+      } else if (HasFallthrough && FallthroughDest) {
+        assert(!FallthroughIsBranchThrough);
+        EmitBlock(FallthroughDest);
+
+      // Case 3: a fallthrough source exists and should branch to the
+      // cleanup and then through to the next.
+      } else if (HasFallthrough) {
+        // Everything is already set up for this.
+
+      // Case 4: no fallthrough source exists.
+      } else {
+        Builder.ClearInsertionPoint();
+      }
+
+      // VI.  Assorted cleaning.
+
+      // Check whether we can merge NormalEntry into a single predecessor.
+      // This might invalidate (non-IR) pointers to NormalEntry.
+      llvm::BasicBlock *NewNormalEntry =
+        SimplifyCleanupEntry(*this, NormalEntry);
+
+      // If it did invalidate those pointers, and NormalEntry was the same
+      // as NormalExit, go back and patch up the fixups.
+      if (NewNormalEntry != NormalEntry && NormalEntry == NormalExit)
+        for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+               I < E; ++I)
+          EHStack.getBranchFixup(I).OptimisticBranchBlock = NewNormalEntry;
+    }
+  }
+
+  assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
+
+  // Emit the EH cleanup if required.
+  if (RequiresEHCleanup) {
+    CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+
+    EmitBlock(EHEntry);
+    EmitCleanup(*this, Fn, /*ForEH*/ true, EHActiveFlag);
+
+    // Append the prepared cleanup prologue from above.
+    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
+    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
+      EHExit->getInstList().push_back(EHInstsToAppend[I]);
+
+    Builder.restoreIP(SavedIP);
+
+    SimplifyCleanupEntry(*this, EHEntry);
+  }
+}
+
+/// Terminate the current block by emitting a branch which might leave
+/// the current cleanup-protected scope.  The target scope may not yet
+/// be known, in which case this will require a fixup.
+///
+/// As a side-effect, this method clears the insertion point.
+void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
+  assert(Dest.getScopeDepth().encloses(EHStack.getInnermostNormalCleanup())
+         && "stale jump destination");
+
+  if (!HaveInsertPoint())
+    return;
+
+  // Create the branch.
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
+
+  // Calculate the innermost active normal cleanup.
+  EHScopeStack::stable_iterator
+    TopCleanup = EHStack.getInnermostActiveNormalCleanup();
+
+  // If we're not in an active normal cleanup scope, or if the
+  // destination scope is within the innermost active normal cleanup
+  // scope, we don't need to worry about fixups.
+  if (TopCleanup == EHStack.stable_end() ||
+      TopCleanup.encloses(Dest.getScopeDepth())) { // works for invalid
+    Builder.ClearInsertionPoint();
+    return;
+  }
+
+  // If we can't resolve the destination cleanup scope, just add this
+  // to the current cleanup scope as a branch fixup.
+  if (!Dest.getScopeDepth().isValid()) {
+    BranchFixup &Fixup = EHStack.addBranchFixup();
+    Fixup.Destination = Dest.getBlock();
+    Fixup.DestinationIndex = Dest.getDestIndex();
+    Fixup.InitialBranch = BI;
+    Fixup.OptimisticBranchBlock = 0;
+
+    Builder.ClearInsertionPoint();
+    return;
+  }
+
+  // Otherwise, thread through all the normal cleanups in scope.
+
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI);
+
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(TopCleanup));
+    BI->setSuccessor(0, CreateNormalEntry(*this, Scope));
+  }
+
+  // Add this destination to all the scopes involved.
+  EHScopeStack::stable_iterator I = TopCleanup;
+  EHScopeStack::stable_iterator E = Dest.getScopeDepth();
+  if (E.strictlyEncloses(I)) {
+    while (true) {
+      EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+      assert(Scope.isNormalCleanup());
+      I = Scope.getEnclosingNormalCleanup();
+
+      // If this is the last cleanup we're propagating through, tell it
+      // that there's a resolved jump moving through it.
+      if (!E.strictlyEncloses(I)) {
+        Scope.addBranchAfter(Index, Dest.getBlock());
+        break;
+      }
+
+      // Otherwise, tell the scope that there's a jump propoagating
+      // through it.  If this isn't new information, all the rest of
+      // the work has been done before.
+      if (!Scope.addBranchThrough(Dest.getBlock()))
+        break;
+    }
+  }
+  
+  Builder.ClearInsertionPoint();
+}
+
+void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
+  // We should never get invalid scope depths for an UnwindDest; that
+  // implies that the destination wasn't set up correctly.
+  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
+
+  if (!HaveInsertPoint())
+    return;
+
+  // Create the branch.
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
+
+  // Calculate the innermost active cleanup.
+  EHScopeStack::stable_iterator
+    InnermostCleanup = EHStack.getInnermostActiveEHCleanup();
+
+  // If the destination is in the same EH cleanup scope as us, we
+  // don't need to thread through anything.
+  if (InnermostCleanup.encloses(Dest.getScopeDepth())) {
+    Builder.ClearInsertionPoint();
+    return;
+  }
+  assert(InnermostCleanup != EHStack.stable_end());
+
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
+
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(InnermostCleanup));
+    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
+  }
+  
+  // Add this destination to all the scopes involved.
+  for (EHScopeStack::stable_iterator
+         I = InnermostCleanup, E = Dest.getScopeDepth(); ; ) {
+    assert(E.strictlyEncloses(I));
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+    assert(Scope.isEHCleanup());
+    I = Scope.getEnclosingEHCleanup();
+
+    // If this is the last cleanup we're propagating through, add this
+    // as a branch-after.
+    if (I == E) {
+      Scope.addEHBranchAfter(Index, Dest.getBlock());
+      break;
+    }
+
+    // Otherwise, add it as a branch-through.  If this isn't new
+    // information, all the rest of the work has been done before.
+    if (!Scope.addEHBranchThrough(Dest.getBlock()))
+      break;
+  }
+  
+  Builder.ClearInsertionPoint();
+}
+
+static bool IsUsedAsNormalCleanup(EHScopeStack &EHStack,
+                                  EHScopeStack::stable_iterator C) {
+  // If we needed a normal block for any reason, that counts.
+  if (cast<EHCleanupScope>(*EHStack.find(C)).getNormalBlock())
+    return true;
+
+  // Check whether any enclosed cleanups were needed.
+  for (EHScopeStack::stable_iterator
+         I = EHStack.getInnermostNormalCleanup();
+         I != C; ) {
+    assert(C.strictlyEncloses(I));
+    EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
+    if (S.getNormalBlock()) return true;
+    I = S.getEnclosingNormalCleanup();
+  }
+
+  return false;
+}
+
+static bool IsUsedAsEHCleanup(EHScopeStack &EHStack,
+                              EHScopeStack::stable_iterator C) {
+  // If we needed an EH block for any reason, that counts.
+  if (cast<EHCleanupScope>(*EHStack.find(C)).getEHBlock())
+    return true;
+
+  // Check whether any enclosed cleanups were needed.
+  for (EHScopeStack::stable_iterator
+         I = EHStack.getInnermostEHCleanup(); I != C; ) {
+    assert(C.strictlyEncloses(I));
+    EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
+    if (S.getEHBlock()) return true;
+    I = S.getEnclosingEHCleanup();
+  }
+
+  return false;
+}
+
+enum ForActivation_t {
+  ForActivation,
+  ForDeactivation
+};
+
+/// The given cleanup block is changing activation state.  Configure a
+/// cleanup variable if necessary.
+///
+/// It would be good if we had some way of determining if there were
+/// extra uses *after* the change-over point.
+static void SetupCleanupBlockActivation(CodeGenFunction &CGF,
+                                        EHScopeStack::stable_iterator C,
+                                        ForActivation_t Kind) {
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*CGF.EHStack.find(C));
+
+  // We always need the flag if we're activating the cleanup, because
+  // we have to assume that the current location doesn't necessarily
+  // dominate all future uses of the cleanup.
+  bool NeedFlag = (Kind == ForActivation);
+
+  // Calculate whether the cleanup was used:
+
+  //   - as a normal cleanup
+  if (Scope.isNormalCleanup() && IsUsedAsNormalCleanup(CGF.EHStack, C)) {
+    Scope.setTestFlagInNormalCleanup();
+    NeedFlag = true;
+  }
+
+  //  - as an EH cleanup
+  if (Scope.isEHCleanup() && IsUsedAsEHCleanup(CGF.EHStack, C)) {
+    Scope.setTestFlagInEHCleanup();
+    NeedFlag = true;
+  }
+
+  // If it hasn't yet been used as either, we're done.
+  if (!NeedFlag) return;
+
+  llvm::AllocaInst *Var = Scope.getActiveFlag();
+  if (!Var) {
+    Var = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(), "cleanup.isactive");
+    Scope.setActiveFlag(Var);
+
+    // Initialize to true or false depending on whether it was
+    // active up to this point.
+    CGF.InitTempAlloca(Var, CGF.Builder.getInt1(Kind == ForDeactivation));
+  }
+
+  CGF.Builder.CreateStore(CGF.Builder.getInt1(Kind == ForActivation), Var);
+}
+
+/// Activate a cleanup that was created in an inactivated state.
+void CodeGenFunction::ActivateCleanupBlock(EHScopeStack::stable_iterator C) {
+  assert(C != EHStack.stable_end() && "activating bottom of stack?");
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(C));
+  assert(!Scope.isActive() && "double activation");
+
+  SetupCleanupBlockActivation(*this, C, ForActivation);
+
+  Scope.setActive(true);
+}
+
+/// Deactive a cleanup that was created in an active state.
+void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C) {
+  assert(C != EHStack.stable_end() && "deactivating bottom of stack?");
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(C));
+  assert(Scope.isActive() && "double deactivation");
+
+  // If it's the top of the stack, just pop it.
+  if (C == EHStack.stable_begin()) {
+    // If it's a normal cleanup, we need to pretend that the
+    // fallthrough is unreachable.
+    CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+    PopCleanupBlock();
+    Builder.restoreIP(SavedIP);
+    return;
+  }
+
+  // Otherwise, follow the general case.
+  SetupCleanupBlockActivation(*this, C, ForDeactivation);
+
+  Scope.setActive(false);
+}
+
+llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
+  if (!NormalCleanupDest)
+    NormalCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
+  return NormalCleanupDest;
+}
+
+llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
+  if (!EHCleanupDest)
+    EHCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
+  return EHCleanupDest;
+}
diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h
new file mode 100644
index 000000000000..c93ec5bb76a9
--- /dev/null
+++ b/lib/CodeGen/CGCleanup.h
@@ -0,0 +1,560 @@
+//===-- CGCleanup.h - Classes for cleanups IR generation --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// These classes support the generation of LLVM IR for cleanups.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CODEGEN_CGCLEANUP_H
+#define CLANG_CODEGEN_CGCLEANUP_H
+
+/// EHScopeStack is defined in CodeGenFunction.h, but its
+/// implementation is in this file and in CGCleanup.cpp.
+#include "CodeGenFunction.h"
+
+namespace llvm {
+  class Value;
+  class BasicBlock;
+}
+
+namespace clang {
+namespace CodeGen {
+
+/// A protected scope for zero-cost EH handling.
+class EHScope {
+  llvm::BasicBlock *CachedLandingPad;
+
+  unsigned K : 2;
+
+protected:
+  enum { BitsRemaining = 30 };
+
+public:
+  enum Kind { Cleanup, Catch, Terminate, Filter };
+
+  EHScope(Kind K) : CachedLandingPad(0), K(K) {}
+
+  Kind getKind() const { return static_cast<Kind>(K); }
+
+  llvm::BasicBlock *getCachedLandingPad() const {
+    return CachedLandingPad;
+  }
+
+  void setCachedLandingPad(llvm::BasicBlock *Block) {
+    CachedLandingPad = Block;
+  }
+};
+
+/// A scope which attempts to handle some, possibly all, types of
+/// exceptions.
+///
+/// Objective C @finally blocks are represented using a cleanup scope
+/// after the catch scope.
+class EHCatchScope : public EHScope {
+  unsigned NumHandlers : BitsRemaining;
+
+  // In effect, we have a flexible array member
+  //   Handler Handlers[0];
+  // But that's only standard in C99, not C++, so we have to do
+  // annoying pointer arithmetic instead.
+
+public:
+  struct Handler {
+    /// A type info value, or null (C++ null, not an LLVM null pointer)
+    /// for a catch-all.
+    llvm::Value *Type;
+
+    /// The catch handler for this type.
+    llvm::BasicBlock *Block;
+
+    /// The unwind destination index for this handler.
+    unsigned Index;
+  };
+
+private:
+  friend class EHScopeStack;
+
+  Handler *getHandlers() {
+    return reinterpret_cast<Handler*>(this+1);
+  }
+
+  const Handler *getHandlers() const {
+    return reinterpret_cast<const Handler*>(this+1);
+  }
+
+public:
+  static size_t getSizeForNumHandlers(unsigned N) {
+    return sizeof(EHCatchScope) + N * sizeof(Handler);
+  }
+
+  EHCatchScope(unsigned NumHandlers)
+    : EHScope(Catch), NumHandlers(NumHandlers) {
+  }
+
+  unsigned getNumHandlers() const {
+    return NumHandlers;
+  }
+
+  void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) {
+    setHandler(I, /*catchall*/ 0, Block);
+  }
+
+  void setHandler(unsigned I, llvm::Value *Type, llvm::BasicBlock *Block) {
+    assert(I < getNumHandlers());
+    getHandlers()[I].Type = Type;
+    getHandlers()[I].Block = Block;
+  }
+
+  const Handler &getHandler(unsigned I) const {
+    assert(I < getNumHandlers());
+    return getHandlers()[I];
+  }
+
+  typedef const Handler *iterator;
+  iterator begin() const { return getHandlers(); }
+  iterator end() const { return getHandlers() + getNumHandlers(); }
+
+  static bool classof(const EHScope *Scope) {
+    return Scope->getKind() == Catch;
+  }
+};
+
+/// A cleanup scope which generates the cleanup blocks lazily.
+class EHCleanupScope : public EHScope {
+  /// Whether this cleanup needs to be run along normal edges.
+  bool IsNormalCleanup : 1;
+
+  /// Whether this cleanup needs to be run along exception edges.
+  bool IsEHCleanup : 1;
+
+  /// Whether this cleanup is currently active.
+  bool IsActive : 1;
+
+  /// Whether the normal cleanup should test the activation flag.
+  bool TestFlagInNormalCleanup : 1;
+
+  /// Whether the EH cleanup should test the activation flag.
+  bool TestFlagInEHCleanup : 1;
+
+  /// The amount of extra storage needed by the Cleanup.
+  /// Always a multiple of the scope-stack alignment.
+  unsigned CleanupSize : 12;
+
+  /// The number of fixups required by enclosing scopes (not including
+  /// this one).  If this is the top cleanup scope, all the fixups
+  /// from this index onwards belong to this scope.
+  unsigned FixupDepth : BitsRemaining - 17; // currently 13
+
+  /// The nearest normal cleanup scope enclosing this one.
+  EHScopeStack::stable_iterator EnclosingNormal;
+
+  /// The nearest EH cleanup scope enclosing this one.
+  EHScopeStack::stable_iterator EnclosingEH;
+
+  /// The dual entry/exit block along the normal edge.  This is lazily
+  /// created if needed before the cleanup is popped.
+  llvm::BasicBlock *NormalBlock;
+
+  /// The dual entry/exit block along the EH edge.  This is lazily
+  /// created if needed before the cleanup is popped.
+  llvm::BasicBlock *EHBlock;
+
+  /// An optional i1 variable indicating whether this cleanup has been
+  /// activated yet.
+  llvm::AllocaInst *ActiveFlag;
+
+  /// Extra information required for cleanups that have resolved
+  /// branches through them.  This has to be allocated on the side
+  /// because everything on the cleanup stack has be trivially
+  /// movable.
+  struct ExtInfo {
+    /// The destinations of normal branch-afters and branch-throughs.
+    llvm::SmallPtrSet<llvm::BasicBlock*, 4> Branches;
+
+    /// Normal branch-afters.
+    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
+      BranchAfters;
+
+    /// The destinations of EH branch-afters and branch-throughs.
+    /// TODO: optimize for the extremely common case of a single
+    /// branch-through.
+    llvm::SmallPtrSet<llvm::BasicBlock*, 4> EHBranches;
+
+    /// EH branch-afters.
+    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
+    EHBranchAfters;
+  };
+  mutable struct ExtInfo *ExtInfo;
+
+  struct ExtInfo &getExtInfo() {
+    if (!ExtInfo) ExtInfo = new struct ExtInfo();
+    return *ExtInfo;
+  }
+
+  const struct ExtInfo &getExtInfo() const {
+    if (!ExtInfo) ExtInfo = new struct ExtInfo();
+    return *ExtInfo;
+  }
+
+public:
+  /// Gets the size required for a lazy cleanup scope with the given
+  /// cleanup-data requirements.
+  static size_t getSizeForCleanupSize(size_t Size) {
+    return sizeof(EHCleanupScope) + Size;
+  }
+
+  size_t getAllocatedSize() const {
+    return sizeof(EHCleanupScope) + CleanupSize;
+  }
+
+  EHCleanupScope(bool IsNormal, bool IsEH, bool IsActive,
+                 unsigned CleanupSize, unsigned FixupDepth,
+                 EHScopeStack::stable_iterator EnclosingNormal,
+                 EHScopeStack::stable_iterator EnclosingEH)
+    : EHScope(EHScope::Cleanup),
+      IsNormalCleanup(IsNormal), IsEHCleanup(IsEH), IsActive(IsActive),
+      TestFlagInNormalCleanup(false), TestFlagInEHCleanup(false),
+      CleanupSize(CleanupSize), FixupDepth(FixupDepth),
+      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
+      NormalBlock(0), EHBlock(0), ActiveFlag(0), ExtInfo(0)
+  {
+    assert(this->CleanupSize == CleanupSize && "cleanup size overflow");
+  }
+
+  ~EHCleanupScope() {
+    delete ExtInfo;
+  }
+
+  bool isNormalCleanup() const { return IsNormalCleanup; }
+  llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
+  void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
+
+  bool isEHCleanup() const { return IsEHCleanup; }
+  llvm::BasicBlock *getEHBlock() const { return EHBlock; }
+  void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
+
+  bool isActive() const { return IsActive; }
+  void setActive(bool A) { IsActive = A; }
+
+  llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; }
+  void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; }
+
+  void setTestFlagInNormalCleanup() { TestFlagInNormalCleanup = true; }
+  bool shouldTestFlagInNormalCleanup() const { return TestFlagInNormalCleanup; }
+
+  void setTestFlagInEHCleanup() { TestFlagInEHCleanup = true; }
+  bool shouldTestFlagInEHCleanup() const { return TestFlagInEHCleanup; }
+
+  unsigned getFixupDepth() const { return FixupDepth; }
+  EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
+    return EnclosingNormal;
+  }
+  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
+    return EnclosingEH;
+  }
+
+  size_t getCleanupSize() const { return CleanupSize; }
+  void *getCleanupBuffer() { return this + 1; }
+
+  EHScopeStack::Cleanup *getCleanup() {
+    return reinterpret_cast<EHScopeStack::Cleanup*>(getCleanupBuffer());
+  }
+
+  /// True if this cleanup scope has any branch-afters or branch-throughs.
+  bool hasBranches() const { return ExtInfo && !ExtInfo->Branches.empty(); }
+
+  /// Add a branch-after to this cleanup scope.  A branch-after is a
+  /// branch from a point protected by this (normal) cleanup to a
+  /// point in the normal cleanup scope immediately containing it.
+  /// For example,
+  ///   for (;;) { A a; break; }
+  /// contains a branch-after.
+  ///
+  /// Branch-afters each have their own destination out of the
+  /// cleanup, guaranteed distinct from anything else threaded through
+  /// it.  Therefore branch-afters usually force a switch after the
+  /// cleanup.
+  void addBranchAfter(llvm::ConstantInt *Index,
+                      llvm::BasicBlock *Block) {
+    struct ExtInfo &ExtInfo = getExtInfo();
+    if (ExtInfo.Branches.insert(Block))
+      ExtInfo.BranchAfters.push_back(std::make_pair(Block, Index));
+  }
+
+  /// Return the number of unique branch-afters on this scope.
+  unsigned getNumBranchAfters() const {
+    return ExtInfo ? ExtInfo->BranchAfters.size() : 0;
+  }
+
+  llvm::BasicBlock *getBranchAfterBlock(unsigned I) const {
+    assert(I < getNumBranchAfters());
+    return ExtInfo->BranchAfters[I].first;
+  }
+
+  llvm::ConstantInt *getBranchAfterIndex(unsigned I) const {
+    assert(I < getNumBranchAfters());
+    return ExtInfo->BranchAfters[I].second;
+  }
+
+  /// Add a branch-through to this cleanup scope.  A branch-through is
+  /// a branch from a scope protected by this (normal) cleanup to an
+  /// enclosing scope other than the immediately-enclosing normal
+  /// cleanup scope.
+  ///
+  /// In the following example, the branch through B's scope is a
+  /// branch-through, while the branch through A's scope is a
+  /// branch-after:
+  ///   for (;;) { A a; B b; break; }
+  ///
+  /// All branch-throughs have a common destination out of the
+  /// cleanup, one possibly shared with the fall-through.  Therefore
+  /// branch-throughs usually don't force a switch after the cleanup.
+  ///
+  /// \return true if the branch-through was new to this scope
+  bool addBranchThrough(llvm::BasicBlock *Block) {
+    return getExtInfo().Branches.insert(Block);
+  }
+
+  /// Determines if this cleanup scope has any branch throughs.
+  bool hasBranchThroughs() const {
+    if (!ExtInfo) return false;
+    return (ExtInfo->BranchAfters.size() != ExtInfo->Branches.size());
+  }
+
+  // Same stuff, only for EH branches instead of normal branches.
+  // It's quite possible that we could find a better representation
+  // for this.
+
+  bool hasEHBranches() const { return ExtInfo && !ExtInfo->EHBranches.empty(); }
+  void addEHBranchAfter(llvm::ConstantInt *Index,
+                        llvm::BasicBlock *Block) {
+    struct ExtInfo &ExtInfo = getExtInfo();
+    if (ExtInfo.EHBranches.insert(Block))
+      ExtInfo.EHBranchAfters.push_back(std::make_pair(Block, Index));
+  }
+
+  unsigned getNumEHBranchAfters() const {
+    return ExtInfo ? ExtInfo->EHBranchAfters.size() : 0;
+  }
+
+  llvm::BasicBlock *getEHBranchAfterBlock(unsigned I) const {
+    assert(I < getNumEHBranchAfters());
+    return ExtInfo->EHBranchAfters[I].first;
+  }
+
+  llvm::ConstantInt *getEHBranchAfterIndex(unsigned I) const {
+    assert(I < getNumEHBranchAfters());
+    return ExtInfo->EHBranchAfters[I].second;
+  }
+
+  bool addEHBranchThrough(llvm::BasicBlock *Block) {
+    return getExtInfo().EHBranches.insert(Block);
+  }
+
+  bool hasEHBranchThroughs() const {
+    if (!ExtInfo) return false;
+    return (ExtInfo->EHBranchAfters.size() != ExtInfo->EHBranches.size());
+  }
+
+  static bool classof(const EHScope *Scope) {
+    return (Scope->getKind() == Cleanup);
+  }
+};
+
+/// An exceptions scope which filters exceptions thrown through it.
+/// Only exceptions matching the filter types will be permitted to be
+/// thrown.
+///
+/// This is used to implement C++ exception specifications.
+class EHFilterScope : public EHScope {
+  unsigned NumFilters : BitsRemaining;
+
+  // Essentially ends in a flexible array member:
+  // llvm::Value *FilterTypes[0];
+
+  llvm::Value **getFilters() {
+    return reinterpret_cast<llvm::Value**>(this+1);
+  }
+
+  llvm::Value * const *getFilters() const {
+    return reinterpret_cast<llvm::Value* const *>(this+1);
+  }
+
+public:
+  EHFilterScope(unsigned NumFilters) :
+    EHScope(Filter), NumFilters(NumFilters) {}
+
+  static size_t getSizeForNumFilters(unsigned NumFilters) {
+    return sizeof(EHFilterScope) + NumFilters * sizeof(llvm::Value*);
+  }
+
+  unsigned getNumFilters() const { return NumFilters; }
+
+  void setFilter(unsigned I, llvm::Value *FilterValue) {
+    assert(I < getNumFilters());
+    getFilters()[I] = FilterValue;
+  }
+
+  llvm::Value *getFilter(unsigned I) const {
+    assert(I < getNumFilters());
+    return getFilters()[I];
+  }
+
+  static bool classof(const EHScope *Scope) {
+    return Scope->getKind() == Filter;
+  }
+};
+
+/// An exceptions scope which calls std::terminate if any exception
+/// reaches it.
+class EHTerminateScope : public EHScope {
+  unsigned DestIndex : BitsRemaining;
+public:
+  EHTerminateScope(unsigned Index) : EHScope(Terminate), DestIndex(Index) {}
+  static size_t getSize() { return sizeof(EHTerminateScope); }
+
+  unsigned getDestIndex() const { return DestIndex; }
+
+  static bool classof(const EHScope *Scope) {
+    return Scope->getKind() == Terminate;
+  }
+};
+
+/// A non-stable pointer into the scope stack.
+class EHScopeStack::iterator {
+  char *Ptr;
+
+  friend class EHScopeStack;
+  explicit iterator(char *Ptr) : Ptr(Ptr) {}
+
+public:
+  iterator() : Ptr(0) {}
+
+  EHScope *get() const { 
+    return reinterpret_cast<EHScope*>(Ptr);
+  }
+
+  EHScope *operator->() const { return get(); }
+  EHScope &operator*() const { return *get(); }
+
+  iterator &operator++() {
+    switch (get()->getKind()) {
+    case EHScope::Catch:
+      Ptr += EHCatchScope::getSizeForNumHandlers(
+          static_cast<const EHCatchScope*>(get())->getNumHandlers());
+      break;
+
+    case EHScope::Filter:
+      Ptr += EHFilterScope::getSizeForNumFilters(
+          static_cast<const EHFilterScope*>(get())->getNumFilters());
+      break;
+
+    case EHScope::Cleanup:
+      Ptr += static_cast<const EHCleanupScope*>(get())
+        ->getAllocatedSize();
+      break;
+
+    case EHScope::Terminate:
+      Ptr += EHTerminateScope::getSize();
+      break;
+    }
+
+    return *this;
+  }
+
+  iterator next() {
+    iterator copy = *this;
+    ++copy;
+    return copy;
+  }
+
+  iterator operator++(int) {
+    iterator copy = *this;
+    operator++();
+    return copy;
+  }
+
+  bool encloses(iterator other) const { return Ptr >= other.Ptr; }
+  bool strictlyEncloses(iterator other) const { return Ptr > other.Ptr; }
+
+  bool operator==(iterator other) const { return Ptr == other.Ptr; }
+  bool operator!=(iterator other) const { return Ptr != other.Ptr; }
+};
+
+inline EHScopeStack::iterator EHScopeStack::begin() const {
+  return iterator(StartOfData);
+}
+
+inline EHScopeStack::iterator EHScopeStack::end() const {
+  return iterator(EndOfBuffer);
+}
+
+inline void EHScopeStack::popCatch() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  assert(isa<EHCatchScope>(*begin()));
+  StartOfData += EHCatchScope::getSizeForNumHandlers(
+                          cast<EHCatchScope>(*begin()).getNumHandlers());
+
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
+  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
+  CatchDepth--;
+}
+
+inline void EHScopeStack::popTerminate() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  assert(isa<EHTerminateScope>(*begin()));
+  StartOfData += EHTerminateScope::getSize();
+
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
+  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
+  CatchDepth--;
+}
+
+inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const {
+  assert(sp.isValid() && "finding invalid savepoint");
+  assert(sp.Size <= stable_begin().Size && "finding savepoint after pop");
+  return iterator(EndOfBuffer - sp.Size);
+}
+
+inline EHScopeStack::stable_iterator
+EHScopeStack::stabilize(iterator ir) const {
+  assert(StartOfData <= ir.Ptr && ir.Ptr <= EndOfBuffer);
+  return stable_iterator(EndOfBuffer - ir.Ptr);
+}
+
+inline EHScopeStack::stable_iterator
+EHScopeStack::getInnermostActiveNormalCleanup() const {
+  for (EHScopeStack::stable_iterator
+         I = getInnermostNormalCleanup(), E = stable_end(); I != E; ) {
+    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
+    if (S.isActive()) return I;
+    I = S.getEnclosingNormalCleanup();
+  }
+  return stable_end();
+}
+
+inline EHScopeStack::stable_iterator
+EHScopeStack::getInnermostActiveEHCleanup() const {
+  for (EHScopeStack::stable_iterator
+         I = getInnermostEHCleanup(), E = stable_end(); I != E; ) {
+    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
+    if (S.isActive()) return I;
+    I = S.getEnclosingEHCleanup();
+  }
+  return stable_end();
+}
+
+}
+}
+
+#endif
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 406db886eeed..469b4605d7ca 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -14,6 +14,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGBlocks.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclObjC.h"
@@ -32,13 +33,14 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Dwarf.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 using namespace clang;
 using namespace clang::CodeGen;
 
 CGDebugInfo::CGDebugInfo(CodeGenModule &CGM)
-  : CGM(CGM), DebugFactory(CGM.getModule()),
+  : CGM(CGM), DBuilder(CGM.getModule()),
     BlockLiteralGenericSet(false) {
   CreateCompileUnit();
 }
@@ -53,28 +55,27 @@ void CGDebugInfo::setLocation(SourceLocation Loc) {
 }
 
 /// getContextDescriptor - Get context info for the decl.
-llvm::DIDescriptor CGDebugInfo::getContextDescriptor(const Decl *Context,
-                                              llvm::DIDescriptor &CompileUnit) {
+llvm::DIDescriptor CGDebugInfo::getContextDescriptor(const Decl *Context) {
   if (!Context)
-    return CompileUnit;
+    return TheCU;
 
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator
     I = RegionMap.find(Context);
   if (I != RegionMap.end())
-    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(I->second));
+    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(&*I->second));
 
   // Check namespace.
   if (const NamespaceDecl *NSDecl = dyn_cast<NamespaceDecl>(Context))
-    return llvm::DIDescriptor(getOrCreateNameSpace(NSDecl, CompileUnit));
+    return llvm::DIDescriptor(getOrCreateNameSpace(NSDecl));
 
   if (const RecordDecl *RDecl = dyn_cast<RecordDecl>(Context)) {
     if (!RDecl->isDependentType()) {
-      llvm::DIType Ty = getOrCreateType(CGM.getContext().getTypeDeclType(RDecl), 
-                                        llvm::DIFile(CompileUnit));
+      llvm::DIType Ty = getOrCreateType(CGM.getContext().getTypeDeclType(RDecl),
+                                        getOrCreateMainFile());
       return llvm::DIDescriptor(Ty);
     }
   }
-  return CompileUnit;
+  return TheCU;
 }
 
 /// getFunctionName - Get function name for the given FunctionDecl. If the
@@ -100,9 +101,14 @@ llvm::StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
   llvm::raw_svector_ostream OS(MethodName);
   OS << (OMD->isInstanceMethod() ? '-' : '+') << '[';
   const DeclContext *DC = OMD->getDeclContext();
-  if (const ObjCImplementationDecl *OID = dyn_cast<const ObjCImplementationDecl>(DC)) {
+  if (const ObjCImplementationDecl *OID = 
+      dyn_cast<const ObjCImplementationDecl>(DC)) {
      OS << OID->getName();
-  } else if (const ObjCCategoryImplDecl *OCD = dyn_cast<const ObjCCategoryImplDecl>(DC)){
+  } else if (const ObjCInterfaceDecl *OID = 
+             dyn_cast<const ObjCInterfaceDecl>(DC)) {
+      OS << OID->getName();
+  } else if (const ObjCCategoryImplDecl *OCD = 
+             dyn_cast<const ObjCCategoryImplDecl>(DC)){
       OS << ((NamedDecl *)OCD)->getIdentifier()->getNameStart() << '(' <<
           OCD->getIdentifier()->getNameStart() << ')';
   }
@@ -131,8 +137,8 @@ CGDebugInfo::getClassName(RecordDecl *RD) {
     NumArgs = TST->getNumArgs();
   } else {
     const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-    Args = TemplateArgs.getFlatArgumentList();
-    NumArgs = TemplateArgs.flat_size();
+    Args = TemplateArgs.data();
+    NumArgs = TemplateArgs.size();
   }
   Buffer = RD->getIdentifier()->getNameStart();
   PrintingPolicy Policy(CGM.getLangOptions());
@@ -144,18 +150,21 @@ CGDebugInfo::getClassName(RecordDecl *RD) {
   char *StrPtr = DebugInfoNames.Allocate<char>(Buffer.length());
   memcpy(StrPtr, Buffer.data(), Buffer.length());
   return llvm::StringRef(StrPtr, Buffer.length());
-
 }
 
 /// getOrCreateFile - Get the file debug info descriptor for the input location.
 llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
   if (!Loc.isValid())
     // If Location is not valid then use main input file.
-    return DebugFactory.CreateFile(TheCU.getFilename(), TheCU.getDirectory(),
-                                   TheCU);
+    return DBuilder.CreateFile(TheCU.getFilename(), TheCU.getDirectory());
+
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
 
+  if (PLoc.isInvalid() || llvm::StringRef(PLoc.getFilename()).empty())
+    // If the location is not valid then use main input file.
+    return DBuilder.CreateFile(TheCU.getFilename(), TheCU.getDirectory());
+
   // Cache the results.
   const char *fname = PLoc.getFilename();
   llvm::DenseMap<const char *, llvm::WeakVH>::iterator it =
@@ -167,21 +176,25 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
       return llvm::DIFile(cast<llvm::MDNode>(it->second));
   }
 
-  llvm::DIFile F = DebugFactory.CreateFile(PLoc.getFilename(),
-                                           getCurrentDirname(), TheCU);
+  llvm::DIFile F = DBuilder.CreateFile(PLoc.getFilename(), getCurrentDirname());
 
   DIFileCache[fname] = F;
   return F;
 
 }
 
+/// getOrCreateMainFile - Get the file info for main compile unit.
+llvm::DIFile CGDebugInfo::getOrCreateMainFile() {
+  return DBuilder.CreateFile(TheCU.getFilename(), TheCU.getDirectory());
+}
+
 /// getLineNumber - Get line number for the location. If location is invalid
 /// then use current location.
 unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) {
   assert (CurLoc.isValid() && "Invalid current location!");
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc.isValid() ? Loc : CurLoc);
-  return PLoc.getLine();
+  return PLoc.isValid()? PLoc.getLine() : 0;
 }
 
 /// getColumnNumber - Get column number for the location. If location is 
@@ -190,7 +203,7 @@ unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc) {
   assert (CurLoc.isValid() && "Invalid current location!");
   SourceManager &SM = CGM.getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc.isValid() ? Loc : CurLoc);
-  return PLoc.getColumn();
+  return PLoc.isValid()? PLoc.getColumn() : 0;
 }
 
 llvm::StringRef CGDebugInfo::getCurrentDirname() {
@@ -251,16 +264,17 @@ void CGDebugInfo::CreateCompileUnit() {
     RuntimeVers = LO.ObjCNonFragileABI ? 2 : 1;
 
   // Create new compile unit.
-  TheCU = DebugFactory.CreateCompileUnit(
+  DBuilder.CreateCompileUnit(
     LangTag, Filename, getCurrentDirname(),
-    Producer, true,
+    Producer,
     LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers);
+  // FIXME - Eliminate TheCU.
+  TheCU = llvm::DICompileUnit(DBuilder.getCU());
 }
 
 /// CreateType - Get the Basic type from the cache or create a new
 /// one if necessary.
-llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
-                                     llvm::DIFile Unit) {
+llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
   unsigned Encoding = 0;
   const char *BTName = NULL;
   switch (BT->getKind()) {
@@ -268,10 +282,10 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
   case BuiltinType::Void:
     return llvm::DIType();
   case BuiltinType::ObjCClass:
-    return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
-                                            Unit, "objc_class", Unit, 0, 0, 0, 0,
-                                            llvm::DIType::FlagFwdDecl, 
-                                            llvm::DIType(), llvm::DIArray());
+    return DBuilder.CreateStructType(TheCU, "objc_class", 
+                                     getOrCreateMainFile(), 0, 0, 0,
+                                     llvm::DIDescriptor::FlagFwdDecl, 
+                                     llvm::DIArray());
   case BuiltinType::ObjCId: {
     // typedef struct objc_class *Class;
     // typedef struct objc_object {
@@ -279,31 +293,31 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
     // } *id;
 
     llvm::DIType OCTy = 
-      DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
-                                       Unit, "objc_class", Unit, 0, 0, 0, 0,
-                                       llvm::DIType::FlagFwdDecl, 
-                                       llvm::DIType(), llvm::DIArray());
+      DBuilder.CreateStructType(TheCU, "objc_class", 
+                                getOrCreateMainFile(), 0, 0, 0,
+                                llvm::DIDescriptor::FlagFwdDecl, 
+                                llvm::DIArray());
     unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
     
-    llvm::DIType ISATy = 
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
-                                     Unit, "", Unit,
-                                     0, Size, 0, 0, 0, OCTy);
-
-    llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
+    llvm::DIType ISATy = DBuilder.CreatePointerType(OCTy, Size);
 
+    llvm::SmallVector<llvm::Value *, 16> EltTys;
     llvm::DIType FieldTy = 
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                     "isa", Unit,
-                                     0,Size, 0, 0, 0, ISATy);
+      DBuilder.CreateMemberType("isa", getOrCreateMainFile(),
+                                0,Size, 0, 0, 0, ISATy);
     EltTys.push_back(FieldTy);
     llvm::DIArray Elements =
-      DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+      DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
     
-    return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
-                                            Unit, "objc_object", Unit, 0, 0, 0, 0,
-                                            0,
-                                            llvm::DIType(), Elements);
+    return DBuilder.CreateStructType(TheCU, "objc_object", 
+                                     getOrCreateMainFile(),
+                                     0, 0, 0, 0, Elements);
+  }
+  case BuiltinType::ObjCSel: {
+    return  DBuilder.CreateStructType(TheCU, "objc_selector", 
+                                      getOrCreateMainFile(), 0, 0, 0,
+                                      llvm::DIDescriptor::FlagFwdDecl, 
+                                      llvm::DIArray());
   }
   case BuiltinType::UChar:
   case BuiltinType::Char_U: Encoding = llvm::dwarf::DW_ATE_unsigned_char; break;
@@ -335,17 +349,12 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
   // Bit size, align and offset of the type.
   uint64_t Size = CGM.getContext().getTypeSize(BT);
   uint64_t Align = CGM.getContext().getTypeAlign(BT);
-  uint64_t Offset = 0;
-  
   llvm::DIType DbgTy = 
-    DebugFactory.CreateBasicType(Unit, BTName,
-                                 Unit, 0, Size, Align,
-                                 Offset, /*flags*/ 0, Encoding);
+    DBuilder.CreateBasicType(BTName, Size, Align, Encoding);
   return DbgTy;
 }
 
-llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty,
-                                     llvm::DIFile Unit) {
+llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) {
   // Bit size, align and offset of the type.
   unsigned Encoding = llvm::dwarf::DW_ATE_complex_float;
   if (Ty->isComplexIntegerType())
@@ -353,12 +362,9 @@ llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty,
 
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
-  uint64_t Offset = 0;
-
   llvm::DIType DbgTy = 
-    DebugFactory.CreateBasicType(Unit, "complex",
-                                 Unit, 0, Size, Align,
-                                 Offset, /*flags*/ 0, Encoding);
+    DBuilder.CreateBasicType("complex", Size, Align, Encoding);
+
   return DbgTy;
 }
 
@@ -389,13 +395,12 @@ llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) {
     return getOrCreateType(QualType(T, 0), Unit);
   }
 
-  llvm::DIType FromTy = getOrCreateType(Qc.apply(T), Unit);
+  llvm::DIType FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit);
 
   // No need to fill in the Name, Line, Size, Alignment, Offset in case of
   // CVR derived types.
-  llvm::DIType DbgTy =
-    DebugFactory.CreateDerivedType(Tag, Unit, "", Unit,
-                                   0, 0, 0, 0, 0, FromTy);
+  llvm::DIType DbgTy = DBuilder.CreateQualifiedType(Tag, FromTy);
+  
   return DbgTy;
 }
 
@@ -413,24 +418,56 @@ llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty,
                                Ty->getPointeeType(), Unit);
 }
 
+/// CreatePointeeType - Create PointTee type. If Pointee is a record
+/// then emit record's fwd if debug info size reduction is enabled.
+llvm::DIType CGDebugInfo::CreatePointeeType(QualType PointeeTy,
+                                            llvm::DIFile Unit) {
+  if (!CGM.getCodeGenOpts().LimitDebugInfo)
+    return getOrCreateType(PointeeTy, Unit);
+  
+  if (const RecordType *RTy = dyn_cast<RecordType>(PointeeTy)) {
+    RecordDecl *RD = RTy->getDecl();
+    llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
+    unsigned Line = getLineNumber(RD->getLocation());
+    llvm::DIDescriptor FDContext =
+      getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()));
+
+    if (RD->isStruct())
+      return DBuilder.CreateStructType(FDContext, RD->getName(), DefUnit,
+                                       Line, 0, 0, llvm::DIType::FlagFwdDecl,
+                                       llvm::DIArray());
+    else if (RD->isUnion())
+      return DBuilder.CreateUnionType(FDContext, RD->getName(), DefUnit,
+                                      Line, 0, 0, llvm::DIType::FlagFwdDecl,
+                                      llvm::DIArray());
+    else {
+      assert(RD->isClass() && "Unknown RecordType!");
+      return DBuilder.CreateClassType(FDContext, RD->getName(), DefUnit,
+                                      Line, 0, 0, 0, llvm::DIType::FlagFwdDecl,
+                                      llvm::DIType(), llvm::DIArray());
+    }
+  }
+  return getOrCreateType(PointeeTy, Unit);
+
+}
+
 llvm::DIType CGDebugInfo::CreatePointerLikeType(unsigned Tag,
                                                 const Type *Ty, 
                                                 QualType PointeeTy,
                                                 llvm::DIFile Unit) {
-  llvm::DIType EltTy = getOrCreateType(PointeeTy, Unit);
 
+  if (Tag == llvm::dwarf::DW_TAG_reference_type)
+    return DBuilder.CreateReferenceType(CreatePointeeType(PointeeTy, Unit));
+                                    
   // Bit size, align and offset of the type.
-  
   // Size is always the size of a pointer. We can't use getTypeSize here
   // because that does not return the correct value for references.
   uint64_t Size = 
     CGM.getContext().Target.getPointerWidth(PointeeTy.getAddressSpace());
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
-  return
-    DebugFactory.CreateDerivedType(Tag, Unit, "", Unit,
-                                   0, Size, Align, 0, 0, EltTy);
-  
+  return 
+    DBuilder.CreatePointerType(CreatePointeeType(PointeeTy, Unit), Size, Align);
 }
 
 llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
@@ -438,16 +475,11 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
   if (BlockLiteralGenericSet)
     return BlockLiteralGeneric;
 
-  unsigned Tag = llvm::dwarf::DW_TAG_structure_type;
-
-  llvm::SmallVector<llvm::DIDescriptor, 5> EltTys;
-
+  llvm::SmallVector<llvm::Value *, 8> EltTys;
   llvm::DIType FieldTy;
-
   QualType FType;
   uint64_t FieldSize, FieldOffset;
   unsigned FieldAlign;
-
   llvm::DIArray Elements;
   llvm::DIType EltTy, DescTy;
 
@@ -456,23 +488,20 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
   EltTys.push_back(CreateMemberType(Unit, FType, "reserved", &FieldOffset));
   EltTys.push_back(CreateMemberType(Unit, FType, "Size", &FieldOffset));
 
-  Elements = DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+  Elements = DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
   EltTys.clear();
 
-  unsigned Flags = llvm::DIType::FlagAppleBlock;
+  unsigned Flags = llvm::DIDescriptor::FlagAppleBlock;
   unsigned LineNo = getLineNumber(CurLoc);
 
-  EltTy = DebugFactory.CreateCompositeType(Tag, Unit, "__block_descriptor",
-                                           Unit, LineNo, FieldOffset, 0, 0, 
-                                           Flags, llvm::DIType(), Elements);
+  EltTy = DBuilder.CreateStructType(Unit, "__block_descriptor",
+                                    Unit, LineNo, FieldOffset, 0,
+                                    Flags, Elements);
 
   // Bit size, align and offset of the type.
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
-  uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
-  DescTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
-                                          Unit, "", Unit,
-                                          LineNo, Size, Align, 0, 0, EltTy);
+  DescTy = DBuilder.CreatePointerType(EltTy, Size);
 
   FieldOffset = 0;
   FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
@@ -487,24 +516,20 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty,
   FieldTy = DescTy;
   FieldSize = CGM.getContext().getTypeSize(Ty);
   FieldAlign = CGM.getContext().getTypeAlign(Ty);
-  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                           "__descriptor", Unit,
-                                           LineNo, FieldSize, FieldAlign,
-                                           FieldOffset, 0, FieldTy);
+  FieldTy = DBuilder.CreateMemberType("__descriptor", Unit,
+                                      LineNo, FieldSize, FieldAlign,
+                                      FieldOffset, 0, FieldTy);
   EltTys.push_back(FieldTy);
 
   FieldOffset += FieldSize;
-  Elements = DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+  Elements = DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
 
-  EltTy = DebugFactory.CreateCompositeType(Tag, Unit, "__block_literal_generic",
-                                           Unit, LineNo, FieldOffset, 0, 0, 
-                                           Flags, llvm::DIType(), Elements);
+  EltTy = DBuilder.CreateStructType(Unit, "__block_literal_generic",
+                                    Unit, LineNo, FieldOffset, 0,
+                                    Flags, Elements);
 
   BlockLiteralGenericSet = true;
-  BlockLiteralGeneric
-    = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type, Unit,
-                                     "", Unit,
-                                     LineNo, Size, Align, 0, 0, EltTy);
+  BlockLiteralGeneric = DBuilder.CreatePointerType(EltTy, Size);
   return BlockLiteralGeneric;
 }
 
@@ -513,46 +538,36 @@ llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty,
   // Typedefs are derived from some other type.  If we have a typedef of a
   // typedef, make sure to emit the whole chain.
   llvm::DIType Src = getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit);
-
+  if (!Src.Verify())
+    return llvm::DIType();
   // We don't set size information, but do specify where the typedef was
   // declared.
   unsigned Line = getLineNumber(Ty->getDecl()->getLocation());
-
-  llvm::DIDescriptor TyContext 
-    = getContextDescriptor(dyn_cast<Decl>(Ty->getDecl()->getDeclContext()),
-                           Unit);
-  llvm::DIType DbgTy = 
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_typedef, 
-                                   TyContext,
-                                   Ty->getDecl()->getName(), Unit,
-                                   Line, 0, 0, 0, 0, Src);
+  llvm::DIType DbgTy = DBuilder.CreateTypedef(Src, Ty->getDecl()->getName(),
+                                              Unit, Line);
   return DbgTy;
 }
 
 llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
                                      llvm::DIFile Unit) {
-  llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
+  llvm::SmallVector<llvm::Value *, 16> EltTys;
 
   // Add the result type at least.
   EltTys.push_back(getOrCreateType(Ty->getResultType(), Unit));
 
   // Set up remainder of arguments if there is a prototype.
   // FIXME: IF NOT, HOW IS THIS REPRESENTED?  llvm-gcc doesn't represent '...'!
-  if (const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(Ty)) {
+  if (isa<FunctionNoProtoType>(Ty))
+    EltTys.push_back(DBuilder.CreateUnspecifiedParameter());
+  else if (const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(Ty)) {
     for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
       EltTys.push_back(getOrCreateType(FTP->getArgType(i), Unit));
-  } else {
-    // FIXME: Handle () case in C.  llvm-gcc doesn't do it either.
   }
 
   llvm::DIArray EltTypeArray =
-    DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+    DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
 
-  llvm::DIType DbgTy =
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_subroutine_type,
-                                     Unit, "", Unit,
-                                     0, 0, 0, 0, 0,
-                                     llvm::DIType(), EltTypeArray);
+  llvm::DIType DbgTy = DBuilder.CreateSubroutineType(Unit, EltTypeArray);
   return DbgTy;
 }
 
@@ -560,7 +575,7 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty,
 /// record fields. This is used while creating debug info entry for a Record.
 void CGDebugInfo::
 CollectRecordFields(const RecordDecl *RD, llvm::DIFile Unit,
-                    llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys) {
+                    llvm::SmallVectorImpl<llvm::Value *> &EltTys) {
   unsigned FieldNo = 0;
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
   for (RecordDecl::field_iterator I = RD->field_begin(),
@@ -595,17 +610,13 @@ CollectRecordFields(const RecordDecl *RD, llvm::DIFile Unit,
     unsigned Flags = 0;
     AccessSpecifier Access = I->getAccess();
     if (Access == clang::AS_private)
-      Flags |= llvm::DIType::FlagPrivate;
+      Flags |= llvm::DIDescriptor::FlagPrivate;
     else if (Access == clang::AS_protected)
-      Flags |= llvm::DIType::FlagProtected;
-
-    // Create a DW_TAG_member node to remember the offset of this field in the
-    // struct.  FIXME: This is an absolutely insane way to capture this
-    // information.  When we gut debug info, this should be fixed.
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             FieldName, FieldDefUnit,
-                                             FieldLine, FieldSize, FieldAlign,
-                                             FieldOffset, Flags, FieldTy);
+      Flags |= llvm::DIDescriptor::FlagProtected;
+
+    FieldTy = DBuilder.CreateMemberType(FieldName, FieldDefUnit,
+                                        FieldLine, FieldSize, FieldAlign,
+                                        FieldOffset, Flags, FieldTy);
     EltTys.push_back(FieldTy);
   }
 }
@@ -621,19 +632,12 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
                                0),
                       Unit);
   
-  unsigned BFlags=0;
-  AccessSpecifier Access = Method->getAccess();
-  if (Access == clang::AS_private)
-    BFlags |= llvm::DIType::FlagPrivate;
-  else if (Access == clang::AS_protected)
-    BFlags |= llvm::DIType::FlagProtected;
-
   // Add "this" pointer.
 
   llvm::DIArray Args = llvm::DICompositeType(FnTy).getTypeArray();
   assert (Args.getNumElements() && "Invalid number of arguments!");
 
-  llvm::SmallVector<llvm::DIDescriptor, 16> Elts;
+  llvm::SmallVector<llvm::Value *, 16> Elts;
 
   // First element is always return type. For 'void' functions it is NULL.
   Elts.push_back(Args.getElement(0));
@@ -645,7 +649,7 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
         QualType ThisPtr =
           Context.getPointerType(Context.getTagDeclType(Method->getParent()));
         llvm::DIType ThisPtrType =
-          DebugFactory.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
+          DBuilder.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
 
         TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;
         Elts.push_back(ThisPtrType);
@@ -656,15 +660,22 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
     Elts.push_back(Args.getElement(i));
 
   llvm::DIArray EltTypeArray =
-    DebugFactory.GetOrCreateArray(Elts.data(), Elts.size());
+    DBuilder.GetOrCreateArray(Elts.data(), Elts.size());
 
-  return
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_subroutine_type,
-                                     Unit, "", Unit,
-                                     0, 0, 0, 0, 0,
-                                     llvm::DIType(), EltTypeArray);
+  return DBuilder.CreateSubroutineType(Unit, EltTypeArray);
 }
 
+/// isFunctionLocalClass - Return true if CXXRecordDecl is defined 
+/// inside a function.
+static bool isFunctionLocalClass(const CXXRecordDecl *RD) {
+  if (const CXXRecordDecl *NRD = 
+      dyn_cast<CXXRecordDecl>(RD->getDeclContext()))
+    return isFunctionLocalClass(NRD);
+  else if (isa<FunctionDecl>(RD->getDeclContext()))
+    return true;
+  return false;
+  
+}
 /// CreateCXXMemberFunction - A helper function to create a DISubprogram for
 /// a single member function GlobalDecl.
 llvm::DISubprogram
@@ -680,7 +691,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
   // Since a single ctor/dtor corresponds to multiple functions, it doesn't
   // make sense to give a single ctor/dtor a linkage name.
   llvm::StringRef MethodLinkageName;
-  if (!IsCtorOrDtor)
+  if (!IsCtorOrDtor && !isFunctionLocalClass(Method->getParent()))
     MethodLinkageName = CGM.getMangledName(Method);
 
   // Get the location for the method.
@@ -705,15 +716,32 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
     ContainingType = RecordTy;
   }
 
+  unsigned Flags = 0;
+  if (Method->isImplicit())
+    Flags |= llvm::DIDescriptor::FlagArtificial;
+  AccessSpecifier Access = Method->getAccess();
+  if (Access == clang::AS_private)
+    Flags |= llvm::DIDescriptor::FlagPrivate;
+  else if (Access == clang::AS_protected)
+    Flags |= llvm::DIDescriptor::FlagProtected;
+  if (const CXXConstructorDecl *CXXC = dyn_cast<CXXConstructorDecl>(Method)) {
+    if (CXXC->isExplicit())
+      Flags |= llvm::DIDescriptor::FlagExplicit;
+  } else if (const CXXConversionDecl *CXXC = 
+             dyn_cast<CXXConversionDecl>(Method)) {
+    if (CXXC->isExplicit())
+      Flags |= llvm::DIDescriptor::FlagExplicit;
+  }
+  if (Method->hasPrototype())
+    Flags |= llvm::DIDescriptor::FlagPrototyped;
+    
   llvm::DISubprogram SP =
-    DebugFactory.CreateSubprogram(RecordTy , MethodName, MethodName, 
-                                  MethodLinkageName,
-                                  MethodDefUnit, MethodLine,
-                                  MethodTy, /*isLocalToUnit=*/false, 
-                                  /* isDefintion=*/ false,
-                                  Virtuality, VIndex, ContainingType,
-                                  Method->isImplicit(),
-                                  CGM.getLangOptions().Optimize);
+    DBuilder.CreateMethod(RecordTy , MethodName, MethodLinkageName, 
+                          MethodDefUnit, MethodLine,
+                          MethodTy, /*isLocalToUnit=*/false, 
+                          /* isDefinition=*/ false,
+                          Virtuality, VIndex, ContainingType,
+                          Flags, CGM.getLangOptions().Optimize);
   
   // Don't cache ctors or dtors since we have to emit multiple functions for
   // a single ctor or dtor.
@@ -728,7 +756,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
 /// a Record.
 void CGDebugInfo::
 CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                          llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                          llvm::SmallVectorImpl<llvm::Value *> &EltTys,
                           llvm::DIType RecordTy) {
   for(CXXRecordDecl::method_iterator I = RD->method_begin(),
         E = RD->method_end(); I != E; ++I) {
@@ -746,26 +774,15 @@ CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
 /// a Record.
 void CGDebugInfo::
 CollectCXXFriends(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                llvm::SmallVectorImpl<llvm::Value *> &EltTys,
                 llvm::DIType RecordTy) {
 
   for (CXXRecordDecl::friend_iterator BI =  RD->friend_begin(),
          BE = RD->friend_end(); BI != BE; ++BI) {
-
-    TypeSourceInfo *TInfo = (*BI)->getFriendType();
-    if(TInfo)
-    {
-        llvm::DIType Ty = getOrCreateType(TInfo->getType(), Unit);
-
-            llvm::DIType DTy =
-          DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_friend,
-                                         RecordTy, llvm::StringRef(),
-                                         Unit, 0, 0, 0,
-                                         0, 0, Ty);
-
-        EltTys.push_back(DTy);
-    }
-
+    if (TypeSourceInfo *TInfo = (*BI)->getFriendType())
+      EltTys.push_back(DBuilder.CreateFriend(RecordTy, 
+                                             getOrCreateType(TInfo->getType(), 
+                                                             Unit)));
   }
 }
 
@@ -774,7 +791,7 @@ CollectCXXFriends(const CXXRecordDecl *RD, llvm::DIFile Unit,
 /// a Record.
 void CGDebugInfo::
 CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                llvm::SmallVectorImpl<llvm::Value *> &EltTys,
                 llvm::DIType RecordTy) {
 
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
@@ -790,23 +807,20 @@ CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
       // virtual base offset offset is -ve. The code generator emits dwarf
       // expression where it expects +ve number.
       BaseOffset = 0 - CGM.getVTables().getVirtualBaseOffsetOffset(RD, Base);
-      BFlags = llvm::DIType::FlagVirtual;
+      BFlags = llvm::DIDescriptor::FlagVirtual;
     } else
-      BaseOffset = RL.getBaseClassOffset(Base);
+      BaseOffset = RL.getBaseClassOffsetInBits(Base);
     
     AccessSpecifier Access = BI->getAccessSpecifier();
     if (Access == clang::AS_private)
-      BFlags |= llvm::DIType::FlagPrivate;
+      BFlags |= llvm::DIDescriptor::FlagPrivate;
     else if (Access == clang::AS_protected)
-      BFlags |= llvm::DIType::FlagProtected;
+      BFlags |= llvm::DIDescriptor::FlagProtected;
     
-    llvm::DIType DTy =
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_inheritance,
-                                     RecordTy, llvm::StringRef(), 
-                                     Unit, 0, 0, 0,
-                                     BaseOffset, BFlags,
-                                     getOrCreateType(BI->getType(),
-                                                     Unit));
+    llvm::DIType DTy = 
+      DBuilder.CreateInheritance(RecordTy,                                     
+                                 getOrCreateType(BI->getType(), Unit),
+                                 BaseOffset, BFlags);
     EltTys.push_back(DTy);
   }
 }
@@ -819,23 +833,13 @@ llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile Unit) {
   ASTContext &Context = CGM.getContext();
 
   /* Function type */
-  llvm::DIDescriptor STy = getOrCreateType(Context.IntTy, Unit);
-  llvm::DIArray SElements = DebugFactory.GetOrCreateArray(&STy, 1);
-  llvm::DIType SubTy =
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_subroutine_type,
-                                     Unit, "", Unit,
-                                     0, 0, 0, 0, 0, llvm::DIType(), SElements);
-
+  llvm::Value *STy = getOrCreateType(Context.IntTy, Unit);
+  llvm::DIArray SElements = DBuilder.GetOrCreateArray(&STy, 1);
+  llvm::DIType SubTy = DBuilder.CreateSubroutineType(Unit, SElements);
   unsigned Size = Context.getTypeSize(Context.VoidPtrTy);
-  llvm::DIType vtbl_ptr_type 
-    = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
-                                     Unit, "__vtbl_ptr_type", Unit,
-                                     0, Size, 0, 0, 0, SubTy);
-
-  VTablePtrType = 
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
-                                   Unit, "", Unit,
-                                   0, Size, 0, 0, 0, vtbl_ptr_type);
+  llvm::DIType vtbl_ptr_type = DBuilder.CreatePointerType(SubTy, Size, 0,
+                                                          "__vtbl_ptr_type");
+  VTablePtrType = DBuilder.CreatePointerType(vtbl_ptr_type, Size);
   return VTablePtrType;
 }
 
@@ -855,7 +859,7 @@ llvm::StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) {
 /// debug info entry in EltTys vector.
 void CGDebugInfo::
 CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
-                  llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys) {
+                  llvm::SmallVectorImpl<llvm::Value *> &EltTys) {
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
 
   // If there is a primary base then it will hold vtable info.
@@ -868,27 +872,24 @@ CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
 
   unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
   llvm::DIType VPTR
-    = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                     getVTableName(RD), Unit,
-                                     0, Size, 0, 0, 0, 
-                                     getOrCreateVTablePtrType(Unit));
+    = DBuilder.CreateMemberType(getVTableName(RD), Unit,
+                                0, Size, 0, 0, 0, 
+                                getOrCreateVTablePtrType(Unit));
   EltTys.push_back(VPTR);
 }
 
+/// getOrCreateRecordType - Emit record type's standalone debug info. 
+llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy, 
+                                                SourceLocation Loc) {
+  llvm::DIType T =  getOrCreateType(RTy, getOrCreateFile(Loc));
+  DBuilder.RetainType(T);
+  return T;
+}
+
 /// CreateType - get structure or union type.
-llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
-                                     llvm::DIFile Unit) {
+llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) {
   RecordDecl *RD = Ty->getDecl();
-
-  unsigned Tag;
-  if (RD->isStruct())
-    Tag = llvm::dwarf::DW_TAG_structure_type;
-  else if (RD->isUnion())
-    Tag = llvm::dwarf::DW_TAG_union_type;
-  else {
-    assert(RD->isClass() && "Unknown RecordType!");
-    Tag = llvm::dwarf::DW_TAG_class_type;
-  }
+  llvm::DIFile Unit = getOrCreateFile(RD->getLocation());
 
   // Get overall information about the record type for the debug info.
   llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
@@ -901,21 +902,21 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
   llvm::DIDescriptor FDContext =
-    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()), Unit);
+    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()));
 
   // If this is just a forward declaration, construct an appropriately
   // marked node and just return it.
   if (!RD->getDefinition()) {
-    llvm::DICompositeType FwdDecl =
-      DebugFactory.CreateCompositeType(Tag, FDContext, RD->getName(),
-                                       DefUnit, Line, 0, 0, 0,
-                                       llvm::DIType::FlagFwdDecl,
-                                       llvm::DIType(), llvm::DIArray());
+    llvm::DIType FwdDecl =
+      DBuilder.CreateStructType(FDContext, RD->getName(),
+                                DefUnit, Line, 0, 0,
+                                llvm::DIDescriptor::FlagFwdDecl,
+                                llvm::DIArray());
 
       return FwdDecl;
   }
 
-  llvm::DIType FwdDecl = DebugFactory.CreateTemporaryType();
+  llvm::DIType FwdDecl = DBuilder.CreateTemporaryType(DefUnit);
 
   llvm::MDNode *MN = FwdDecl;
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = MN;
@@ -927,7 +928,7 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
   RegionMap[Ty->getDecl()] = llvm::WeakVH(FwdDecl);
 
   // Convert all the elements.
-  llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
+  llvm::SmallVector<llvm::Value *, 16> EltTys;
 
   const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD);
   if (CXXDecl) {
@@ -951,36 +952,41 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
           llvm::DIType VTy = getOrCreateType(V->getType(), VUnit);
           // Do not use DIGlobalVariable for enums.
           if (VTy.getTag() != llvm::dwarf::DW_TAG_enumeration_type) {
-            DebugFactory.CreateGlobalVariable(FwdDecl, VName, VName, VName, VUnit,
-                                              getLineNumber(V->getLocation()),
-                                              VTy, true, true, CI);
+            DBuilder.CreateStaticVariable(FwdDecl, VName, VName, VUnit,
+                                          getLineNumber(V->getLocation()),
+                                          VTy, true, CI);
           }
         }
       }
     }
 
   CollectRecordFields(RD, Unit, EltTys);
-  llvm::MDNode *ContainingType = NULL;
+  llvm::SmallVector<llvm::Value *, 16> TemplateParams;
   if (CXXDecl) {
     CollectCXXMemberFunctions(CXXDecl, Unit, EltTys, FwdDecl);
     CollectCXXFriends(CXXDecl, Unit, EltTys, FwdDecl);
-
-    // A class's primary base or the class itself contains the vtable.
-    const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
-    if (const CXXRecordDecl *PBase = RL.getPrimaryBase())
-      ContainingType = 
-        getOrCreateType(QualType(PBase->getTypeForDecl(), 0), Unit);
-    else if (CXXDecl->isDynamicClass()) 
-      ContainingType = FwdDecl;
+    if (ClassTemplateSpecializationDecl *TSpecial
+        = dyn_cast<ClassTemplateSpecializationDecl>(RD)) {
+      const TemplateArgumentList &TAL = TSpecial->getTemplateArgs();
+      for (unsigned i = 0, e = TAL.size(); i != e; ++i) {
+        const TemplateArgument &TA = TAL[i];
+        if (TA.getKind() == TemplateArgument::Type) {
+          llvm::DIType TTy = getOrCreateType(TA.getAsType(), Unit);
+          llvm::DITemplateTypeParameter TTP =
+            DBuilder.CreateTemplateTypeParameter(TheCU, TTy.getName(), TTy);
+          TemplateParams.push_back(TTP);
+        } else if (TA.getKind() == TemplateArgument::Integral) {
+          llvm::DIType TTy = getOrCreateType(TA.getIntegralType(), Unit);
+          // FIXME: Get parameter name, instead of parameter type name.
+          llvm::DITemplateValueParameter TVP =
+            DBuilder.CreateTemplateValueParameter(TheCU, TTy.getName(), TTy,
+                                                  TA.getAsIntegral()->getZExtValue());
+          TemplateParams.push_back(TVP);          
+        }
+      }
+    }
   }
 
-  llvm::DIArray Elements =
-    DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
-
-  // Bit size, align and offset of the type.
-  uint64_t Size = CGM.getContext().getTypeSize(Ty);
-  uint64_t Align = CGM.getContext().getTypeAlign(Ty);
-
   RegionStack.pop_back();
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator RI = 
     RegionMap.find(Ty->getDecl());
@@ -988,25 +994,54 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
     RegionMap.erase(RI);
 
   llvm::DIDescriptor RDContext =  
-    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()), Unit);
-
+    getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()));
   llvm::StringRef RDName = RD->getName();
-  // If this is a class, include the template arguments also.
-  if (Tag == llvm::dwarf::DW_TAG_class_type) 
+  uint64_t Size = CGM.getContext().getTypeSize(Ty);
+  uint64_t Align = CGM.getContext().getTypeAlign(Ty);
+  llvm::DIArray Elements =
+    DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
+  llvm::MDNode *RealDecl = NULL;
+
+  if (RD->isStruct()) 
+    RealDecl = DBuilder.CreateStructType(RDContext, RDName, DefUnit, Line,
+                                         Size, Align, 0, Elements);
+  else if (RD->isUnion())
+    RealDecl = DBuilder.CreateUnionType(RDContext, RDName, DefUnit, Line,
+                                         Size, Align, 0, Elements);
+  else {
+    assert(RD->isClass() && "Unknown RecordType!");
     RDName = getClassName(RD);
-  
-  llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, RDContext,
-                                     RDName,
-                                     DefUnit, Line, Size, Align, 0, 0, 
-                                     llvm::DIType(), Elements, 
-                                     0, ContainingType);
+     // A class's primary base or the class itself contains the vtable.
+    llvm::MDNode *ContainingType = NULL;
+    const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+    if (const CXXRecordDecl *PBase = RL.getPrimaryBase()) {
+      // Seek non virtual primary base root.
+      while (1) {
+        const ASTRecordLayout &BRL = CGM.getContext().getASTRecordLayout(PBase);
+        const CXXRecordDecl *PBT = BRL.getPrimaryBase();
+        if (PBT && !BRL.isPrimaryBaseVirtual())
+          PBase = PBT;
+        else 
+          break;
+      }
+      ContainingType = 
+        getOrCreateType(QualType(PBase->getTypeForDecl(), 0), Unit);
+    }
+    else if (CXXDecl->isDynamicClass()) 
+      ContainingType = FwdDecl;
+    llvm::DIArray TParamsArray = 
+      DBuilder.GetOrCreateArray(TemplateParams.data(), TemplateParams.size());
+   RealDecl = DBuilder.CreateClassType(RDContext, RDName, DefUnit, Line,
+                                       Size, Align, 0, 0, llvm::DIType(),
+                                       Elements, ContainingType,
+                                       TParamsArray);
+  }
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
   llvm::DIType(FwdDeclNode).replaceAllUsesWith(RealDecl);
   RegionMap[RD] = llvm::WeakVH(RealDecl);
-  return RealDecl;
+  return llvm::DIType(RealDecl);
 }
 
 /// CreateType - get objective-c object type.
@@ -1020,7 +1055,8 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCObjectType *Ty,
 llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
                                      llvm::DIFile Unit) {
   ObjCInterfaceDecl *ID = Ty->getDecl();
-  unsigned Tag = llvm::dwarf::DW_TAG_structure_type;
+  if (!ID)
+    return llvm::DIType();
 
   // Get overall information about the record type for the debug info.
   llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation());
@@ -1030,11 +1066,10 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // If this is just a forward declaration, return a special forward-declaration
   // debug type.
   if (ID->isForwardDecl()) {
-    llvm::DICompositeType FwdDecl =
-      DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(),
-                                       DefUnit, Line, 0, 0, 0, 0,
-                                       llvm::DIType(), llvm::DIArray(),
-                                       RuntimeLang);
+    llvm::DIType FwdDecl =
+      DBuilder.CreateStructType(Unit, ID->getName(),
+                                DefUnit, Line, 0, 0, 0,
+                                llvm::DIArray(), RuntimeLang);
     return FwdDecl;
   }
 
@@ -1044,7 +1079,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   // its members.  Finally, we create a descriptor for the complete type (which
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
-  llvm::DIType FwdDecl = DebugFactory.CreateTemporaryType();
+  llvm::DIType FwdDecl = DBuilder.CreateTemporaryType(DefUnit);
 
   llvm::MDNode *MN = FwdDecl;
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = MN;
@@ -1056,27 +1091,29 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   RegionMap[Ty->getDecl()] = llvm::WeakVH(FwdDecl);
 
   // Convert all the elements.
-  llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
+  llvm::SmallVector<llvm::Value *, 16> EltTys;
 
   ObjCInterfaceDecl *SClass = ID->getSuperClass();
   if (SClass) {
     llvm::DIType SClassTy =
       getOrCreateType(CGM.getContext().getObjCInterfaceType(SClass), Unit);
+    if (!SClassTy.isValid())
+      return llvm::DIType();
+    
     llvm::DIType InhTag =
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_inheritance,
-                                     Unit, "", Unit, 0, 0, 0,
-                                     0 /* offset */, 0, SClassTy);
+      DBuilder.CreateInheritance(FwdDecl, SClassTy, 0, 0);
     EltTys.push_back(InhTag);
   }
 
   const ASTRecordLayout &RL = CGM.getContext().getASTObjCInterfaceLayout(ID);
 
   unsigned FieldNo = 0;
-  for (ObjCInterfaceDecl::ivar_iterator I = ID->ivar_begin(),
-         E = ID->ivar_end();  I != E; ++I, ++FieldNo) {
-    ObjCIvarDecl *Field = *I;
+  for (ObjCIvarDecl *Field = ID->all_declared_ivar_begin(); Field;
+       Field = Field->getNextIvar(), ++FieldNo) {
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
-
+    if (!FieldTy.isValid())
+      return llvm::DIType();
+    
     llvm::StringRef FieldName = Field->getName();
 
     // Ignore unnamed fields.
@@ -1105,22 +1142,18 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
 
     unsigned Flags = 0;
     if (Field->getAccessControl() == ObjCIvarDecl::Protected)
-      Flags = llvm::DIType::FlagProtected;
+      Flags = llvm::DIDescriptor::FlagProtected;
     else if (Field->getAccessControl() == ObjCIvarDecl::Private)
-      Flags = llvm::DIType::FlagPrivate;
-
-    // Create a DW_TAG_member node to remember the offset of this field in the
-    // struct.  FIXME: This is an absolutely insane way to capture this
-    // information.  When we gut debug info, this should be fixed.
-    FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                             FieldName, FieldDefUnit,
-                                             FieldLine, FieldSize, FieldAlign,
-                                             FieldOffset, Flags, FieldTy);
+      Flags = llvm::DIDescriptor::FlagPrivate;
+
+    FieldTy = DBuilder.CreateMemberType(FieldName, FieldDefUnit,
+                                        FieldLine, FieldSize, FieldAlign,
+                                        FieldOffset, Flags, FieldTy);
     EltTys.push_back(FieldTy);
   }
 
   llvm::DIArray Elements =
-    DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+    DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
 
   RegionStack.pop_back();
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator RI = 
@@ -1132,10 +1165,10 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
-  llvm::DICompositeType RealDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(), DefUnit,
-                                     Line, Size, Align, 0, 0, llvm::DIType(), 
-                                     Elements, RuntimeLang);
+  llvm::DIType RealDecl =
+    DBuilder.CreateStructType(Unit, ID->getName(), DefUnit,
+                                  Line, Size, Align, 0,
+                                  Elements, RuntimeLang);
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
@@ -1145,18 +1178,11 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   return RealDecl;
 }
 
-llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
-                                     llvm::DIFile Unit) {
-  return CreateEnumType(Ty->getDecl(), Unit);
-
-}
-
-llvm::DIType CGDebugInfo::CreateType(const TagType *Ty,
-                                     llvm::DIFile Unit) {
+llvm::DIType CGDebugInfo::CreateType(const TagType *Ty) {
   if (const RecordType *RT = dyn_cast<RecordType>(Ty))
-    return CreateType(RT, Unit);
+    return CreateType(RT);
   else if (const EnumType *ET = dyn_cast<EnumType>(Ty))
-    return CreateType(ET, Unit);
+    return CreateEnumType(ET->getDecl());
 
   return llvm::DIType();
 }
@@ -1168,17 +1194,14 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty,
   if (NumElems > 0)
     --NumElems;
 
-  llvm::DIDescriptor Subscript = DebugFactory.GetOrCreateSubrange(0, NumElems);
-  llvm::DIArray SubscriptArray = DebugFactory.GetOrCreateArray(&Subscript, 1);
+  llvm::Value *Subscript = DBuilder.GetOrCreateSubrange(0, NumElems);
+  llvm::DIArray SubscriptArray = DBuilder.GetOrCreateArray(&Subscript, 1);
 
   uint64_t Size = CGM.getContext().getTypeSize(Ty);
   uint64_t Align = CGM.getContext().getTypeAlign(Ty);
 
   return
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_vector_type,
-                                     Unit, "", Unit,
-                                     0, Size, Align, 0, 0,
-                                     ElementTy,  SubscriptArray);
+    DBuilder.CreateVectorType(Size, Align, ElementTy, SubscriptArray);
 }
 
 llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
@@ -1204,27 +1227,28 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
   // Add the dimensions of the array.  FIXME: This loses CV qualifiers from
   // interior arrays, do we care?  Why aren't nested arrays represented the
   // obvious/recursive way?
-  llvm::SmallVector<llvm::DIDescriptor, 8> Subscripts;
+  llvm::SmallVector<llvm::Value *, 8> Subscripts;
   QualType EltTy(Ty, 0);
-  while ((Ty = dyn_cast<ArrayType>(EltTy))) {
-    uint64_t Upper = 0;
-    if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
-      if (CAT->getSize().getZExtValue())
-        Upper = CAT->getSize().getZExtValue() - 1;
-    // FIXME: Verify this is right for VLAs.
-    Subscripts.push_back(DebugFactory.GetOrCreateSubrange(0, Upper));
+  if (Ty->isIncompleteArrayType())
     EltTy = Ty->getElementType();
+  else {
+    while ((Ty = dyn_cast<ArrayType>(EltTy))) {
+      uint64_t Upper = 0;
+      if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
+        if (CAT->getSize().getZExtValue())
+          Upper = CAT->getSize().getZExtValue() - 1;
+      // FIXME: Verify this is right for VLAs.
+      Subscripts.push_back(DBuilder.GetOrCreateSubrange(0, Upper));
+      EltTy = Ty->getElementType();
+    }
   }
 
   llvm::DIArray SubscriptArray =
-    DebugFactory.GetOrCreateArray(Subscripts.data(), Subscripts.size());
+    DBuilder.GetOrCreateArray(Subscripts.data(), Subscripts.size());
 
   llvm::DIType DbgTy = 
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_array_type,
-                                     Unit, "", Unit,
-                                     0, Size, Align, 0, 0,
-                                     getOrCreateType(EltTy, Unit),
-                                     SubscriptArray);
+    DBuilder.CreateArrayType(Size, Align, getOrCreateType(EltTy, Unit),
+                             SubscriptArray);
   return DbgTy;
 }
 
@@ -1234,6 +1258,12 @@ llvm::DIType CGDebugInfo::CreateType(const LValueReferenceType *Ty,
                                Ty, Ty->getPointeeType(), Unit);
 }
 
+llvm::DIType CGDebugInfo::CreateType(const RValueReferenceType *Ty, 
+                                     llvm::DIFile Unit) {
+  return CreatePointerLikeType(llvm::dwarf::DW_TAG_rvalue_reference_type, 
+                               Ty, Ty->getPointeeType(), Unit);
+}
+
 llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty, 
                                      llvm::DIFile U) {
   QualType PointerDiffTy = CGM.getContext().getPointerDiffType();
@@ -1249,47 +1279,46 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty,
   std::pair<uint64_t, unsigned> Info = CGM.getContext().getTypeInfo(Ty);
 
   uint64_t FieldOffset = 0;
-  llvm::DIDescriptor ElementTypes[2];
+  llvm::Value *ElementTypes[2];
   
   // FIXME: This should probably be a function type instead.
   ElementTypes[0] =
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, U,
-                                   "ptr", U, 0,
-                                   Info.first, Info.second, FieldOffset, 0,
-                                   PointerDiffDITy);
+    DBuilder.CreateMemberType("ptr", U, 0,
+                              Info.first, Info.second, FieldOffset, 0,
+                              PointerDiffDITy);
   FieldOffset += Info.first;
   
   ElementTypes[1] =
-    DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, U,
-                                   "ptr", U, 0,
-                                   Info.first, Info.second, FieldOffset, 0,
-                                   PointerDiffDITy);
+    DBuilder.CreateMemberType("ptr", U, 0,
+                              Info.first, Info.second, FieldOffset, 0,
+                              PointerDiffDITy);
   
   llvm::DIArray Elements = 
-    DebugFactory.GetOrCreateArray(&ElementTypes[0],
-                                  llvm::array_lengthof(ElementTypes));
+    DBuilder.GetOrCreateArray(&ElementTypes[0],
+                              llvm::array_lengthof(ElementTypes));
 
-  return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type, 
-                                          U, llvm::StringRef("test"), 
-                                          U, 0, FieldOffset, 
-                                          0, 0, 0, llvm::DIType(), Elements);
+  return DBuilder.CreateStructType(U, llvm::StringRef("test"), 
+                                   U, 0, FieldOffset, 
+                                   0, 0, Elements);
 }
 
 /// CreateEnumType - get enumeration type.
-llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED, llvm::DIFile Unit){
-  llvm::SmallVector<llvm::DIDescriptor, 32> Enumerators;
+llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED) {
+  llvm::DIFile Unit = getOrCreateFile(ED->getLocation());
+  llvm::SmallVector<llvm::Value *, 16> Enumerators;
 
   // Create DIEnumerator elements for each enumerator.
   for (EnumDecl::enumerator_iterator
          Enum = ED->enumerator_begin(), EnumEnd = ED->enumerator_end();
        Enum != EnumEnd; ++Enum) {
-    Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getName(),
-                                            Enum->getInitVal().getZExtValue()));
+    Enumerators.push_back(
+      DBuilder.CreateEnumerator(Enum->getName(),
+                                Enum->getInitVal().getZExtValue()));
   }
 
   // Return a CompositeType for the enum itself.
   llvm::DIArray EltArray =
-    DebugFactory.GetOrCreateArray(Enumerators.data(), Enumerators.size());
+    DBuilder.GetOrCreateArray(Enumerators.data(), Enumerators.size());
 
   llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
   unsigned Line = getLineNumber(ED->getLocation());
@@ -1299,11 +1328,11 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED, llvm::DIFile Unit){
     Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
     Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl());
   }
+  llvm::DIDescriptor EnumContext = 
+    getContextDescriptor(dyn_cast<Decl>(ED->getDeclContext()));
   llvm::DIType DbgTy = 
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_enumeration_type,
-                                     Unit, ED->getName(), DefUnit, Line,
-                                     Size, Align, 0, 0,
-                                     llvm::DIType(), EltArray);
+    DBuilder.CreateEnumerationType(EnumContext, ED->getName(), DefUnit, Line,
+                                   Size, Align, EltArray);
   return DbgTy;
 }
 
@@ -1316,20 +1345,23 @@ static QualType UnwrapTypeForDebugInfo(QualType T) {
     case Type::TemplateSpecialization:
       T = cast<TemplateSpecializationType>(T)->desugar();
       break;
-    case Type::TypeOfExpr: {
-      TypeOfExprType *Ty = cast<TypeOfExprType>(T);
-      T = Ty->getUnderlyingExpr()->getType();
+    case Type::TypeOfExpr:
+      T = cast<TypeOfExprType>(T)->getUnderlyingExpr()->getType();
       break;
-    }
     case Type::TypeOf:
       T = cast<TypeOfType>(T)->getUnderlyingType();
       break;
     case Type::Decltype:
       T = cast<DecltypeType>(T)->getUnderlyingType();
       break;
+    case Type::Attributed:
+      T = cast<AttributedType>(T)->getEquivalentType();
     case Type::Elaborated:
       T = cast<ElaboratedType>(T)->getNamedType();
       break;
+    case Type::Paren:
+      T = cast<ParenType>(T)->getInnerType();
+      break;
     case Type::SubstTemplateTypeParm:
       T = cast<SubstTemplateTypeParmType>(T)->getReplacementType();
       break;
@@ -1400,15 +1432,15 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
     return CreateType(cast<ObjCObjectType>(Ty), Unit);
   case Type::ObjCInterface:
     return CreateType(cast<ObjCInterfaceType>(Ty), Unit);
-  case Type::Builtin: return CreateType(cast<BuiltinType>(Ty), Unit);
-  case Type::Complex: return CreateType(cast<ComplexType>(Ty), Unit);
+  case Type::Builtin: return CreateType(cast<BuiltinType>(Ty));
+  case Type::Complex: return CreateType(cast<ComplexType>(Ty));
   case Type::Pointer: return CreateType(cast<PointerType>(Ty), Unit);
   case Type::BlockPointer:
     return CreateType(cast<BlockPointerType>(Ty), Unit);
   case Type::Typedef: return CreateType(cast<TypedefType>(Ty), Unit);
   case Type::Record:
   case Type::Enum:
-    return CreateType(cast<TagType>(Ty), Unit);
+    return CreateType(cast<TagType>(Ty));
   case Type::FunctionProto:
   case Type::FunctionNoProto:
     return CreateType(cast<FunctionType>(Ty), Unit);
@@ -1419,29 +1451,29 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty,
 
   case Type::LValueReference:
     return CreateType(cast<LValueReferenceType>(Ty), Unit);
+  case Type::RValueReference:
+    return CreateType(cast<RValueReferenceType>(Ty), Unit);
 
   case Type::MemberPointer:
     return CreateType(cast<MemberPointerType>(Ty), Unit);
 
+  case Type::Attributed:
   case Type::TemplateSpecialization:
   case Type::Elaborated:
+  case Type::Paren:
   case Type::SubstTemplateTypeParm:
   case Type::TypeOfExpr:
   case Type::TypeOf:
   case Type::Decltype:
+  case Type::Auto:
     llvm_unreachable("type should have been unwrapped!");
-    return llvm::DIType();
-      
-  case Type::RValueReference:
-    // FIXME: Implement!
-    Diag = "rvalue references";
-    break;
+    return llvm::DIType();      
   }
   
   assert(Diag && "Fall through without a diagnostic?");
   unsigned DiagID = CGM.getDiags().getCustomDiagID(Diagnostic::Error,
                                "debug information for %0 is not yet supported");
-  CGM.getDiags().Report(FullSourceLoc(), DiagID)
+  CGM.getDiags().Report(DiagID)
     << Diag;
   return llvm::DIType();
 }
@@ -1453,10 +1485,9 @@ llvm::DIType CGDebugInfo::CreateMemberType(llvm::DIFile Unit, QualType FType,
   llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit);
   uint64_t FieldSize = CGM.getContext().getTypeSize(FType);
   unsigned FieldAlign = CGM.getContext().getTypeAlign(FType);
-  llvm::DIType Ty = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member,
-                                                   Unit, Name, Unit, 0,
-                                                   FieldSize, FieldAlign,
-                                                   *Offset, 0, FieldTy);
+  llvm::DIType Ty = DBuilder.CreateMemberType(Name, Unit, 0,
+                                              FieldSize, FieldAlign,
+                                              *Offset, 0, FieldTy);
   *Offset += FieldSize;
   return Ty;
 }
@@ -1473,12 +1504,15 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   FnBeginRegionCount.push_back(RegionStack.size());
 
   const Decl *D = GD.getDecl();
+  unsigned Flags = 0;
+  llvm::DIFile Unit = getOrCreateFile(CurLoc);
+  llvm::DIDescriptor FDContext(Unit);
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // If there is a DISubprogram for  this function available then use it.
     llvm::DenseMap<const FunctionDecl *, llvm::WeakVH>::iterator
       FI = SPCache.find(FD);
     if (FI != SPCache.end()) {
-      llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(FI->second));
+      llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(&*FI->second));
       if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) {
         llvm::MDNode *SPN = SP;
         RegionStack.push_back(SPN);
@@ -1489,13 +1523,20 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
     Name = getFunctionName(FD);
     // Use mangled name as linkage name for c/c++ functions.
     LinkageName = CGM.getMangledName(GD);
+    if (LinkageName == Name)
+      LinkageName = llvm::StringRef();
+    if (FD->hasPrototype())
+      Flags |= llvm::DIDescriptor::FlagPrototyped;
+    if (const NamespaceDecl *NSDecl =
+        dyn_cast_or_null<NamespaceDecl>(FD->getDeclContext()))
+      FDContext = getOrCreateNameSpace(NSDecl);
   } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) {
     Name = getObjCMethodName(OMD);
-    LinkageName = Name;
+    Flags |= llvm::DIDescriptor::FlagPrototyped;
   } else {
-    // Use llvm function name as linkage name.
+    // Use llvm function name.
     Name = Fn->getName();
-    LinkageName = Name;
+    Flags |= llvm::DIDescriptor::FlagPrototyped;
   }
   if (!Name.empty() && Name[0] == '\01')
     Name = Name.substr(1);
@@ -1503,16 +1544,14 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   // It is expected that CurLoc is set before using EmitFunctionStart.
   // Usually, CurLoc points to the left bracket location of compound
   // statement representing function body.
-  llvm::DIFile Unit = getOrCreateFile(CurLoc);
   unsigned LineNo = getLineNumber(CurLoc);
-
+  if (D->isImplicit())
+    Flags |= llvm::DIDescriptor::FlagArtificial;
   llvm::DISubprogram SP =
-    DebugFactory.CreateSubprogram(Unit, Name, Name, LinkageName, Unit, LineNo,
-                                  getOrCreateType(FnType, Unit),
-                                  Fn->hasInternalLinkage(), true/*definition*/,
-                                  0, 0, llvm::DIType(),
-                                  D->isImplicit(),
-                                  CGM.getLangOptions().Optimize, Fn);
+    DBuilder.CreateFunction(FDContext, Name, LinkageName, Unit,
+                            LineNo, getOrCreateType(FnType, Unit),
+                            Fn->hasInternalLinkage(), true/*definition*/,
+                            Flags, CGM.getLangOptions().Optimize, Fn);
 
   // Push function on region stack.
   llvm::MDNode *SPN = SP;
@@ -1556,7 +1595,8 @@ void CGDebugInfo::UpdateLineDirectiveRegion(CGBuilderTy &Builder) {
   PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc);
   PresumedLoc PPLoc = SM.getPresumedLoc(PrevLoc);
 
-  if (!strcmp(PPLoc.getFilename(), PCLoc.getFilename()))
+  if (PCLoc.isInvalid() || PPLoc.isInvalid() ||
+      !strcmp(PPLoc.getFilename(), PCLoc.getFilename()))
     return;
 
   // If #line directive stack is empty then we are entering a new scope.
@@ -1570,9 +1610,10 @@ void CGDebugInfo::UpdateLineDirectiveRegion(CGBuilderTy &Builder) {
           && "error handling  #line regions!");
 
   bool SeenThisFile = false;
+  // Chek if current file is already seen earlier.
   for(std::vector<const char *>::iterator I = LineDirectiveFiles.begin(),
         E = LineDirectiveFiles.end(); I != E; ++I)
-    if (!strcmp(PPLoc.getFilename(), *I)) {
+    if (!strcmp(PCLoc.getFilename(), *I)) {
       SeenThisFile = true;
       break;
     }
@@ -1599,12 +1640,12 @@ void CGDebugInfo::UpdateLineDirectiveRegion(CGBuilderTy &Builder) {
 /// region - "llvm.dbg.region.start.".
 void CGDebugInfo::EmitRegionStart(CGBuilderTy &Builder) {
   llvm::DIDescriptor D =
-    DebugFactory.CreateLexicalBlock(RegionStack.empty() ? 
-                                    llvm::DIDescriptor() : 
-                                    llvm::DIDescriptor(RegionStack.back()),
-                                    getOrCreateFile(CurLoc),
-                                    getLineNumber(CurLoc), 
-                                    getColumnNumber(CurLoc));
+    DBuilder.CreateLexicalBlock(RegionStack.empty() ? 
+                                llvm::DIDescriptor() : 
+                                llvm::DIDescriptor(RegionStack.back()),
+                                getOrCreateFile(CurLoc),
+                                getLineNumber(CurLoc), 
+                                getColumnNumber(CurLoc));
   llvm::MDNode *DN = D;
   RegionStack.push_back(DN);
 }
@@ -1637,8 +1678,7 @@ void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
 llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
                                                        uint64_t *XOffset) {
 
-  llvm::SmallVector<llvm::DIDescriptor, 5> EltTys;
-
+  llvm::SmallVector<llvm::Value *, 5> EltTys;
   QualType FType;
   uint64_t FieldSize, FieldOffset;
   unsigned FieldAlign;
@@ -1654,7 +1694,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
   EltTys.push_back(CreateMemberType(Unit, FType, "__flags", &FieldOffset));
   EltTys.push_back(CreateMemberType(Unit, FType, "__size", &FieldOffset));
 
-  bool HasCopyAndDispose = CGM.BlockRequiresCopying(Type);
+  bool HasCopyAndDispose = CGM.getContext().BlockRequiresCopying(Type);
   if (HasCopyAndDispose) {
     FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy);
     EltTys.push_back(CreateMemberType(Unit, FType, "__copy_helper",
@@ -1685,24 +1725,21 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
   FieldAlign = Align.getQuantity()*8;
 
   *XOffset = FieldOffset;  
-  FieldTy = DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
-                                           VD->getName(), Unit,
-                                           0, FieldSize, FieldAlign,
-                                           FieldOffset, 0, FieldTy);
+  FieldTy = DBuilder.CreateMemberType(VD->getName(), Unit,
+                                      0, FieldSize, FieldAlign,
+                                      FieldOffset, 0, FieldTy);
   EltTys.push_back(FieldTy);
   FieldOffset += FieldSize;
   
   llvm::DIArray Elements = 
-    DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+    DBuilder.GetOrCreateArray(EltTys.data(), EltTys.size());
   
-  unsigned Flags = llvm::DIType::FlagBlockByrefStruct;
-  
-  return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type, 
-                                          Unit, "", Unit,
-                                          0, FieldOffset, 0, 0, Flags,
-                                          llvm::DIType(), Elements);
+  unsigned Flags = llvm::DIDescriptor::FlagBlockByrefStruct;
   
+  return DBuilder.CreateStructType(Unit, "", Unit, 0, FieldOffset, 0, Flags,
+                                   Elements);
 }
+
 /// EmitDeclare - Emit local variable declaration debug info.
 void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
                               llvm::Value *Storage, CGBuilderTy &Builder) {
@@ -1721,37 +1758,119 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
   if (!Ty)
     return;
 
+  if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage)) {
+    // If Storage is an aggregate returned as 'sret' then let debugger know
+    // about this.
+    if (Arg->hasStructRetAttr())
+      Ty = DBuilder.CreateReferenceType(Ty);
+    else if (CXXRecordDecl *Record = VD->getType()->getAsCXXRecordDecl()) {
+      // If an aggregate variable has non trivial destructor or non trivial copy
+      // constructor than it is pass indirectly. Let debug info know about this
+      // by using reference of the aggregate type as a argument type.
+      if (!Record->hasTrivialCopyConstructor() || !Record->hasTrivialDestructor())
+        Ty = DBuilder.CreateReferenceType(Ty);
+    }
+  }
+      
   // Get location information.
   unsigned Line = getLineNumber(VD->getLocation());
   unsigned Column = getColumnNumber(VD->getLocation());
-
-  // Create the descriptor for the variable.
-  llvm::DIVariable D =
-    DebugFactory.CreateVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
-                                VD->getName(),
-                                Unit, Line, Ty, CGM.getLangOptions().Optimize);
-  // Insert an llvm.dbg.declare into the current block.
-  llvm::Instruction *Call =
-    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertBlock());
-
+  unsigned Flags = 0;
+  if (VD->isImplicit())
+    Flags |= llvm::DIDescriptor::FlagArtificial;
   llvm::MDNode *Scope = RegionStack.back();
-  Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+    
+  llvm::StringRef Name = VD->getName();
+  if (!Name.empty()) {
+    if (VD->hasAttr<BlocksAttr>()) {
+      CharUnits offset = CharUnits::fromQuantity(32);
+      llvm::SmallVector<llvm::Value *, 9> addr;
+      const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
+      addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
+      // offset of __forwarding field
+      offset = 
+        CharUnits::fromQuantity(CGM.getContext().Target.getPointerWidth(0)/8);
+      addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
+      addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
+      addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
+      // offset of x field
+      offset = CharUnits::fromQuantity(XOffset/8);
+      addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
+
+      // Create the descriptor for the variable.
+      llvm::DIVariable D =
+        DBuilder.CreateComplexVariable(Tag, 
+                                       llvm::DIDescriptor(RegionStack.back()),
+                                       VD->getName(), Unit, Line, Ty,
+                                       addr.data(), addr.size());
+      
+      // Insert an llvm.dbg.declare into the current block.
+      llvm::Instruction *Call =
+        DBuilder.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+      
+      Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+      return;
+    } 
+      // Create the descriptor for the variable.
+    llvm::DIVariable D =
+      DBuilder.CreateLocalVariable(Tag, llvm::DIDescriptor(Scope), 
+                                   Name, Unit, Line, Ty, 
+                                   CGM.getLangOptions().Optimize, Flags);
+    
+    // Insert an llvm.dbg.declare into the current block.
+    llvm::Instruction *Call =
+      DBuilder.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+    
+    Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+    return;
+  }
+  
+  // If VD is an anonymous union then Storage represents value for
+  // all union fields.
+  if (const RecordType *RT = dyn_cast<RecordType>(VD->getType()))
+    if (const RecordDecl *RD = dyn_cast<RecordDecl>(RT->getDecl()))
+      if (RD->isUnion()) {
+        for (RecordDecl::field_iterator I = RD->field_begin(),
+               E = RD->field_end();
+             I != E; ++I) {
+          FieldDecl *Field = *I;
+          llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
+          llvm::StringRef FieldName = Field->getName();
+          
+          // Ignore unnamed fields. Do not ignore unnamed records.
+          if (FieldName.empty() && !isa<RecordType>(Field->getType()))
+            continue;
+          
+          // Use VarDecl's Tag, Scope and Line number.
+          llvm::DIVariable D =
+            DBuilder.CreateLocalVariable(Tag, llvm::DIDescriptor(Scope),
+                                         FieldName, Unit, Line, FieldTy, 
+                                         CGM.getLangOptions().Optimize, Flags);
+          
+          // Insert an llvm.dbg.declare into the current block.
+          llvm::Instruction *Call =
+            DBuilder.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+          
+          Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
+        }
+      }
 }
 
 /// EmitDeclare - Emit local variable declaration debug info.
-void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
+void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
                               llvm::Value *Storage, CGBuilderTy &Builder,
-                              CodeGenFunction *CGF) {
-  const ValueDecl *VD = BDRE->getDecl();
+                              const CGBlockInfo &blockInfo) {
   assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
 
   if (Builder.GetInsertBlock() == 0)
     return;
 
+  bool isByRef = VD->hasAttr<BlocksAttr>();
+
   uint64_t XOffset = 0;
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
   llvm::DIType Ty;
-  if (VD->hasAttr<BlocksAttr>())
+  if (isByRef)
     Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset);
   else 
     Ty = getOrCreateType(VD->getType(), Unit);
@@ -1760,20 +1879,25 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
   unsigned Line = getLineNumber(VD->getLocation());
   unsigned Column = getColumnNumber(VD->getLocation());
 
-  CharUnits offset = CGF->BlockDecls[VD];
+  const llvm::TargetData &target = CGM.getTargetData();
+
+  CharUnits offset = CharUnits::fromQuantity(
+    target.getStructLayout(blockInfo.StructureType)
+          ->getElementOffset(blockInfo.getCapture(VD).getIndex()));
+
   llvm::SmallVector<llvm::Value *, 9> addr;
   const llvm::Type *Int64Ty = llvm::Type::getInt64Ty(CGM.getLLVMContext());
-  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
-  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
+  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
+  addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
   addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
-  if (BDRE->isByRef()) {
-    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
-    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
+  if (isByRef) {
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
     // offset of __forwarding field
-    offset = CharUnits::fromQuantity(CGF->LLVMPointerWidth/8);
+    offset = CharUnits::fromQuantity(target.getPointerSize()/8);
     addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
-    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpDeref));
-    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIFactory::OpPlus));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
+    addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
     // offset of x field
     offset = CharUnits::fromQuantity(XOffset/8);
     addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
@@ -1781,13 +1905,12 @@ void CGDebugInfo::EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag,
 
   // Create the descriptor for the variable.
   llvm::DIVariable D =
-    DebugFactory.CreateComplexVariable(Tag,
-                                       llvm::DIDescriptor(RegionStack.back()),
-                                       VD->getName(), Unit, Line, Ty,
-                                       addr);
+    DBuilder.CreateComplexVariable(Tag, llvm::DIDescriptor(RegionStack.back()),
+                                   VD->getName(), Unit, Line, Ty,
+                                   addr.data(), addr.size());
   // Insert an llvm.dbg.declare into the current block.
   llvm::Instruction *Call = 
-    DebugFactory.InsertDeclare(Storage, D, Builder.GetInsertBlock());
+    DBuilder.InsertDeclare(Storage, D, Builder.GetInsertBlock());
   
   llvm::MDNode *Scope = RegionStack.back();
   Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope));
@@ -1800,9 +1923,10 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
 }
 
 void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
-  const BlockDeclRefExpr *BDRE, llvm::Value *Storage, CGBuilderTy &Builder,
-  CodeGenFunction *CGF) {
-  EmitDeclare(BDRE, llvm::dwarf::DW_TAG_auto_variable, Storage, Builder, CGF);
+  const VarDecl *variable, llvm::Value *Storage, CGBuilderTy &Builder,
+  const CGBlockInfo &blockInfo) {
+  EmitDeclare(variable, llvm::dwarf::DW_TAG_auto_variable, Storage, Builder,
+              blockInfo);
 }
 
 /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
@@ -1836,14 +1960,16 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
   }
   llvm::StringRef DeclName = D->getName();
   llvm::StringRef LinkageName;
-  if (D->getDeclContext() && !isa<FunctionDecl>(D->getDeclContext()))
+  if (D->getDeclContext() && !isa<FunctionDecl>(D->getDeclContext())
+      && !isa<ObjCMethodDecl>(D->getDeclContext()))
     LinkageName = Var->getName();
+  if (LinkageName == DeclName)
+    LinkageName = llvm::StringRef();
   llvm::DIDescriptor DContext = 
-    getContextDescriptor(dyn_cast<Decl>(D->getDeclContext()), Unit);
-  DebugFactory.CreateGlobalVariable(DContext, DeclName, DeclName, LinkageName,
-                                    Unit, LineNo, getOrCreateType(T, Unit),
-                                    Var->hasInternalLinkage(),
-                                    true/*definition*/, Var);
+    getContextDescriptor(dyn_cast<Decl>(D->getDeclContext()));
+  DBuilder.CreateStaticVariable(DContext, DeclName, LinkageName,
+                                Unit, LineNo, getOrCreateType(T, Unit),
+                                Var->hasInternalLinkage(), Var);
 }
 
 /// EmitGlobalVariable - Emit information about an objective-c interface.
@@ -1868,49 +1994,45 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                            ArrayType::Normal, 0);
   }
 
-  DebugFactory.CreateGlobalVariable(Unit, Name, Name, Name, Unit, LineNo,
-                                    getOrCreateType(T, Unit),
-                                    Var->hasInternalLinkage(),
-                                    true/*definition*/, Var);
+  DBuilder.CreateGlobalVariable(Name, Unit, LineNo,
+                                getOrCreateType(T, Unit),
+                                Var->hasInternalLinkage(), Var);
 }
 
 /// EmitGlobalVariable - Emit global variable's debug info.
 void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, 
-                                     llvm::ConstantInt *Init,
-                                     CGBuilderTy &Builder) {
+                                     llvm::Constant *Init) {
   // Create the descriptor for the variable.
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
   llvm::StringRef Name = VD->getName();
   llvm::DIType Ty = getOrCreateType(VD->getType(), Unit);
   if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) {
     if (const EnumDecl *ED = dyn_cast<EnumDecl>(ECD->getDeclContext()))
-      Ty = CreateEnumType(ED, Unit);
+      Ty = CreateEnumType(ED);
   }
   // Do not use DIGlobalVariable for enums.
   if (Ty.getTag() == llvm::dwarf::DW_TAG_enumeration_type)
     return;
-  DebugFactory.CreateGlobalVariable(Unit, Name, Name, Name, Unit,
-                                    getLineNumber(VD->getLocation()),
-                                    Ty, true, true, Init);
+  DBuilder.CreateStaticVariable(Unit, Name, Name, Unit,
+                                getLineNumber(VD->getLocation()),
+                                Ty, true, Init);
 }
 
 /// getOrCreateNamesSpace - Return namespace descriptor for the given
 /// namespace decl.
 llvm::DINameSpace 
-CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl, 
-                                  llvm::DIDescriptor Unit) {
+CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) {
   llvm::DenseMap<const NamespaceDecl *, llvm::WeakVH>::iterator I = 
     NameSpaceCache.find(NSDecl);
   if (I != NameSpaceCache.end())
     return llvm::DINameSpace(cast<llvm::MDNode>(I->second));
   
   unsigned LineNo = getLineNumber(NSDecl->getLocation());
-
+  llvm::DIFile FileD = getOrCreateFile(NSDecl->getLocation());
   llvm::DIDescriptor Context = 
-    getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()), Unit);
+    getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()));
   llvm::DINameSpace NS =
-    DebugFactory.CreateNameSpace(Context, NSDecl->getName(), 
-                                 llvm::DIFile(Unit), LineNo);
+    DBuilder.CreateNameSpace(Context, NSDecl->getName(), FileD, LineNo);
   NameSpaceCache[NSDecl] = llvm::WeakVH(NS);
   return NS;
 }
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index a1ad012353e5..6a9ab9c58b52 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -19,6 +19,7 @@
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Analysis/DebugInfo.h"
+#include "llvm/Analysis/DIBuilder.h"
 #include "llvm/Support/ValueHandle.h"
 #include "llvm/Support/Allocator.h"
 
@@ -36,13 +37,14 @@ namespace CodeGen {
   class CodeGenModule;
   class CodeGenFunction;
   class GlobalDecl;
+  class CGBlockInfo;
 
 /// CGDebugInfo - This class gathers all debug information during compilation
 /// and is responsible for emitting to llvm globals or pass directly to
 /// the backend.
 class CGDebugInfo {
   CodeGenModule &CGM;
-  llvm::DIFactory DebugFactory;
+  llvm::DIBuilder DBuilder;
   llvm::DICompileUnit TheCU;
   SourceLocation CurLoc, PrevLoc;
   llvm::DIType VTablePtrType;
@@ -74,8 +76,8 @@ class CGDebugInfo {
   llvm::DenseMap<const NamespaceDecl *, llvm::WeakVH> NameSpaceCache;
 
   /// Helper functions for getOrCreateType.
-  llvm::DIType CreateType(const BuiltinType *Ty, llvm::DIFile F);
-  llvm::DIType CreateType(const ComplexType *Ty, llvm::DIFile F);
+  llvm::DIType CreateType(const BuiltinType *Ty);
+  llvm::DIType CreateType(const ComplexType *Ty);
   llvm::DIType CreateQualifiedType(QualType Ty, llvm::DIFile F);
   llvm::DIType CreateType(const TypedefType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const ObjCObjectPointerType *Ty,
@@ -83,22 +85,21 @@ class CGDebugInfo {
   llvm::DIType CreateType(const PointerType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const BlockPointerType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const FunctionType *Ty, llvm::DIFile F);
-  llvm::DIType CreateType(const TagType *Ty, llvm::DIFile F);
-  llvm::DIType CreateType(const RecordType *Ty, llvm::DIFile F);
+  llvm::DIType CreateType(const TagType *Ty);
+  llvm::DIType CreateType(const RecordType *Ty);
   llvm::DIType CreateType(const ObjCInterfaceType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const ObjCObjectType *Ty, llvm::DIFile F);
-  llvm::DIType CreateType(const EnumType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const VectorType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const ArrayType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F);
+  llvm::DIType CreateType(const RValueReferenceType *Ty, llvm::DIFile Unit);
   llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F);
-  llvm::DIType CreateEnumType(const EnumDecl *ED, llvm::DIFile Unit);
+  llvm::DIType CreateEnumType(const EnumDecl *ED);
   llvm::DIType getOrCreateMethodType(const CXXMethodDecl *Method,
                                      llvm::DIFile F);
   llvm::DIType getOrCreateVTablePtrType(llvm::DIFile F);
-  llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N, 
-                                         llvm::DIDescriptor Unit);
-
+  llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N);
+  llvm::DIType CreatePointeeType(QualType PointeeTy, llvm::DIFile F);
   llvm::DIType CreatePointerLikeType(unsigned Tag,
                                      const Type *Ty, QualType PointeeTy,
                                      llvm::DIFile F);
@@ -109,26 +110,26 @@ class CGDebugInfo {
   
   void CollectCXXMemberFunctions(const CXXRecordDecl *Decl,
                                  llvm::DIFile F,
-                                 llvm::SmallVectorImpl<llvm::DIDescriptor> &E,
+                                 llvm::SmallVectorImpl<llvm::Value *> &E,
                                  llvm::DIType T);
 
   void CollectCXXFriends(const CXXRecordDecl *Decl,
                        llvm::DIFile F,
-                       llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                       llvm::SmallVectorImpl<llvm::Value *> &EltTys,
                        llvm::DIType RecordTy);
 
   void CollectCXXBases(const CXXRecordDecl *Decl,
                        llvm::DIFile F,
-                       llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                       llvm::SmallVectorImpl<llvm::Value *> &EltTys,
                        llvm::DIType RecordTy);
 
 
   void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F,
-                           llvm::SmallVectorImpl<llvm::DIDescriptor> &E);
+                           llvm::SmallVectorImpl<llvm::Value *> &E);
 
   void CollectVTableInfo(const CXXRecordDecl *Decl,
                          llvm::DIFile F,
-                         llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys);
+                         llvm::SmallVectorImpl<llvm::Value *> &EltTys);
 
 public:
   CGDebugInfo(CodeGenModule &CGM);
@@ -169,10 +170,10 @@ public:
 
   /// EmitDeclareOfBlockDeclRefVariable - Emit call to llvm.dbg.declare for an
   /// imported variable declaration in a block.
-  void EmitDeclareOfBlockDeclRefVariable(const BlockDeclRefExpr *BDRE,
-                                         llvm::Value *AI,
+  void EmitDeclareOfBlockDeclRefVariable(const VarDecl *variable,
+                                         llvm::Value *storage,
                                          CGBuilderTy &Builder,
-                                         CodeGenFunction *CGF);
+                                         const CGBlockInfo &blockInfo);
 
   /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument
   /// variable declaration.
@@ -186,17 +187,19 @@ public:
   void EmitGlobalVariable(llvm::GlobalVariable *GV, ObjCInterfaceDecl *Decl);
 
   /// EmitGlobalVariable - Emit global variable's debug info.
-  void EmitGlobalVariable(const ValueDecl *VD, llvm::ConstantInt *Init, 
-                          CGBuilderTy &Builder);
+  void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init);
 
+  /// getOrCreateRecordType - Emit record type's standalone debug info. 
+  llvm::DIType getOrCreateRecordType(QualType Ty, SourceLocation L);
 private:
   /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration.
   void EmitDeclare(const VarDecl *decl, unsigned Tag, llvm::Value *AI,
                    CGBuilderTy &Builder);
 
-  /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration.
-  void EmitDeclare(const BlockDeclRefExpr *BDRE, unsigned Tag, llvm::Value *AI,
-                   CGBuilderTy &Builder, CodeGenFunction *CGF);
+  /// EmitDeclare - Emit call to llvm.dbg.declare for a variable
+  /// declaration from an enclosing block.
+  void EmitDeclare(const VarDecl *decl, unsigned Tag, llvm::Value *AI,
+                   CGBuilderTy &Builder, const CGBlockInfo &blockInfo);
 
   // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.  
   // See BuildByRefType.
@@ -204,8 +207,7 @@ private:
                                             uint64_t *OffSet);
 
   /// getContextDescriptor - Get context info for the decl.
-  llvm::DIDescriptor getContextDescriptor(const Decl *Decl,
-                                          llvm::DIDescriptor &CU);
+  llvm::DIDescriptor getContextDescriptor(const Decl *Decl);
 
   /// getCurrentDirname - Return current directory name.
   llvm::StringRef getCurrentDirname();
@@ -217,6 +219,9 @@ private:
   /// location.
   llvm::DIFile getOrCreateFile(SourceLocation Loc);
 
+  /// getOrCreateMainFile - Get the file info for main compile unit.
+  llvm::DIFile getOrCreateMainFile();
+
   /// getOrCreateType - Get the type from the cache or create a new type if
   /// necessary.
   llvm::DIType getOrCreateType(QualType Ty, llvm::DIFile F);
@@ -232,6 +237,7 @@ private:
   /// name is constructred on demand (e.g. C++ destructor) then the name
   /// is stored on the side.
   llvm::StringRef getFunctionName(const FunctionDecl *FD);
+
   /// getObjCMethodName - Returns the unmangled name of an Objective-C method.
   /// This is the display name for the debugging info.  
   llvm::StringRef getObjCMethodName(const ObjCMethodDecl *FD);
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 57e5236c67e5..a87dfaed9d5e 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -14,6 +14,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGBlocks.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
@@ -44,6 +45,7 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   case Decl::CXXDestructor:
   case Decl::CXXConversion:
   case Decl::Field:
+  case Decl::IndirectField:
   case Decl::ObjCIvar:
   case Decl::ObjCAtDefsField:      
   case Decl::ParmVar:
@@ -68,7 +70,6 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   case Decl::Friend:
   case Decl::FriendTemplate:
   case Decl::Block:
-    
     assert(0 && "Declaration not should not be in declstmts!");
   case Decl::Function:  // void X();
   case Decl::Record:    // struct/union/class X;
@@ -80,14 +81,15 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   case Decl::UsingDirective: // using namespace X; [C++]
   case Decl::NamespaceAlias:
   case Decl::StaticAssert: // static_assert(X, ""); [C++0x]
+  case Decl::Label:        // __label__ x;
     // None of these decls require codegen support.
     return;
 
   case Decl::Var: {
     const VarDecl &VD = cast<VarDecl>(D);
-    assert(VD.isBlockVarDecl() &&
+    assert(VD.isLocalVarDecl() &&
            "Should not see file-scope variables inside a function!");
-    return EmitBlockVarDecl(VD);
+    return EmitVarDecl(VD);
   }
 
   case Decl::Typedef: {   // typedef int X;
@@ -100,17 +102,14 @@ void CodeGenFunction::EmitDecl(const Decl &D) {
   }
 }
 
-/// EmitBlockVarDecl - This method handles emission of any variable declaration
+/// EmitVarDecl - This method handles emission of any variable declaration
 /// inside a function, including static vars etc.
-void CodeGenFunction::EmitBlockVarDecl(const VarDecl &D) {
-  if (D.hasAttr<AsmLabelAttr>())
-    CGM.ErrorUnsupported(&D, "__asm__");
-
+void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
   switch (D.getStorageClass()) {
   case SC_None:
   case SC_Auto:
   case SC_Register:
-    return EmitLocalBlockVarDecl(D);
+    return EmitAutoVarDecl(D);
   case SC_Static: {
     llvm::GlobalValue::LinkageTypes Linkage = 
       llvm::GlobalValue::InternalLinkage;
@@ -124,7 +123,7 @@ void CodeGenFunction::EmitBlockVarDecl(const VarDecl &D) {
       if (llvm::GlobalValue::isWeakForLinker(CurFn->getLinkage()))
         Linkage = CurFn->getLinkage();
     
-    return EmitStaticBlockVarDecl(D, Linkage);
+    return EmitStaticVarDecl(D, Linkage);
   }
   case SC_Extern:
   case SC_PrivateExtern:
@@ -144,22 +143,32 @@ static std::string GetStaticDeclName(CodeGenFunction &CGF, const VarDecl &D,
   }
   
   std::string ContextName;
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CGF.CurFuncDecl)) {
+  if (!CGF.CurFuncDecl) {
+    // Better be in a block declared in global scope.
+    const NamedDecl *ND = cast<NamedDecl>(&D);
+    const DeclContext *DC = ND->getDeclContext();
+    if (const BlockDecl *BD = dyn_cast<BlockDecl>(DC)) {
+      MangleBuffer Name;
+      CGM.getBlockMangledName(GlobalDecl(), Name, BD);
+      ContextName = Name.getString();
+    }
+    else
+      assert(0 && "Unknown context for block static var decl");
+  } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CGF.CurFuncDecl)) {
     llvm::StringRef Name = CGM.getMangledName(FD);
     ContextName = Name.str();
   } else if (isa<ObjCMethodDecl>(CGF.CurFuncDecl))
     ContextName = CGF.CurFn->getName();
   else
-    // FIXME: What about in a block??
-    assert(0 && "Unknown context for block var decl");
+    assert(0 && "Unknown context for static var decl");
   
   return ContextName + Separator + D.getNameAsString();
 }
 
 llvm::GlobalVariable *
-CodeGenFunction::CreateStaticBlockVarDecl(const VarDecl &D,
-                                          const char *Separator,
-                                      llvm::GlobalValue::LinkageTypes Linkage) {
+CodeGenFunction::CreateStaticVarDecl(const VarDecl &D,
+                                     const char *Separator,
+                                     llvm::GlobalValue::LinkageTypes Linkage) {
   QualType Ty = D.getType();
   assert(Ty->isConstantSizeType() && "VLAs can't be static");
 
@@ -172,16 +181,18 @@ CodeGenFunction::CreateStaticBlockVarDecl(const VarDecl &D,
                              CGM.EmitNullConstant(D.getType()), Name, 0,
                              D.isThreadSpecified(), Ty.getAddressSpace());
   GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
+  if (Linkage != llvm::GlobalValue::InternalLinkage)
+    GV->setVisibility(CurFn->getVisibility());
   return GV;
 }
 
-/// AddInitializerToGlobalBlockVarDecl - Add the initializer for 'D' to the
+/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
 /// global variable that has already been created for it.  If the initializer
 /// has a different type than GV does, this may free GV and return a different
 /// one.  Otherwise it just returns GV.
 llvm::GlobalVariable *
-CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
-                                                    llvm::GlobalVariable *GV) {
+CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
+                                               llvm::GlobalVariable *GV) {
   llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(), D.getType(), this);
 
   // If constant emission failed, then this should be a C++ static
@@ -189,12 +200,12 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
   if (!Init) {
     if (!getContext().getLangOptions().CPlusPlus)
       CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
-    else {
+    else if (Builder.GetInsertBlock()) {
       // Since we have a static initializer, this global variable can't 
       // be constant.
       GV->setConstant(false);
-      
-      EmitStaticCXXBlockVarDeclInit(D, GV);
+
+      EmitCXXGuardedInit(D, GV);
     }
     return GV;
   }
@@ -209,8 +220,10 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
     GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
                                   OldGV->isConstant(),
                                   OldGV->getLinkage(), Init, "",
-                                  0, D.isThreadSpecified(),
+                                  /*InsertBefore*/ OldGV,
+                                  D.isThreadSpecified(),
                                   D.getType().getAddressSpace());
+    GV->setVisibility(OldGV->getVisibility());
     
     // Steal the name of the old global
     GV->takeName(OldGV);
@@ -228,12 +241,12 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
   return GV;
 }
 
-void CodeGenFunction::EmitStaticBlockVarDecl(const VarDecl &D,
+void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
                                       llvm::GlobalValue::LinkageTypes Linkage) {
   llvm::Value *&DMEntry = LocalDeclMap[&D];
   assert(DMEntry == 0 && "Decl already exists in localdeclmap!");
 
-  llvm::GlobalVariable *GV = CreateStaticBlockVarDecl(D, ".", Linkage);
+  llvm::GlobalVariable *GV = CreateStaticVarDecl(D, ".", Linkage);
 
   // Store into LocalDeclMap before generating initializer to handle
   // circular references.
@@ -244,10 +257,14 @@ void CodeGenFunction::EmitStaticBlockVarDecl(const VarDecl &D,
   // Make sure to evaluate VLA bounds now so that we have them for later.
   if (D.getType()->isVariablyModifiedType())
     EmitVLASize(D.getType());
+  
+  // Local static block variables must be treated as globals as they may be
+  // referenced in their RHS initializer block-literal expresion.
+  CGM.setStaticLocalDeclAddress(&D, GV);
 
   // If this value has an initializer, emit it.
   if (D.getInit())
-    GV = AddInitializerToGlobalBlockVarDecl(D, GV);
+    GV = AddInitializerToStaticVarDecl(D, GV);
 
   GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
 
@@ -266,17 +283,13 @@ void CodeGenFunction::EmitStaticBlockVarDecl(const VarDecl &D,
   if (D.hasAttr<UsedAttr>())
     CGM.AddUsedGlobal(GV);
 
-  if (getContext().getLangOptions().CPlusPlus)
-    CGM.setStaticLocalDeclAddress(&D, GV);
-  
   // We may have to cast the constant because of the initializer
   // mismatch above.
   //
   // FIXME: It is really dangerous to store this in the map; if anyone
   // RAUW's the GV uses of this constant will be invalid.
   const llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(D.getType());
-  const llvm::Type *LPtrTy =
-    llvm::PointerType::get(LTy, D.getType().getAddressSpace());
+  const llvm::Type *LPtrTy = LTy->getPointerTo(D.getType().getAddressSpace());
   DMEntry = llvm::ConstantExpr::getBitCast(GV, LPtrTy);
 
   // Emit global variable debug descriptor for static vars.
@@ -293,6 +306,15 @@ unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
   return ByRefValueInfo.find(VD)->second.second;
 }
 
+llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr,
+                                                     const VarDecl *V) {
+  llvm::Value *Loc = Builder.CreateStructGEP(BaseAddr, 1, "forwarding");
+  Loc = Builder.CreateLoad(Loc);
+  Loc = Builder.CreateStructGEP(Loc, getByRefValueLLVMField(V),
+                                V->getNameAsString());
+  return Loc;
+}
+
 /// BuildByRefType - This routine changes a __block variable declared as T x
 ///   into:
 ///
@@ -307,7 +329,7 @@ unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const {
 ///        T x;
 ///      } x
 ///
-const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
+const llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
   std::pair<const llvm::Type *, unsigned> &Info = ByRefValueInfo[D];
   if (Info.first)
     return Info.first;
@@ -316,9 +338,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
 
   std::vector<const llvm::Type *> Types;
   
-  const llvm::PointerType *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-
-  llvm::PATypeHolder ByRefTypeHolder = llvm::OpaqueType::get(VMContext);
+  llvm::PATypeHolder ByRefTypeHolder = llvm::OpaqueType::get(getLLVMContext());
   
   // void *__isa;
   Types.push_back(Int8PtrTy);
@@ -332,7 +352,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
   // int32_t __size;
   Types.push_back(Int32Ty);
 
-  bool HasCopyAndDispose = BlockRequiresCopying(Ty);
+  bool HasCopyAndDispose = getContext().BlockRequiresCopying(Ty);
   if (HasCopyAndDispose) {
     /// void *__copy_helper;
     Types.push_back(Int8PtrTy);
@@ -343,7 +363,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
 
   bool Packed = false;
   CharUnits Align = getContext().getDeclAlign(D);
-  if (Align > CharUnits::fromQuantity(Target.getPointerAlign(0) / 8)) {
+  if (Align > getContext().toCharUnitsFromBits(Target.getPointerAlign(0))) {
     // We have to insert padding.
     
     // The struct above has 2 32-bit integers.
@@ -359,7 +379,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
     
     unsigned NumPaddingBytes = AlignedOffsetInBytes - CurrentOffsetInBytes;
     if (NumPaddingBytes > 0) {
-      const llvm::Type *Ty = llvm::Type::getInt8Ty(VMContext);
+      const llvm::Type *Ty = llvm::Type::getInt8Ty(getLLVMContext());
       // FIXME: We need a sema error for alignment larger than the minimum of
       // the maximal stack alignmint and the alignment of malloc on the system.
       if (NumPaddingBytes > 1)
@@ -375,7 +395,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
   // T x;
   Types.push_back(ConvertTypeForMem(Ty));
   
-  const llvm::Type *T = llvm::StructType::get(VMContext, Types, Packed);
+  const llvm::Type *T = llvm::StructType::get(getLLVMContext(), Types, Packed);
   
   cast<llvm::OpaqueType>(ByRefTypeHolder.get())->refineAbstractTypeTo(T);
   CGM.getModule().addTypeName("struct.__block_byref_" + D->getNameAsString(), 
@@ -484,18 +504,101 @@ namespace {
     CallBlockRelease(llvm::Value *Addr) : Addr(Addr) {}
 
     void Emit(CodeGenFunction &CGF, bool IsForEH) {
-      llvm::Value *V = CGF.Builder.CreateStructGEP(Addr, 1, "forwarding");
-      V = CGF.Builder.CreateLoad(V);
-      CGF.BuildBlockRelease(V);
+      CGF.BuildBlockRelease(Addr, BLOCK_FIELD_IS_BYREF);
     }
   };
 }
 
-/// EmitLocalBlockVarDecl - Emit code and set up an entry in LocalDeclMap for a
+
+/// canEmitInitWithFewStoresAfterMemset - Decide whether we can emit the
+/// non-zero parts of the specified initializer with equal or fewer than
+/// NumStores scalar stores.
+static bool canEmitInitWithFewStoresAfterMemset(llvm::Constant *Init,
+                                                unsigned &NumStores) {
+  // Zero and Undef never requires any extra stores.
+  if (isa<llvm::ConstantAggregateZero>(Init) ||
+      isa<llvm::ConstantPointerNull>(Init) ||
+      isa<llvm::UndefValue>(Init))
+    return true;
+  if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
+      isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
+      isa<llvm::ConstantExpr>(Init))
+    return Init->isNullValue() || NumStores--;
+
+  // See if we can emit each element.
+  if (isa<llvm::ConstantArray>(Init) || isa<llvm::ConstantStruct>(Init)) {
+    for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
+      llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
+      if (!canEmitInitWithFewStoresAfterMemset(Elt, NumStores))
+        return false;
+    }
+    return true;
+  }
+  
+  // Anything else is hard and scary.
+  return false;
+}
+
+/// emitStoresForInitAfterMemset - For inits that
+/// canEmitInitWithFewStoresAfterMemset returned true for, emit the scalar
+/// stores that would be required.
+static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
+                                         CGBuilderTy &Builder) {
+  // Zero doesn't require any stores.
+  if (isa<llvm::ConstantAggregateZero>(Init) ||
+      isa<llvm::ConstantPointerNull>(Init) ||
+      isa<llvm::UndefValue>(Init))
+    return;
+  
+  if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
+      isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
+      isa<llvm::ConstantExpr>(Init)) {
+    if (!Init->isNullValue())
+      Builder.CreateStore(Init, Loc);
+    return;
+  }
+  
+  assert((isa<llvm::ConstantStruct>(Init) || isa<llvm::ConstantArray>(Init)) &&
+         "Unknown value type!");
+  
+  for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
+    llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
+    if (Elt->isNullValue()) continue;
+    
+    // Otherwise, get a pointer to the element and emit it.
+    emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
+                                 Builder);
+  }
+}
+
+
+/// shouldUseMemSetPlusStoresToInitialize - Decide whether we should use memset
+/// plus some stores to initialize a local variable instead of using a memcpy
+/// from a constant global.  It is beneficial to use memset if the global is all
+/// zeros, or mostly zeros and large.
+static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init,
+                                                  uint64_t GlobalSize) {
+  // If a global is all zeros, always use a memset.
+  if (isa<llvm::ConstantAggregateZero>(Init)) return true;
+
+
+  // If a non-zero global is <= 32 bytes, always use a memcpy.  If it is large,
+  // do it if it will require 6 or fewer scalar stores.
+  // TODO: Should budget depends on the size?  Avoiding a large global warrants
+  // plopping in more stores.
+  unsigned StoreBudget = 6;
+  uint64_t SizeLimit = 32;
+  
+  return GlobalSize > SizeLimit && 
+         canEmitInitWithFewStoresAfterMemset(Init, StoreBudget);
+}
+
+
+/// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
-void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
-                                            SpecialInitFn *SpecialInit) {
+void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D,
+                                      SpecialInitFn *SpecialInit) {
   QualType Ty = D.getType();
   unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
   bool isByRef = D.hasAttr<BlocksAttr>();
@@ -520,7 +623,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
         // If this variable is marked 'const', emit the value as a global.
         if (CGM.getCodeGenOpts().MergeAllConstants &&
             Ty.isConstant(getContext())) {
-          EmitStaticBlockVarDecl(D, llvm::GlobalValue::InternalLinkage);
+          EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage);
           return;
         }
         
@@ -542,9 +645,9 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
             // Create a flag that is used to indicate when the NRVO was applied
             // to this variable. Set it to zero to indicate that NRVO was not 
             // applied.
-            const llvm::Type *BoolTy = llvm::Type::getInt1Ty(VMContext);
-            llvm::Value *Zero = llvm::ConstantInt::get(BoolTy, 0);
-            NRVOFlag = CreateTempAlloca(BoolTy, "nrvo");            
+            llvm::Value *Zero = Builder.getFalse();
+            NRVOFlag = CreateTempAlloca(Zero->getType(), "nrvo");
+            EnsureInsertPoint();
             Builder.CreateStore(Zero, NRVOFlag);
             
             // Record the NRVO flag for this variable.
@@ -561,7 +664,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
         Align = getContext().getDeclAlign(&D);
         if (isByRef)
           Align = std::max(Align, 
-              CharUnits::fromQuantity(Target.getPointerAlign(0) / 8));
+              getContext().toCharUnitsFromBits(Target.getPointerAlign(0)));
         Alloc->setAlignment(Align.getQuantity());
         DeclPtr = Alloc;
       }
@@ -569,9 +672,8 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       // Targets that don't support recursion emit locals as globals.
       const char *Class =
         D.getStorageClass() == SC_Register ? ".reg." : ".auto.";
-      DeclPtr = CreateStaticBlockVarDecl(D, Class,
-                                         llvm::GlobalValue
-                                         ::InternalLinkage);
+      DeclPtr = CreateStaticVarDecl(D, Class,
+                                    llvm::GlobalValue::InternalLinkage);
     }
 
     // FIXME: Can this happen?
@@ -582,8 +684,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
 
     if (!DidCallStackSave) {
       // Save the stack.
-      const llvm::Type *LTy = llvm::Type::getInt8PtrTy(VMContext);
-      llvm::Value *Stack = CreateTempAlloca(LTy, "saved_stack");
+      llvm::Value *Stack = CreateTempAlloca(Int8PtrTy, "saved_stack");
 
       llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave);
       llvm::Value *V = Builder.CreateCall(F);
@@ -593,19 +694,19 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       DidCallStackSave = true;
 
       // Push a cleanup block and restore the stack there.
+      // FIXME: in general circumstances, this should be an EH cleanup.
       EHStack.pushCleanup<CallStackRestore>(NormalCleanup, Stack);
     }
 
     // Get the element type.
     const llvm::Type *LElemTy = ConvertTypeForMem(Ty);
-    const llvm::Type *LElemPtrTy =
-      llvm::PointerType::get(LElemTy, Ty.getAddressSpace());
+    const llvm::Type *LElemPtrTy = LElemTy->getPointerTo(Ty.getAddressSpace());
 
     llvm::Value *VLASize = EmitVLASize(Ty);
 
     // Allocate memory for the array.
     llvm::AllocaInst *VLA = 
-      Builder.CreateAlloca(llvm::Type::getInt8Ty(VMContext), VLASize, "vla");
+      Builder.CreateAlloca(llvm::Type::getInt8Ty(getLLVMContext()), VLASize, "vla");
     VLA->setAlignment(getContext().getDeclAlign(&D).getQuantity());
 
     DeclPtr = Builder.CreateBitCast(VLA, LElemPtrTy, "tmp");
@@ -639,59 +740,65 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
   }
 
   if (isByRef) {
-    const llvm::PointerType *PtrToInt8Ty = llvm::Type::getInt8PtrTy(VMContext);
-
     EnsureInsertPoint();
-    llvm::Value *isa_field = Builder.CreateStructGEP(DeclPtr, 0);
-    llvm::Value *forwarding_field = Builder.CreateStructGEP(DeclPtr, 1);
-    llvm::Value *flags_field = Builder.CreateStructGEP(DeclPtr, 2);
-    llvm::Value *size_field = Builder.CreateStructGEP(DeclPtr, 3);
     llvm::Value *V;
-    int flag = 0;
-    int flags = 0;
+
+    BlockFieldFlags fieldFlags;
+    bool fieldNeedsCopyDispose = false;
 
     needsDispose = true;
 
     if (Ty->isBlockPointerType()) {
-      flag |= BLOCK_FIELD_IS_BLOCK;
-      flags |= BLOCK_HAS_COPY_DISPOSE;
-    } else if (BlockRequiresCopying(Ty)) {
-      flag |= BLOCK_FIELD_IS_OBJECT;
-      flags |= BLOCK_HAS_COPY_DISPOSE;
+      fieldFlags |= BLOCK_FIELD_IS_BLOCK;
+      fieldNeedsCopyDispose = true;
+    } else if (getContext().isObjCNSObjectType(Ty) || 
+               Ty->isObjCObjectPointerType()) {
+      fieldFlags |= BLOCK_FIELD_IS_OBJECT;
+      fieldNeedsCopyDispose = true;
+    } else if (getLangOptions().CPlusPlus) {
+      if (getContext().getBlockVarCopyInits(&D))
+        fieldNeedsCopyDispose = true;
+      else if (const CXXRecordDecl *record = D.getType()->getAsCXXRecordDecl())
+        fieldNeedsCopyDispose = !record->hasTrivialDestructor();
     }
 
     // FIXME: Someone double check this.
     if (Ty.isObjCGCWeak())
-      flag |= BLOCK_FIELD_IS_WEAK;
+      fieldFlags |= BLOCK_FIELD_IS_WEAK;
 
     int isa = 0;
-    if (flag&BLOCK_FIELD_IS_WEAK)
+    if (fieldFlags & BLOCK_FIELD_IS_WEAK)
       isa = 1;
-    V = llvm::ConstantInt::get(Int32Ty, isa);
-    V = Builder.CreateIntToPtr(V, PtrToInt8Ty, "isa");
-    Builder.CreateStore(V, isa_field);
+    V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");
+    Builder.CreateStore(V, Builder.CreateStructGEP(DeclPtr, 0, "byref.isa"));
 
-    Builder.CreateStore(DeclPtr, forwarding_field);
+    Builder.CreateStore(DeclPtr, Builder.CreateStructGEP(DeclPtr, 1,
+                                                         "byref.forwarding"));
 
-    V = llvm::ConstantInt::get(Int32Ty, flags);
-    Builder.CreateStore(V, flags_field);
+    // Blocks ABI:
+    //   c) the flags field is set to either 0 if no helper functions are
+    //      needed or BLOCK_HAS_COPY_DISPOSE if they are,
+    BlockFlags flags;
+    if (fieldNeedsCopyDispose) flags |= BLOCK_HAS_COPY_DISPOSE;
+    Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()),
+                        Builder.CreateStructGEP(DeclPtr, 2, "byref.flags"));
 
     const llvm::Type *V1;
     V1 = cast<llvm::PointerType>(DeclPtr->getType())->getElementType();
-    V = llvm::ConstantInt::get(Int32Ty,
-                               CGM.GetTargetTypeStoreSize(V1).getQuantity());
-    Builder.CreateStore(V, size_field);
+    V = llvm::ConstantInt::get(IntTy, CGM.GetTargetTypeStoreSize(V1).getQuantity());
+    Builder.CreateStore(V, Builder.CreateStructGEP(DeclPtr, 3, "byref.size"));
 
-    if (flags & BLOCK_HAS_COPY_DISPOSE) {
-      BlockHasCopyDispose = true;
+    if (fieldNeedsCopyDispose) {
       llvm::Value *copy_helper = Builder.CreateStructGEP(DeclPtr, 4);
-      Builder.CreateStore(BuildbyrefCopyHelper(DeclPtr->getType(), flag, 
-                                               Align.getQuantity()),
+      Builder.CreateStore(CGM.BuildbyrefCopyHelper(DeclPtr->getType(),
+                                                   fieldFlags, 
+                                                   Align.getQuantity(), &D),
                           copy_helper);
 
       llvm::Value *destroy_helper = Builder.CreateStructGEP(DeclPtr, 5);
-      Builder.CreateStore(BuildbyrefDestroyHelper(DeclPtr->getType(), flag,
-                                                  Align.getQuantity()),
+      Builder.CreateStore(CGM.BuildbyrefDestroyHelper(DeclPtr->getType(),
+                                                      fieldFlags,
+                                                      Align.getQuantity(), &D),
                           destroy_helper);
     }
   }
@@ -700,9 +807,6 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
     SpecialInit(*this, D, DeclPtr);
   } else if (Init) {
     llvm::Value *Loc = DeclPtr;
-    if (isByRef)
-      Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
-                                    D.getNameAsString());
     
     bool isVolatile = getContext().getCanonicalType(Ty).isVolatileQualified();
     
@@ -712,27 +816,25 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(), Ty,this);
       assert(Init != 0 && "Wasn't a simple constant init?");
       
-      llvm::Value *AlignVal = 
-      llvm::ConstantInt::get(Int32Ty, Align.getQuantity());
-      const llvm::Type *IntPtr =
-      llvm::IntegerType::get(VMContext, LLVMPointerWidth);
       llvm::Value *SizeVal =
-      llvm::ConstantInt::get(IntPtr, 
+        llvm::ConstantInt::get(IntPtrTy, 
                              getContext().getTypeSizeInChars(Ty).getQuantity());
       
-      const llvm::Type *BP = llvm::Type::getInt8PtrTy(VMContext);
+      const llvm::Type *BP = Int8PtrTy;
       if (Loc->getType() != BP)
         Loc = Builder.CreateBitCast(Loc, BP, "tmp");
-      
-      llvm::Value *NotVolatile =
-        llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext), 0);
-
-      // If the initializer is all zeros, codegen with memset.
-      if (isa<llvm::ConstantAggregateZero>(Init)) {
-        llvm::Value *Zero =
-          llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext), 0);
-        Builder.CreateCall5(CGM.getMemSetFn(Loc->getType(), SizeVal->getType()),
-                            Loc, Zero, SizeVal, AlignVal, NotVolatile);
+
+      // If the initializer is all or mostly zeros, codegen with memset then do
+      // a few stores afterward.
+      if (shouldUseMemSetPlusStoresToInitialize(Init, 
+                      CGM.getTargetData().getTypeAllocSize(Init->getType()))) {
+        Builder.CreateMemSet(Loc, Builder.getInt8(0), SizeVal,
+                             Align.getQuantity(), false);
+        if (!Init->isNullValue()) {
+          Loc = Builder.CreateBitCast(Loc, Init->getType()->getPointerTo());
+          emitStoresForInitAfterMemset(Init, Loc, Builder);
+        }
+        
       } else {
         // Otherwise, create a temporary global with the initializer then 
         // memcpy from the global to the alloca.
@@ -747,20 +849,36 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
         if (SrcPtr->getType() != BP)
           SrcPtr = Builder.CreateBitCast(SrcPtr, BP, "tmp");
 
-        Builder.CreateCall5(CGM.getMemCpyFn(Loc->getType(), SrcPtr->getType(),
-                                            SizeVal->getType()),
-                            Loc, SrcPtr, SizeVal, AlignVal, NotVolatile);
+        Builder.CreateMemCpy(Loc, SrcPtr, SizeVal, Align.getQuantity(), false);
       }
     } else if (Ty->isReferenceType()) {
       RValue RV = EmitReferenceBindingToExpr(Init, &D);
+      if (isByRef)
+        Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
+                                      D.getNameAsString());
       EmitStoreOfScalar(RV.getScalarVal(), Loc, false, Alignment, Ty);
     } else if (!hasAggregateLLVMType(Init->getType())) {
       llvm::Value *V = EmitScalarExpr(Init);
+      if (isByRef) {
+        // When RHS has side-effect, must go through "forwarding' field
+        // to get to the address of the __block variable descriptor.
+        if (Init->HasSideEffects(getContext()))
+          Loc = BuildBlockByrefAddress(DeclPtr, &D);
+        else
+          Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
+                                        D.getNameAsString());
+      }
       EmitStoreOfScalar(V, Loc, isVolatile, Alignment, Ty);
     } else if (Init->getType()->isAnyComplexType()) {
+      if (isByRef)
+        Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
+                                      D.getNameAsString());
       EmitComplexExprIntoAddr(Init, Loc, isVolatile);
     } else {
-      EmitAggExpr(Init, Loc, isVolatile);
+      if (isByRef)
+        Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
+                                      D.getNameAsString());
+      EmitAggExpr(Init, AggValueSlot::forAddr(Loc, isVolatile, true, false));
     }
   }
 
@@ -805,9 +923,8 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
   }
 
   // If this is a block variable, clean it up.
-  // FIXME: this should be an EH cleanup as well.  rdar://problem/8224178
   if (needsDispose && CGM.getLangOptions().getGCMode() != LangOptions::GCOnly)
-    EHStack.pushCleanup<CallBlockRelease>(NormalCleanup, DeclPtr);
+    EHStack.pushCleanup<CallBlockRelease>(NormalAndEHCleanup, DeclPtr);
 }
 
 /// Emit an alloca (or GlobalValue depending on target)
@@ -817,7 +934,6 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg) {
   assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) &&
          "Invalid argument to EmitParmDecl");
   QualType Ty = D.getType();
-  CanQualType CTy = getContext().getCanonicalType(Ty);
 
   llvm::Value *DeclPtr;
   // If this is an aggregate or variable sized value, reuse the input pointer.
@@ -829,8 +945,9 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg) {
     DeclPtr = CreateMemTemp(Ty, D.getName() + ".addr");
 
     // Store the initial value into the alloca.
-    unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
-    EmitStoreOfScalar(Arg, DeclPtr, CTy.isVolatileQualified(), Alignment, Ty);
+    EmitStoreOfScalar(Arg, DeclPtr, Ty.isVolatileQualified(),
+                      getContext().getDeclAlign(&D).getQuantity(), Ty,
+                      CGM.getTBAAInfo(Ty));
   }
   Arg->setName(D.getName());
 
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index e2f197522b26..e295267896eb 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
+#include "CGObjCRuntime.h"
 #include "CGCXXABI.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Intrinsics.h"
@@ -34,14 +35,24 @@ static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
   unsigned Alignment = Context.getDeclAlign(&D).getQuantity();
   if (!CGF.hasAggregateLLVMType(T)) {
     llvm::Value *V = CGF.EmitScalarExpr(Init);
-    CGF.EmitStoreOfScalar(V, DeclPtr, isVolatile, Alignment, T);
+    CodeGenModule &CGM = CGF.CGM;
+    Qualifiers::GC GCAttr = CGM.getContext().getObjCGCAttrKind(T);
+    if (GCAttr == Qualifiers::Strong)
+      CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, V, DeclPtr,
+                                                D.isThreadSpecified());
+    else if (GCAttr == Qualifiers::Weak)
+      CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, V, DeclPtr);
+    else
+      CGF.EmitStoreOfScalar(V, DeclPtr, isVolatile, Alignment, T);
   } else if (T->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(Init, DeclPtr, isVolatile);
   } else {
-    CGF.EmitAggExpr(Init, DeclPtr, isVolatile);
+    CGF.EmitAggExpr(Init, AggValueSlot::forAddr(DeclPtr, isVolatile, true));
   }
 }
 
+/// Emit code to cause the destruction of the given variable with
+/// static storage duration.
 static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
                             llvm::Constant *DeclPtr) {
   CodeGenModule &CGM = CGF.CGM;
@@ -106,15 +117,13 @@ CodeGenFunction::EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
     return;
   }
 
-  const llvm::Type *Int8PtrTy = 
-    llvm::Type::getInt8Ty(VMContext)->getPointerTo();
-
   std::vector<const llvm::Type *> Params;
   Params.push_back(Int8PtrTy);
 
   // Get the destructor function type
   const llvm::Type *DtorFnTy =
-    llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), Params, false);
+    llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
+                            Params, false);
   DtorFnTy = llvm::PointerType::getUnqual(DtorFnTy);
 
   Params.clear();
@@ -138,6 +147,11 @@ CodeGenFunction::EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
   Builder.CreateCall(AtExitFn, &Args[0], llvm::array_endof(Args));
 }
 
+void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
+                                         llvm::GlobalVariable *DeclPtr) {
+  CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr);
+}
+
 static llvm::Function *
 CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
                                    const llvm::FunctionType *FTy,
@@ -145,20 +159,22 @@ CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
   llvm::Function *Fn =
     llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
                            Name, &CGM.getModule());
+  if (!CGM.getContext().getLangOptions().AppleKext) {
+    // Set the section if needed.
+    if (const char *Section = 
+          CGM.getContext().Target.getStaticInitSectionSpecifier())
+      Fn->setSection(Section);
+  }
 
-  // Set the section if needed.
-  if (const char *Section = 
-        CGM.getContext().Target.getStaticInitSectionSpecifier())
-    Fn->setSection(Section);
-
-  if (!CGM.getLangOptions().Exceptions)
+  if (!CGM.getLangOptions().areExceptionsEnabled())
     Fn->setDoesNotThrow();
 
   return Fn;
 }
 
 void
-CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D) {
+CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
+                                            llvm::GlobalVariable *Addr) {
   const llvm::FunctionType *FTy
     = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                               false);
@@ -167,7 +183,7 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D) {
   llvm::Function *Fn =
     CreateGlobalInitOrDestructFunction(*this, FTy, "__cxx_global_var_init");
 
-  CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D);
+  CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr);
 
   if (D->hasAttr<InitPriorityAttr>()) {
     unsigned int order = D->getAttr<InitPriorityAttr>()->getPriority();
@@ -240,13 +256,20 @@ void CodeGenModule::EmitCXXGlobalDtorFunc() {
   AddGlobalDtor(Fn);
 }
 
+/// Emit the code necessary to initialize the given global variable.
 void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
-                                                       const VarDecl *D) {
+                                                       const VarDecl *D,
+                                                 llvm::GlobalVariable *Addr) {
   StartFunction(GlobalDecl(), getContext().VoidTy, Fn, FunctionArgList(),
                 SourceLocation());
 
-  llvm::Constant *DeclPtr = CGM.GetAddrOfGlobalVar(D);
-  EmitCXXGlobalVarDeclInit(*D, DeclPtr);
+  // Use guarded initialization if the global variable is weak due to
+  // being a class template's static data member.
+  if (Addr->hasWeakLinkage() && D->getInstantiatedFromStaticDataMember()) {
+    EmitCXXGuardedInit(*D, Addr);
+  } else {
+    EmitCXXGlobalVarDeclInit(*D, Addr);
+  }
 
   FinishFunction();
 }
@@ -283,143 +306,6 @@ void CodeGenFunction::GenerateCXXGlobalDtorFunc(llvm::Function *Fn,
   FinishFunction();
 }
 
-static llvm::Constant *getGuardAcquireFn(CodeGenFunction &CGF) {
-  // int __cxa_guard_acquire(__int64_t *guard_object);
-  
-  const llvm::Type *Int64PtrTy = 
-    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
-  
-  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
-  
-  const llvm::FunctionType *FTy =
-    llvm::FunctionType::get(CGF.ConvertType(CGF.getContext().IntTy),
-                            Args, /*isVarArg=*/false);
-  
-  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire");
-}
-
-static llvm::Constant *getGuardReleaseFn(CodeGenFunction &CGF) {
-  // void __cxa_guard_release(__int64_t *guard_object);
-  
-  const llvm::Type *Int64PtrTy = 
-    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
-  
-  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
-  
-  const llvm::FunctionType *FTy =
-  llvm::FunctionType::get(llvm::Type::getVoidTy(CGF.getLLVMContext()),
-                          Args, /*isVarArg=*/false);
-  
-  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release");
-}
-
-static llvm::Constant *getGuardAbortFn(CodeGenFunction &CGF) {
-  // void __cxa_guard_abort(__int64_t *guard_object);
-  
-  const llvm::Type *Int64PtrTy = 
-    llvm::Type::getInt64PtrTy(CGF.getLLVMContext());
-  
-  std::vector<const llvm::Type*> Args(1, Int64PtrTy);
-  
-  const llvm::FunctionType *FTy =
-  llvm::FunctionType::get(llvm::Type::getVoidTy(CGF.getLLVMContext()),
-                          Args, /*isVarArg=*/false);
-  
-  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort");
-}
-
-namespace {
-  struct CallGuardAbort : EHScopeStack::Cleanup {
-    llvm::GlobalVariable *Guard;
-    CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
-
-    void Emit(CodeGenFunction &CGF, bool IsForEH) {
-      // It shouldn't be possible for this to throw, but if it can,
-      // this should allow for the possibility of an invoke.
-      CGF.Builder.CreateCall(getGuardAbortFn(CGF), Guard)
-        ->setDoesNotThrow();
-    }
-  };
-}
-
-void
-CodeGenFunction::EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
-                                               llvm::GlobalVariable *GV) {
-  // Bail out early if this initializer isn't reachable.
-  if (!Builder.GetInsertBlock()) return;
-
-  bool ThreadsafeStatics = getContext().getLangOptions().ThreadsafeStatics;
-  
-  llvm::SmallString<256> GuardVName;
-  CGM.getCXXABI().getMangleContext().mangleGuardVariable(&D, GuardVName);
-
-  // Create the guard variable.
-  llvm::GlobalVariable *GuardVariable =
-    new llvm::GlobalVariable(CGM.getModule(), Int64Ty,
-                             false, GV->getLinkage(),
-                             llvm::Constant::getNullValue(Int64Ty),
-                             GuardVName.str());
-
-  // Load the first byte of the guard variable.
-  const llvm::Type *PtrTy
-    = llvm::PointerType::get(llvm::Type::getInt8Ty(VMContext), 0);
-  llvm::Value *V = 
-    Builder.CreateLoad(Builder.CreateBitCast(GuardVariable, PtrTy), "tmp");
-
-  llvm::BasicBlock *InitCheckBlock = createBasicBlock("init.check");
-  llvm::BasicBlock *EndBlock = createBasicBlock("init.end");
-
-  // Check if the first byte of the guard variable is zero.
-  Builder.CreateCondBr(Builder.CreateIsNull(V, "tobool"), 
-                       InitCheckBlock, EndBlock);
-
-  EmitBlock(InitCheckBlock);
-
-  // Variables used when coping with thread-safe statics and exceptions.
-  if (ThreadsafeStatics) {    
-    // Call __cxa_guard_acquire.
-    V = Builder.CreateCall(getGuardAcquireFn(*this), GuardVariable);
-               
-    llvm::BasicBlock *InitBlock = createBasicBlock("init");
-  
-    Builder.CreateCondBr(Builder.CreateIsNotNull(V, "tobool"),
-                         InitBlock, EndBlock);
-  
-    // Call __cxa_guard_abort along the exceptional edge.
-    if (Exceptions)
-      EHStack.pushCleanup<CallGuardAbort>(EHCleanup, GuardVariable);
-    
-    EmitBlock(InitBlock);
-  }
-
-  if (D.getType()->isReferenceType()) {
-    unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
-    QualType T = D.getType();
-    RValue RV = EmitReferenceBindingToExpr(D.getInit(), &D);
-    EmitStoreOfScalar(RV.getScalarVal(), GV, /*Volatile=*/false, Alignment, T);
-  } else
-    EmitDeclInit(*this, D, GV);
-
-  if (ThreadsafeStatics) {
-    // Pop the guard-abort cleanup if we pushed one.
-    if (Exceptions)
-      PopCleanupBlock();
-
-    // Call __cxa_guard_release.  This cannot throw.
-    Builder.CreateCall(getGuardReleaseFn(*this), GuardVariable);    
-  } else {
-    llvm::Value *One = 
-      llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext), 1);
-    Builder.CreateStore(One, Builder.CreateBitCast(GuardVariable, PtrTy));
-  }
-
-  // Register the call to the destructor.
-  if (!D.getType()->isReferenceType())
-    EmitDeclDestroy(*this, D, GV);
-  
-  EmitBlock(EndBlock);
-}
-
 /// GenerateCXXAggrDestructorHelper - Generates a helper function which when
 /// invoked, calls the default destructor on array elements in reverse order of
 /// construction.
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index 7fb616e5a150..6181965748cc 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -14,163 +14,18 @@
 #include "clang/AST/StmtCXX.h"
 
 #include "llvm/Intrinsics.h"
+#include "llvm/IntrinsicInst.h"
 #include "llvm/Support/CallSite.h"
 
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CGException.h"
+#include "CGCleanup.h"
 #include "TargetInfo.h"
 
 using namespace clang;
 using namespace CodeGen;
 
-/// Push an entry of the given size onto this protected-scope stack.
-char *EHScopeStack::allocate(size_t Size) {
-  if (!StartOfBuffer) {
-    unsigned Capacity = 1024;
-    while (Capacity < Size) Capacity *= 2;
-    StartOfBuffer = new char[Capacity];
-    StartOfData = EndOfBuffer = StartOfBuffer + Capacity;
-  } else if (static_cast<size_t>(StartOfData - StartOfBuffer) < Size) {
-    unsigned CurrentCapacity = EndOfBuffer - StartOfBuffer;
-    unsigned UsedCapacity = CurrentCapacity - (StartOfData - StartOfBuffer);
-
-    unsigned NewCapacity = CurrentCapacity;
-    do {
-      NewCapacity *= 2;
-    } while (NewCapacity < UsedCapacity + Size);
-
-    char *NewStartOfBuffer = new char[NewCapacity];
-    char *NewEndOfBuffer = NewStartOfBuffer + NewCapacity;
-    char *NewStartOfData = NewEndOfBuffer - UsedCapacity;
-    memcpy(NewStartOfData, StartOfData, UsedCapacity);
-    delete [] StartOfBuffer;
-    StartOfBuffer = NewStartOfBuffer;
-    EndOfBuffer = NewEndOfBuffer;
-    StartOfData = NewStartOfData;
-  }
-
-  assert(StartOfBuffer + Size <= StartOfData);
-  StartOfData -= Size;
-  return StartOfData;
-}
-
-EHScopeStack::stable_iterator
-EHScopeStack::getEnclosingEHCleanup(iterator it) const {
-  assert(it != end());
-  do {
-    if (isa<EHCleanupScope>(*it)) {
-      if (cast<EHCleanupScope>(*it).isEHCleanup())
-        return stabilize(it);
-      return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
-    }
-    ++it;
-  } while (it != end());
-  return stable_end();
-}
-
-
-void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
-  assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned");
-  char *Buffer = allocate(EHCleanupScope::getSizeForCleanupSize(Size));
-  bool IsNormalCleanup = Kind & NormalCleanup;
-  bool IsEHCleanup = Kind & EHCleanup;
-  bool IsActive = !(Kind & InactiveCleanup);
-  EHCleanupScope *Scope =
-    new (Buffer) EHCleanupScope(IsNormalCleanup,
-                                IsEHCleanup,
-                                IsActive,
-                                Size,
-                                BranchFixups.size(),
-                                InnermostNormalCleanup,
-                                InnermostEHCleanup);
-  if (IsNormalCleanup)
-    InnermostNormalCleanup = stable_begin();
-  if (IsEHCleanup)
-    InnermostEHCleanup = stable_begin();
-
-  return Scope->getCleanupBuffer();
-}
-
-void EHScopeStack::popCleanup() {
-  assert(!empty() && "popping exception stack when not empty");
-
-  assert(isa<EHCleanupScope>(*begin()));
-  EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
-  InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
-  StartOfData += Cleanup.getAllocatedSize();
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  // Destroy the cleanup.
-  Cleanup.~EHCleanupScope();
-
-  // Check whether we can shrink the branch-fixups stack.
-  if (!BranchFixups.empty()) {
-    // If we no longer have any normal cleanups, all the fixups are
-    // complete.
-    if (!hasNormalCleanups())
-      BranchFixups.clear();
-
-    // Otherwise we can still trim out unnecessary nulls.
-    else
-      popNullFixups();
-  }
-}
-
-EHFilterScope *EHScopeStack::pushFilter(unsigned NumFilters) {
-  char *Buffer = allocate(EHFilterScope::getSizeForNumFilters(NumFilters));
-  CatchDepth++;
-  return new (Buffer) EHFilterScope(NumFilters);
-}
-
-void EHScopeStack::popFilter() {
-  assert(!empty() && "popping exception stack when not empty");
-
-  EHFilterScope &Filter = cast<EHFilterScope>(*begin());
-  StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched filter push/pop");
-  CatchDepth--;
-}
-
-EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
-  char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
-  CatchDepth++;
-  EHCatchScope *Scope = new (Buffer) EHCatchScope(NumHandlers);
-  for (unsigned I = 0; I != NumHandlers; ++I)
-    Scope->getHandlers()[I].Index = getNextEHDestIndex();
-  return Scope;
-}
-
-void EHScopeStack::pushTerminate() {
-  char *Buffer = allocate(EHTerminateScope::getSize());
-  CatchDepth++;
-  new (Buffer) EHTerminateScope(getNextEHDestIndex());
-}
-
-/// Remove any 'null' fixups on the stack.  However, we can't pop more
-/// fixups than the fixup depth on the innermost normal cleanup, or
-/// else fixups that we try to add to that cleanup will end up in the
-/// wrong place.  We *could* try to shrink fixup depths, but that's
-/// actually a lot of work for little benefit.
-void EHScopeStack::popNullFixups() {
-  // We expect this to only be called when there's still an innermost
-  // normal cleanup;  otherwise there really shouldn't be any fixups.
-  assert(hasNormalCleanups());
-
-  EHScopeStack::iterator it = find(InnermostNormalCleanup);
-  unsigned MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
-  assert(BranchFixups.size() >= MinSize && "fixup stack out of order");
-
-  while (BranchFixups.size() > MinSize &&
-         BranchFixups.back().Destination == 0)
-    BranchFixups.pop_back();
-}
-
 static llvm::Constant *getAllocateExceptionFn(CodeGenFunction &CGF) {
   // void *__cxa_allocate_exception(size_t thrown_size);
   const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
@@ -287,7 +142,7 @@ static llvm::Constant *getTerminateFn(CodeGenFunction &CGF) {
 }
 
 static llvm::Constant *getCatchallRethrowFn(CodeGenFunction &CGF,
-                                            const char *Name) {
+                                            llvm::StringRef Name) {
   const llvm::Type *Int8PtrTy =
     llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   std::vector<const llvm::Type*> Args(1, Int8PtrTy);
@@ -299,6 +154,7 @@ static llvm::Constant *getCatchallRethrowFn(CodeGenFunction &CGF,
 }
 
 const EHPersonality EHPersonality::GNU_C("__gcc_personality_v0");
+const EHPersonality EHPersonality::GNU_C_SJLJ("__gcc_personality_sj0");
 const EHPersonality EHPersonality::NeXT_ObjC("__objc_personality_v0");
 const EHPersonality EHPersonality::GNU_CPlusPlus("__gxx_personality_v0");
 const EHPersonality EHPersonality::GNU_CPlusPlus_SJLJ("__gxx_personality_sj0");
@@ -306,6 +162,8 @@ const EHPersonality EHPersonality::GNU_ObjC("__gnu_objc_personality_v0",
                                             "objc_exception_throw");
 
 static const EHPersonality &getCPersonality(const LangOptions &L) {
+  if (L.SjLjExceptions)
+    return EHPersonality::GNU_C_SJLJ;
   return EHPersonality::GNU_C;
 }
 
@@ -357,24 +215,102 @@ const EHPersonality &EHPersonality::get(const LangOptions &L) {
     return getCPersonality(L);
 }
 
-static llvm::Constant *getPersonalityFn(CodeGenFunction &CGF,
+static llvm::Constant *getPersonalityFn(CodeGenModule &CGM,
                                         const EHPersonality &Personality) {
-  const char *Name = Personality.getPersonalityFnName();
-
   llvm::Constant *Fn =
-    CGF.CGM.CreateRuntimeFunction(llvm::FunctionType::get(
-                                    llvm::Type::getInt32Ty(
-                                      CGF.CGM.getLLVMContext()),
-                                    true),
-                            Name);
-  return llvm::ConstantExpr::getBitCast(Fn, CGF.CGM.PtrToInt8Ty);
+    CGM.CreateRuntimeFunction(llvm::FunctionType::get(
+                                llvm::Type::getInt32Ty(CGM.getLLVMContext()),
+                                true),
+                              Personality.getPersonalityFnName());
+  return Fn;
+}
+
+static llvm::Constant *getOpaquePersonalityFn(CodeGenModule &CGM,
+                                        const EHPersonality &Personality) {
+  llvm::Constant *Fn = getPersonalityFn(CGM, Personality);
+  return llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
+}
+
+/// Check whether a personality function could reasonably be swapped
+/// for a C++ personality function.
+static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
+  for (llvm::Constant::use_iterator
+         I = Fn->use_begin(), E = Fn->use_end(); I != E; ++I) {
+    llvm::User *User = *I;
+
+    // Conditionally white-list bitcasts.
+    if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(User)) {
+      if (CE->getOpcode() != llvm::Instruction::BitCast) return false;
+      if (!PersonalityHasOnlyCXXUses(CE))
+        return false;
+      continue;
+    }
+
+    // Otherwise, it has to be a selector call.
+    if (!isa<llvm::EHSelectorInst>(User)) return false;
+
+    llvm::EHSelectorInst *Selector = cast<llvm::EHSelectorInst>(User);
+    for (unsigned I = 2, E = Selector->getNumArgOperands(); I != E; ++I) {
+      // Look for something that would've been returned by the ObjC
+      // runtime's GetEHType() method.
+      llvm::GlobalVariable *GV
+        = dyn_cast<llvm::GlobalVariable>(Selector->getArgOperand(I));
+      if (!GV) continue;
+
+      // ObjC EH selector entries are always global variables with
+      // names starting like this.
+      if (GV->getName().startswith("OBJC_EHTYPE"))
+        return false;
+    }
+  }
+
+  return true;
+}
+
+/// Try to use the C++ personality function in ObjC++.  Not doing this
+/// can cause some incompatibilities with gcc, which is more
+/// aggressive about only using the ObjC++ personality in a function
+/// when it really needs it.
+void CodeGenModule::SimplifyPersonality() {
+  // For now, this is really a Darwin-specific operation.
+  if (Context.Target.getTriple().getOS() != llvm::Triple::Darwin)
+    return;
+
+  // If we're not in ObjC++ -fexceptions, there's nothing to do.
+  if (!Features.CPlusPlus || !Features.ObjC1 || !Features.Exceptions)
+    return;
+
+  const EHPersonality &ObjCXX = EHPersonality::get(Features);
+  const EHPersonality &CXX = getCXXPersonality(Features);
+  if (&ObjCXX == &CXX ||
+      ObjCXX.getPersonalityFnName() == CXX.getPersonalityFnName())
+    return;
+
+  llvm::Function *Fn =
+    getModule().getFunction(ObjCXX.getPersonalityFnName());
+
+  // Nothing to do if it's unused.
+  if (!Fn || Fn->use_empty()) return;
+  
+  // Can't do the optimization if it has non-C++ uses.
+  if (!PersonalityHasOnlyCXXUses(Fn)) return;
+
+  // Create the C++ personality function and kill off the old
+  // function.
+  llvm::Constant *CXXFn = getPersonalityFn(*this, CXX);
+
+  // This can happen if the user is screwing with us.
+  if (Fn->getType() != CXXFn->getType()) return;
+
+  Fn->replaceAllUsesWith(CXXFn);
+  Fn->eraseFromParent();
 }
 
 /// Returns the value to inject into a selector to indicate the
 /// presence of a catch-all.
 static llvm::Constant *getCatchAllValue(CodeGenFunction &CGF) {
   // Possibly we should use @llvm.eh.catch.all.value here.
-  return llvm::ConstantPointerNull::get(CGF.CGM.PtrToInt8Ty);
+  return llvm::ConstantPointerNull::get(CGF.Int8PtrTy);
 }
 
 /// Returns the value to inject into a selector to indicate the
@@ -386,26 +322,11 @@ static llvm::Constant *getCleanupValue(CodeGenFunction &CGF) {
 namespace {
   /// A cleanup to free the exception object if its initialization
   /// throws.
-  struct FreeExceptionCleanup : EHScopeStack::Cleanup {
-    FreeExceptionCleanup(llvm::Value *ShouldFreeVar,
-                         llvm::Value *ExnLocVar)
-      : ShouldFreeVar(ShouldFreeVar), ExnLocVar(ExnLocVar) {}
-
-    llvm::Value *ShouldFreeVar;
-    llvm::Value *ExnLocVar;
-    
-    void Emit(CodeGenFunction &CGF, bool IsForEH) {
-      llvm::BasicBlock *FreeBB = CGF.createBasicBlock("free-exnobj");
-      llvm::BasicBlock *DoneBB = CGF.createBasicBlock("free-exnobj.done");
-
-      llvm::Value *ShouldFree = CGF.Builder.CreateLoad(ShouldFreeVar,
-                                                       "should-free-exnobj");
-      CGF.Builder.CreateCondBr(ShouldFree, FreeBB, DoneBB);
-      CGF.EmitBlock(FreeBB);
-      llvm::Value *ExnLocLocal = CGF.Builder.CreateLoad(ExnLocVar, "exnobj");
-      CGF.Builder.CreateCall(getFreeExceptionFn(CGF), ExnLocLocal)
+  struct FreeException {
+    static void Emit(CodeGenFunction &CGF, bool forEH,
+                     llvm::Value *exn) {
+      CGF.Builder.CreateCall(getFreeExceptionFn(CGF), exn)
         ->setDoesNotThrow();
-      CGF.EmitBlock(DoneBB);
     }
   };
 }
@@ -414,41 +335,17 @@ namespace {
 // differs from EmitAnyExprToMem only in that, if a final copy-ctor
 // call is required, an exception within that copy ctor causes
 // std::terminate to be invoked.
-static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E, 
-                             llvm::Value *ExnLoc) {
-  // We want to release the allocated exception object if this
-  // expression throws.  We do this by pushing an EH-only cleanup
-  // block which, furthermore, deactivates itself after the expression
-  // is complete.
-  llvm::AllocaInst *ShouldFreeVar =
-    CGF.CreateTempAlloca(llvm::Type::getInt1Ty(CGF.getLLVMContext()),
-                         "should-free-exnobj.var");
-  CGF.InitTempAlloca(ShouldFreeVar,
-                     llvm::ConstantInt::getFalse(CGF.getLLVMContext()));
-
-  // A variable holding the exception pointer.  This is necessary
-  // because the throw expression does not necessarily dominate the
-  // cleanup, for example if it appears in a conditional expression.
-  llvm::AllocaInst *ExnLocVar =
-    CGF.CreateTempAlloca(ExnLoc->getType(), "exnobj.var");
-
+static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e,
+                             llvm::Value *addr) {
   // Make sure the exception object is cleaned up if there's an
   // exception during initialization.
-  // FIXME: stmt expressions might require this to be a normal
-  // cleanup, too.
-  CGF.EHStack.pushCleanup<FreeExceptionCleanup>(EHCleanup,
-                                                ShouldFreeVar,
-                                                ExnLocVar);
-  EHScopeStack::stable_iterator Cleanup = CGF.EHStack.stable_begin();
-
-  CGF.Builder.CreateStore(ExnLoc, ExnLocVar);
-  CGF.Builder.CreateStore(llvm::ConstantInt::getTrue(CGF.getLLVMContext()),
-                          ShouldFreeVar);
+  CGF.pushFullExprCleanup<FreeException>(EHCleanup, addr);
+  EHScopeStack::stable_iterator cleanup = CGF.EHStack.stable_begin();
 
   // __cxa_allocate_exception returns a void*;  we need to cast this
   // to the appropriate type for the object.
-  const llvm::Type *Ty = CGF.ConvertType(E->getType())->getPointerTo();
-  llvm::Value *TypedExnLoc = CGF.Builder.CreateBitCast(ExnLoc, Ty);
+  const llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo();
+  llvm::Value *typedAddr = CGF.Builder.CreateBitCast(addr, ty);
 
   // FIXME: this isn't quite right!  If there's a final unelided call
   // to a copy constructor, then according to [except.terminate]p1 we
@@ -457,22 +354,10 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
   // evaluated but before the exception is caught.  But the best way
   // to handle that is to teach EmitAggExpr to do the final copy
   // differently if it can't be elided.
-  CGF.EmitAnyExprToMem(E, TypedExnLoc, /*Volatile*/ false);
-
-  CGF.Builder.CreateStore(llvm::ConstantInt::getFalse(CGF.getLLVMContext()),
-                          ShouldFreeVar);
+  CGF.EmitAnyExprToMem(e, typedAddr, /*Volatile*/ false, /*IsInit*/ true);
 
-  // Technically, the exception object is like a temporary; it has to
-  // be cleaned up when its full-expression is complete.
-  // Unfortunately, the AST represents full-expressions by creating a
-  // CXXExprWithTemporaries, which it only does when there are actually
-  // temporaries.
-  //
-  // If any cleanups have been added since we pushed ours, they must
-  // be from temporaries;  this will get popped at the same time.
-  // Otherwise we need to pop ours off.  FIXME: this is very brittle.
-  if (Cleanup == CGF.EHStack.stable_begin())
-    CGF.PopCleanupBlock();
+  // Deactivate the cleanup block.
+  CGF.DeactivateCleanupBlock(cleanup);
 }
 
 llvm::Value *CodeGenFunction::getExceptionSlot() {
@@ -495,8 +380,10 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
       Builder.CreateUnreachable();
     }
 
-    // Clear the insertion point to indicate we are in unreachable code.
-    Builder.ClearInsertionPoint();
+    // throw is an expression, and the expression emitters expect us
+    // to leave ourselves at a valid insertion point.
+    EmitBlock(createBasicBlock("throw.cont"));
+
     return;
   }
 
@@ -517,7 +404,8 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
 
   // Now throw the exception.
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(getLLVMContext());
-  llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, true);
+  llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, 
+                                                         /*ForEH=*/true);
 
   // The address of the destructor.  If the exception type has a
   // trivial destructor (or isn't a record), we just pass null.
@@ -545,16 +433,13 @@ void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E) {
     Builder.CreateUnreachable();
   }
 
-  // Clear the insertion point to indicate we are in unreachable code.
-  Builder.ClearInsertionPoint();
-
-  // FIXME: For now, emit a dummy basic block because expr emitters in generally
-  // are not ready to handle emitting expressions at unreachable points.
-  EnsureInsertPoint();
+  // throw is an expression, and the expression emitters expect us
+  // to leave ourselves at a valid insertion point.
+  EmitBlock(createBasicBlock("throw.cont"));
 }
 
 void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
-  if (!Exceptions)
+  if (!CGM.getLangOptions().areExceptionsEnabled())
     return;
   
   const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(D);
@@ -575,13 +460,14 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) {
   for (unsigned I = 0; I != NumExceptions; ++I) {
     QualType Ty = Proto->getExceptionType(I);
     QualType ExceptType = Ty.getNonReferenceType().getUnqualifiedType();
-    llvm::Value *EHType = CGM.GetAddrOfRTTIDescriptor(ExceptType, true);
+    llvm::Value *EHType = CGM.GetAddrOfRTTIDescriptor(ExceptType,
+                                                      /*ForEH=*/true);
     Filter->setFilter(I, EHType);
   }
 }
 
 void CodeGenFunction::EmitEndEHSpec(const Decl *D) {
-  if (!Exceptions)
+  if (!CGM.getLangOptions().areExceptionsEnabled())
     return;
   
   const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(D);
@@ -625,7 +511,7 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
       if (CaughtType->isObjCObjectPointerType())
         TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType);
       else
-        TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, true);
+        TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, /*ForEH=*/true);
       CatchScope->setHandler(I, TypeInfo, Handler);
     } else {
       // No exception decl indicates '...', a catch-all.
@@ -655,7 +541,7 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
   assert(EHStack.requiresLandingPad());
   assert(!EHStack.empty());
 
-  if (!Exceptions)
+  if (!CGM.getLangOptions().areExceptionsEnabled())
     return 0;
 
   // Check the innermost scope for a cached landing pad.  If this is
@@ -739,8 +625,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   // Save the current IR generation state.
   CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
 
-  const EHPersonality &Personality =
-    EHPersonality::get(CGF.CGM.getLangOptions());
+  const EHPersonality &Personality = EHPersonality::get(getLangOptions());
 
   // Create and configure the landing pad.
   llvm::BasicBlock *LP = createBasicBlock("lpad");
@@ -757,7 +642,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   // Build the selector arguments.
   llvm::SmallVector<llvm::Value*, 8> EHSelector;
   EHSelector.push_back(Exn);
-  EHSelector.push_back(getPersonalityFn(*this, Personality));
+  EHSelector.push_back(getOpaquePersonalityFn(CGM, Personality));
 
   // Accumulate all the handlers in scope.
   llvm::DenseMap<llvm::Value*, UnwindDest> EHHandlers;
@@ -837,7 +722,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
   // If we have a catch-all, add null to the selector.
   if (CatchAll.isValid()) {
-    EHSelector.push_back(getCatchAllValue(CGF));
+    EHSelector.push_back(getCatchAllValue(*this));
 
   // If we have an EH filter, we need to add those handlers in the
   // right place in the selector, which is to say, at the end.
@@ -853,14 +738,14 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
     // Also check whether we need a cleanup.
     if (UseInvokeInlineHack || HasEHCleanup)
       EHSelector.push_back(UseInvokeInlineHack
-                           ? getCatchAllValue(CGF)
-                           : getCleanupValue(CGF));
+                           ? getCatchAllValue(*this)
+                           : getCleanupValue(*this));
 
   // Otherwise, signal that we at least have cleanups.
   } else if (UseInvokeInlineHack || HasEHCleanup) {
     EHSelector.push_back(UseInvokeInlineHack
-                         ? getCatchAllValue(CGF)
-                         : getCleanupValue(CGF));
+                         ? getCatchAllValue(*this)
+                         : getCleanupValue(*this));
   } else {
     assert(LastToEmitInLoop > 2);
     LastToEmitInLoop--;
@@ -902,7 +787,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
     // Check whether the exception matches.
     llvm::CallInst *Id
       = Builder.CreateCall(llvm_eh_typeid_for,
-                           Builder.CreateBitCast(Type, CGM.PtrToInt8Ty));
+                           Builder.CreateBitCast(Type, Int8PtrTy));
     Id->setDoesNotThrow();
     Builder.CreateCondBr(Builder.CreateICmpEQ(Selection, Id),
                          Match, Next);
@@ -1141,55 +1026,54 @@ static void InitCatchParam(CodeGenFunction &CGF,
     return;
   }
 
-  // FIXME: this *really* needs to be done via a proper, Sema-emitted
-  // initializer expression.
-
-  CXXRecordDecl *RD = CatchType.getTypePtr()->getAsCXXRecordDecl();
-  assert(RD && "aggregate catch type was not a record!");
+  assert(isa<RecordType>(CatchType) && "unexpected catch type!");
 
   const llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok
 
-  if (RD->hasTrivialCopyConstructor()) {
-    llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, true);
-    llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
-    CGF.EmitAggregateCopy(ParamAddr, Cast, CatchType);
+  // Check for a copy expression.  If we don't have a copy expression,
+  // that means a trivial copy is okay.
+  const Expr *copyExpr = CatchParam.getInit();
+  if (!copyExpr) {
+    llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true);
+    llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
+    CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType);
     return;
   }
 
   // We have to call __cxa_get_exception_ptr to get the adjusted
   // pointer before copying.
-  llvm::CallInst *AdjustedExn =
+  llvm::CallInst *rawAdjustedExn =
     CGF.Builder.CreateCall(getGetExceptionPtrFn(CGF), Exn);
-  AdjustedExn->setDoesNotThrow();
-  llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
+  rawAdjustedExn->setDoesNotThrow();
 
-  CXXConstructorDecl *CD = RD->getCopyConstructor(CGF.getContext(), 0);
-  assert(CD && "record has no copy constructor!");
-  llvm::Value *CopyCtor = CGF.CGM.GetAddrOfCXXConstructor(CD, Ctor_Complete);
+  // Cast that to the appropriate type.
+  llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy);
 
-  CallArgList CallArgs;
-  CallArgs.push_back(std::make_pair(RValue::get(ParamAddr),
-                                    CD->getThisType(CGF.getContext())));
-  CallArgs.push_back(std::make_pair(RValue::get(Cast),
-                                    CD->getParamDecl(0)->getType()));
-
-  const FunctionProtoType *FPT
-    = CD->getType()->getAs<FunctionProtoType>();
+  // The copy expression is defined in terms of an OpaqueValueExpr.
+  // Find it and map it to the adjusted expression.
+  CodeGenFunction::OpaqueValueMapping
+    opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr),
+           CGF.MakeAddrLValue(adjustedExn, CatchParam.getType()));
 
   // Call the copy ctor in a terminate scope.
   CGF.EHStack.pushTerminate();
-  CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(CallArgs, FPT),
-               CopyCtor, ReturnValueSlot(), CallArgs, CD);
+
+  // Perform the copy construction.
+  CGF.EmitAggExpr(copyExpr, AggValueSlot::forAddr(ParamAddr, false, false));
+
+  // Leave the terminate scope.
   CGF.EHStack.popTerminate();
 
+  // Undo the opaque value mapping.
+  opaque.pop();
+
   // Finally we can call __cxa_begin_catch.
   CallBeginCatch(CGF, Exn, true);
 }
 
 /// Begins a catch statement by initializing the catch variable and
 /// calling __cxa_begin_catch.
-static void BeginCatch(CodeGenFunction &CGF,
-                       const CXXCatchStmt *S) {
+static void BeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *S) {
   // We have to be very careful with the ordering of cleanups here:
   //   C++ [except.throw]p4:
   //     The destruction [of the exception temporary] occurs
@@ -1221,7 +1105,7 @@ static void BeginCatch(CodeGenFunction &CGF,
   }
 
   // Emit the local.
-  CGF.EmitLocalBlockVarDecl(*CatchParam, &InitCatchParam);
+  CGF.EmitAutoVarDecl(*CatchParam, &InitCatchParam);
 }
 
 namespace {
@@ -1428,7 +1312,7 @@ CodeGenFunction::EnterFinallyBlock(const Stmt *Body,
   JumpDest RethrowDest = getJumpDestInCurrentScope(getUnreachableBlock());
 
   // Whether the finally block is being executed for EH purposes.
-  llvm::AllocaInst *ForEHVar = CreateTempAlloca(CGF.Builder.getInt1Ty(),
+  llvm::AllocaInst *ForEHVar = CreateTempAlloca(Builder.getInt1Ty(),
                                                 "finally.for-eh");
   InitTempAlloca(ForEHVar, llvm::ConstantInt::getFalse(getLLVMContext()));
 
@@ -1502,7 +1386,7 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
   
   // Tell the backend what the exception table should be:
   // nothing but a catch-all.
-  llvm::Value *Args[3] = { Exn, getPersonalityFn(*this, Personality),
+  llvm::Value *Args[3] = { Exn, getOpaquePersonalityFn(CGM, Personality),
                            getCatchAllValue(*this) };
   Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
                      Args, Args+3, "eh.selector")
@@ -1511,7 +1395,7 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
   llvm::CallInst *TerminateCall = Builder.CreateCall(getTerminateFn(*this));
   TerminateCall->setDoesNotReturn();
   TerminateCall->setDoesNotThrow();
-  CGF.Builder.CreateUnreachable();
+  Builder.CreateUnreachable();
 
   // Restore the saved insertion state.
   Builder.restoreIP(SavedIP);
@@ -1553,8 +1437,9 @@ CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
 
   // This can always be a call because we necessarily didn't find
   // anything on the EH stack which needs our help.
+  llvm::StringRef RethrowName = Personality.getCatchallRethrowFnName();
   llvm::Constant *RethrowFn;
-  if (const char *RethrowName = Personality.getCatchallRethrowFnName())
+  if (!RethrowName.empty())
     RethrowFn = getCatchallRethrowFn(*this, RethrowName);
   else
     RethrowFn = getUnwindResumeOrRethrowFn();
@@ -1569,6 +1454,3 @@ CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
   return RethrowBlock;
 }
 
-EHScopeStack::Cleanup::~Cleanup() {
-  llvm_unreachable("Cleanup is indestructable");
-}
diff --git a/lib/CodeGen/CGException.h b/lib/CodeGen/CGException.h
index f1294746a42d..1f9b8964dcae 100644
--- a/lib/CodeGen/CGException.h
+++ b/lib/CodeGen/CGException.h
@@ -15,578 +15,40 @@
 #ifndef CLANG_CODEGEN_CGEXCEPTION_H
 #define CLANG_CODEGEN_CGEXCEPTION_H
 
-/// EHScopeStack is defined in CodeGenFunction.h, but its
-/// implementation is in this file and in CGException.cpp.
-#include "CodeGenFunction.h"
-
-namespace llvm {
-  class Value;
-  class BasicBlock;
-}
+#include "llvm/ADT/StringRef.h"
 
 namespace clang {
+class LangOptions;
+
 namespace CodeGen {
 
 /// The exceptions personality for a function.  When 
 class EHPersonality {
-  const char *PersonalityFn;
+  llvm::StringRef PersonalityFn;
 
   // If this is non-null, this personality requires a non-standard
   // function for rethrowing an exception after a catchall cleanup.
   // This function must have prototype void(void*).
-  const char *CatchallRethrowFn;
+  llvm::StringRef CatchallRethrowFn;
 
-  EHPersonality(const char *PersonalityFn,
-                const char *CatchallRethrowFn = 0)
+  EHPersonality(llvm::StringRef PersonalityFn,
+                llvm::StringRef CatchallRethrowFn = llvm::StringRef())
     : PersonalityFn(PersonalityFn),
       CatchallRethrowFn(CatchallRethrowFn) {}
 
 public:
   static const EHPersonality &get(const LangOptions &Lang);
   static const EHPersonality GNU_C;
+  static const EHPersonality GNU_C_SJLJ;
   static const EHPersonality GNU_ObjC;
   static const EHPersonality NeXT_ObjC;
   static const EHPersonality GNU_CPlusPlus;
   static const EHPersonality GNU_CPlusPlus_SJLJ;
 
-  const char *getPersonalityFnName() const { return PersonalityFn; }
-  const char *getCatchallRethrowFnName() const { return CatchallRethrowFn; }
-};
-
-/// A protected scope for zero-cost EH handling.
-class EHScope {
-  llvm::BasicBlock *CachedLandingPad;
-
-  unsigned K : 2;
-
-protected:
-  enum { BitsRemaining = 30 };
-
-public:
-  enum Kind { Cleanup, Catch, Terminate, Filter };
-
-  EHScope(Kind K) : CachedLandingPad(0), K(K) {}
-
-  Kind getKind() const { return static_cast<Kind>(K); }
-
-  llvm::BasicBlock *getCachedLandingPad() const {
-    return CachedLandingPad;
-  }
-
-  void setCachedLandingPad(llvm::BasicBlock *Block) {
-    CachedLandingPad = Block;
-  }
-};
-
-/// A scope which attempts to handle some, possibly all, types of
-/// exceptions.
-///
-/// Objective C @finally blocks are represented using a cleanup scope
-/// after the catch scope.
-class EHCatchScope : public EHScope {
-  unsigned NumHandlers : BitsRemaining;
-
-  // In effect, we have a flexible array member
-  //   Handler Handlers[0];
-  // But that's only standard in C99, not C++, so we have to do
-  // annoying pointer arithmetic instead.
-
-public:
-  struct Handler {
-    /// A type info value, or null (C++ null, not an LLVM null pointer)
-    /// for a catch-all.
-    llvm::Value *Type;
-
-    /// The catch handler for this type.
-    llvm::BasicBlock *Block;
-
-    /// The unwind destination index for this handler.
-    unsigned Index;
-  };
-
-private:
-  friend class EHScopeStack;
-
-  Handler *getHandlers() {
-    return reinterpret_cast<Handler*>(this+1);
-  }
-
-  const Handler *getHandlers() const {
-    return reinterpret_cast<const Handler*>(this+1);
-  }
-
-public:
-  static size_t getSizeForNumHandlers(unsigned N) {
-    return sizeof(EHCatchScope) + N * sizeof(Handler);
-  }
-
-  EHCatchScope(unsigned NumHandlers)
-    : EHScope(Catch), NumHandlers(NumHandlers) {
-  }
-
-  unsigned getNumHandlers() const {
-    return NumHandlers;
-  }
-
-  void setCatchAllHandler(unsigned I, llvm::BasicBlock *Block) {
-    setHandler(I, /*catchall*/ 0, Block);
-  }
-
-  void setHandler(unsigned I, llvm::Value *Type, llvm::BasicBlock *Block) {
-    assert(I < getNumHandlers());
-    getHandlers()[I].Type = Type;
-    getHandlers()[I].Block = Block;
-  }
-
-  const Handler &getHandler(unsigned I) const {
-    assert(I < getNumHandlers());
-    return getHandlers()[I];
-  }
-
-  typedef const Handler *iterator;
-  iterator begin() const { return getHandlers(); }
-  iterator end() const { return getHandlers() + getNumHandlers(); }
-
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Catch;
-  }
-};
-
-/// A cleanup scope which generates the cleanup blocks lazily.
-class EHCleanupScope : public EHScope {
-  /// Whether this cleanup needs to be run along normal edges.
-  bool IsNormalCleanup : 1;
-
-  /// Whether this cleanup needs to be run along exception edges.
-  bool IsEHCleanup : 1;
-
-  /// Whether this cleanup was activated before all normal uses.
-  bool ActivatedBeforeNormalUse : 1;
-
-  /// Whether this cleanup was activated before all EH uses.
-  bool ActivatedBeforeEHUse : 1;
-
-  /// The amount of extra storage needed by the Cleanup.
-  /// Always a multiple of the scope-stack alignment.
-  unsigned CleanupSize : 12;
-
-  /// The number of fixups required by enclosing scopes (not including
-  /// this one).  If this is the top cleanup scope, all the fixups
-  /// from this index onwards belong to this scope.
-  unsigned FixupDepth : BitsRemaining - 16;
-
-  /// The nearest normal cleanup scope enclosing this one.
-  EHScopeStack::stable_iterator EnclosingNormal;
-
-  /// The nearest EH cleanup scope enclosing this one.
-  EHScopeStack::stable_iterator EnclosingEH;
-
-  /// The dual entry/exit block along the normal edge.  This is lazily
-  /// created if needed before the cleanup is popped.
-  llvm::BasicBlock *NormalBlock;
-
-  /// The dual entry/exit block along the EH edge.  This is lazily
-  /// created if needed before the cleanup is popped.
-  llvm::BasicBlock *EHBlock;
-
-  /// An optional i1 variable indicating whether this cleanup has been
-  /// activated yet.  This has one of three states:
-  ///   - it is null if the cleanup is inactive
-  ///   - it is activeSentinel() if the cleanup is active and was not
-  ///     required before activation
-  ///   - it points to a valid variable
-  llvm::AllocaInst *ActiveVar;
-
-  /// Extra information required for cleanups that have resolved
-  /// branches through them.  This has to be allocated on the side
-  /// because everything on the cleanup stack has be trivially
-  /// movable.
-  struct ExtInfo {
-    /// The destinations of normal branch-afters and branch-throughs.
-    llvm::SmallPtrSet<llvm::BasicBlock*, 4> Branches;
-
-    /// Normal branch-afters.
-    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
-      BranchAfters;
-
-    /// The destinations of EH branch-afters and branch-throughs.
-    /// TODO: optimize for the extremely common case of a single
-    /// branch-through.
-    llvm::SmallPtrSet<llvm::BasicBlock*, 4> EHBranches;
-
-    /// EH branch-afters.
-    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
-    EHBranchAfters;
-  };
-  mutable struct ExtInfo *ExtInfo;
-
-  struct ExtInfo &getExtInfo() {
-    if (!ExtInfo) ExtInfo = new struct ExtInfo();
-    return *ExtInfo;
-  }
-
-  const struct ExtInfo &getExtInfo() const {
-    if (!ExtInfo) ExtInfo = new struct ExtInfo();
-    return *ExtInfo;
-  }
-
-public:
-  /// Gets the size required for a lazy cleanup scope with the given
-  /// cleanup-data requirements.
-  static size_t getSizeForCleanupSize(size_t Size) {
-    return sizeof(EHCleanupScope) + Size;
-  }
-
-  size_t getAllocatedSize() const {
-    return sizeof(EHCleanupScope) + CleanupSize;
-  }
-
-  EHCleanupScope(bool IsNormal, bool IsEH, bool IsActive,
-                 unsigned CleanupSize, unsigned FixupDepth,
-                 EHScopeStack::stable_iterator EnclosingNormal,
-                 EHScopeStack::stable_iterator EnclosingEH)
-    : EHScope(EHScope::Cleanup),
-      IsNormalCleanup(IsNormal), IsEHCleanup(IsEH),
-      ActivatedBeforeNormalUse(IsActive),
-      ActivatedBeforeEHUse(IsActive),
-      CleanupSize(CleanupSize), FixupDepth(FixupDepth),
-      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
-      NormalBlock(0), EHBlock(0),
-      ActiveVar(IsActive ? activeSentinel() : 0),
-      ExtInfo(0)
-  {
-    assert(this->CleanupSize == CleanupSize && "cleanup size overflow");
-  }
-
-  ~EHCleanupScope() {
-    delete ExtInfo;
-  }
-
-  bool isNormalCleanup() const { return IsNormalCleanup; }
-  llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
-  void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; }
-
-  bool isEHCleanup() const { return IsEHCleanup; }
-  llvm::BasicBlock *getEHBlock() const { return EHBlock; }
-  void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
-
-  static llvm::AllocaInst *activeSentinel() {
-    return reinterpret_cast<llvm::AllocaInst*>(1);
-  }
-
-  bool isActive() const { return ActiveVar != 0; }
-  llvm::AllocaInst *getActiveVar() const { return ActiveVar; }
-  void setActiveVar(llvm::AllocaInst *Var) { ActiveVar = Var; }
-
-  bool wasActivatedBeforeNormalUse() const { return ActivatedBeforeNormalUse; }
-  void setActivatedBeforeNormalUse(bool B) { ActivatedBeforeNormalUse = B; }
-
-  bool wasActivatedBeforeEHUse() const { return ActivatedBeforeEHUse; }
-  void setActivatedBeforeEHUse(bool B) { ActivatedBeforeEHUse = B; }
-
-  unsigned getFixupDepth() const { return FixupDepth; }
-  EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
-    return EnclosingNormal;
-  }
-  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
-    return EnclosingEH;
-  }
-
-  size_t getCleanupSize() const { return CleanupSize; }
-  void *getCleanupBuffer() { return this + 1; }
-
-  EHScopeStack::Cleanup *getCleanup() {
-    return reinterpret_cast<EHScopeStack::Cleanup*>(getCleanupBuffer());
-  }
-
-  /// True if this cleanup scope has any branch-afters or branch-throughs.
-  bool hasBranches() const { return ExtInfo && !ExtInfo->Branches.empty(); }
-
-  /// Add a branch-after to this cleanup scope.  A branch-after is a
-  /// branch from a point protected by this (normal) cleanup to a
-  /// point in the normal cleanup scope immediately containing it.
-  /// For example,
-  ///   for (;;) { A a; break; }
-  /// contains a branch-after.
-  ///
-  /// Branch-afters each have their own destination out of the
-  /// cleanup, guaranteed distinct from anything else threaded through
-  /// it.  Therefore branch-afters usually force a switch after the
-  /// cleanup.
-  void addBranchAfter(llvm::ConstantInt *Index,
-                      llvm::BasicBlock *Block) {
-    struct ExtInfo &ExtInfo = getExtInfo();
-    if (ExtInfo.Branches.insert(Block))
-      ExtInfo.BranchAfters.push_back(std::make_pair(Block, Index));
-  }
-
-  /// Return the number of unique branch-afters on this scope.
-  unsigned getNumBranchAfters() const {
-    return ExtInfo ? ExtInfo->BranchAfters.size() : 0;
-  }
-
-  llvm::BasicBlock *getBranchAfterBlock(unsigned I) const {
-    assert(I < getNumBranchAfters());
-    return ExtInfo->BranchAfters[I].first;
-  }
-
-  llvm::ConstantInt *getBranchAfterIndex(unsigned I) const {
-    assert(I < getNumBranchAfters());
-    return ExtInfo->BranchAfters[I].second;
-  }
-
-  /// Add a branch-through to this cleanup scope.  A branch-through is
-  /// a branch from a scope protected by this (normal) cleanup to an
-  /// enclosing scope other than the immediately-enclosing normal
-  /// cleanup scope.
-  ///
-  /// In the following example, the branch through B's scope is a
-  /// branch-through, while the branch through A's scope is a
-  /// branch-after:
-  ///   for (;;) { A a; B b; break; }
-  ///
-  /// All branch-throughs have a common destination out of the
-  /// cleanup, one possibly shared with the fall-through.  Therefore
-  /// branch-throughs usually don't force a switch after the cleanup.
-  ///
-  /// \return true if the branch-through was new to this scope
-  bool addBranchThrough(llvm::BasicBlock *Block) {
-    return getExtInfo().Branches.insert(Block);
-  }
-
-  /// Determines if this cleanup scope has any branch throughs.
-  bool hasBranchThroughs() const {
-    if (!ExtInfo) return false;
-    return (ExtInfo->BranchAfters.size() != ExtInfo->Branches.size());
-  }
-
-  // Same stuff, only for EH branches instead of normal branches.
-  // It's quite possible that we could find a better representation
-  // for this.
-
-  bool hasEHBranches() const { return ExtInfo && !ExtInfo->EHBranches.empty(); }
-  void addEHBranchAfter(llvm::ConstantInt *Index,
-                        llvm::BasicBlock *Block) {
-    struct ExtInfo &ExtInfo = getExtInfo();
-    if (ExtInfo.EHBranches.insert(Block))
-      ExtInfo.EHBranchAfters.push_back(std::make_pair(Block, Index));
-  }
-
-  unsigned getNumEHBranchAfters() const {
-    return ExtInfo ? ExtInfo->EHBranchAfters.size() : 0;
-  }
-
-  llvm::BasicBlock *getEHBranchAfterBlock(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].first;
-  }
-
-  llvm::ConstantInt *getEHBranchAfterIndex(unsigned I) const {
-    assert(I < getNumEHBranchAfters());
-    return ExtInfo->EHBranchAfters[I].second;
-  }
-
-  bool addEHBranchThrough(llvm::BasicBlock *Block) {
-    return getExtInfo().EHBranches.insert(Block);
-  }
-
-  bool hasEHBranchThroughs() const {
-    if (!ExtInfo) return false;
-    return (ExtInfo->EHBranchAfters.size() != ExtInfo->EHBranches.size());
-  }
-
-  static bool classof(const EHScope *Scope) {
-    return (Scope->getKind() == Cleanup);
-  }
-};
-
-/// An exceptions scope which filters exceptions thrown through it.
-/// Only exceptions matching the filter types will be permitted to be
-/// thrown.
-///
-/// This is used to implement C++ exception specifications.
-class EHFilterScope : public EHScope {
-  unsigned NumFilters : BitsRemaining;
-
-  // Essentially ends in a flexible array member:
-  // llvm::Value *FilterTypes[0];
-
-  llvm::Value **getFilters() {
-    return reinterpret_cast<llvm::Value**>(this+1);
-  }
-
-  llvm::Value * const *getFilters() const {
-    return reinterpret_cast<llvm::Value* const *>(this+1);
-  }
-
-public:
-  EHFilterScope(unsigned NumFilters) :
-    EHScope(Filter), NumFilters(NumFilters) {}
-
-  static size_t getSizeForNumFilters(unsigned NumFilters) {
-    return sizeof(EHFilterScope) + NumFilters * sizeof(llvm::Value*);
-  }
-
-  unsigned getNumFilters() const { return NumFilters; }
-
-  void setFilter(unsigned I, llvm::Value *FilterValue) {
-    assert(I < getNumFilters());
-    getFilters()[I] = FilterValue;
-  }
-
-  llvm::Value *getFilter(unsigned I) const {
-    assert(I < getNumFilters());
-    return getFilters()[I];
-  }
-
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Filter;
-  }
+  llvm::StringRef getPersonalityFnName() const { return PersonalityFn; }
+  llvm::StringRef getCatchallRethrowFnName() const { return CatchallRethrowFn; }
 };
 
-/// An exceptions scope which calls std::terminate if any exception
-/// reaches it.
-class EHTerminateScope : public EHScope {
-  unsigned DestIndex : BitsRemaining;
-public:
-  EHTerminateScope(unsigned Index) : EHScope(Terminate), DestIndex(Index) {}
-  static size_t getSize() { return sizeof(EHTerminateScope); }
-
-  unsigned getDestIndex() const { return DestIndex; }
-
-  static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Terminate;
-  }
-};
-
-/// A non-stable pointer into the scope stack.
-class EHScopeStack::iterator {
-  char *Ptr;
-
-  friend class EHScopeStack;
-  explicit iterator(char *Ptr) : Ptr(Ptr) {}
-
-public:
-  iterator() : Ptr(0) {}
-
-  EHScope *get() const { 
-    return reinterpret_cast<EHScope*>(Ptr);
-  }
-
-  EHScope *operator->() const { return get(); }
-  EHScope &operator*() const { return *get(); }
-
-  iterator &operator++() {
-    switch (get()->getKind()) {
-    case EHScope::Catch:
-      Ptr += EHCatchScope::getSizeForNumHandlers(
-          static_cast<const EHCatchScope*>(get())->getNumHandlers());
-      break;
-
-    case EHScope::Filter:
-      Ptr += EHFilterScope::getSizeForNumFilters(
-          static_cast<const EHFilterScope*>(get())->getNumFilters());
-      break;
-
-    case EHScope::Cleanup:
-      Ptr += static_cast<const EHCleanupScope*>(get())
-        ->getAllocatedSize();
-      break;
-
-    case EHScope::Terminate:
-      Ptr += EHTerminateScope::getSize();
-      break;
-    }
-
-    return *this;
-  }
-
-  iterator next() {
-    iterator copy = *this;
-    ++copy;
-    return copy;
-  }
-
-  iterator operator++(int) {
-    iterator copy = *this;
-    operator++();
-    return copy;
-  }
-
-  bool encloses(iterator other) const { return Ptr >= other.Ptr; }
-  bool strictlyEncloses(iterator other) const { return Ptr > other.Ptr; }
-
-  bool operator==(iterator other) const { return Ptr == other.Ptr; }
-  bool operator!=(iterator other) const { return Ptr != other.Ptr; }
-};
-
-inline EHScopeStack::iterator EHScopeStack::begin() const {
-  return iterator(StartOfData);
-}
-
-inline EHScopeStack::iterator EHScopeStack::end() const {
-  return iterator(EndOfBuffer);
-}
-
-inline void EHScopeStack::popCatch() {
-  assert(!empty() && "popping exception stack when not empty");
-
-  assert(isa<EHCatchScope>(*begin()));
-  StartOfData += EHCatchScope::getSizeForNumHandlers(
-                          cast<EHCatchScope>(*begin()).getNumHandlers());
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
-}
-
-inline void EHScopeStack::popTerminate() {
-  assert(!empty() && "popping exception stack when not empty");
-
-  assert(isa<EHTerminateScope>(*begin()));
-  StartOfData += EHTerminateScope::getSize();
-
-  if (empty()) NextEHDestIndex = FirstEHDestIndex;
-
-  assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
-  CatchDepth--;
-}
-
-inline EHScopeStack::iterator EHScopeStack::find(stable_iterator sp) const {
-  assert(sp.isValid() && "finding invalid savepoint");
-  assert(sp.Size <= stable_begin().Size && "finding savepoint after pop");
-  return iterator(EndOfBuffer - sp.Size);
-}
-
-inline EHScopeStack::stable_iterator
-EHScopeStack::stabilize(iterator ir) const {
-  assert(StartOfData <= ir.Ptr && ir.Ptr <= EndOfBuffer);
-  return stable_iterator(EndOfBuffer - ir.Ptr);
-}
-
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveNormalCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostNormalCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingNormalCleanup();
-  }
-  return stable_end();
-}
-
-inline EHScopeStack::stable_iterator
-EHScopeStack::getInnermostActiveEHCleanup() const {
-  for (EHScopeStack::stable_iterator
-         I = getInnermostEHCleanup(), E = stable_end(); I != E; ) {
-    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
-    if (S.isActive()) return I;
-    I = S.getEnclosingEHCleanup();
-  }
-  return stable_end();
-}
-
 }
 }
 
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 3750ab80c3fc..1b7e7a007ed2 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -29,6 +29,18 @@ using namespace CodeGen;
 //                        Miscellaneous Helper Methods
 //===--------------------------------------------------------------------===//
 
+llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
+  unsigned addressSpace =
+    cast<llvm::PointerType>(value->getType())->getAddressSpace();
+
+  const llvm::PointerType *destType = Int8PtrTy;
+  if (addressSpace)
+    destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
+
+  if (value->getType() == destType) return value;
+  return Builder.CreateBitCast(value, destType);
+}
+
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block.
 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(const llvm::Type *Ty,
@@ -68,7 +80,7 @@ llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
     llvm::Value *MemPtr = EmitScalarExpr(E);
-    return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr, MPT);
+    return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
   }
 
   QualType BoolTy = getContext().BoolTy;
@@ -78,35 +90,40 @@ llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
   return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
 }
 
-/// EmitAnyExpr - Emit code to compute the specified expression which can have
-/// any type.  The result is returned as an RValue struct.  If this is an
-/// aggregate expression, the aggloc/agglocvolatile arguments indicate where the
+/// EmitIgnoredExpr - Emit code to compute the specified expression,
+/// ignoring the result.
+void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
+  if (E->isRValue())
+    return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
+
+  // Just emit it as an l-value and drop the result.
+  EmitLValue(E);
+}
+
+/// EmitAnyExpr - Emit code to compute the specified expression which
+/// can have any type.  The result is returned as an RValue struct.
+/// If this is an aggregate expression, AggSlot indicates where the
 /// result should be returned.
-RValue CodeGenFunction::EmitAnyExpr(const Expr *E, llvm::Value *AggLoc,
-                                    bool IsAggLocVolatile, bool IgnoreResult,
-                                    bool IsInitializer) {
+RValue CodeGenFunction::EmitAnyExpr(const Expr *E, AggValueSlot AggSlot,
+                                    bool IgnoreResult) {
   if (!hasAggregateLLVMType(E->getType()))
     return RValue::get(EmitScalarExpr(E, IgnoreResult));
   else if (E->getType()->isAnyComplexType())
-    return RValue::getComplex(EmitComplexExpr(E, false, false,
-                                              IgnoreResult, IgnoreResult));
+    return RValue::getComplex(EmitComplexExpr(E, IgnoreResult, IgnoreResult));
 
-  EmitAggExpr(E, AggLoc, IsAggLocVolatile, IgnoreResult, IsInitializer);
-  return RValue::getAggregate(AggLoc, IsAggLocVolatile);
+  EmitAggExpr(E, AggSlot, IgnoreResult);
+  return AggSlot.asRValue();
 }
 
 /// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
 /// always be accessible even if no aggregate location is provided.
-RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E,
-                                          bool IsAggLocVolatile,
-                                          bool IsInitializer) {
-  llvm::Value *AggLoc = 0;
+RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
+  AggValueSlot AggSlot = AggValueSlot::ignored();
 
   if (hasAggregateLLVMType(E->getType()) &&
       !E->getType()->isAnyComplexType())
-    AggLoc = CreateMemTemp(E->getType(), "agg.tmp");
-  return EmitAnyExpr(E, AggLoc, IsAggLocVolatile, /*IgnoreResult=*/false,
-                     IsInitializer);
+    AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
+  return EmitAnyExpr(E, AggSlot);
 }
 
 /// EmitAnyExprToMem - Evaluate an expression into a given memory
@@ -118,7 +135,7 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
   if (E->getType()->isComplexType())
     EmitComplexExprIntoAddr(E, Location, IsLocationVolatile);
   else if (hasAggregateLLVMType(E->getType()))
-    EmitAggExpr(E, Location, IsLocationVolatile, /*Ignore*/ false, IsInit);
+    EmitAggExpr(E, AggValueSlot::forAddr(Location, IsLocationVolatile, IsInit));
   else {
     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
     LValue LV = MakeAddrLValue(Location, E->getType());
@@ -126,34 +143,34 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
   }
 }
 
+namespace {
 /// \brief An adjustment to be made to the temporary created when emitting a
 /// reference binding, which accesses a particular subobject of that temporary.
-struct SubobjectAdjustment {
-  enum { DerivedToBaseAdjustment, FieldAdjustment } Kind;
-  
-  union {
-    struct {
-      const CastExpr *BasePath;
-      const CXXRecordDecl *DerivedClass;
-    } DerivedToBase;
-    
-    FieldDecl *Field;
+  struct SubobjectAdjustment {
+    enum { DerivedToBaseAdjustment, FieldAdjustment } Kind;
+
+    union {
+      struct {
+        const CastExpr *BasePath;
+        const CXXRecordDecl *DerivedClass;
+      } DerivedToBase;
+
+      FieldDecl *Field;
+    };
+
+    SubobjectAdjustment(const CastExpr *BasePath,
+                        const CXXRecordDecl *DerivedClass)
+      : Kind(DerivedToBaseAdjustment) {
+      DerivedToBase.BasePath = BasePath;
+      DerivedToBase.DerivedClass = DerivedClass;
+    }
+
+    SubobjectAdjustment(FieldDecl *Field)
+      : Kind(FieldAdjustment) {
+      this->Field = Field;
+    }
   };
-  
-  SubobjectAdjustment(const CastExpr *BasePath, 
-                      const CXXRecordDecl *DerivedClass)
-    : Kind(DerivedToBaseAdjustment) 
-  {
-    DerivedToBase.BasePath = BasePath;
-    DerivedToBase.DerivedClass = DerivedClass;
-  }
-  
-  SubobjectAdjustment(FieldDecl *Field)
-    : Kind(FieldAdjustment)
-  { 
-    this->Field = Field;
-  }
-};
+}
 
 static llvm::Value *
 CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
@@ -161,8 +178,10 @@ CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
   if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
     if (VD->hasGlobalStorage()) {
       llvm::SmallString<256> Name;
-      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Name);
-      
+      llvm::raw_svector_ostream Out(Name);
+      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
+      Out.flush();
+
       const llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
   
       // Create the reference temporary.
@@ -180,14 +199,14 @@ CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
 }
 
 static llvm::Value *
-EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
+EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
                             llvm::Value *&ReferenceTemporary,
                             const CXXDestructorDecl *&ReferenceTemporaryDtor,
                             const NamedDecl *InitializedDecl) {
   if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
     E = DAE->getExpr();
   
-  if (const CXXExprWithTemporaries *TE = dyn_cast<CXXExprWithTemporaries>(E)) {
+  if (const ExprWithCleanups *TE = dyn_cast<ExprWithCleanups>(E)) {
     CodeGenFunction::RunCleanupsScope Scope(CGF);
 
     return EmitExprForReferenceBinding(CGF, TE->getSubExpr(), 
@@ -197,10 +216,9 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
   }
 
   RValue RV;
-  if (E->isLvalue(CGF.getContext()) == Expr::LV_Valid) {
+  if (E->isGLValue()) {
     // Emit the expression as an lvalue.
     LValue LV = CGF.EmitLValue(E);
-
     if (LV.isSimple())
       return LV.getAddress();
     
@@ -232,8 +250,8 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
           continue;
         }
       } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
-        if (ME->getBase()->isLvalue(CGF.getContext()) != Expr::LV_Valid &&
-            ME->getBase()->getType()->isRecordType()) {
+        if (!ME->isArrow() && ME->getBase()->isRValue()) {
+          assert(ME->getBase()->getType()->isRecordType());
           if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
             E = ME->getBase();
             Adjustments.push_back(SubobjectAdjustment(Field));
@@ -247,13 +265,16 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
     }
     
     // Create a reference temporary if necessary.
+    AggValueSlot AggSlot = AggValueSlot::ignored();
     if (CGF.hasAggregateLLVMType(E->getType()) &&
-        !E->getType()->isAnyComplexType())
+        !E->getType()->isAnyComplexType()) {
       ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(), 
                                                     InitializedDecl);
+      AggSlot = AggValueSlot::forAddr(ReferenceTemporary, false,
+                                      InitializedDecl != 0);
+    }
       
-    RV = CGF.EmitAnyExpr(E, ReferenceTemporary, /*IsAggLocVolatile=*/false,
-                         /*IgnoreResult=*/false, InitializedDecl);
+    RV = CGF.EmitAnyExpr(E, AggSlot);
 
     if (InitializedDecl) {
       // Get the destructor for the reference temporary.
@@ -328,7 +349,7 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
 }
 
 RValue
-CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
+CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
                                             const NamedDecl *InitializedDecl) {
   llvm::Value *ReferenceTemporary = 0;
   const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
@@ -343,8 +364,8 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
     if (VD->hasGlobalStorage()) {
       llvm::Constant *DtorFn = 
         CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
-      CGF.EmitCXXGlobalDtorRegistration(DtorFn, 
-                                      cast<llvm::Constant>(ReferenceTemporary));
+      EmitCXXGlobalDtorRegistration(DtorFn, 
+                                    cast<llvm::Constant>(ReferenceTemporary));
       
       return RValue::get(Value);
     }
@@ -370,14 +391,14 @@ void CodeGenFunction::EmitCheck(llvm::Value *Address, unsigned Size) {
   if (!CatchUndefined)
     return;
 
-  Address = Builder.CreateBitCast(Address, PtrToInt8Ty);
+  // This needs to be to the standard address space.
+  Address = Builder.CreateBitCast(Address, Int8PtrTy);
 
   const llvm::Type *IntPtrT = IntPtrTy;
   llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, &IntPtrT, 1);
-  const llvm::IntegerType *Int1Ty = llvm::Type::getInt1Ty(VMContext);
 
   // In time, people may want to control this and use a 1 here.
-  llvm::Value *Arg = llvm::ConstantInt::get(Int1Ty, 0);
+  llvm::Value *Arg = Builder.getFalse();
   llvm::Value *C = Builder.CreateCall2(F, Address, Arg);
   llvm::BasicBlock *Cont = createBasicBlock();
   llvm::BasicBlock *Check = createBasicBlock();
@@ -468,7 +489,8 @@ LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E) {
   LValue LV = EmitLValue(E);
   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
-    EmitCheck(LV.getAddress(), getContext().getTypeSize(E->getType()) / 8);
+    EmitCheck(LV.getAddress(), 
+              getContext().getTypeSizeInChars(E->getType()).getQuantity());
   return LV;
 }
 
@@ -498,7 +520,9 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
   case Expr::BinaryOperatorClass:
     return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
   case Expr::CompoundAssignOperatorClass:
-    return EmitCompoundAssignOperatorLValue(cast<CompoundAssignOperator>(E));
+    if (!E->getType()->isAnyComplexType())
+      return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
+    return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
   case Expr::CallExprClass:
   case Expr::CXXMemberCallExprClass:
   case Expr::CXXOperatorCallExprClass:
@@ -523,8 +547,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
   case Expr::CXXBindTemporaryExprClass:
     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
-  case Expr::CXXExprWithTemporariesClass:
-    return EmitCXXExprWithTemporariesLValue(cast<CXXExprWithTemporaries>(E));
+  case Expr::ExprWithCleanupsClass:
+    return EmitExprWithCleanupsLValue(cast<ExprWithCleanups>(E));
   case Expr::CXXScalarValueInitExprClass:
     return EmitNullInitializationLValue(cast<CXXScalarValueInitExpr>(E));
   case Expr::CXXDefaultArgExprClass:
@@ -538,11 +562,6 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
   case Expr::ObjCPropertyRefExprClass:
     return EmitObjCPropertyRefLValue(cast<ObjCPropertyRefExpr>(E));
-  case Expr::ObjCImplicitSetterGetterRefExprClass:
-    return EmitObjCKVCRefLValue(cast<ObjCImplicitSetterGetterRefExpr>(E));
-  case Expr::ObjCSuperExprClass:
-    return EmitObjCSuperExprLValue(cast<ObjCSuperExpr>(E));
-
   case Expr::StmtExprClass:
     return EmitStmtExprLValue(cast<StmtExpr>(E));
   case Expr::UnaryOperatorClass:
@@ -557,8 +576,12 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
   case Expr::ConditionalOperatorClass:
     return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
+  case Expr::BinaryConditionalOperatorClass:
+    return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
   case Expr::ChooseExprClass:
     return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(getContext()));
+  case Expr::OpaqueValueExprClass:
+    return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
   case Expr::ImplicitCastExprClass:
   case Expr::CStyleCastExprClass:
   case Expr::CXXFunctionalCastExprClass:
@@ -571,35 +594,58 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
 }
 
 llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
-                                              unsigned Alignment, QualType Ty) {
+                                              unsigned Alignment, QualType Ty,
+                                              llvm::MDNode *TBAAInfo) {
   llvm::LoadInst *Load = Builder.CreateLoad(Addr, "tmp");
   if (Volatile)
     Load->setVolatile(true);
   if (Alignment)
     Load->setAlignment(Alignment);
+  if (TBAAInfo)
+    CGM.DecorateInstruction(Load, TBAAInfo);
 
-  // Bool can have different representation in memory than in registers.
-  llvm::Value *V = Load;
-  if (Ty->isBooleanType())
-    if (V->getType() != llvm::Type::getInt1Ty(VMContext))
-      V = Builder.CreateTrunc(V, llvm::Type::getInt1Ty(VMContext), "tobool");
+  return EmitFromMemory(Load, Ty);
+}
 
-  return V;
+static bool isBooleanUnderlyingType(QualType Ty) {
+  if (const EnumType *ET = dyn_cast<EnumType>(Ty))
+    return ET->getDecl()->getIntegerType()->isBooleanType();
+  return false;
 }
 
-void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
-                                        bool Volatile, unsigned Alignment,
-                                        QualType Ty) {
+llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
+  // Bool has a different representation in memory than in registers.
+  if (Ty->isBooleanType() || isBooleanUnderlyingType(Ty)) {
+    // This should really always be an i1, but sometimes it's already
+    // an i8, and it's awkward to track those cases down.
+    if (Value->getType()->isIntegerTy(1))
+      return Builder.CreateZExt(Value, Builder.getInt8Ty(), "frombool");
+    assert(Value->getType()->isIntegerTy(8) && "value rep of bool not i1/i8");
+  }
 
-  if (Ty->isBooleanType()) {
-    // Bool can have different representation in memory than in registers.
-    const llvm::PointerType *DstPtr = cast<llvm::PointerType>(Addr->getType());
-    Value = Builder.CreateIntCast(Value, DstPtr->getElementType(), false);
+  return Value;
+}
+
+llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
+  // Bool has a different representation in memory than in registers.
+  if (Ty->isBooleanType() || isBooleanUnderlyingType(Ty)) {
+    assert(Value->getType()->isIntegerTy(8) && "memory rep of bool not i8");
+    return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
   }
 
+  return Value;
+}
+
+void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
+                                        bool Volatile, unsigned Alignment,
+                                        QualType Ty,
+                                        llvm::MDNode *TBAAInfo) {
+  Value = EmitToMemory(Value, Ty);
   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
   if (Alignment)
     Store->setAlignment(Alignment);
+  if (TBAAInfo)
+    CGM.DecorateInstruction(Store, TBAAInfo);
 }
 
 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
@@ -622,7 +668,8 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, QualType ExprType) {
 
     // Everything needs a load.
     return RValue::get(EmitLoadOfScalar(Ptr, LV.isVolatileQualified(),
-                                        LV.getAlignment(), ExprType));
+                                        LV.getAlignment(), ExprType,
+                                        LV.getTBAAInfo()));
 
   }
 
@@ -641,11 +688,8 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, QualType ExprType) {
   if (LV.isBitField())
     return EmitLoadOfBitfieldLValue(LV, ExprType);
 
-  if (LV.isPropertyRef())
-    return EmitLoadOfPropertyRefLValue(LV, ExprType);
-
-  assert(LV.isKVCRef() && "Unknown LValue type!");
-  return EmitLoadOfKVCRefLValue(LV, ExprType);
+  assert(LV.isPropertyRef() && "Unknown LValue type!");
+  return EmitLoadOfPropertyRefLValue(LV);
 }
 
 RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
@@ -671,13 +715,13 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
 
     // Offset by the byte offset, if used.
     if (AI.FieldByteOffset) {
-      const llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(VMContext);
-      Ptr = Builder.CreateBitCast(Ptr, i8PTy);
+      Ptr = EmitCastToVoidPtr(Ptr);
       Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset,"bf.field.offs");
     }
 
     // Cast to the access type.
-    const llvm::Type *PTy = llvm::Type::getIntNPtrTy(VMContext, AI.AccessWidth,
+    const llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(),
+                                                     AI.AccessWidth,
                                                     ExprType.getAddressSpace());
     Ptr = Builder.CreateBitCast(Ptr, PTy);
 
@@ -720,16 +764,6 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
   return RValue::get(Res);
 }
 
-RValue CodeGenFunction::EmitLoadOfPropertyRefLValue(LValue LV,
-                                                    QualType ExprType) {
-  return EmitObjCPropertyGet(LV.getPropertyRefExpr());
-}
-
-RValue CodeGenFunction::EmitLoadOfKVCRefLValue(LValue LV,
-                                               QualType ExprType) {
-  return EmitObjCPropertyGet(LV.getKVCRefExpr());
-}
-
 // If this is a reference to a subset of the elements of a vector, create an
 // appropriate shufflevector.
 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV,
@@ -757,9 +791,8 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV,
     Mask.push_back(llvm::ConstantInt::get(Int32Ty, InIdx));
   }
 
-  llvm::Value *MaskV = llvm::ConstantVector::get(&Mask[0], Mask.size());
-  Vec = Builder.CreateShuffleVector(Vec,
-                                    llvm::UndefValue::get(Vec->getType()),
+  llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
+  Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
                                     MaskV, "tmp");
   return RValue::get(Vec);
 }
@@ -790,11 +823,8 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
     if (Dst.isBitField())
       return EmitStoreThroughBitfieldLValue(Src, Dst, Ty);
 
-    if (Dst.isPropertyRef())
-      return EmitStoreThroughPropertyRefLValue(Src, Dst, Ty);
-
-    assert(Dst.isKVCRef() && "Unknown LValue type");
-    return EmitStoreThroughKVCRefLValue(Src, Dst, Ty);
+    assert(Dst.isPropertyRef() && "Unknown LValue type");
+    return EmitStoreThroughPropertyRefLValue(Src, Dst);
   }
 
   if (Dst.isObjCWeak() && !Dst.isNonGC()) {
@@ -831,7 +861,8 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
 
   assert(Src.isScalar() && "Can't emit an agg store with this method");
   EmitStoreOfScalar(Src.getScalarVal(), Dst.getAddress(),
-                    Dst.isVolatileQualified(), Dst.getAlignment(), Ty);
+                    Dst.isVolatileQualified(), Dst.getAlignment(), Ty,
+                    Dst.getTBAAInfo());
 }
 
 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
@@ -877,6 +908,8 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
 
     // Get the field pointer.
     llvm::Value *Ptr = Dst.getBitFieldBaseAddr();
+    unsigned addressSpace =
+      cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
 
     // Only offset by the field index if used, so that incoming values are not
     // required to be structures.
@@ -885,14 +918,15 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
 
     // Offset by the byte offset, if used.
     if (AI.FieldByteOffset) {
-      const llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(VMContext);
-      Ptr = Builder.CreateBitCast(Ptr, i8PTy);
+      Ptr = EmitCastToVoidPtr(Ptr);
       Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset,"bf.field.offs");
     }
 
     // Cast to the access type.
-    const llvm::Type *PTy = llvm::Type::getIntNPtrTy(VMContext, AI.AccessWidth,
-                                                     Ty.getAddressSpace());
+    const llvm::Type *AccessLTy =
+      llvm::Type::getIntNTy(getLLVMContext(), AI.AccessWidth);
+
+    const llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
     Ptr = Builder.CreateBitCast(Ptr, PTy);
 
     // Extract the piece of the bit-field value to write in this access, limited
@@ -904,8 +938,6 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
                                                             AI.TargetBitWidth));
 
     // Extend or truncate to the access size.
-    const llvm::Type *AccessLTy =
-      llvm::Type::getIntNTy(VMContext, AI.AccessWidth);
     if (ResSizeInBits < AI.AccessWidth)
       Val = Builder.CreateZExt(Val, AccessLTy);
     else if (ResSizeInBits > AI.AccessWidth)
@@ -938,18 +970,6 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
   }
 }
 
-void CodeGenFunction::EmitStoreThroughPropertyRefLValue(RValue Src,
-                                                        LValue Dst,
-                                                        QualType Ty) {
-  EmitObjCPropertySet(Dst.getPropertyRefExpr(), Src);
-}
-
-void CodeGenFunction::EmitStoreThroughKVCRefLValue(RValue Src,
-                                                   LValue Dst,
-                                                   QualType Ty) {
-  EmitObjCPropertySet(Dst.getKVCRefExpr(), Src);
-}
-
 void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
                                                                LValue Dst,
                                                                QualType Ty) {
@@ -975,7 +995,7 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
         Mask[InIdx] = llvm::ConstantInt::get(Int32Ty, i);
       }
 
-      llvm::Value *MaskV = llvm::ConstantVector::get(&Mask[0], Mask.size());
+      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
       Vec = Builder.CreateShuffleVector(SrcVal,
                                         llvm::UndefValue::get(Vec->getType()),
                                         MaskV, "tmp");
@@ -990,8 +1010,7 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
         ExtMask.push_back(llvm::ConstantInt::get(Int32Ty, i));
       for (; i != NumDstElts; ++i)
         ExtMask.push_back(llvm::UndefValue::get(Int32Ty));
-      llvm::Value *ExtMaskV = llvm::ConstantVector::get(&ExtMask[0],
-                                                        ExtMask.size());
+      llvm::Value *ExtMaskV = llvm::ConstantVector::get(ExtMask);
       llvm::Value *ExtSrcVal =
         Builder.CreateShuffleVector(SrcVal,
                                     llvm::UndefValue::get(SrcVal->getType()),
@@ -1006,7 +1025,7 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
         unsigned Idx = getAccessedFieldNo(i, Elts);
         Mask[Idx] = llvm::ConstantInt::get(Int32Ty, i+NumDstElts);
       }
-      llvm::Value *MaskV = llvm::ConstantVector::get(&Mask[0], Mask.size());
+      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
       Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV, "tmp");
     } else {
       // We should never shorten the vector
@@ -1040,8 +1059,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
   
   if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
     if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
-      if ((VD->isBlockVarDecl() && !VD->hasLocalStorage()) ||
-          VD->isFileVarDecl()) {
+      if (VD->hasGlobalStorage()) {
         LV.setGlobalObjCRef(true);
         LV.setThreadLocalRef(VD->isThreadSpecified());
       }
@@ -1116,7 +1134,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
 
 static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
                                       const Expr *E, const FunctionDecl *FD) {
-  llvm::Value* V = CGF.CGM.GetAddrOfFunction(FD);
+  llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD);
   if (!FD->hasPrototype()) {
     if (const FunctionProtoType *Proto =
             FD->getType()->getAs<FunctionProtoType>()) {
@@ -1135,10 +1153,10 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
 
 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   const NamedDecl *ND = E->getDecl();
-  unsigned Alignment = CGF.getContext().getDeclAlign(ND).getQuantity();
+  unsigned Alignment = getContext().getDeclAlign(ND).getQuantity();
 
   if (ND->hasAttr<WeakRefAttr>()) {
-    const ValueDecl* VD = cast<ValueDecl>(ND);
+    const ValueDecl *VD = cast<ValueDecl>(ND);
     llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
     return MakeAddrLValue(Aliasee, E->getType(), Alignment);
   }
@@ -1149,20 +1167,18 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     if (VD->hasExternalStorage() || VD->isFileVarDecl()) 
       return EmitGlobalVarDeclLValue(*this, E, VD);
 
-    bool NonGCable = VD->hasLocalStorage() && !VD->hasAttr<BlocksAttr>();
+    bool NonGCable = VD->hasLocalStorage() &&
+                     !VD->getType()->isReferenceType() &&
+                     !VD->hasAttr<BlocksAttr>();
 
     llvm::Value *V = LocalDeclMap[VD];
-    if (!V && getContext().getLangOptions().CPlusPlus &&
-        VD->isStaticLocal()) 
+    if (!V && VD->isStaticLocal()) 
       V = CGM.getStaticLocalDeclAddress(VD);
     assert(V && "DeclRefExpr not entered in LocalDeclMap?");
 
-    if (VD->hasAttr<BlocksAttr>()) {
-      V = Builder.CreateStructGEP(V, 1, "forwarding");
-      V = Builder.CreateLoad(V);
-      V = Builder.CreateStructGEP(V, getByRefValueLLVMField(VD),
-                                  VD->getNameAsString());
-    }
+    if (VD->hasAttr<BlocksAttr>())
+      V = BuildBlockByrefAddress(V, VD);
+    
     if (VD->getType()->isReferenceType())
       V = Builder.CreateLoad(V, "tmp");
 
@@ -1174,24 +1190,10 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     setObjCGCLValueClass(getContext(), E, LV);
     return LV;
   }
-  
-  // If we're emitting an instance method as an independent lvalue,
-  // we're actually emitting a member pointer.
-  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(ND))
-    if (MD->isInstance()) {
-      llvm::Value *V = CGM.getCXXABI().EmitMemberPointer(MD);
-      return MakeAddrLValue(V, MD->getType(), Alignment);
-    }
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
-    return EmitFunctionDeclLValue(*this, E, FD);
-  
-  // If we're emitting a field as an independent lvalue, we're
-  // actually emitting a member pointer.
-  if (const FieldDecl *FD = dyn_cast<FieldDecl>(ND)) {
-    llvm::Value *V = CGM.getCXXABI().EmitMemberPointer(FD);
-    return MakeAddrLValue(V, FD->getType(), Alignment);
-  }
-  
+
+  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
+    return EmitFunctionDeclLValue(*this, E, fn);
+
   assert(false && "Unhandled DeclRefExpr");
   
   // an invalid LValue, but the assert will
@@ -1201,7 +1203,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
 
 LValue CodeGenFunction::EmitBlockDeclRefLValue(const BlockDeclRefExpr *E) {
   unsigned Alignment =
-    CGF.getContext().getDeclAlign(E->getDecl()).getQuantity();
+    getContext().getDeclAlign(E->getDecl()).getQuantity();
   return MakeAddrLValue(GetAddrOfBlockDecl(E), E->getType(), Alignment);
 }
 
@@ -1233,9 +1235,22 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
   case UO_Real:
   case UO_Imag: {
     LValue LV = EmitLValue(E->getSubExpr());
+    assert(LV.isSimple() && "real/imag on non-ordinary l-value");
+    llvm::Value *Addr = LV.getAddress();
+
+    // real and imag are valid on scalars.  This is a faster way of
+    // testing that.
+    if (!cast<llvm::PointerType>(Addr->getType())
+           ->getElementType()->isStructTy()) {
+      assert(E->getSubExpr()->getType()->isArithmeticType());
+      return LV;
+    }
+
+    assert(E->getSubExpr()->getType()->isAnyComplexType());
+
     unsigned Idx = E->getOpcode() == UO_Imag;
     return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
-                                                    Idx, "idx"),
+                                                  Idx, "idx"),
                           ExprTy);
   }
   case UO_PreInc:
@@ -1297,7 +1312,9 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
       CurDecl = getContext().getTranslationUnitDecl();
 
     std::string FunctionName =
-      PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurDecl);
+        (isa<BlockDecl>(CurDecl)
+         ? FnName.str()
+         : PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurDecl));
 
     llvm::Constant *C =
       CGM.GetAddrOfConstantCString(FunctionName, GlobalVarName.c_str());
@@ -1363,15 +1380,14 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
     // Emit the vector as an lvalue to get its address.
     LValue LHS = EmitLValue(E->getBase());
     assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
-    Idx = Builder.CreateIntCast(Idx, CGF.Int32Ty, IdxSigned, "vidx");
+    Idx = Builder.CreateIntCast(Idx, Int32Ty, IdxSigned, "vidx");
     return LValue::MakeVectorElt(LHS.getAddress(), Idx,
                                  E->getBase()->getType().getCVRQualifiers());
   }
 
   // Extend or truncate the index type to 32 or 64-bits.
-  if (!Idx->getType()->isIntegerTy(LLVMPointerWidth))
-    Idx = Builder.CreateIntCast(Idx, IntPtrTy,
-                                IdxSigned, "idxprom");
+  if (Idx->getType() != IntPtrTy)
+    Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
   
   // FIXME: As llvm implements the object size checking, this can come out.
   if (CatchUndefined) {
@@ -1401,17 +1417,12 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
 
     Idx = Builder.CreateMul(Idx, VLASize);
 
-    QualType BaseType = getContext().getBaseElementType(VAT);
-
-    CharUnits BaseTypeSize = getContext().getTypeSizeInChars(BaseType);
-    Idx = Builder.CreateUDiv(Idx,
-                             llvm::ConstantInt::get(Idx->getType(),
-                                 BaseTypeSize.getQuantity()));
-    
     // The base must be a pointer, which is not an aggregate.  Emit it.
     llvm::Value *Base = EmitScalarExpr(E->getBase());
-    
-    Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
+
+    Address = EmitCastToVoidPtr(Base);
+    Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
+    Address = Builder.CreateBitCast(Address, Base->getType());
   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
     // Indexing over an interface, as in "NSString *P; P[4];"
     llvm::Value *InterfaceSize =
@@ -1420,12 +1431,10 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
 
     Idx = Builder.CreateMul(Idx, InterfaceSize);
 
-    const llvm::Type *i8PTy = llvm::Type::getInt8PtrTy(VMContext);
-    
     // The base must be a pointer, which is not an aggregate.  Emit it.
     llvm::Value *Base = EmitScalarExpr(E->getBase());
-    Address = Builder.CreateGEP(Builder.CreateBitCast(Base, i8PTy),
-                                Idx, "arrayidx");
+    Address = EmitCastToVoidPtr(Base);
+    Address = Builder.CreateGEP(Address, Idx, "arrayidx");
     Address = Builder.CreateBitCast(Address, Base->getType());
   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
     // If this is A[i] where A is an array, the frontend will have decayed the
@@ -1469,7 +1478,7 @@ llvm::Constant *GenerateConstantVector(llvm::LLVMContext &VMContext,
   for (unsigned i = 0, e = Elts.size(); i != e; ++i)
     CElts.push_back(llvm::ConstantInt::get(Int32Ty, Elts[i]));
 
-  return llvm::ConstantVector::get(&CElts[0], CElts.size());
+  return llvm::ConstantVector::get(CElts);
 }
 
 LValue CodeGenFunction::
@@ -1485,7 +1494,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
     Base = MakeAddrLValue(Ptr, PT->getPointeeType());
     Base.getQuals().removeObjCGCAttr();
-  } else if (E->getBase()->isLvalue(getContext()) == Expr::LV_Valid) {
+  } else if (E->getBase()->isGLValue()) {
     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
     // emit the base as an lvalue.
     assert(E->getBase()->getType()->isVectorType());
@@ -1507,7 +1516,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
   E->getEncodedElementAccess(Indices);
 
   if (Base.isSimple()) {
-    llvm::Constant *CV = GenerateConstantVector(VMContext, Indices);
+    llvm::Constant *CV = GenerateConstantVector(getLLVMContext(), Indices);
     return LValue::MakeExtVectorElt(Base.getAddress(), CV,
                                     Base.getVRQualifiers());
   }
@@ -1522,7 +1531,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     else
       CElts.push_back(cast<llvm::Constant>(BaseElts->getOperand(Indices[i])));
   }
-  llvm::Constant *CV = llvm::ConstantVector::get(&CElts[0], CElts.size());
+  llvm::Constant *CV = llvm::ConstantVector::get(CElts);
   return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV,
                                   Base.getVRQualifiers());
 }
@@ -1539,12 +1548,6 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
     const PointerType *PTy =
       BaseExpr->getType()->getAs<PointerType>();
     BaseQuals = PTy->getPointeeType().getQualifiers();
-  } else if (isa<ObjCPropertyRefExpr>(BaseExpr->IgnoreParens()) ||
-             isa<ObjCImplicitSetterGetterRefExpr>(
-               BaseExpr->IgnoreParens())) {
-    RValue RV = EmitObjCPropertyGet(BaseExpr);
-    BaseValue = RV.getAggregateAddr();
-    BaseQuals = BaseExpr->getType().getQualifiers();
   } else {
     LValue BaseLV = EmitLValue(BaseExpr);
     if (BaseLV.isNonGC())
@@ -1574,8 +1577,8 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
   return LValue();
 }
 
-LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value* BaseValue,
-                                              const FieldDecl* Field,
+LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value *BaseValue,
+                                              const FieldDecl *Field,
                                               unsigned CVRQualifiers) {
   const CGRecordLayout &RL =
     CGM.getTypes().getCGRecordLayout(Field->getParent());
@@ -1589,23 +1592,13 @@ LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value* BaseValue,
 /// that the base value is a pointer to the enclosing record, derive
 /// an lvalue for the ultimate field.
 LValue CodeGenFunction::EmitLValueForAnonRecordField(llvm::Value *BaseValue,
-                                                     const FieldDecl *Field,
+                                             const IndirectFieldDecl *Field,
                                                      unsigned CVRQualifiers) {
-  llvm::SmallVector<const FieldDecl *, 8> Path;
-  Path.push_back(Field);
-
-  while (Field->getParent()->isAnonymousStructOrUnion()) {
-    const ValueDecl *VD = Field->getParent()->getAnonymousStructOrUnionObject();
-    if (!isa<FieldDecl>(VD)) break;
-    Field = cast<FieldDecl>(VD);
-    Path.push_back(Field);
-  }
-
-  llvm::SmallVectorImpl<const FieldDecl*>::reverse_iterator
-    I = Path.rbegin(), E = Path.rend();
+  IndirectFieldDecl::chain_iterator I = Field->chain_begin(),
+    IEnd = Field->chain_end();
   while (true) {
-    LValue LV = EmitLValueForField(BaseValue, *I, CVRQualifiers);
-    if (++I == E) return LV;
+    LValue LV = EmitLValueForField(BaseValue, cast<FieldDecl>(*I), CVRQualifiers);
+    if (++I == IEnd) return LV;
 
     assert(LV.isSimple());
     BaseValue = LV.getAddress();
@@ -1613,8 +1606,8 @@ LValue CodeGenFunction::EmitLValueForAnonRecordField(llvm::Value *BaseValue,
   }
 }
 
-LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
-                                           const FieldDecl* Field,
+LValue CodeGenFunction::EmitLValueForField(llvm::Value *BaseValue,
+                                           const FieldDecl *Field,
                                            unsigned CVRQualifiers) {
   if (Field->isBitField())
     return EmitLValueForBitfield(BaseValue, Field, CVRQualifiers);
@@ -1628,7 +1621,7 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
   if (Field->getParent()->isUnion()) {
     const llvm::Type *FieldTy =
       CGM.getTypes().ConvertTypeForMem(Field->getType());
-    const llvm::PointerType * BaseTy =
+    const llvm::PointerType *BaseTy =
       cast<llvm::PointerType>(BaseValue->getType());
     unsigned AS = BaseTy->getAddressSpace();
     V = Builder.CreateBitCast(V,
@@ -1650,8 +1643,8 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
 }
 
 LValue 
-CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value* BaseValue, 
-                                                  const FieldDecl* Field,
+CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value *BaseValue, 
+                                                  const FieldDecl *Field,
                                                   unsigned CVRQualifiers) {
   QualType FieldType = Field->getType();
   
@@ -1669,71 +1662,74 @@ CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value* BaseValue,
   return MakeAddrLValue(V, FieldType, Alignment);
 }
 
-LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr* E){
+LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
   llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
-  const Expr* InitExpr = E->getInitializer();
+  const Expr *InitExpr = E->getInitializer();
   LValue Result = MakeAddrLValue(DeclPtr, E->getType());
 
-  EmitAnyExprToMem(InitExpr, DeclPtr, /*Volatile*/ false);
+  EmitAnyExprToMem(InitExpr, DeclPtr, /*Volatile*/ false, /*Init*/ true);
 
   return Result;
 }
 
-LValue 
-CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
-  if (E->isLvalue(getContext()) == Expr::LV_Valid) {
-    if (int Cond = ConstantFoldsToSimpleInteger(E->getCond())) {
-      Expr *Live = Cond == 1 ? E->getLHS() : E->getRHS();
-      if (Live)
-        return EmitLValue(Live);
-    }
+LValue CodeGenFunction::
+EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
+  if (!expr->isGLValue()) {
+    // ?: here should be an aggregate.
+    assert((hasAggregateLLVMType(expr->getType()) &&
+            !expr->getType()->isAnyComplexType()) &&
+           "Unexpected conditional operator!");
+    return EmitAggExprToLValue(expr);
+  }
 
-    if (!E->getLHS())
-      return EmitUnsupportedLValue(E, "conditional operator with missing LHS");
+  const Expr *condExpr = expr->getCond();
 
-    llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
-    llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
-    llvm::BasicBlock *ContBlock = createBasicBlock("cond.end");
-    
-    EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
+  if (int condValue = ConstantFoldsToSimpleInteger(condExpr)) {
+    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
+    if (condValue == -1) std::swap(live, dead);
+
+    if (!ContainsLabel(dead))
+      return EmitLValue(live);
+  }
+
+  OpaqueValueMapping binding(*this, expr);
+
+  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
+  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
+  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
+
+  ConditionalEvaluation eval(*this);
+  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
     
-    // Any temporaries created here are conditional.
-    BeginConditionalBranch();
-    EmitBlock(LHSBlock);
-    LValue LHS = EmitLValue(E->getLHS());
-    EndConditionalBranch();
+  // Any temporaries created here are conditional.
+  EmitBlock(lhsBlock);
+  eval.begin(*this);
+  LValue lhs = EmitLValue(expr->getTrueExpr());
+  eval.end(*this);
     
-    if (!LHS.isSimple())
-      return EmitUnsupportedLValue(E, "conditional operator");
+  if (!lhs.isSimple())
+    return EmitUnsupportedLValue(expr, "conditional operator");
 
-    // FIXME: We shouldn't need an alloca for this.
-    llvm::Value *Temp = CreateTempAlloca(LHS.getAddress()->getType(),"condtmp");
-    Builder.CreateStore(LHS.getAddress(), Temp);
-    EmitBranch(ContBlock);
-    
-    // Any temporaries created here are conditional.
-    BeginConditionalBranch();
-    EmitBlock(RHSBlock);
-    LValue RHS = EmitLValue(E->getRHS());
-    EndConditionalBranch();
-    if (!RHS.isSimple())
-      return EmitUnsupportedLValue(E, "conditional operator");
-
-    Builder.CreateStore(RHS.getAddress(), Temp);
-    EmitBranch(ContBlock);
-
-    EmitBlock(ContBlock);
+  lhsBlock = Builder.GetInsertBlock();
+  Builder.CreateBr(contBlock);
     
-    Temp = Builder.CreateLoad(Temp, "lv");
-    return MakeAddrLValue(Temp, E->getType());
-  }
-  
-  // ?: here should be an aggregate.
-  assert((hasAggregateLLVMType(E->getType()) &&
-          !E->getType()->isAnyComplexType()) &&
-         "Unexpected conditional operator!");
+  // Any temporaries created here are conditional.
+  EmitBlock(rhsBlock);
+  eval.begin(*this);
+  LValue rhs = EmitLValue(expr->getFalseExpr());
+  eval.end(*this);
+  if (!rhs.isSimple())
+    return EmitUnsupportedLValue(expr, "conditional operator");
+  rhsBlock = Builder.GetInsertBlock();
 
-  return EmitAggExprToLValue(E);
+  EmitBlock(contBlock);
+
+  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(),
+                                         "cond-lvalue");
+  phi->reserveOperandSpace(2);
+  phi->addIncoming(lhs.getAddress(), lhsBlock);
+  phi->addIncoming(rhs.getAddress(), rhsBlock);
+  return MakeAddrLValue(phi, expr->getType());
 }
 
 /// EmitCastLValue - Casts are never lvalues unless that cast is a dynamic_cast.
@@ -1747,36 +1743,57 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   switch (E->getCastKind()) {
   case CK_ToVoid:
     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
-   
-  case CK_NoOp:
-    if (E->getSubExpr()->Classify(getContext()).getKind() 
-                                          != Expr::Classification::CL_PRValue) {
-      LValue LV = EmitLValue(E->getSubExpr());
-      if (LV.isPropertyRef() || LV.isKVCRef()) {
-        QualType QT = E->getSubExpr()->getType();
-        RValue RV = 
-          LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT) 
-                             : EmitLoadOfKVCRefLValue(LV, QT);
-        assert(!RV.isScalar() && "EmitCastLValue-scalar cast of property ref");
-        llvm::Value *V = RV.getAggregateAddr();
-        return MakeAddrLValue(V, QT);
-      }
-      return LV;
+
+  case CK_Dependent:
+    llvm_unreachable("dependent cast kind in IR gen!");
+
+  case CK_GetObjCProperty: {
+    LValue LV = EmitLValue(E->getSubExpr());
+    assert(LV.isPropertyRef());
+    RValue RV = EmitLoadOfPropertyRefLValue(LV);
+
+    // Property is an aggregate r-value.
+    if (RV.isAggregate()) {
+      return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
     }
+
+    // Implicit property returns an l-value.
+    assert(RV.isScalar());
+    return MakeAddrLValue(RV.getScalarVal(), E->getSubExpr()->getType());
+  }
+
+  case CK_NoOp:
+  case CK_LValueToRValue:
+    if (!E->getSubExpr()->Classify(getContext()).isPRValue() 
+        || E->getType()->isRecordType())
+      return EmitLValue(E->getSubExpr());
     // Fall through to synthesize a temporary.
-      
-  case CK_Unknown:
+
   case CK_BitCast:
   case CK_ArrayToPointerDecay:
   case CK_FunctionToPointerDecay:
   case CK_NullToMemberPointer:
+  case CK_NullToPointer:
   case CK_IntegralToPointer:
   case CK_PointerToIntegral:
+  case CK_PointerToBoolean:
   case CK_VectorSplat:
   case CK_IntegralCast:
+  case CK_IntegralToBoolean:
   case CK_IntegralToFloating:
   case CK_FloatingToIntegral:
+  case CK_FloatingToBoolean:
   case CK_FloatingCast:
+  case CK_FloatingRealToComplex:
+  case CK_FloatingComplexToReal:
+  case CK_FloatingComplexToBoolean:
+  case CK_FloatingComplexCast:
+  case CK_FloatingComplexToIntegralComplex:
+  case CK_IntegralRealToComplex:
+  case CK_IntegralComplexToReal:
+  case CK_IntegralComplexToBoolean:
+  case CK_IntegralComplexCast:
+  case CK_IntegralComplexToFloatingComplex:
   case CK_DerivedToBaseMemberPointer:
   case CK_BaseToDerivedMemberPointer:
   case CK_MemberPointerToBoolean:
@@ -1810,17 +1827,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
       cast<CXXRecordDecl>(DerivedClassTy->getDecl());
     
     LValue LV = EmitLValue(E->getSubExpr());
-    llvm::Value *This;
-    if (LV.isPropertyRef() || LV.isKVCRef()) {
-      QualType QT = E->getSubExpr()->getType();
-      RValue RV = 
-        LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT)
-                           : EmitLoadOfKVCRefLValue(LV, QT);
-      assert (!RV.isScalar() && "EmitCastLValue");
-      This = RV.getAggregateAddr();
-    }
-    else
-      This = LV.getAddress();
+    llvm::Value *This = LV.getAddress();
     
     // Perform the derived-to-base conversion
     llvm::Value *Base = 
@@ -1876,6 +1883,11 @@ LValue CodeGenFunction::EmitNullInitializationLValue(
   return LV;
 }
 
+LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
+  assert(e->isGLValue() || e->getType()->isRecordType());
+  return getOpaqueLValueMapping(e);
+}
+
 //===--------------------------------------------------------------------===//
 //                             Expression Emission
 //===--------------------------------------------------------------------===//
@@ -1922,7 +1934,7 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
   // Comma expressions just emit their LHS then their RHS as an l-value.
   if (E->getOpcode() == BO_Comma) {
-    EmitAnyExpr(E->getLHS());
+    EmitIgnoredExpr(E->getLHS());
     EnsureInsertPoint();
     return EmitLValue(E->getRHS());
   }
@@ -1930,20 +1942,20 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
   if (E->getOpcode() == BO_PtrMemD ||
       E->getOpcode() == BO_PtrMemI)
     return EmitPointerToDataMemberBinaryExpr(E);
-  
-  // Can only get l-value for binary operator expressions which are a
-  // simple assignment of aggregate type.
-  if (E->getOpcode() != BO_Assign)
-    return EmitUnsupportedLValue(E, "binary l-value expression");
 
+  assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
+  
   if (!hasAggregateLLVMType(E->getType())) {
-    // Emit the LHS as an l-value.
+    // __block variables need the RHS evaluated first.
+    RValue RV = EmitAnyExpr(E->getRHS());
     LValue LV = EmitLValue(E->getLHS());
-    // Store the value through the l-value.
-    EmitStoreThroughLValue(EmitAnyExpr(E->getRHS()), LV, E->getType());
+    EmitStoreThroughLValue(RV, LV, E->getType());
     return LV;
   }
-  
+
+  if (E->getType()->isAnyComplexType())
+    return EmitComplexAssignmentLValue(E);
+
   return EmitAggExprToLValue(E);
 }
 
@@ -1966,9 +1978,11 @@ LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
 }
 
 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
-  llvm::Value *Temp = CreateMemTemp(E->getType(), "tmp");
-  EmitCXXConstructExpr(Temp, E);
-  return MakeAddrLValue(Temp, E->getType());
+  assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
+         && "binding l-value to type which needs a temporary");
+  AggValueSlot Slot = CreateAggTemp(E->getType(), "tmp");
+  EmitCXXConstructExpr(E, Slot);
+  return MakeAddrLValue(Slot.getAddr(), E->getType());
 }
 
 LValue
@@ -1978,9 +1992,11 @@ CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
 
 LValue
 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
-  LValue LV = EmitLValue(E->getSubExpr());
-  EmitCXXTemporary(E->getTemporary(), LV.getAddress());
-  return LV;
+  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
+  Slot.setLifetimeExternallyManaged();
+  EmitAggExpr(E->getSubExpr(), Slot);
+  EmitCXXTemporary(E->getTemporary(), Slot.getAddr());
+  return MakeAddrLValue(Slot.getAddr(), E->getType());
 }
 
 LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
@@ -2040,24 +2056,6 @@ LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
   return LV;
 }
 
-LValue
-CodeGenFunction::EmitObjCPropertyRefLValue(const ObjCPropertyRefExpr *E) {
-  // This is a special l-value that just issues sends when we load or store
-  // through it.
-  return LValue::MakePropertyRef(E, E->getType().getCVRQualifiers());
-}
-
-LValue CodeGenFunction::EmitObjCKVCRefLValue(
-                                const ObjCImplicitSetterGetterRefExpr *E) {
-  // This is a special l-value that just issues sends when we load or store
-  // through it.
-  return LValue::MakeKVCRef(E, E->getType().getCVRQualifiers());
-}
-
-LValue CodeGenFunction::EmitObjCSuperExprLValue(const ObjCSuperExpr *E) {
-  return EmitUnsupportedLValue(E, "use of super");
-}
-
 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
   // Can only get l-value for message expression returning aggregate type
   RValue RV = EmitAnyExprToTemp(E);
@@ -2078,7 +2076,6 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
 
   const FunctionType *FnType
     = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
-  QualType ResultType = FnType->getResultType();
 
   CallArgList Args;
   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
@@ -2105,4 +2102,3 @@ EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
 
   return MakeAddrLValue(AddV, MPT->getPointeeType());
 }
-
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 28c8b3545b38..f992dc7c9cb9 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -32,27 +32,28 @@ namespace  {
 class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   CodeGenFunction &CGF;
   CGBuilderTy &Builder;
-  llvm::Value *DestPtr;
-  bool VolatileDest;
+  AggValueSlot Dest;
   bool IgnoreResult;
-  bool IsInitializer;
-  bool RequiresGCollection;
 
   ReturnValueSlot getReturnValueSlot() const {
     // If the destination slot requires garbage collection, we can't
     // use the real return value slot, because we have to use the GC
     // API.
-    if (RequiresGCollection) return ReturnValueSlot();
+    if (Dest.requiresGCollection()) return ReturnValueSlot();
 
-    return ReturnValueSlot(DestPtr, VolatileDest);
+    return ReturnValueSlot(Dest.getAddr(), Dest.isVolatile());
+  }
+
+  AggValueSlot EnsureSlot(QualType T) {
+    if (!Dest.isIgnored()) return Dest;
+    return CGF.CreateAggTemp(T, "agg.tmp.ensured");
   }
 
 public:
-  AggExprEmitter(CodeGenFunction &cgf, llvm::Value *destPtr, bool v,
-                 bool ignore, bool isinit, bool requiresGCollection)
-    : CGF(cgf), Builder(CGF.Builder),
-      DestPtr(destPtr), VolatileDest(v), IgnoreResult(ignore),
-      IsInitializer(isinit), RequiresGCollection(requiresGCollection) {
+  AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest,
+                 bool ignore)
+    : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
+      IgnoreResult(ignore) {
   }
 
   //===--------------------------------------------------------------------===//
@@ -114,9 +115,8 @@ public:
     EmitAggLoadOfLValue(E);
   }
   void VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E);
-  void VisitObjCImplicitSetterGetterRefExpr(ObjCImplicitSetterGetterRefExpr *E);
 
-  void VisitConditionalOperator(const ConditionalOperator *CO);
+  void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
   void VisitChooseExpr(const ChooseExpr *CE);
   void VisitInitListExpr(InitListExpr *E);
   void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
@@ -125,10 +125,12 @@ public:
   }
   void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
   void VisitCXXConstructExpr(const CXXConstructExpr *E);
-  void VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E);
+  void VisitExprWithCleanups(ExprWithCleanups *E);
   void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
   void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
 
+  void VisitOpaqueValueExpr(OpaqueValueExpr *E);
+
   void VisitVAArgExpr(VAArgExpr *E);
 
   void EmitInitializationToLValue(Expr *E, LValue Address, QualType T);
@@ -176,13 +178,13 @@ bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
 /// directly into the return value slot.  If GC does interfere, a final
 /// move will be performed.
 void AggExprEmitter::EmitGCMove(const Expr *E, RValue Src) {
-  if (RequiresGCollection) {
+  if (Dest.requiresGCollection()) {
     std::pair<uint64_t, unsigned> TypeInfo = 
       CGF.getContext().getTypeInfo(E->getType());
     unsigned long size = TypeInfo.first/8;
     const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
     llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size);
-    CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, DestPtr,
+    CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, Dest.getAddr(),
                                                     Src.getAggregateAddr(),
                                                     SizeVal);
   }
@@ -192,13 +194,13 @@ void AggExprEmitter::EmitGCMove(const Expr *E, RValue Src) {
 void AggExprEmitter::EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore) {
   assert(Src.isAggregate() && "value must be aggregate value!");
 
-  // If DestPtr is null, then we're evaluating an aggregate expression
+  // If Dest is ignored, then we're evaluating an aggregate expression
   // in a context (like an expression statement) that doesn't care
   // about the result.  C says that an lvalue-to-rvalue conversion is
   // performed in these cases; C++ says that it is not.  In either
   // case, we don't actually need to do anything unless the value is
   // volatile.
-  if (DestPtr == 0) {
+  if (Dest.isIgnored()) {
     if (!Src.isVolatileQualified() ||
         CGF.CGM.getLangOptions().CPlusPlus ||
         (IgnoreResult && Ignore))
@@ -206,26 +208,27 @@ void AggExprEmitter::EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore) {
 
     // If the source is volatile, we must read from it; to do that, we need
     // some place to put it.
-    DestPtr = CGF.CreateMemTemp(E->getType(), "agg.tmp");
+    Dest = CGF.CreateAggTemp(E->getType(), "agg.tmp");
   }
 
-  if (RequiresGCollection) {
+  if (Dest.requiresGCollection()) {
     std::pair<uint64_t, unsigned> TypeInfo = 
     CGF.getContext().getTypeInfo(E->getType());
     unsigned long size = TypeInfo.first/8;
     const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
     llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size);
     CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
-                                              DestPtr, Src.getAggregateAddr(),
-                                              SizeVal);
+                                                      Dest.getAddr(),
+                                                      Src.getAggregateAddr(),
+                                                      SizeVal);
     return;
   }
   // If the result of the assignment is used, copy the LHS there also.
   // FIXME: Pass VolatileDest as well.  I think we also need to merge volatile
   // from the source as well, as we can't eliminate it if either operand
   // is volatile, unless copy has volatile for both source and destination..
-  CGF.EmitAggregateCopy(DestPtr, Src.getAggregateAddr(), E->getType(),
-                        VolatileDest|Src.isVolatileQualified());
+  CGF.EmitAggregateCopy(Dest.getAddr(), Src.getAggregateAddr(), E->getType(),
+                        Dest.isVolatile()|Src.isVolatileQualified());
 }
 
 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
@@ -241,15 +244,17 @@ void AggExprEmitter::EmitFinalDestCopy(const Expr *E, LValue Src, bool Ignore) {
 //                            Visitor Methods
 //===----------------------------------------------------------------------===//
 
+void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
+  EmitFinalDestCopy(e, CGF.getOpaqueLValueMapping(e));
+}
+
 void AggExprEmitter::VisitCastExpr(CastExpr *E) {
-  if (!DestPtr && E->getCastKind() != CK_Dynamic) {
+  if (Dest.isIgnored() && E->getCastKind() != CK_Dynamic) {
     Visit(E->getSubExpr());
     return;
   }
 
   switch (E->getCastKind()) {
-  default: assert(0 && "Unhandled cast kind!");
-
   case CK_Dynamic: {
     assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
     LValue LV = CGF.EmitCheckedLValue(E->getSubExpr());
@@ -259,8 +264,8 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     else
       CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
     
-    if (DestPtr)
-      CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");      
+    if (!Dest.isIgnored())
+      CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
     break;
   }
       
@@ -268,7 +273,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     // GCC union extension
     QualType Ty = E->getSubExpr()->getType();
     QualType PtrTy = CGF.getContext().getPointerType(Ty);
-    llvm::Value *CastPtr = Builder.CreateBitCast(DestPtr,
+    llvm::Value *CastPtr = Builder.CreateBitCast(Dest.getAddr(),
                                                  CGF.ConvertType(PtrTy));
     EmitInitializationToLValue(E->getSubExpr(), CGF.MakeAddrLValue(CastPtr, Ty),
                                Ty);
@@ -283,8 +288,15 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     break;
   }
 
-  // FIXME: Remove the CK_Unknown check here.
-  case CK_Unknown:
+  case CK_GetObjCProperty: {
+    LValue LV = CGF.EmitLValue(E->getSubExpr());
+    assert(LV.isPropertyRef());
+    RValue RV = CGF.EmitLoadOfPropertyRefLValue(LV, getReturnValueSlot());
+    EmitGCMove(E, RV);
+    break;
+  }
+
+  case CK_LValueToRValue: // hope for downstream optimization
   case CK_NoOp:
   case CK_UserDefinedConversion:
   case CK_ConstructorConversion:
@@ -293,10 +305,45 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
            "Implicit cast types must be compatible");
     Visit(E->getSubExpr());
     break;
-
+      
   case CK_LValueBitCast:
-    llvm_unreachable("there are no lvalue bit-casts on aggregates");
+    llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
     break;
+      
+  case CK_Dependent:
+  case CK_BitCast:
+  case CK_ArrayToPointerDecay:
+  case CK_FunctionToPointerDecay:
+  case CK_NullToPointer:
+  case CK_NullToMemberPointer:
+  case CK_BaseToDerivedMemberPointer:
+  case CK_DerivedToBaseMemberPointer:
+  case CK_MemberPointerToBoolean:
+  case CK_IntegralToPointer:
+  case CK_PointerToIntegral:
+  case CK_PointerToBoolean:
+  case CK_ToVoid:
+  case CK_VectorSplat:
+  case CK_IntegralCast:
+  case CK_IntegralToBoolean:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingToBoolean:
+  case CK_FloatingCast:
+  case CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
+  case CK_ObjCObjectLValueCast:
+  case CK_FloatingRealToComplex:
+  case CK_FloatingComplexToReal:
+  case CK_FloatingComplexToBoolean:
+  case CK_FloatingComplexCast:
+  case CK_FloatingComplexToIntegralComplex:
+  case CK_IntegralRealToComplex:
+  case CK_IntegralComplexToReal:
+  case CK_IntegralComplexToBoolean:
+  case CK_IntegralComplexCast:
+  case CK_IntegralComplexToFloatingComplex:
+    llvm_unreachable("cast kind invalid for aggregate types");
   }
 }
 
@@ -316,24 +363,18 @@ void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
 }
 
 void AggExprEmitter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
-  RValue RV = CGF.EmitObjCPropertyGet(E, getReturnValueSlot());
-  EmitGCMove(E, RV);
-}
-
-void AggExprEmitter::VisitObjCImplicitSetterGetterRefExpr(
-                                   ObjCImplicitSetterGetterRefExpr *E) {
-  RValue RV = CGF.EmitObjCPropertyGet(E, getReturnValueSlot());
-  EmitGCMove(E, RV);
+  llvm_unreachable("direct property access not surrounded by "
+                   "lvalue-to-rvalue cast");
 }
 
 void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
-  CGF.EmitAnyExpr(E->getLHS(), 0, false, true);
-  CGF.EmitAggExpr(E->getRHS(), DestPtr, VolatileDest,
-                  /*IgnoreResult=*/false, IsInitializer);
+  CGF.EmitIgnoredExpr(E->getLHS());
+  Visit(E->getRHS());
 }
 
 void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
-  CGF.EmitCompoundStmt(*E->getSubStmt(), true, DestPtr, VolatileDest);
+  CodeGenFunction::StmtExprEvaluation eval(CGF);
+  CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);
 }
 
 void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
@@ -355,64 +396,62 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
                                                  E->getRHS()->getType())
          && "Invalid assignment");
+
+  // FIXME:  __block variables need the RHS evaluated first!
   LValue LHS = CGF.EmitLValue(E->getLHS());
 
   // We have to special case property setters, otherwise we must have
   // a simple lvalue (no aggregates inside vectors, bitfields).
   if (LHS.isPropertyRef()) {
-    llvm::Value *AggLoc = DestPtr;
-    if (!AggLoc)
-      AggLoc = CGF.CreateMemTemp(E->getRHS()->getType());
-    CGF.EmitAggExpr(E->getRHS(), AggLoc, VolatileDest);
-    CGF.EmitObjCPropertySet(LHS.getPropertyRefExpr(),
-                            RValue::getAggregate(AggLoc, VolatileDest));
-  } else if (LHS.isKVCRef()) {
-    llvm::Value *AggLoc = DestPtr;
-    if (!AggLoc)
-      AggLoc = CGF.CreateMemTemp(E->getRHS()->getType());
-    CGF.EmitAggExpr(E->getRHS(), AggLoc, VolatileDest);
-    CGF.EmitObjCPropertySet(LHS.getKVCRefExpr(),
-                            RValue::getAggregate(AggLoc, VolatileDest));
+    AggValueSlot Slot = EnsureSlot(E->getRHS()->getType());
+    CGF.EmitAggExpr(E->getRHS(), Slot);
+    CGF.EmitStoreThroughPropertyRefLValue(Slot.asRValue(), LHS);
   } else {
-    bool RequiresGCollection = false;
+    bool GCollection = false;
     if (CGF.getContext().getLangOptions().getGCMode())
-      RequiresGCollection = TypeRequiresGCollection(E->getLHS()->getType());
+      GCollection = TypeRequiresGCollection(E->getLHS()->getType());
 
     // Codegen the RHS so that it stores directly into the LHS.
-    CGF.EmitAggExpr(E->getRHS(), LHS.getAddress(), LHS.isVolatileQualified(),
-                    false, false, RequiresGCollection);
+    AggValueSlot LHSSlot = AggValueSlot::forLValue(LHS, true, 
+                                                   GCollection);
+    CGF.EmitAggExpr(E->getRHS(), LHSSlot, false);
     EmitFinalDestCopy(E, LHS, true);
   }
 }
 
-void AggExprEmitter::VisitConditionalOperator(const ConditionalOperator *E) {
-  if (!E->getLHS()) {
-    CGF.ErrorUnsupported(E, "conditional operator with missing LHS");
-    return;
-  }
-
+void AggExprEmitter::
+VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
 
+  // Bind the common expression if necessary.
+  CodeGenFunction::OpaqueValueMapping binding(CGF, E);
+
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
 
-  CGF.BeginConditionalBranch();
+  // Save whether the destination's lifetime is externally managed.
+  bool DestLifetimeManaged = Dest.isLifetimeExternallyManaged();
+
+  eval.begin(CGF);
   CGF.EmitBlock(LHSBlock);
+  Visit(E->getTrueExpr());
+  eval.end(CGF);
 
-  // Handle the GNU extension for missing LHS.
-  assert(E->getLHS() && "Must have LHS for aggregate value");
+  assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
+  CGF.Builder.CreateBr(ContBlock);
 
-  Visit(E->getLHS());
-  CGF.EndConditionalBranch();
-  CGF.EmitBranch(ContBlock);
+  // If the result of an agg expression is unused, then the emission
+  // of the LHS might need to create a destination slot.  That's fine
+  // with us, and we can safely emit the RHS into the same slot, but
+  // we shouldn't claim that its lifetime is externally managed.
+  Dest.setLifetimeExternallyManaged(DestLifetimeManaged);
 
-  CGF.BeginConditionalBranch();
+  eval.begin(CGF);
   CGF.EmitBlock(RHSBlock);
-
-  Visit(E->getRHS());
-  CGF.EndConditionalBranch();
-  CGF.EmitBranch(ContBlock);
+  Visit(E->getFalseExpr());
+  eval.end(CGF);
 
   CGF.EmitBlock(ContBlock);
 }
@@ -434,65 +473,78 @@ void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
 }
 
 void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
-  llvm::Value *Val = DestPtr;
-
-  if (!Val) {
-    // Create a temporary variable.
-    Val = CGF.CreateMemTemp(E->getType(), "tmp");
-
-    // FIXME: volatile
-    CGF.EmitAggExpr(E->getSubExpr(), Val, false);
-  } else
-    Visit(E->getSubExpr());
-
-  // Don't make this a live temporary if we're emitting an initializer expr.
-  if (!IsInitializer)
-    CGF.EmitCXXTemporary(E->getTemporary(), Val);
+  // Ensure that we have a slot, but if we already do, remember
+  // whether its lifetime was externally managed.
+  bool WasManaged = Dest.isLifetimeExternallyManaged();
+  Dest = EnsureSlot(E->getType());
+  Dest.setLifetimeExternallyManaged();
+
+  Visit(E->getSubExpr());
+
+  // Set up the temporary's destructor if its lifetime wasn't already
+  // being managed.
+  if (!WasManaged)
+    CGF.EmitCXXTemporary(E->getTemporary(), Dest.getAddr());
 }
 
 void
 AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
-  llvm::Value *Val = DestPtr;
-
-  if (!Val) // Create a temporary variable.
-    Val = CGF.CreateMemTemp(E->getType(), "tmp");
-
-  CGF.EmitCXXConstructExpr(Val, E);
+  AggValueSlot Slot = EnsureSlot(E->getType());
+  CGF.EmitCXXConstructExpr(E, Slot);
 }
 
-void AggExprEmitter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
-  llvm::Value *Val = DestPtr;
-
-  CGF.EmitCXXExprWithTemporaries(E, Val, VolatileDest, IsInitializer);
+void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
+  CGF.EmitExprWithCleanups(E, Dest);
 }
 
 void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
-  llvm::Value *Val = DestPtr;
-
-  if (!Val) {
-    // Create a temporary variable.
-    Val = CGF.CreateMemTemp(E->getType(), "tmp");
-  }
-  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Val, E->getType()),
-                                 E->getType());
+  QualType T = E->getType();
+  AggValueSlot Slot = EnsureSlot(T);
+  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T), T);
 }
 
 void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
-  llvm::Value *Val = DestPtr;
+  QualType T = E->getType();
+  AggValueSlot Slot = EnsureSlot(T);
+  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddr(), T), T);
+}
 
-  if (!Val) {
-    // Create a temporary variable.
-    Val = CGF.CreateMemTemp(E->getType(), "tmp");
-  }
-  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Val, E->getType()),
-                                 E->getType());
+/// isSimpleZero - If emitting this value will obviously just cause a store of
+/// zero to memory, return true.  This can return false if uncertain, so it just
+/// handles simple cases.
+static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
+  // (0)
+  if (const ParenExpr *PE = dyn_cast<ParenExpr>(E))
+    return isSimpleZero(PE->getSubExpr(), CGF);
+  // 0
+  if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E))
+    return IL->getValue() == 0;
+  // +0.0
+  if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E))
+    return FL->getValue().isPosZero();
+  // int()
+  if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) &&
+      CGF.getTypes().isZeroInitializable(E->getType()))
+    return true;
+  // (int*)0 - Null pointer expressions.
+  if (const CastExpr *ICE = dyn_cast<CastExpr>(E))
+    return ICE->getCastKind() == CK_NullToPointer;
+  // '\0'
+  if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E))
+    return CL->getValue() == 0;
+  
+  // Otherwise, hard case: conservatively return false.
+  return false;
 }
 
+
 void 
 AggExprEmitter::EmitInitializationToLValue(Expr* E, LValue LV, QualType T) {
   // FIXME: Ignore result?
   // FIXME: Are initializers affected by volatile?
-  if (isa<ImplicitValueInitExpr>(E)) {
+  if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
+    // Storing "i32 0" to a zero'd memory location is a noop.
+  } else if (isa<ImplicitValueInitExpr>(E)) {
     EmitNullInitializationToLValue(LV, T);
   } else if (T->isReferenceType()) {
     RValue RV = CGF.EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
@@ -500,13 +552,19 @@ AggExprEmitter::EmitInitializationToLValue(Expr* E, LValue LV, QualType T) {
   } else if (T->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(E, LV.getAddress(), false);
   } else if (CGF.hasAggregateLLVMType(T)) {
-    CGF.EmitAnyExpr(E, LV.getAddress(), false);
+    CGF.EmitAggExpr(E, AggValueSlot::forAddr(LV.getAddress(), false, true,
+                                             false, Dest.isZeroed()));
   } else {
-    CGF.EmitStoreThroughLValue(CGF.EmitAnyExpr(E), LV, T);
+    CGF.EmitStoreThroughLValue(RValue::get(CGF.EmitScalarExpr(E)), LV, T);
   }
 }
 
 void AggExprEmitter::EmitNullInitializationToLValue(LValue LV, QualType T) {
+  // If the destination slot is already zeroed out before the aggregate is
+  // copied into it, we don't have to emit any zeros here.
+  if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T))
+    return;
+  
   if (!CGF.hasAggregateLLVMType(T)) {
     // For non-aggregates, we can store zero
     llvm::Value *Null = llvm::Constant::getNullValue(CGF.ConvertType(T));
@@ -534,9 +592,10 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     return;
   }
 #endif
-  if (E->hadArrayRangeDesignator()) {
+  if (E->hadArrayRangeDesignator())
     CGF.ErrorUnsupported(E, "GNU array range designator extension");
-  }
+
+  llvm::Value *DestPtr = Dest.getAddr();
 
   // Handle initialization of an array.
   if (E->getType()->isArrayType()) {
@@ -563,13 +622,27 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     // FIXME: were we intentionally ignoring address spaces and GC attributes?
 
     for (uint64_t i = 0; i != NumArrayElements; ++i) {
+      // If we're done emitting initializers and the destination is known-zeroed
+      // then we're done.
+      if (i == NumInitElements &&
+          Dest.isZeroed() &&
+          CGF.getTypes().isZeroInitializable(ElementType))
+        break;
+
       llvm::Value *NextVal = Builder.CreateStructGEP(DestPtr, i, ".array");
       LValue LV = CGF.MakeAddrLValue(NextVal, ElementType);
+      
       if (i < NumInitElements)
         EmitInitializationToLValue(E->getInit(i), LV, ElementType);
-
       else
         EmitNullInitializationToLValue(LV, ElementType);
+      
+      // If the GEP didn't get used because of a dead zero init or something
+      // else, clean it up for -O0 builds and general tidiness.
+      if (llvm::GetElementPtrInst *GEP =
+            dyn_cast<llvm::GetElementPtrInst>(NextVal))
+        if (GEP->use_empty())
+          GEP->eraseFromParent();
     }
     return;
   }
@@ -583,16 +656,6 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
   unsigned NumInitElements = E->getNumInits();
   RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl();
   
-  // If we're initializing the whole aggregate, just do it in place.
-  // FIXME: This is a hack around an AST bug (PR6537).
-  if (NumInitElements == 1 && E->getType() == E->getInit(0)->getType()) {
-    EmitInitializationToLValue(E->getInit(0),
-                               CGF.MakeAddrLValue(DestPtr, E->getType()),
-                               E->getType());
-    return;
-  }
-  
-  
   if (E->getType()->isUnionType()) {
     // Only initialize one field of a union. The field itself is
     // specified by the initializer list.
@@ -612,13 +675,13 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 
     // FIXME: volatility
     FieldDecl *Field = E->getInitializedFieldInUnion();
-    LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, Field, 0);
 
+    LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, Field, 0);
     if (NumInitElements) {
       // Store the initializer into the field
       EmitInitializationToLValue(E->getInit(0), FieldLoc, Field->getType());
     } else {
-      // Default-initialize to null
+      // Default-initialize to null.
       EmitNullInitializationToLValue(FieldLoc, Field->getType());
     }
 
@@ -638,10 +701,16 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     if (Field->isUnnamedBitfield())
       continue;
 
+    // Don't emit GEP before a noop store of zero.
+    if (CurInitVal == NumInitElements && Dest.isZeroed() &&
+        CGF.getTypes().isZeroInitializable(E->getType()))
+      break;
+    
     // FIXME: volatility
     LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, *Field, 0);
     // We never generate write-barries for initialized fields.
     FieldLoc.setNonGC(true);
+    
     if (CurInitVal < NumInitElements) {
       // Store the initializer into the field.
       EmitInitializationToLValue(E->getInit(CurInitVal++), FieldLoc,
@@ -650,6 +719,14 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
       // We're out of initalizers; default-initialize to null
       EmitNullInitializationToLValue(FieldLoc, Field->getType());
     }
+    
+    // If the GEP didn't get used because of a dead zero init or something
+    // else, clean it up for -O0 builds and general tidiness.
+    if (FieldLoc.isSimple())
+      if (llvm::GetElementPtrInst *GEP =
+            dyn_cast<llvm::GetElementPtrInst>(FieldLoc.getAddress()))
+        if (GEP->use_empty())
+          GEP->eraseFromParent();
   }
 }
 
@@ -657,31 +734,126 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 //                        Entry Points into this File
 //===----------------------------------------------------------------------===//
 
+/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
+/// non-zero bytes that will be stored when outputting the initializer for the
+/// specified initializer expression.
+static uint64_t GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
+  if (const ParenExpr *PE = dyn_cast<ParenExpr>(E))
+    return GetNumNonZeroBytesInInit(PE->getSubExpr(), CGF);
+
+  // 0 and 0.0 won't require any non-zero stores!
+  if (isSimpleZero(E, CGF)) return 0;
+
+  // If this is an initlist expr, sum up the size of sizes of the (present)
+  // elements.  If this is something weird, assume the whole thing is non-zero.
+  const InitListExpr *ILE = dyn_cast<InitListExpr>(E);
+  if (ILE == 0 || !CGF.getTypes().isZeroInitializable(ILE->getType()))
+    return CGF.getContext().getTypeSize(E->getType())/8;
+  
+  // InitListExprs for structs have to be handled carefully.  If there are
+  // reference members, we need to consider the size of the reference, not the
+  // referencee.  InitListExprs for unions and arrays can't have references.
+  if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
+    if (!RT->isUnionType()) {
+      RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl();
+      uint64_t NumNonZeroBytes = 0;
+      
+      unsigned ILEElement = 0;
+      for (RecordDecl::field_iterator Field = SD->field_begin(),
+           FieldEnd = SD->field_end(); Field != FieldEnd; ++Field) {
+        // We're done once we hit the flexible array member or run out of
+        // InitListExpr elements.
+        if (Field->getType()->isIncompleteArrayType() ||
+            ILEElement == ILE->getNumInits())
+          break;
+        if (Field->isUnnamedBitfield())
+          continue;
+
+        const Expr *E = ILE->getInit(ILEElement++);
+        
+        // Reference values are always non-null and have the width of a pointer.
+        if (Field->getType()->isReferenceType())
+          NumNonZeroBytes += CGF.getContext().Target.getPointerWidth(0);
+        else
+          NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
+      }
+      
+      return NumNonZeroBytes;
+    }
+  }
+  
+  
+  uint64_t NumNonZeroBytes = 0;
+  for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
+    NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF);
+  return NumNonZeroBytes;
+}
+
+/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
+/// zeros in it, emit a memset and avoid storing the individual zeros.
+///
+static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
+                                     CodeGenFunction &CGF) {
+  // If the slot is already known to be zeroed, nothing to do.  Don't mess with
+  // volatile stores.
+  if (Slot.isZeroed() || Slot.isVolatile() || Slot.getAddr() == 0) return;
+  
+  // If the type is 16-bytes or smaller, prefer individual stores over memset.
+  std::pair<uint64_t, unsigned> TypeInfo =
+    CGF.getContext().getTypeInfo(E->getType());
+  if (TypeInfo.first/8 <= 16)
+    return;
+
+  // Check to see if over 3/4 of the initializer are known to be zero.  If so,
+  // we prefer to emit memset + individual stores for the rest.
+  uint64_t NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
+  if (NumNonZeroBytes*4 > TypeInfo.first/8)
+    return;
+  
+  // Okay, it seems like a good idea to use an initial memset, emit the call.
+  llvm::Constant *SizeVal = CGF.Builder.getInt64(TypeInfo.first/8);
+  unsigned Align = TypeInfo.second/8;
+
+  llvm::Value *Loc = Slot.getAddr();
+  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  
+  Loc = CGF.Builder.CreateBitCast(Loc, BP);
+  CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, Align, false);
+  
+  // Tell the AggExprEmitter that the slot is known zero.
+  Slot.setZeroed();
+}
+
+
+
+
 /// EmitAggExpr - Emit the computation of the specified expression of aggregate
 /// type.  The result is computed into DestPtr.  Note that if DestPtr is null,
 /// the value of the aggregate expression is not needed.  If VolatileDest is
 /// true, DestPtr cannot be 0.
+///
+/// \param IsInitializer - true if this evaluation is initializing an
+/// object whose lifetime is already being managed.
 //
 // FIXME: Take Qualifiers object.
-void CodeGenFunction::EmitAggExpr(const Expr *E, llvm::Value *DestPtr,
-                                  bool VolatileDest, bool IgnoreResult,
-                                  bool IsInitializer,
-                                  bool RequiresGCollection) {
+void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot,
+                                  bool IgnoreResult) {
   assert(E && hasAggregateLLVMType(E->getType()) &&
          "Invalid aggregate expression to emit");
-  assert ((DestPtr != 0 || VolatileDest == false)
-          && "volatile aggregate can't be 0");
+  assert((Slot.getAddr() != 0 || Slot.isIgnored()) &&
+         "slot has bits but no address");
 
-  AggExprEmitter(*this, DestPtr, VolatileDest, IgnoreResult, IsInitializer,
-                 RequiresGCollection)
-    .Visit(const_cast<Expr*>(E));
+  // Optimize the slot if possible.
+  CheckAggExprForMemSetUse(Slot, E, *this);
+ 
+  AggExprEmitter(*this, Slot, IgnoreResult).Visit(const_cast<Expr*>(E));
 }
 
 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
   assert(hasAggregateLLVMType(E->getType()) && "Invalid argument!");
   llvm::Value *Temp = CreateMemTemp(E->getType());
   LValue LV = MakeAddrLValue(Temp, E->getType());
-  EmitAggExpr(E, Temp, LV.isVolatileQualified());
+  EmitAggExpr(E, AggValueSlot::forAddr(Temp, LV.isVolatileQualified(), false));
   return LV;
 }
 
@@ -734,12 +906,12 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
 
   const llvm::PointerType *DPT = cast<llvm::PointerType>(DestPtr->getType());
   const llvm::Type *DBP =
-    llvm::Type::getInt8PtrTy(VMContext, DPT->getAddressSpace());
+    llvm::Type::getInt8PtrTy(getLLVMContext(), DPT->getAddressSpace());
   DestPtr = Builder.CreateBitCast(DestPtr, DBP, "tmp");
 
   const llvm::PointerType *SPT = cast<llvm::PointerType>(SrcPtr->getType());
   const llvm::Type *SBP =
-    llvm::Type::getInt8PtrTy(VMContext, SPT->getAddressSpace());
+    llvm::Type::getInt8PtrTy(getLLVMContext(), SPT->getAddressSpace());
   SrcPtr = Builder.CreateBitCast(SrcPtr, SBP, "tmp");
 
   if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
@@ -766,11 +938,7 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
     }
   }
   
-  Builder.CreateCall5(CGM.getMemCpyFn(DestPtr->getType(), SrcPtr->getType(),
-                                      IntPtrTy),
-                      DestPtr, SrcPtr,
-                      // TypeInfo.first describes size in bits.
-                      llvm::ConstantInt::get(IntPtrTy, TypeInfo.first/8),
-                      Builder.getInt32(TypeInfo.second/8),
-                      Builder.getInt1(isVolatile));
+  Builder.CreateMemCpy(DestPtr, SrcPtr,
+                       llvm::ConstantInt::get(IntPtrTy, TypeInfo.first/8),
+                       TypeInfo.second/8, isVolatile);
 }
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 9a98281771fa..bba7864bff98 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -11,9 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Frontend/CodeGenOptions.h"
 #include "CodeGenFunction.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
+#include "CGDebugInfo.h"
 #include "llvm/Intrinsics.h"
 using namespace clang;
 using namespace CodeGen;
@@ -51,9 +53,65 @@ RValue CodeGenFunction::EmitCXXMemberCall(const CXXMethodDecl *MD,
                   Callee, ReturnValue, Args, MD);
 }
 
+static const CXXRecordDecl *getMostDerivedClassDecl(const Expr *Base) {
+  const Expr *E = Base;
+  
+  while (true) {
+    E = E->IgnoreParens();
+    if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
+      if (CE->getCastKind() == CK_DerivedToBase || 
+          CE->getCastKind() == CK_UncheckedDerivedToBase ||
+          CE->getCastKind() == CK_NoOp) {
+        E = CE->getSubExpr();
+        continue;
+      }
+    }
+
+    break;
+  }
+
+  QualType DerivedType = E->getType();
+  if (const PointerType *PTy = DerivedType->getAs<PointerType>())
+    DerivedType = PTy->getPointeeType();
+
+  return cast<CXXRecordDecl>(DerivedType->castAs<RecordType>()->getDecl());
+}
+
 /// canDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given
 /// expr can be devirtualized.
-static bool canDevirtualizeMemberFunctionCalls(const Expr *Base) {
+static bool canDevirtualizeMemberFunctionCalls(ASTContext &Context,
+                                               const Expr *Base, 
+                                               const CXXMethodDecl *MD) {
+  
+  // When building with -fapple-kext, all calls must go through the vtable since
+  // the kernel linker can do runtime patching of vtables.
+  if (Context.getLangOptions().AppleKext)
+    return false;
+
+  // If the most derived class is marked final, we know that no subclass can
+  // override this member function and so we can devirtualize it. For example:
+  //
+  // struct A { virtual void f(); }
+  // struct B final : A { };
+  //
+  // void f(B *b) {
+  //   b->f();
+  // }
+  //
+  const CXXRecordDecl *MostDerivedClassDecl = getMostDerivedClassDecl(Base);
+  if (MostDerivedClassDecl->hasAttr<FinalAttr>())
+    return true;
+
+  // If the member function is marked 'final', we know that it can't be
+  // overridden and can therefore devirtualize it.
+  if (MD->hasAttr<FinalAttr>())
+    return true;
+
+  // Similarly, if the class itself is marked 'final' it can't be overridden
+  // and we can therefore devirtualize the member function call.
+  if (MD->getParent()->hasAttr<FinalAttr>())
+    return true;
+
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Base)) {
     if (const VarDecl *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
       // This is a record decl. We know the type and can devirtualize it.
@@ -74,11 +132,13 @@ static bool canDevirtualizeMemberFunctionCalls(const Expr *Base) {
   // Check if this is a call expr that returns a record type.
   if (const CallExpr *CE = dyn_cast<CallExpr>(Base))
     return CE->getCallReturnType()->isRecordType();
-  
+
   // We can't devirtualize the call.
   return false;
 }
 
+// Note: This function also emit constructor calls to support a MSVC
+// extensions allowing explicit constructor function call.
 RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
                                               ReturnValueSlot ReturnValue) {
   if (isa<BinaryOperator>(CE->getCallee()->IgnoreParens())) 
@@ -87,6 +147,16 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   const MemberExpr *ME = cast<MemberExpr>(CE->getCallee()->IgnoreParens());
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
 
+  CGDebugInfo *DI = getDebugInfo();
+  if (DI && CGM.getCodeGenOpts().LimitDebugInfo
+      && !isa<CallExpr>(ME->getBase())) {
+    QualType PQTy = ME->getBase()->IgnoreParenImpCasts()->getType();
+    if (const PointerType * PTy = dyn_cast<PointerType>(PQTy)) {
+      DI->getOrCreateRecordType(PTy->getPointeeType(), 
+                                MD->getParent()->getLocation());
+    }
+  }
+
   if (MD->isStatic()) {
     // The method is static, emit it as we would a regular call.
     llvm::Value *Callee = CGM.GetAddrOfFunction(MD);
@@ -98,32 +168,47 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   llvm::Value *This;
   if (ME->isArrow())
     This = EmitScalarExpr(ME->getBase());
-  else {
-    LValue BaseLV = EmitLValue(ME->getBase());
-    This = BaseLV.getAddress();
-  }
+  else
+    This = EmitLValue(ME->getBase()).getAddress();
 
   if (MD->isTrivial()) {
     if (isa<CXXDestructorDecl>(MD)) return RValue::get(0);
-
-    assert(MD->isCopyAssignment() && "unknown trivial member function");
-    // We don't like to generate the trivial copy assignment operator when
-    // it isn't necessary; just produce the proper effect here.
-    llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
-    EmitAggregateCopy(This, RHS, CE->getType());
-    return RValue::get(This);
+    if (isa<CXXConstructorDecl>(MD) && 
+        cast<CXXConstructorDecl>(MD)->isDefaultConstructor())
+      return RValue::get(0);
+
+    if (MD->isCopyAssignmentOperator()) {
+      // We don't like to generate the trivial copy assignment operator when
+      // it isn't necessary; just produce the proper effect here.
+      llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
+      EmitAggregateCopy(This, RHS, CE->getType());
+      return RValue::get(This);
+    }
+    
+    if (isa<CXXConstructorDecl>(MD) && 
+        cast<CXXConstructorDecl>(MD)->isCopyConstructor()) {
+      llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
+      EmitSynthesizedCXXCopyCtorCall(cast<CXXConstructorDecl>(MD), This, RHS,
+                                     CE->arg_begin(), CE->arg_end());
+      return RValue::get(This);
+    }
+    llvm_unreachable("unknown trivial member function");
   }
 
   // Compute the function type we're calling.
-  const CGFunctionInfo &FInfo =
-    (isa<CXXDestructorDecl>(MD)
-     ? CGM.getTypes().getFunctionInfo(cast<CXXDestructorDecl>(MD),
-                                      Dtor_Complete)
-     : CGM.getTypes().getFunctionInfo(MD));
+  const CGFunctionInfo *FInfo = 0;
+  if (isa<CXXDestructorDecl>(MD))
+    FInfo = &CGM.getTypes().getFunctionInfo(cast<CXXDestructorDecl>(MD),
+                                           Dtor_Complete);
+  else if (isa<CXXConstructorDecl>(MD))
+    FInfo = &CGM.getTypes().getFunctionInfo(cast<CXXConstructorDecl>(MD),
+                                            Ctor_Complete);
+  else
+    FInfo = &CGM.getTypes().getFunctionInfo(MD);
 
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
   const llvm::Type *Ty
-    = CGM.getTypes().GetFunctionType(FInfo, FPT->isVariadic());
+    = CGM.getTypes().GetFunctionType(*FInfo, FPT->isVariadic());
 
   // C++ [class.virtual]p12:
   //   Explicit qualification with the scope operator (5.1) suppresses the
@@ -131,20 +216,34 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   //
   // We also don't emit a virtual call if the base expression has a record type
   // because then we know what the type is.
-  bool UseVirtualCall = MD->isVirtual() && !ME->hasQualifier()
-                     && !canDevirtualizeMemberFunctionCalls(ME->getBase());
-
+  bool UseVirtualCall;
+  UseVirtualCall = MD->isVirtual() && !ME->hasQualifier()
+                   && !canDevirtualizeMemberFunctionCalls(getContext(),
+                                                          ME->getBase(), MD);
   llvm::Value *Callee;
   if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(MD)) {
     if (UseVirtualCall) {
       Callee = BuildVirtualCall(Dtor, Dtor_Complete, This, Ty);
     } else {
-      Callee = CGM.GetAddrOfFunction(GlobalDecl(Dtor, Dtor_Complete), Ty);
+      if (getContext().getLangOptions().AppleKext &&
+          MD->isVirtual() &&
+          ME->hasQualifier())
+        Callee = BuildAppleKextVirtualCall(MD, ME->getQualifier(), Ty);
+      else
+        Callee = CGM.GetAddrOfFunction(GlobalDecl(Dtor, Dtor_Complete), Ty);
     }
+  } else if (const CXXConstructorDecl *Ctor =
+               dyn_cast<CXXConstructorDecl>(MD)) {
+    Callee = CGM.GetAddrOfFunction(GlobalDecl(Ctor, Ctor_Complete), Ty);
   } else if (UseVirtualCall) {
-    Callee = BuildVirtualCall(MD, This, Ty); 
+      Callee = BuildVirtualCall(MD, This, Ty); 
   } else {
-    Callee = CGM.GetAddrOfFunction(MD, Ty);
+    if (getContext().getLangOptions().AppleKext &&
+        MD->isVirtual() &&
+        ME->hasQualifier())
+      Callee = BuildAppleKextVirtualCall(MD, ME->getQualifier(), Ty);
+    else 
+      Callee = CGM.GetAddrOfFunction(MD, Ty);
   }
 
   return EmitCXXMemberCall(MD, Callee, ReturnValue, This, /*VTT=*/0,
@@ -180,7 +279,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
 
   // Ask the ABI to load the callee.  Note that This is modified.
   llvm::Value *Callee =
-    CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(CGF, This, MemFnPtr, MPT);
+    CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, This, MemFnPtr, MPT);
   
   CallArgList Args;
 
@@ -203,29 +302,14 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
                                                ReturnValueSlot ReturnValue) {
   assert(MD->isInstance() &&
          "Trying to emit a member call expr on a static method!");
-  if (MD->isCopyAssignment()) {
+  LValue LV = EmitLValue(E->getArg(0));
+  llvm::Value *This = LV.getAddress();
+
+  if (MD->isCopyAssignmentOperator()) {
     const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(MD->getDeclContext());
     if (ClassDecl->hasTrivialCopyAssignment()) {
       assert(!ClassDecl->hasUserDeclaredCopyAssignment() &&
              "EmitCXXOperatorMemberCallExpr - user declared copy assignment");
-      LValue LV = EmitLValue(E->getArg(0));
-      llvm::Value *This;
-      if (LV.isPropertyRef() || LV.isKVCRef()) {
-        llvm::Value *AggLoc  = CreateMemTemp(E->getArg(1)->getType());
-        EmitAggExpr(E->getArg(1), AggLoc, false /*VolatileDest*/);
-        if (LV.isPropertyRef())
-          EmitObjCPropertySet(LV.getPropertyRefExpr(),
-                              RValue::getAggregate(AggLoc, 
-                                                   false /*VolatileDest*/));
-        else
-          EmitObjCPropertySet(LV.getKVCRefExpr(),
-                              RValue::getAggregate(AggLoc, 
-                                                   false /*VolatileDest*/));
-        return RValue::getAggregate(0, false);
-      }
-      else
-        This = LV.getAddress();
-      
       llvm::Value *Src = EmitLValue(E->getArg(1)).getAddress();
       QualType Ty = E->getType();
       EmitAggregateCopy(This, Src, Ty);
@@ -237,21 +321,10 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
   const llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
                                    FPT->isVariadic());
-  LValue LV = EmitLValue(E->getArg(0));
-  llvm::Value *This;
-  if (LV.isPropertyRef() || LV.isKVCRef()) {
-    QualType QT = E->getArg(0)->getType();
-    RValue RV = 
-      LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT)
-                         : EmitLoadOfKVCRefLValue(LV, QT);
-    assert (!RV.isScalar() && "EmitCXXOperatorMemberCallExpr");
-    This = RV.getAggregateAddr();
-  }
-  else
-    This = LV.getAddress();
-
   llvm::Value *Callee;
-  if (MD->isVirtual() && !canDevirtualizeMemberFunctionCalls(E->getArg(0)))
+  if (MD->isVirtual() && 
+      !canDevirtualizeMemberFunctionCalls(getContext(),
+                                           E->getArg(0), MD))
     Callee = BuildVirtualCall(MD, This, Ty);
   else
     Callee = CGM.GetAddrOfFunction(MD, Ty);
@@ -261,28 +334,29 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
 }
 
 void
-CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
-                                      const CXXConstructExpr *E) {
-  assert(Dest && "Must have a destination!");
+CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
+                                      AggValueSlot Dest) {
+  assert(!Dest.isIgnored() && "Must have a destination!");
   const CXXConstructorDecl *CD = E->getConstructor();
   
   // If we require zero initialization before (or instead of) calling the
   // constructor, as can be the case with a non-user-provided default
   // constructor, emit the zero initialization now.
   if (E->requiresZeroInitialization())
-    EmitNullInitialization(Dest, E->getType());
-
+    EmitNullInitialization(Dest.getAddr(), E->getType());
   
   // If this is a call to a trivial default constructor, do nothing.
   if (CD->isTrivial() && CD->isDefaultConstructor())
     return;
   
-  // Code gen optimization to eliminate copy constructor and return
-  // its first argument instead, if in fact that argument is a temporary 
-  // object.
+  // Elide the constructor if we're constructing from a temporary.
+  // The temporary check is required because Sema sets this on NRVO
+  // returns.
   if (getContext().getLangOptions().ElideConstructors && E->isElidable()) {
-    if (const Expr *Arg = E->getArg(0)->getTemporaryObject()) {
-      EmitAggExpr(Arg, Dest, false);
+    assert(getContext().hasSameUnqualifiedType(E->getType(),
+                                               E->getArg(0)->getType()));
+    if (E->getArg(0)->isTemporaryObject(getContext(), CD->getParent())) {
+      EmitAggExpr(E->getArg(0), Dest);
       return;
     }
   }
@@ -294,7 +368,7 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
     const llvm::Type *BasePtr = ConvertType(BaseElementTy);
     BasePtr = llvm::PointerType::getUnqual(BasePtr);
     llvm::Value *BaseAddrPtr =
-      Builder.CreateBitCast(Dest, BasePtr);
+      Builder.CreateBitCast(Dest.getAddr(), BasePtr);
     
     EmitCXXAggrConstructorCall(CD, Array, BaseAddrPtr, 
                                E->arg_begin(), E->arg_end());
@@ -307,11 +381,36 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
       E->getConstructionKind() == CXXConstructExpr::CK_VirtualBase;
     
     // Call the constructor.
-    EmitCXXConstructorCall(CD, Type, ForVirtualBase, Dest,
+    EmitCXXConstructorCall(CD, Type, ForVirtualBase, Dest.getAddr(),
                            E->arg_begin(), E->arg_end());
   }
 }
 
+void
+CodeGenFunction::EmitSynthesizedCXXCopyCtor(llvm::Value *Dest, 
+                                            llvm::Value *Src,
+                                            const Expr *Exp) {
+  if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Exp))
+    Exp = E->getSubExpr();
+  assert(isa<CXXConstructExpr>(Exp) && 
+         "EmitSynthesizedCXXCopyCtor - unknown copy ctor expr");
+  const CXXConstructExpr* E = cast<CXXConstructExpr>(Exp);
+  const CXXConstructorDecl *CD = E->getConstructor();
+  RunCleanupsScope Scope(*this);
+  
+  // If we require zero initialization before (or instead of) calling the
+  // constructor, as can be the case with a non-user-provided default
+  // constructor, emit the zero initialization now.
+  // FIXME. Do I still need this for a copy ctor synthesis?
+  if (E->requiresZeroInitialization())
+    EmitNullInitialization(Dest, E->getType());
+  
+  assert(!getContext().getAsConstantArrayType(E->getType())
+         && "EmitSynthesizedCXXCopyCtor - Copied-in Array");
+  EmitSynthesizedCXXCopyCtorCall(CD, Dest, Src,
+                                 E->arg_begin(), E->arg_end());
+}
+
 /// Check whether the given operator new[] is the global placement
 /// operator new[].
 static bool IsPlacementOperatorNewArray(ASTContext &Ctx,
@@ -341,7 +440,7 @@ static CharUnits CalculateCookiePadding(CodeGenFunction &CGF,
   if (IsPlacementOperatorNewArray(CGF.getContext(), OperatorNew))
     return CharUnits::Zero();
 
-  return CGF.CGM.getCXXABI().GetArrayCookieSize(E->getAllocatedType());
+  return CGF.CGM.getCXXABI().GetArrayCookieSize(E);
 }
 
 static llvm::Value *EmitCXXNewAllocSize(ASTContext &Context,
@@ -387,7 +486,7 @@ static llvm::Value *EmitCXXNewAllocSize(ASTContext &Context,
     unsigned SizeWidth = NEC.getBitWidth();
 
     // Determine if there is an overflow here by doing an extended multiply.
-    NEC.zext(SizeWidth*2);
+    NEC = NEC.zext(SizeWidth*2);
     llvm::APInt SC(SizeWidth*2, TypeSize.getQuantity());
     SC *= NEC;
 
@@ -396,8 +495,7 @@ static llvm::Value *EmitCXXNewAllocSize(ASTContext &Context,
       // overflow's already happened because SizeWithoutCookie isn't
       // used if the allocator returns null or throws, as it should
       // always do on an overflow.
-      llvm::APInt SWC = SC;
-      SWC.trunc(SizeWidth);
+      llvm::APInt SWC = SC.trunc(SizeWidth);
       SizeWithoutCookie = llvm::ConstantInt::get(SizeTy, SWC);
 
       // Add the cookie size.
@@ -405,7 +503,7 @@ static llvm::Value *EmitCXXNewAllocSize(ASTContext &Context,
     }
     
     if (SC.countLeadingZeros() >= SizeWidth) {
-      SC.trunc(SizeWidth);
+      SC = SC.trunc(SizeWidth);
       Size = llvm::ConstantInt::get(SizeTy, SC);
     } else {
       // On overflow, produce a -1 so operator new throws.
@@ -531,8 +629,11 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const CXXNewExpr *E,
   else if (AllocType->isAnyComplexType())
     CGF.EmitComplexExprIntoAddr(Init, NewPtr, 
                                 AllocType.isVolatileQualified());
-  else
-    CGF.EmitAggExpr(Init, NewPtr, AllocType.isVolatileQualified());
+  else {
+    AggValueSlot Slot
+      = AggValueSlot::forAddr(NewPtr, AllocType.isVolatileQualified(), true);
+    CGF.EmitAggExpr(Init, Slot);
+  }
 }
 
 void
@@ -591,18 +692,10 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
 
 static void EmitZeroMemSet(CodeGenFunction &CGF, QualType T,
                            llvm::Value *NewPtr, llvm::Value *Size) {
-  llvm::LLVMContext &VMContext = CGF.CGM.getLLVMContext();
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(VMContext);
-  if (NewPtr->getType() != BP)
-    NewPtr = CGF.Builder.CreateBitCast(NewPtr, BP, "tmp");
-  
-  CGF.Builder.CreateCall5(CGF.CGM.getMemSetFn(BP, CGF.IntPtrTy), NewPtr,
-                llvm::Constant::getNullValue(llvm::Type::getInt8Ty(VMContext)),
-                          Size,
-                    llvm::ConstantInt::get(CGF.Int32Ty, 
-                                           CGF.getContext().getTypeAlign(T)/8),
-                          llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext),
-                                                 0));
+  CGF.EmitCastToVoidPtr(NewPtr);
+  CharUnits Alignment = CGF.getContext().getTypeAlignInChars(T);
+  CGF.Builder.CreateMemSet(NewPtr, CGF.Builder.getInt8(0), Size,
+                           Alignment.getQuantity(), false);
 }
                        
 static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
@@ -669,6 +762,163 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
   StoreAnyExprIntoOneUnit(CGF, E, NewPtr);
 }
 
+namespace {
+  /// A cleanup to call the given 'operator delete' function upon
+  /// abnormal exit from a new expression.
+  class CallDeleteDuringNew : public EHScopeStack::Cleanup {
+    size_t NumPlacementArgs;
+    const FunctionDecl *OperatorDelete;
+    llvm::Value *Ptr;
+    llvm::Value *AllocSize;
+
+    RValue *getPlacementArgs() { return reinterpret_cast<RValue*>(this+1); }
+
+  public:
+    static size_t getExtraSize(size_t NumPlacementArgs) {
+      return NumPlacementArgs * sizeof(RValue);
+    }
+
+    CallDeleteDuringNew(size_t NumPlacementArgs,
+                        const FunctionDecl *OperatorDelete,
+                        llvm::Value *Ptr,
+                        llvm::Value *AllocSize) 
+      : NumPlacementArgs(NumPlacementArgs), OperatorDelete(OperatorDelete),
+        Ptr(Ptr), AllocSize(AllocSize) {}
+
+    void setPlacementArg(unsigned I, RValue Arg) {
+      assert(I < NumPlacementArgs && "index out of range");
+      getPlacementArgs()[I] = Arg;
+    }
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const FunctionProtoType *FPT
+        = OperatorDelete->getType()->getAs<FunctionProtoType>();
+      assert(FPT->getNumArgs() == NumPlacementArgs + 1 ||
+             (FPT->getNumArgs() == 2 && NumPlacementArgs == 0));
+
+      CallArgList DeleteArgs;
+
+      // The first argument is always a void*.
+      FunctionProtoType::arg_type_iterator AI = FPT->arg_type_begin();
+      DeleteArgs.push_back(std::make_pair(RValue::get(Ptr), *AI++));
+
+      // A member 'operator delete' can take an extra 'size_t' argument.
+      if (FPT->getNumArgs() == NumPlacementArgs + 2)
+        DeleteArgs.push_back(std::make_pair(RValue::get(AllocSize), *AI++));
+
+      // Pass the rest of the arguments, which must match exactly.
+      for (unsigned I = 0; I != NumPlacementArgs; ++I)
+        DeleteArgs.push_back(std::make_pair(getPlacementArgs()[I], *AI++));
+
+      // Call 'operator delete'.
+      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(DeleteArgs, FPT),
+                   CGF.CGM.GetAddrOfFunction(OperatorDelete),
+                   ReturnValueSlot(), DeleteArgs, OperatorDelete);
+    }
+  };
+
+  /// A cleanup to call the given 'operator delete' function upon
+  /// abnormal exit from a new expression when the new expression is
+  /// conditional.
+  class CallDeleteDuringConditionalNew : public EHScopeStack::Cleanup {
+    size_t NumPlacementArgs;
+    const FunctionDecl *OperatorDelete;
+    DominatingValue<RValue>::saved_type Ptr;
+    DominatingValue<RValue>::saved_type AllocSize;
+
+    DominatingValue<RValue>::saved_type *getPlacementArgs() {
+      return reinterpret_cast<DominatingValue<RValue>::saved_type*>(this+1);
+    }
+
+  public:
+    static size_t getExtraSize(size_t NumPlacementArgs) {
+      return NumPlacementArgs * sizeof(DominatingValue<RValue>::saved_type);
+    }
+
+    CallDeleteDuringConditionalNew(size_t NumPlacementArgs,
+                                   const FunctionDecl *OperatorDelete,
+                                   DominatingValue<RValue>::saved_type Ptr,
+                              DominatingValue<RValue>::saved_type AllocSize)
+      : NumPlacementArgs(NumPlacementArgs), OperatorDelete(OperatorDelete),
+        Ptr(Ptr), AllocSize(AllocSize) {}
+
+    void setPlacementArg(unsigned I, DominatingValue<RValue>::saved_type Arg) {
+      assert(I < NumPlacementArgs && "index out of range");
+      getPlacementArgs()[I] = Arg;
+    }
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const FunctionProtoType *FPT
+        = OperatorDelete->getType()->getAs<FunctionProtoType>();
+      assert(FPT->getNumArgs() == NumPlacementArgs + 1 ||
+             (FPT->getNumArgs() == 2 && NumPlacementArgs == 0));
+
+      CallArgList DeleteArgs;
+
+      // The first argument is always a void*.
+      FunctionProtoType::arg_type_iterator AI = FPT->arg_type_begin();
+      DeleteArgs.push_back(std::make_pair(Ptr.restore(CGF), *AI++));
+
+      // A member 'operator delete' can take an extra 'size_t' argument.
+      if (FPT->getNumArgs() == NumPlacementArgs + 2) {
+        RValue RV = AllocSize.restore(CGF);
+        DeleteArgs.push_back(std::make_pair(RV, *AI++));
+      }
+
+      // Pass the rest of the arguments, which must match exactly.
+      for (unsigned I = 0; I != NumPlacementArgs; ++I) {
+        RValue RV = getPlacementArgs()[I].restore(CGF);
+        DeleteArgs.push_back(std::make_pair(RV, *AI++));
+      }
+
+      // Call 'operator delete'.
+      CGF.EmitCall(CGF.CGM.getTypes().getFunctionInfo(DeleteArgs, FPT),
+                   CGF.CGM.GetAddrOfFunction(OperatorDelete),
+                   ReturnValueSlot(), DeleteArgs, OperatorDelete);
+    }
+  };
+}
+
+/// Enter a cleanup to call 'operator delete' if the initializer in a
+/// new-expression throws.
+static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
+                                  const CXXNewExpr *E,
+                                  llvm::Value *NewPtr,
+                                  llvm::Value *AllocSize,
+                                  const CallArgList &NewArgs) {
+  // If we're not inside a conditional branch, then the cleanup will
+  // dominate and we can do the easier (and more efficient) thing.
+  if (!CGF.isInConditionalBranch()) {
+    CallDeleteDuringNew *Cleanup = CGF.EHStack
+      .pushCleanupWithExtra<CallDeleteDuringNew>(EHCleanup,
+                                                 E->getNumPlacementArgs(),
+                                                 E->getOperatorDelete(),
+                                                 NewPtr, AllocSize);
+    for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I)
+      Cleanup->setPlacementArg(I, NewArgs[I+1].first);
+
+    return;
+  }
+
+  // Otherwise, we need to save all this stuff.
+  DominatingValue<RValue>::saved_type SavedNewPtr =
+    DominatingValue<RValue>::save(CGF, RValue::get(NewPtr));
+  DominatingValue<RValue>::saved_type SavedAllocSize =
+    DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));
+
+  CallDeleteDuringConditionalNew *Cleanup = CGF.EHStack
+    .pushCleanupWithExtra<CallDeleteDuringConditionalNew>(InactiveEHCleanup,
+                                                 E->getNumPlacementArgs(),
+                                                 E->getOperatorDelete(),
+                                                 SavedNewPtr,
+                                                 SavedAllocSize);
+  for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I)
+    Cleanup->setPlacementArg(I,
+                     DominatingValue<RValue>::save(CGF, NewArgs[I+1].first));
+
+  CGF.ActivateCleanupBlock(CGF.EHStack.stable_begin());
+}
+
 llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   QualType AllocType = E->getAllocatedType();
   if (AllocType->isArrayType())
@@ -757,13 +1007,22 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
          CalculateCookiePadding(*this, E).isZero());
   if (AllocSize != AllocSizeWithoutCookie) {
     assert(E->isArray());
-    NewPtr = CGM.getCXXABI().InitializeArrayCookie(CGF, NewPtr, NumElements,
-                                                   AllocType);
+    NewPtr = CGM.getCXXABI().InitializeArrayCookie(*this, NewPtr, NumElements,
+                                                   E, AllocType);
+  }
+
+  // If there's an operator delete, enter a cleanup to call it if an
+  // exception is thrown.
+  EHScopeStack::stable_iterator CallOperatorDelete;
+  if (E->getOperatorDelete()) {
+    EnterNewDeleteCleanup(*this, E, NewPtr, AllocSize, NewArgs);
+    CallOperatorDelete = EHStack.stable_begin();
   }
 
   const llvm::Type *ElementPtrTy
     = ConvertTypeForMem(AllocType)->getPointerTo(AS);
   NewPtr = Builder.CreateBitCast(NewPtr, ElementPtrTy);
+
   if (E->isArray()) {
     EmitNewInitializer(*this, E, NewPtr, NumElements, AllocSizeWithoutCookie);
 
@@ -776,6 +1035,11 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   } else {
     EmitNewInitializer(*this, E, NewPtr, NumElements, AllocSizeWithoutCookie);
   }
+
+  // Deactivate the 'operator delete' cleanup if we finished
+  // initialization.
+  if (CallOperatorDelete.isValid())
+    DeactivateCleanupBlock(CallOperatorDelete);
   
   if (NullCheckResult) {
     Builder.CreateBr(NewEnd);
@@ -876,6 +1140,8 @@ static void EmitObjectDelete(CodeGenFunction &CGF,
   }
 
   // Make sure that we call delete even if the dtor throws.
+  // This doesn't have to a conditional cleanup because we're going
+  // to pop it off in a second.
   CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
                                             Ptr, OperatorDelete, ElementType);
 
@@ -950,18 +1216,19 @@ namespace {
 
 /// Emit the code for deleting an array of objects.
 static void EmitArrayDelete(CodeGenFunction &CGF,
-                            const FunctionDecl *OperatorDelete,
+                            const CXXDeleteExpr *E,
                             llvm::Value *Ptr,
                             QualType ElementType) {
   llvm::Value *NumElements = 0;
   llvm::Value *AllocatedPtr = 0;
   CharUnits CookieSize;
-  CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr, ElementType,
+  CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr, E, ElementType,
                                       NumElements, AllocatedPtr, CookieSize);
 
   assert(AllocatedPtr && "ReadArrayCookie didn't set AllocatedPtr");
 
   // Make sure that we call delete even if one of the dtors throws.
+  const FunctionDecl *OperatorDelete = E->getOperatorDelete();
   CGF.EHStack.pushCleanup<CallArrayDelete>(NormalAndEHCleanup,
                                            AllocatedPtr, OperatorDelete,
                                            NumElements, ElementType,
@@ -1031,7 +1298,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
          cast<llvm::PointerType>(Ptr->getType())->getElementType());
 
   if (E->isArrayForm()) {
-    EmitArrayDelete(*this, E->getOperatorDelete(), Ptr, DeleteTy);
+    EmitArrayDelete(*this, E, Ptr, DeleteTy);
   } else {
     EmitObjectDelete(*this, E->getOperatorDelete(), Ptr, DeleteTy);
   }
@@ -1039,7 +1306,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   EmitBlock(DeleteEnd);
 }
 
-llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
+llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
   QualType Ty = E->getType();
   const llvm::Type *LTy = ConvertType(Ty)->getPointerTo();
   
@@ -1059,8 +1326,6 @@ llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
       // FIXME: if subE is an lvalue do
       LValue Obj = EmitLValue(subE);
       llvm::Value *This = Obj.getAddress();
-      LTy = LTy->getPointerTo()->getPointerTo();
-      llvm::Value *V = Builder.CreateBitCast(This, LTy);
       // We need to do a zero check for *p, unless it has NonNullAttr.
       // FIXME: PointerType->hasAttr<NonNullAttr>()
       bool CanBeZero = false;
@@ -1071,12 +1336,12 @@ llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
         llvm::BasicBlock *NonZeroBlock = createBasicBlock();
         llvm::BasicBlock *ZeroBlock = createBasicBlock();
         
-        llvm::Value *Zero = llvm::Constant::getNullValue(LTy);
-        Builder.CreateCondBr(Builder.CreateICmpNE(V, Zero),
+        llvm::Value *Zero = llvm::Constant::getNullValue(This->getType());
+        Builder.CreateCondBr(Builder.CreateICmpNE(This, Zero),
                              NonZeroBlock, ZeroBlock);
         EmitBlock(ZeroBlock);
         /// Call __cxa_bad_typeid
-        const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
+        const llvm::Type *ResultType = llvm::Type::getVoidTy(getLLVMContext());
         const llvm::FunctionType *FTy;
         FTy = llvm::FunctionType::get(ResultType, false);
         llvm::Value *F = CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid");
@@ -1084,7 +1349,7 @@ llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
         Builder.CreateUnreachable();
         EmitBlock(NonZeroBlock);
       }
-      V = Builder.CreateLoad(V, "vtable");
+      llvm::Value *V = GetVTablePtr(This, LTy->getPointerTo());
       V = Builder.CreateConstInBoundsGEP1_64(V, -1ULL);
       V = Builder.CreateLoad(V);
       return V;
@@ -1141,23 +1406,20 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *V,
   // See if this is a dynamic_cast(void*)
   if (ToVoid) {
     llvm::Value *This = V;
-    V = Builder.CreateBitCast(This, PtrDiffTy->getPointerTo()->getPointerTo());
-    V = Builder.CreateLoad(V, "vtable");
+    V = GetVTablePtr(This, PtrDiffTy->getPointerTo());
     V = Builder.CreateConstInBoundsGEP1_64(V, -2ULL);
     V = Builder.CreateLoad(V, "offset to top");
-    This = Builder.CreateBitCast(This, llvm::Type::getInt8PtrTy(VMContext));
+    This = EmitCastToVoidPtr(This);
     V = Builder.CreateInBoundsGEP(This, V);
     V = Builder.CreateBitCast(V, LTy);
   } else {
     /// Call __dynamic_cast
-    const llvm::Type *ResultType = llvm::Type::getInt8PtrTy(VMContext);
+    const llvm::Type *ResultType = Int8PtrTy;
     const llvm::FunctionType *FTy;
     std::vector<const llvm::Type*> ArgTys;
-    const llvm::Type *PtrToInt8Ty
-      = llvm::Type::getInt8Ty(VMContext)->getPointerTo();
-    ArgTys.push_back(PtrToInt8Ty);
-    ArgTys.push_back(PtrToInt8Ty);
-    ArgTys.push_back(PtrToInt8Ty);
+    ArgTys.push_back(Int8PtrTy);
+    ArgTys.push_back(Int8PtrTy);
+    ArgTys.push_back(Int8PtrTy);
     ArgTys.push_back(PtrDiffTy);
     FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
 
@@ -1172,7 +1434,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *V,
     llvm::Value *DestArg
       = CGM.GetAddrOfRTTIDescriptor(DestTy.getUnqualifiedType());
     
-    V = Builder.CreateBitCast(V, PtrToInt8Ty);
+    V = Builder.CreateBitCast(V, Int8PtrTy);
     V = Builder.CreateCall4(CGM.CreateRuntimeFunction(FTy, "__dynamic_cast"),
                             V, SrcArg, DestArg, hint);
     V = Builder.CreateBitCast(V, LTy);
@@ -1182,7 +1444,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(llvm::Value *V,
       Builder.CreateCondBr(Builder.CreateIsNotNull(V), ContBlock, BadCastBlock);
       EmitBlock(BadCastBlock);
       /// Invoke __cxa_bad_cast
-      ResultType = llvm::Type::getVoidTy(VMContext);
+      ResultType = llvm::Type::getVoidTy(getLLVMContext());
       const llvm::FunctionType *FBadTy;
       FBadTy = llvm::FunctionType::get(ResultType, false);
       llvm::Value *F = CGM.CreateRuntimeFunction(FBadTy, "__cxa_bad_cast");
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 79e9dd42ee28..7b0292b8f225 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -35,14 +35,9 @@ class ComplexExprEmitter
   // True is we should ignore the value of a
   bool IgnoreReal;
   bool IgnoreImag;
-  // True if we should ignore the value of a=b
-  bool IgnoreRealAssign;
-  bool IgnoreImagAssign;
 public:
-  ComplexExprEmitter(CodeGenFunction &cgf, bool ir=false, bool ii=false,
-                     bool irn=false, bool iin=false)
-    : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii),
-    IgnoreRealAssign(irn), IgnoreImagAssign(iin) {
+  ComplexExprEmitter(CodeGenFunction &cgf, bool ir=false, bool ii=false)
+    : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii) {
   }
 
 
@@ -60,36 +55,36 @@ public:
     IgnoreImag = false;
     return I;
   }
-  bool TestAndClearIgnoreRealAssign() {
-    bool I = IgnoreRealAssign;
-    IgnoreRealAssign = false;
-    return I;
-  }
-  bool TestAndClearIgnoreImagAssign() {
-    bool I = IgnoreImagAssign;
-    IgnoreImagAssign = false;
-    return I;
-  }
 
   /// EmitLoadOfLValue - Given an expression with complex type that represents a
   /// value l-value, this method emits the address of the l-value, then loads
   /// and returns the result.
   ComplexPairTy EmitLoadOfLValue(const Expr *E) {
-    LValue LV = CGF.EmitLValue(E);
+    return EmitLoadOfLValue(CGF.EmitLValue(E));
+  }
+
+  ComplexPairTy EmitLoadOfLValue(LValue LV) {
     if (LV.isSimple())
       return EmitLoadOfComplex(LV.getAddress(), LV.isVolatileQualified());
 
-    if (LV.isPropertyRef())
-      return CGF.EmitObjCPropertyGet(LV.getPropertyRefExpr()).getComplexVal();
-
-    assert(LV.isKVCRef() && "Unknown LValue type!");
-    return CGF.EmitObjCPropertyGet(LV.getKVCRefExpr()).getComplexVal();
+    assert(LV.isPropertyRef() && "Unknown LValue type!");
+    return CGF.EmitLoadOfPropertyRefLValue(LV).getComplexVal();
   }
 
   /// EmitLoadOfComplex - Given a pointer to a complex value, emit code to load
   /// the real and imaginary pieces.
   ComplexPairTy EmitLoadOfComplex(llvm::Value *SrcPtr, bool isVolatile);
 
+  /// EmitStoreThroughLValue - Given an l-value of complex type, store
+  /// a complex number into it.
+  void EmitStoreThroughLValue(ComplexPairTy Val, LValue LV) {
+    if (LV.isSimple())
+      return EmitStoreOfComplex(Val, LV.getAddress(), LV.isVolatileQualified());
+
+    assert(LV.isPropertyRef() && "Unknown LValue type!");
+    CGF.EmitStoreThroughPropertyRefLValue(RValue::getComplex(Val), LV);
+  }
+
   /// EmitStoreOfComplex - Store the specified real/imag parts into the
   /// specified value pointer.
   void EmitStoreOfComplex(ComplexPairTy Val, llvm::Value *ResPtr, bool isVol);
@@ -102,6 +97,10 @@ public:
   //                            Visitor Methods
   //===--------------------------------------------------------------------===//
 
+  ComplexPairTy Visit(Expr *E) {
+    return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E);
+  }
+    
   ComplexPairTy VisitStmt(Stmt *S) {
     S->dump(CGF.getContext().getSourceManager());
     assert(0 && "Stmt can't have complex result type!");
@@ -117,10 +116,7 @@ public:
     return EmitLoadOfLValue(E);
   }
   ComplexPairTy VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
-    return EmitLoadOfLValue(E);
-  }
-  ComplexPairTy VisitObjCImplicitSetterGetterRefExpr(
-                               ObjCImplicitSetterGetterRefExpr *E) {
+    assert(E->getObjectKind() == OK_Ordinary);
     return EmitLoadOfLValue(E);
   }
   ComplexPairTy VisitObjCMessageExpr(ObjCMessageExpr *E) {
@@ -128,6 +124,11 @@ public:
   }
   ComplexPairTy VisitArraySubscriptExpr(Expr *E) { return EmitLoadOfLValue(E); }
   ComplexPairTy VisitMemberExpr(const Expr *E) { return EmitLoadOfLValue(E); }
+  ComplexPairTy VisitOpaqueValueExpr(OpaqueValueExpr *E) {
+    if (E->isGLValue())
+      return EmitLoadOfLValue(CGF.getOpaqueLValueMapping(E));
+    return CGF.getOpaqueRValueMapping(E).getComplexVal();
+  }
 
   // FIXME: CompoundLiteralExpr
 
@@ -165,8 +166,6 @@ public:
   ComplexPairTy VisitUnaryPlus     (const UnaryOperator *E) {
     TestAndClearIgnoreReal();
     TestAndClearIgnoreImag();
-    TestAndClearIgnoreRealAssign();
-    TestAndClearIgnoreImagAssign();
     return Visit(E->getSubExpr());
   }
   ComplexPairTy VisitUnaryMinus    (const UnaryOperator *E);
@@ -178,8 +177,8 @@ public:
   ComplexPairTy VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
     return Visit(DAE->getExpr());
   }
-  ComplexPairTy VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
-    return CGF.EmitCXXExprWithTemporaries(E).getComplexVal();
+  ComplexPairTy VisitExprWithCleanups(ExprWithCleanups *E) {
+    return CGF.EmitExprWithCleanups(E).getComplexVal();
   }
   ComplexPairTy VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
     assert(E->getType()->isAnyComplexType() && "Expected complex type!");
@@ -202,6 +201,10 @@ public:
   };
 
   BinOpInfo EmitBinOps(const BinaryOperator *E);
+  LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
+                                  ComplexPairTy (ComplexExprEmitter::*Func)
+                                  (const BinOpInfo &),
+                                  ComplexPairTy &Val);
   ComplexPairTy EmitCompoundAssign(const CompoundAssignOperator *E,
                                    ComplexPairTy (ComplexExprEmitter::*Func)
                                    (const BinOpInfo &));
@@ -211,15 +214,15 @@ public:
   ComplexPairTy EmitBinMul(const BinOpInfo &Op);
   ComplexPairTy EmitBinDiv(const BinOpInfo &Op);
 
-  ComplexPairTy VisitBinMul(const BinaryOperator *E) {
-    return EmitBinMul(EmitBinOps(E));
-  }
   ComplexPairTy VisitBinAdd(const BinaryOperator *E) {
     return EmitBinAdd(EmitBinOps(E));
   }
   ComplexPairTy VisitBinSub(const BinaryOperator *E) {
     return EmitBinSub(EmitBinOps(E));
   }
+  ComplexPairTy VisitBinMul(const BinaryOperator *E) {
+    return EmitBinMul(EmitBinOps(E));
+  }
   ComplexPairTy VisitBinDiv(const BinaryOperator *E) {
     return EmitBinDiv(EmitBinOps(E));
   }
@@ -242,11 +245,15 @@ public:
   // Logical and/or always return int, never complex.
 
   // No comparisons produce a complex result.
+
+  LValue EmitBinAssignLValue(const BinaryOperator *E,
+                             ComplexPairTy &Val);
   ComplexPairTy VisitBinAssign     (const BinaryOperator *E);
   ComplexPairTy VisitBinComma      (const BinaryOperator *E);
 
 
-  ComplexPairTy VisitConditionalOperator(const ConditionalOperator *CO);
+  ComplexPairTy
+  VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
   ComplexPairTy VisitChooseExpr(ChooseExpr *CE);
 
   ComplexPairTy VisitInitListExpr(InitListExpr *E);
@@ -265,13 +272,13 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfComplex(llvm::Value *SrcPtr,
                                                     bool isVolatile) {
   llvm::Value *Real=0, *Imag=0;
 
-  if (!IgnoreReal) {
+  if (!IgnoreReal || isVolatile) {
     llvm::Value *RealP = Builder.CreateStructGEP(SrcPtr, 0,
                                                  SrcPtr->getName() + ".realp");
     Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr->getName() + ".real");
   }
 
-  if (!IgnoreImag) {
+  if (!IgnoreImag || isVolatile) {
     llvm::Value *ImagP = Builder.CreateStructGEP(SrcPtr, 1,
                                                  SrcPtr->getName() + ".imagp");
     Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr->getName() + ".imag");
@@ -307,8 +314,7 @@ ComplexPairTy ComplexExprEmitter::VisitExpr(Expr *E) {
 ComplexPairTy ComplexExprEmitter::
 VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
   llvm::Value *Imag = CGF.EmitScalarExpr(IL->getSubExpr());
-  return
-        ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
+  return ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
 }
 
 
@@ -320,6 +326,7 @@ ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) {
 }
 
 ComplexPairTy ComplexExprEmitter::VisitStmtExpr(const StmtExpr *E) {
+  CodeGenFunction::StmtExprEvaluation eval(CGF);
   return CGF.EmitCompoundStmt(*E->getSubStmt(), true).getComplexVal();
 }
 
@@ -341,6 +348,22 @@ ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val,
 
 ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op, 
                                            QualType DestTy) {
+  switch (CK) {
+  case CK_GetObjCProperty: {
+    LValue LV = CGF.EmitLValue(Op);
+    assert(LV.isPropertyRef() && "Unknown LValue type!");
+    return CGF.EmitLoadOfPropertyRefLValue(LV).getComplexVal();
+  }
+
+  case CK_NoOp:
+  case CK_LValueToRValue:
+    return Visit(Op);
+
+  // TODO: do all of these
+  default:
+    break;
+  }
+
   // Two cases here: cast from (complex to complex) and (scalar to complex).
   if (Op->getType()->isAnyComplexType())
     return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy);
@@ -372,8 +395,6 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op,
 ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
   TestAndClearIgnoreReal();
   TestAndClearIgnoreImag();
-  TestAndClearIgnoreRealAssign();
-  TestAndClearIgnoreImagAssign();
   ComplexPairTy Op = Visit(E->getSubExpr());
 
   llvm::Value *ResR, *ResI;
@@ -390,8 +411,6 @@ ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
 ComplexPairTy ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
   TestAndClearIgnoreReal();
   TestAndClearIgnoreImag();
-  TestAndClearIgnoreRealAssign();
-  TestAndClearIgnoreImagAssign();
   // ~(a+ib) = a + i*-b
   ComplexPairTy Op = Visit(E->getSubExpr());
   llvm::Value *ResI;
@@ -505,8 +524,6 @@ ComplexExprEmitter::BinOpInfo
 ComplexExprEmitter::EmitBinOps(const BinaryOperator *E) {
   TestAndClearIgnoreReal();
   TestAndClearIgnoreImag();
-  TestAndClearIgnoreRealAssign();
-  TestAndClearIgnoreImagAssign();
   BinOpInfo Ops;
   Ops.LHS = Visit(E->getLHS());
   Ops.RHS = Visit(E->getRHS());
@@ -515,37 +532,31 @@ ComplexExprEmitter::EmitBinOps(const BinaryOperator *E) {
 }
 
 
-// Compound assignments.
-ComplexPairTy ComplexExprEmitter::
-EmitCompoundAssign(const CompoundAssignOperator *E,
-                   ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
+LValue ComplexExprEmitter::
+EmitCompoundAssignLValue(const CompoundAssignOperator *E,
+          ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&),
+                         ComplexPairTy &Val) {
   TestAndClearIgnoreReal();
   TestAndClearIgnoreImag();
-  bool ignreal = TestAndClearIgnoreRealAssign();
-  bool ignimag = TestAndClearIgnoreImagAssign();
-  QualType LHSTy = E->getLHS()->getType(), RHSTy = E->getRHS()->getType();
+  QualType LHSTy = E->getLHS()->getType();
 
   BinOpInfo OpInfo;
 
   // Load the RHS and LHS operands.
   // __block variables need to have the rhs evaluated first, plus this should
-  // improve codegen a little.  It is possible for the RHS to be complex or
-  // scalar.
+  // improve codegen a little.
   OpInfo.Ty = E->getComputationResultType();
-  OpInfo.RHS = EmitCast(CK_Unknown, E->getRHS(), OpInfo.Ty);
+
+  // The RHS should have been converted to the computation type.
+  assert(OpInfo.Ty->isAnyComplexType());
+  assert(CGF.getContext().hasSameUnqualifiedType(OpInfo.Ty,
+                                                 E->getRHS()->getType()));
+  OpInfo.RHS = Visit(E->getRHS());
   
   LValue LHS = CGF.EmitLValue(E->getLHS());
-  // We know the LHS is a complex lvalue.
-  ComplexPairTy LHSComplexPair;
-  if (LHS.isPropertyRef())
-    LHSComplexPair =
-      CGF.EmitObjCPropertyGet(LHS.getPropertyRefExpr()).getComplexVal();
-  else if (LHS.isKVCRef())
-    LHSComplexPair =
-      CGF.EmitObjCPropertyGet(LHS.getKVCRefExpr()).getComplexVal();
-  else
-    LHSComplexPair = EmitLoadOfComplex(LHS.getAddress(),
-                                       LHS.isVolatileQualified());
+
+  // Load from the l-value.
+  ComplexPairTy LHSComplexPair = EmitLoadOfLValue(LHS);
   
   OpInfo.LHS = EmitComplexToComplexCast(LHSComplexPair, LHSTy, OpInfo.Ty);
 
@@ -554,108 +565,107 @@ EmitCompoundAssign(const CompoundAssignOperator *E,
 
   // Truncate the result back to the LHS type.
   Result = EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy);
+  Val = Result;
 
   // Store the result value into the LHS lvalue.
-  if (LHS.isPropertyRef())
-    CGF.EmitObjCPropertySet(LHS.getPropertyRefExpr(),
-                            RValue::getComplex(Result));
-  else if (LHS.isKVCRef())
-    CGF.EmitObjCPropertySet(LHS.getKVCRefExpr(), RValue::getComplex(Result));
-  else
-    EmitStoreOfComplex(Result, LHS.getAddress(), LHS.isVolatileQualified());
-
-  // Restore the Ignore* flags.
-  IgnoreReal = ignreal;
-  IgnoreImag = ignimag;
-  IgnoreRealAssign = ignreal;
-  IgnoreImagAssign = ignimag;
- 
+  EmitStoreThroughLValue(Result, LHS);
+
+  return LHS;
+}
+
+// Compound assignments.
+ComplexPairTy ComplexExprEmitter::
+EmitCompoundAssign(const CompoundAssignOperator *E,
+                   ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
+  ComplexPairTy Val;
+  LValue LV = EmitCompoundAssignLValue(E, Func, Val);
+
+  // The result of an assignment in C is the assigned r-value.
+  if (!CGF.getContext().getLangOptions().CPlusPlus)
+    return Val;
+
   // Objective-C property assignment never reloads the value following a store.
-  if (LHS.isPropertyRef() || LHS.isKVCRef())
-    return Result;
+  if (LV.isPropertyRef())
+    return Val;
 
-  // Otherwise, reload the value.
-  return EmitLoadOfComplex(LHS.getAddress(), LHS.isVolatileQualified());
+  // If the lvalue is non-volatile, return the computed value of the assignment.
+  if (!LV.isVolatileQualified())
+    return Val;
+
+  return EmitLoadOfComplex(LV.getAddress(), LV.isVolatileQualified());
 }
 
-ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
-  TestAndClearIgnoreReal();
-  TestAndClearIgnoreImag();
-  bool ignreal = TestAndClearIgnoreRealAssign();
-  bool ignimag = TestAndClearIgnoreImagAssign();
+LValue ComplexExprEmitter::EmitBinAssignLValue(const BinaryOperator *E,
+                                               ComplexPairTy &Val) {
   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(), 
                                                  E->getRHS()->getType()) &&
          "Invalid assignment");
-  // Emit the RHS.
-  ComplexPairTy Val = Visit(E->getRHS());
+  TestAndClearIgnoreReal();
+  TestAndClearIgnoreImag();
+
+  // Emit the RHS.  __block variables need the RHS evaluated first.
+  Val = Visit(E->getRHS());
 
   // Compute the address to store into.
   LValue LHS = CGF.EmitLValue(E->getLHS());
 
   // Store the result value into the LHS lvalue.
-  if (LHS.isPropertyRef())
-    CGF.EmitObjCPropertySet(LHS.getPropertyRefExpr(), RValue::getComplex(Val));
-  else if (LHS.isKVCRef())
-    CGF.EmitObjCPropertySet(LHS.getKVCRefExpr(), RValue::getComplex(Val));
-  else
-    EmitStoreOfComplex(Val, LHS.getAddress(), LHS.isVolatileQualified());
+  EmitStoreThroughLValue(Val, LHS);
+
+  return LHS;
+}
 
-  // Restore the Ignore* flags.
-  IgnoreReal = ignreal;
-  IgnoreImag = ignimag;
-  IgnoreRealAssign = ignreal;
-  IgnoreImagAssign = ignimag;
+ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
+  ComplexPairTy Val;
+  LValue LV = EmitBinAssignLValue(E, Val);
+
+  // The result of an assignment in C is the assigned r-value.
+  if (!CGF.getContext().getLangOptions().CPlusPlus)
+    return Val;
 
   // Objective-C property assignment never reloads the value following a store.
-  if (LHS.isPropertyRef() || LHS.isKVCRef())
+  if (LV.isPropertyRef())
+    return Val;
+
+  // If the lvalue is non-volatile, return the computed value of the assignment.
+  if (!LV.isVolatileQualified())
     return Val;
 
-  // Otherwise, reload the value.
-  return EmitLoadOfComplex(LHS.getAddress(), LHS.isVolatileQualified());
+  return EmitLoadOfComplex(LV.getAddress(), LV.isVolatileQualified());
 }
 
 ComplexPairTy ComplexExprEmitter::VisitBinComma(const BinaryOperator *E) {
-  CGF.EmitStmt(E->getLHS());
-  CGF.EnsureInsertPoint();
+  CGF.EmitIgnoredExpr(E->getLHS());
   return Visit(E->getRHS());
 }
 
 ComplexPairTy ComplexExprEmitter::
-VisitConditionalOperator(const ConditionalOperator *E) {
-  if (!E->getLHS()) {
-    CGF.ErrorUnsupported(E, "conditional operator with missing LHS");
-    const llvm::Type *EltTy =
-      CGF.ConvertType(E->getType()->getAs<ComplexType>()->getElementType());
-    llvm::Value *U = llvm::UndefValue::get(EltTy);
-    return ComplexPairTy(U, U);
-  }
-
+VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   TestAndClearIgnoreReal();
   TestAndClearIgnoreImag();
-  TestAndClearIgnoreRealAssign();
-  TestAndClearIgnoreImagAssign();
   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
 
+  // Bind the common expression if necessary.
+  CodeGenFunction::OpaqueValueMapping binding(CGF, E);
+
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
 
+  eval.begin(CGF);
   CGF.EmitBlock(LHSBlock);
-
-  // Handle the GNU extension for missing LHS.
-  assert(E->getLHS() && "Must have LHS for complex value");
-
-  ComplexPairTy LHS = Visit(E->getLHS());
+  ComplexPairTy LHS = Visit(E->getTrueExpr());
   LHSBlock = Builder.GetInsertBlock();
   CGF.EmitBranch(ContBlock);
+  eval.end(CGF);
 
+  eval.begin(CGF);
   CGF.EmitBlock(RHSBlock);
-
-  ComplexPairTy RHS = Visit(E->getRHS());
+  ComplexPairTy RHS = Visit(E->getFalseExpr());
   RHSBlock = Builder.GetInsertBlock();
-  CGF.EmitBranch(ContBlock);
-
   CGF.EmitBlock(ContBlock);
+  eval.end(CGF);
 
   // Create a PHI node for the real part.
   llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), "cond.r");
@@ -716,12 +726,11 @@ ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
 /// EmitComplexExpr - Emit the computation of the specified expression of
 /// complex type, ignoring the result.
 ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal,
-                                               bool IgnoreImag, bool IgnoreRealAssign, bool IgnoreImagAssign) {
+                                               bool IgnoreImag) {
   assert(E && E->getType()->isAnyComplexType() &&
          "Invalid complex expression to emit");
 
-  return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag, IgnoreRealAssign,
-                            IgnoreImagAssign)
+  return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag)
     .Visit(const_cast<Expr*>(E));
 }
 
@@ -749,3 +758,27 @@ ComplexPairTy CodeGenFunction::LoadComplexFromAddr(llvm::Value *SrcAddr,
                                                    bool SrcIsVolatile) {
   return ComplexExprEmitter(*this).EmitLoadOfComplex(SrcAddr, SrcIsVolatile);
 }
+
+LValue CodeGenFunction::EmitComplexAssignmentLValue(const BinaryOperator *E) {
+  assert(E->getOpcode() == BO_Assign);
+  ComplexPairTy Val; // ignored
+  return ComplexExprEmitter(*this).EmitBinAssignLValue(E, Val);
+}
+
+LValue CodeGenFunction::
+EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) {
+  ComplexPairTy(ComplexExprEmitter::*Op)(const ComplexExprEmitter::BinOpInfo &);
+  switch (E->getOpcode()) {
+  case BO_MulAssign: Op = &ComplexExprEmitter::EmitBinMul; break;
+  case BO_DivAssign: Op = &ComplexExprEmitter::EmitBinDiv; break;
+  case BO_SubAssign: Op = &ComplexExprEmitter::EmitBinSub; break;
+  case BO_AddAssign: Op = &ComplexExprEmitter::EmitBinAdd; break;
+
+  default:
+    llvm_unreachable("unexpected complex compound assignment");
+    Op = 0;
+  }
+
+  ComplexPairTy Val; // ignored
+  return ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
+}
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index 9c31c2a3d538..40d7b6c32e4b 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -142,11 +142,11 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
   // constants are cast to bool, and because clang is not enforcing bitfield
   // width limits.
   if (FieldSize > FieldValue.getBitWidth())
-    FieldValue.zext(FieldSize);
+    FieldValue = FieldValue.zext(FieldSize);
 
   // Truncate the size of FieldValue to the bit field size.
   if (FieldSize < FieldValue.getBitWidth())
-    FieldValue.trunc(FieldSize);
+    FieldValue = FieldValue.trunc(FieldSize);
 
   if (FieldOffset < NextFieldOffsetInBytes * 8) {
     // Either part of the field or the entire field can go into the previous
@@ -166,20 +166,20 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
 
       if (CGM.getTargetData().isBigEndian()) {
         Tmp = Tmp.lshr(NewFieldWidth);
-        Tmp.trunc(BitsInPreviousByte);
+        Tmp = Tmp.trunc(BitsInPreviousByte);
 
         // We want the remaining high bits.
-        FieldValue.trunc(NewFieldWidth);
+        FieldValue = FieldValue.trunc(NewFieldWidth);
       } else {
-        Tmp.trunc(BitsInPreviousByte);
+        Tmp = Tmp.trunc(BitsInPreviousByte);
 
         // We want the remaining low bits.
         FieldValue = FieldValue.lshr(BitsInPreviousByte);
-        FieldValue.trunc(NewFieldWidth);
+        FieldValue = FieldValue.trunc(NewFieldWidth);
       }
     }
 
-    Tmp.zext(8);
+    Tmp = Tmp.zext(8);
     if (CGM.getTargetData().isBigEndian()) {
       if (FitsCompletelyInPreviousByte)
         Tmp = Tmp.shl(BitsInPreviousByte - FieldValue.getBitWidth());
@@ -231,13 +231,10 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
 
     if (CGM.getTargetData().isBigEndian()) {
       // We want the high bits.
-      Tmp = FieldValue;
-      Tmp = Tmp.lshr(Tmp.getBitWidth() - 8);
-      Tmp.trunc(8);
+      Tmp = FieldValue.lshr(FieldValue.getBitWidth() - 8).trunc(8);
     } else {
       // We want the low bits.
-      Tmp = FieldValue;
-      Tmp.trunc(8);
+      Tmp = FieldValue.trunc(8);
 
       FieldValue = FieldValue.lshr(8);
     }
@@ -245,7 +242,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
     Elements.push_back(llvm::ConstantInt::get(CGM.getLLVMContext(), Tmp));
     NextFieldOffsetInBytes++;
 
-    FieldValue.trunc(FieldValue.getBitWidth() - 8);
+    FieldValue = FieldValue.trunc(FieldValue.getBitWidth() - 8);
   }
 
   assert(FieldValue.getBitWidth() > 0 &&
@@ -257,10 +254,9 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
     if (CGM.getTargetData().isBigEndian()) {
       unsigned BitWidth = FieldValue.getBitWidth();
 
-      FieldValue.zext(8);
-      FieldValue = FieldValue << (8 - BitWidth);
+      FieldValue = FieldValue.zext(8) << (8 - BitWidth);
     } else
-      FieldValue.zext(8);
+      FieldValue = FieldValue.zext(8);
   }
 
   // Append the last element.
@@ -372,7 +368,7 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
     }
   }
 
-  uint64_t LayoutSizeInBytes = Layout.getSize() / 8;
+  uint64_t LayoutSizeInBytes = Layout.getSize().getQuantity();
 
   if (NextFieldOffsetInBytes > LayoutSizeInBytes) {
     // If the struct is bigger than the size of the record type,
@@ -398,9 +394,9 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
   }
 
   // Append tail padding if necessary.
-  AppendTailPadding(Layout.getSize());
+  AppendTailPadding(CGM.getContext().toBits(Layout.getSize()));
 
-  assert(Layout.getSize() / 8 == NextFieldOffsetInBytes &&
+  assert(Layout.getSize().getQuantity() == NextFieldOffsetInBytes &&
          "Tail padding mismatch!");
 
   return true;
@@ -454,17 +450,10 @@ public:
   llvm::Constant *VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
     return Visit(E->getInitializer());
   }
-    
+
   llvm::Constant *VisitUnaryAddrOf(UnaryOperator *E) {
-    if (const MemberPointerType *MPT = 
-          E->getType()->getAs<MemberPointerType>()) {
-      DeclRefExpr *DRE = cast<DeclRefExpr>(E->getSubExpr());
-      NamedDecl *ND = DRE->getDecl();
-      if (MPT->isMemberFunctionPointer())
-        return CGM.getCXXABI().EmitMemberPointer(cast<CXXMethodDecl>(ND));
-      else 
-        return CGM.getCXXABI().EmitMemberPointer(cast<FieldDecl>(ND));
-    }
+    if (E->getType()->isMemberPointerType())
+      return CGM.getMemberPointerConstant(E);
 
     return 0;
   }
@@ -755,7 +744,7 @@ public:
         if (!VD->hasLocalStorage()) {
           if (VD->isFileVarDecl() || VD->hasExternalStorage())
             return CGM.GetAddrOfGlobalVar(VD);
-          else if (VD->isBlockVarDecl()) {
+          else if (VD->isLocalVarDecl()) {
             assert(CGF && "Can't access static local vars without CGF");
             return CGF->GetAddrOfStaticLocalVar(VD);
           }
@@ -925,7 +914,7 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
         else
           Inits.push_back(llvm::ConstantFP::get(VMContext, Elt.getFloat()));
       }
-      return llvm::ConstantVector::get(&Inits[0], Inits.size());
+      return llvm::ConstantVector::get(Inits);
     }
     }
   }
@@ -938,6 +927,38 @@ llvm::Constant *CodeGenModule::EmitConstantExpr(const Expr *E,
   return C;
 }
 
+static uint64_t getFieldOffset(ASTContext &C, const FieldDecl *field) {
+  const ASTRecordLayout &layout = C.getASTRecordLayout(field->getParent());
+  return layout.getFieldOffset(field->getFieldIndex());
+}
+    
+llvm::Constant *
+CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
+  // Member pointer constants always have a very particular form.
+  const MemberPointerType *type = cast<MemberPointerType>(uo->getType());
+  const ValueDecl *decl = cast<DeclRefExpr>(uo->getSubExpr())->getDecl();
+
+  // A member function pointer.
+  if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(decl))
+    return getCXXABI().EmitMemberPointer(method);
+
+  // Otherwise, a member data pointer.
+  uint64_t fieldOffset;
+  if (const FieldDecl *field = dyn_cast<FieldDecl>(decl))
+    fieldOffset = getFieldOffset(getContext(), field);
+  else {
+    const IndirectFieldDecl *ifield = cast<IndirectFieldDecl>(decl);
+
+    fieldOffset = 0;
+    for (IndirectFieldDecl::chain_iterator ci = ifield->chain_begin(),
+           ce = ifield->chain_end(); ci != ce; ++ci)
+      fieldOffset += getFieldOffset(getContext(), cast<FieldDecl>(*ci));
+  }
+
+  CharUnits chars = getContext().toCharUnitsFromBits((int64_t) fieldOffset);
+  return getCXXABI().EmitMemberDataPointer(type, chars);
+}
+
 static void
 FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
                              std::vector<llvm::Constant *> &Elements,
@@ -964,8 +985,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
     for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
          E = RD->bases_end(); I != E; ++I) {
       if (I->isVirtual()) {
-        // FIXME: We should initialize null pointer to data members in virtual
-        // bases here.
+        // Ignore virtual bases.
         continue;
       }
       
@@ -980,7 +1000,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
       if (CGM.getTypes().isZeroInitializable(BaseDecl))
         continue;
 
-      uint64_t BaseOffset = Layout.getBaseClassOffset(BaseDecl);
+      uint64_t BaseOffset = Layout.getBaseClassOffsetInBits(BaseDecl);
       FillInNullDataMemberPointers(CGM, I->getType(),
                                    Elements, StartOffset + BaseOffset);
     }
@@ -1005,9 +1025,10 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
     uint64_t StartIndex = StartOffset / 8;
     uint64_t EndIndex = StartIndex + CGM.getContext().getTypeSize(T) / 8;
 
+    // FIXME: hardcodes Itanium member pointer representation!
     llvm::Constant *NegativeOne =
       llvm::ConstantInt::get(llvm::Type::getInt8Ty(CGM.getLLVMContext()),
-                             -1ULL, /*isSigned=*/true);
+                             -1ULL, /*isSigned*/true);
 
     // Fill in the null data member pointer.
     for (uint64_t I = StartIndex; I != EndIndex; ++I)
@@ -1015,6 +1036,124 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
   }
 }
 
+static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
+                                               const llvm::Type *baseType,
+                                               const CXXRecordDecl *base);
+
+static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
+                                        const CXXRecordDecl *record,
+                                        bool asCompleteObject) {
+  const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
+  const llvm::StructType *structure =
+    (asCompleteObject ? layout.getLLVMType()
+                      : layout.getBaseSubobjectLLVMType());
+
+  unsigned numElements = structure->getNumElements();
+  std::vector<llvm::Constant *> elements(numElements);
+
+  // Fill in all the bases.
+  for (CXXRecordDecl::base_class_const_iterator
+         I = record->bases_begin(), E = record->bases_end(); I != E; ++I) {
+    if (I->isVirtual()) {
+      // Ignore virtual bases; if we're laying out for a complete
+      // object, we'll lay these out later.
+      continue;
+    }
+
+    const CXXRecordDecl *base = 
+      cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
+
+    // Ignore empty bases.
+    if (base->isEmpty())
+      continue;
+    
+    unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(base);
+    const llvm::Type *baseType = structure->getElementType(fieldIndex);
+    elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
+  }
+
+  // Fill in all the fields.
+  for (RecordDecl::field_iterator I = record->field_begin(),
+         E = record->field_end(); I != E; ++I) {
+    const FieldDecl *field = *I;
+    
+    // Ignore bit fields.
+    if (field->isBitField())
+      continue;
+    
+    unsigned fieldIndex = layout.getLLVMFieldNo(field);
+    elements[fieldIndex] = CGM.EmitNullConstant(field->getType());
+  }
+
+  // Fill in the virtual bases, if we're working with the complete object.
+  if (asCompleteObject) {
+    for (CXXRecordDecl::base_class_const_iterator
+           I = record->vbases_begin(), E = record->vbases_end(); I != E; ++I) {
+      const CXXRecordDecl *base = 
+        cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
+
+      // Ignore empty bases.
+      if (base->isEmpty())
+        continue;
+
+      unsigned fieldIndex = layout.getVirtualBaseIndex(base);
+
+      // We might have already laid this field out.
+      if (elements[fieldIndex]) continue;
+
+      const llvm::Type *baseType = structure->getElementType(fieldIndex);
+      elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
+    }
+  }
+
+  // Now go through all other fields and zero them out.
+  for (unsigned i = 0; i != numElements; ++i) {
+    if (!elements[i])
+      elements[i] = llvm::Constant::getNullValue(structure->getElementType(i));
+  }
+  
+  return llvm::ConstantStruct::get(structure, elements);
+}
+
+/// Emit the null constant for a base subobject.
+static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
+                                               const llvm::Type *baseType,
+                                               const CXXRecordDecl *base) {
+  const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
+
+  // Just zero out bases that don't have any pointer to data members.
+  if (baseLayout.isZeroInitializableAsBase())
+    return llvm::Constant::getNullValue(baseType);
+
+  // If the base type is a struct, we can just use its null constant.
+  if (isa<llvm::StructType>(baseType)) {
+    return EmitNullConstant(CGM, base, /*complete*/ false);
+  }
+
+  // Otherwise, some bases are represented as arrays of i8 if the size
+  // of the base is smaller than its corresponding LLVM type.  Figure
+  // out how many elements this base array has.
+  const llvm::ArrayType *baseArrayType = cast<llvm::ArrayType>(baseType);
+  unsigned numBaseElements = baseArrayType->getNumElements();
+
+  // Fill in null data member pointers.
+  std::vector<llvm::Constant *> baseElements(numBaseElements);
+  FillInNullDataMemberPointers(CGM, CGM.getContext().getTypeDeclType(base),
+                               baseElements, 0);
+
+  // Now go through all other elements and zero them out.
+  if (numBaseElements) {
+    const llvm::Type *i8 = llvm::Type::getInt8Ty(CGM.getLLVMContext());
+    llvm::Constant *i8_zero = llvm::Constant::getNullValue(i8);
+    for (unsigned i = 0; i != numBaseElements; ++i) {
+      if (!baseElements[i])
+        baseElements[i] = i8_zero;
+    }
+  }
+      
+  return llvm::ConstantArray::get(baseArrayType, baseElements);
+}
+
 llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
   if (getTypes().isZeroInitializable(T))
     return llvm::Constant::getNullValue(getTypes().ConvertTypeForMem(T));
@@ -1036,79 +1175,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
 
   if (const RecordType *RT = T->getAs<RecordType>()) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-    const llvm::StructType *STy =
-      cast<llvm::StructType>(getTypes().ConvertTypeForMem(T));
-    unsigned NumElements = STy->getNumElements();
-    std::vector<llvm::Constant *> Elements(NumElements);
-
-    const CGRecordLayout &Layout = getTypes().getCGRecordLayout(RD);
-    
-    // Go through all bases and fill in any null pointer to data members.
-    for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-         E = RD->bases_end(); I != E; ++I) {
-      if (I->isVirtual()) {
-        // FIXME: We should initialize null pointer to data members in virtual
-        // bases here.
-        continue;
-      }
-
-      const CXXRecordDecl *BaseDecl = 
-        cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
-
-      // Ignore empty bases.
-      if (BaseDecl->isEmpty())
-        continue;
-
-      // Ignore bases that don't have any pointer to data members.
-      if (getTypes().isZeroInitializable(BaseDecl))
-        continue;
-
-      // Currently, all bases are arrays of i8. Figure out how many elements
-      // this base array has.
-      unsigned BaseFieldNo = Layout.getNonVirtualBaseLLVMFieldNo(BaseDecl);
-      const llvm::ArrayType *BaseArrayTy =
-        cast<llvm::ArrayType>(STy->getElementType(BaseFieldNo));
-      
-      unsigned NumBaseElements = BaseArrayTy->getNumElements();
-      std::vector<llvm::Constant *> BaseElements(NumBaseElements);
-      
-      // Now fill in null data member pointers.
-      FillInNullDataMemberPointers(*this, I->getType(), BaseElements, 0);
-      
-      // Now go through all other elements and zero them out.
-      if (NumBaseElements) {
-        llvm::Constant *Zero =
-          llvm::ConstantInt::get(llvm::Type::getInt8Ty(getLLVMContext()), 0);
-        
-        for (unsigned I = 0; I != NumBaseElements; ++I) {
-          if (!BaseElements[I])
-            BaseElements[I] = Zero;
-        }
-      }
-      
-      Elements[BaseFieldNo] = llvm::ConstantArray::get(BaseArrayTy, 
-                                                       BaseElements);
-    }
-      
-    for (RecordDecl::field_iterator I = RD->field_begin(),
-         E = RD->field_end(); I != E; ++I) {
-      const FieldDecl *FD = *I;
-      
-      // Ignore bit fields.
-      if (FD->isBitField())
-        continue;
-
-      unsigned FieldNo = Layout.getLLVMFieldNo(FD);
-      Elements[FieldNo] = EmitNullConstant(FD->getType());
-    }
-    
-    // Now go through all other fields and zero them out.
-    for (unsigned i = 0; i != NumElements; ++i) {
-      if (!Elements[i])
-        Elements[i] = llvm::Constant::getNullValue(STy->getElementType(i));
-    }
-    
-    return llvm::ConstantStruct::get(STy, Elements);
+    return ::EmitNullConstant(*this, RD, /*complete object*/ true);
   }
 
   assert(T->isMemberPointerType() && "Should only see member pointers here!");
@@ -1117,6 +1184,5 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
   
   // Itanium C++ ABI 2.3:
   //   A NULL pointer is represented as -1.
-  return llvm::ConstantInt::get(getTypes().ConvertTypeForMem(T), -1ULL, 
-                                /*isSigned=*/true);
+  return getCXXABI().EmitNullMemberPointer(T->castAs<MemberPointerType>());
 }
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 2318cc4e9aeb..3e1debd820b4 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -11,10 +11,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Frontend/CodeGenOptions.h"
 #include "CodeGenFunction.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "CodeGenModule.h"
+#include "CGDebugInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/RecordLayout.h"
@@ -37,6 +39,7 @@ using llvm::Value;
 //                         Scalar Expression Emitter
 //===----------------------------------------------------------------------===//
 
+namespace {
 struct BinOpInfo {
   Value *LHS;
   Value *RHS;
@@ -45,7 +48,13 @@ struct BinOpInfo {
   const Expr *E;      // Entire expr, for error unsupported.  May not be binop.
 };
 
-namespace {
+static bool MustVisitNullValue(const Expr *E) {
+  // If a null pointer expression's type is the C++0x nullptr_t, then
+  // it's not necessarily a simple constant and it must be evaluated
+  // for its potential side effects.
+  return E->getType()->isNullPtrType();
+}
+
 class ScalarExprEmitter
   : public StmtVisitor<ScalarExprEmitter, Value*> {
   CodeGenFunction &CGF;
@@ -101,10 +110,49 @@ public:
   /// EmitNullValue - Emit a value that corresponds to null for the given type.
   Value *EmitNullValue(QualType Ty);
 
+  /// EmitFloatToBoolConversion - Perform an FP to boolean conversion.
+  Value *EmitFloatToBoolConversion(Value *V) {
+    // Compare against 0.0 for fp scalars.
+    llvm::Value *Zero = llvm::Constant::getNullValue(V->getType());
+    return Builder.CreateFCmpUNE(V, Zero, "tobool");
+  }
+
+  /// EmitPointerToBoolConversion - Perform a pointer to boolean conversion.
+  Value *EmitPointerToBoolConversion(Value *V) {
+    Value *Zero = llvm::ConstantPointerNull::get(
+                                      cast<llvm::PointerType>(V->getType()));
+    return Builder.CreateICmpNE(V, Zero, "tobool");
+  }
+
+  Value *EmitIntToBoolConversion(Value *V) {
+    // Because of the type rules of C, we often end up computing a
+    // logical value, then zero extending it to int, then wanting it
+    // as a logical value again.  Optimize this common case.
+    if (llvm::ZExtInst *ZI = dyn_cast<llvm::ZExtInst>(V)) {
+      if (ZI->getOperand(0)->getType() == Builder.getInt1Ty()) {
+        Value *Result = ZI->getOperand(0);
+        // If there aren't any more uses, zap the instruction to save space.
+        // Note that there can be more uses, for example if this
+        // is the result of an assignment.
+        if (ZI->use_empty())
+          ZI->eraseFromParent();
+        return Result;
+      }
+    }
+
+    const llvm::IntegerType *Ty = cast<llvm::IntegerType>(V->getType());
+    Value *Zero = llvm::ConstantInt::get(Ty, 0);
+    return Builder.CreateICmpNE(V, Zero, "tobool");
+  }
+
   //===--------------------------------------------------------------------===//
   //                            Visitor Methods
   //===--------------------------------------------------------------------===//
 
+  Value *Visit(Expr *E) {
+    return StmtVisitor<ScalarExprEmitter, Value*>::Visit(E);
+  }
+    
   Value *VisitStmt(Stmt *S) {
     S->dump(CGF.getContext().getSourceManager());
     assert(0 && "Stmt can't have complex result type!");
@@ -112,7 +160,9 @@ public:
   }
   Value *VisitExpr(Expr *S);
   
-  Value *VisitParenExpr(ParenExpr *PE) { return Visit(PE->getSubExpr()); }
+  Value *VisitParenExpr(ParenExpr *PE) {
+    return Visit(PE->getSubExpr()); 
+  }
 
   // Leaves.
   Value *VisitIntegerLiteral(const IntegerLiteral *E) {
@@ -133,11 +183,6 @@ public:
   Value *VisitGNUNullExpr(const GNUNullExpr *E) {
     return EmitNullValue(E->getType());
   }
-  Value *VisitTypesCompatibleExpr(const TypesCompatibleExpr *E) {
-    return llvm::ConstantInt::get(ConvertType(E->getType()),
-                                  CGF.getContext().typesAreCompatible(
-                                    E->getArgType1(), E->getArgType2()));
-  }
   Value *VisitOffsetOfExpr(OffsetOfExpr *E);
   Value *VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E);
   Value *VisitAddrLabelExpr(const AddrLabelExpr *E) {
@@ -145,17 +190,45 @@ public:
     return Builder.CreateBitCast(V, ConvertType(E->getType()));
   }
 
+  Value *VisitSizeOfPackExpr(SizeOfPackExpr *E) {
+    return llvm::ConstantInt::get(ConvertType(E->getType()), 
+                                  E->getPackLength());
+  }
+
+  Value *VisitOpaqueValueExpr(OpaqueValueExpr *E) {
+    if (E->isGLValue())
+      return EmitLoadOfLValue(CGF.getOpaqueLValueMapping(E), E->getType());
+
+    // Otherwise, assume the mapping is the scalar directly.
+    return CGF.getOpaqueRValueMapping(E).getScalarVal();
+  }
+    
   // l-values.
   Value *VisitDeclRefExpr(DeclRefExpr *E) {
     Expr::EvalResult Result;
-    if (E->Evaluate(Result, CGF.getContext()) && Result.Val.isInt()) {
-      assert(!Result.HasSideEffects && "Constant declref with side-effect?!");
-      llvm::ConstantInt *CI 
-        = llvm::ConstantInt::get(VMContext, Result.Val.getInt());
-      CGF.EmitDeclRefExprDbgValue(E, CI);
-      return CI;
+    if (!E->Evaluate(Result, CGF.getContext()))
+      return EmitLoadOfLValue(E);
+
+    assert(!Result.HasSideEffects && "Constant declref with side-effect?!");
+
+    llvm::Constant *C;
+    if (Result.Val.isInt()) {
+      C = llvm::ConstantInt::get(VMContext, Result.Val.getInt());
+    } else if (Result.Val.isFloat()) {
+      C = llvm::ConstantFP::get(VMContext, Result.Val.getFloat());
+    } else {
+      return EmitLoadOfLValue(E);
     }
-    return EmitLoadOfLValue(E);
+
+    // Make sure we emit a debug reference to the global variable.
+    if (VarDecl *VD = dyn_cast<VarDecl>(E->getDecl())) {
+      if (!CGF.getContext().DeclMustBeEmitted(VD))
+        CGF.EmitDeclRefExprDbgValue(E, C);
+    } else if (isa<EnumConstantDecl>(E->getDecl())) {
+      CGF.EmitDeclRefExprDbgValue(E, C);
+    }
+
+    return C;
   }
   Value *VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
     return CGF.EmitObjCSelectorExpr(E);
@@ -167,10 +240,8 @@ public:
     return EmitLoadOfLValue(E);
   }
   Value *VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
-    return EmitLoadOfLValue(E);
-  }
-  Value *VisitObjCImplicitSetterGetterRefExpr(
-                        ObjCImplicitSetterGetterRefExpr *E) {
+    assert(E->getObjectKind() == OK_Ordinary &&
+           "reached property reference without lvalue-to-rvalue");
     return EmitLoadOfLValue(E);
   }
   Value *VisitObjCMessageExpr(ObjCMessageExpr *E) {
@@ -234,17 +305,26 @@ public:
     return EmitScalarPrePostIncDec(E, LV, true, true);
   }
 
+  llvm::Value *EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
+                                               llvm::Value *InVal,
+                                               llvm::Value *NextVal,
+                                               bool IsInc);
+
   llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
                                        bool isInc, bool isPre);
 
     
   Value *VisitUnaryAddrOf(const UnaryOperator *E) {
-    // If the sub-expression is an instance member reference,
-    // EmitDeclRefLValue will magically emit it with the appropriate
-    // value as the "address".
+    if (isa<MemberPointerType>(E->getType())) // never sugared
+      return CGF.CGM.getMemberPointerConstant(E);
+
     return EmitLValue(E->getSubExpr()).getAddress();
   }
-  Value *VisitUnaryDeref(const Expr *E) { return EmitLoadOfLValue(E); }
+  Value *VisitUnaryDeref(const UnaryOperator *E) {
+    if (E->getType()->isVoidType())
+      return Visit(E->getSubExpr()); // the actual value should be unused
+    return EmitLoadOfLValue(E);
+  }
   Value *VisitUnaryPlus(const UnaryOperator *E) {
     // This differs from gcc, though, most likely due to a bug in gcc.
     TestAndClearIgnoreResultAssign();
@@ -267,8 +347,8 @@ public:
     return CGF.LoadCXXThis();
   }
 
-  Value *VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
-    return CGF.EmitCXXExprWithTemporaries(E).getScalarVal();
+  Value *VisitExprWithCleanups(ExprWithCleanups *E) {
+    return CGF.EmitExprWithCleanups(E).getScalarVal();
   }
   Value *VisitCXXNewExpr(const CXXNewExpr *E) {
     return CGF.EmitCXXNewExpr(E);
@@ -278,8 +358,11 @@ public:
     return 0;
   }
   Value *VisitUnaryTypeTraitExpr(const UnaryTypeTraitExpr *E) {
-    return llvm::ConstantInt::get(Builder.getInt1Ty(),
-                                  E->EvaluateTrait(CGF.getContext()));
+    return llvm::ConstantInt::get(Builder.getInt1Ty(), E->getValue());
+  }
+
+  Value *VisitBinaryTypeTraitExpr(const BinaryTypeTraitExpr *E) {
+    return llvm::ConstantInt::get(ConvertType(E->getType()), E->getValue());
   }
 
   Value *VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E) {
@@ -301,6 +384,10 @@ public:
     return 0;
   }
 
+  Value *VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
+    return llvm::ConstantInt::get(Builder.getInt1Ty(), E->getValue());
+  }
+
   // Binary Operators.
   Value *EmitMul(const BinOpInfo &Ops) {
     if (Ops.Ty->hasSignedIntegerRepresentation()) {
@@ -318,9 +405,23 @@ public:
       return Builder.CreateFMul(Ops.LHS, Ops.RHS, "mul");
     return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul");
   }
+  bool isTrapvOverflowBehavior() {
+    return CGF.getContext().getLangOptions().getSignedOverflowBehavior() 
+               == LangOptions::SOB_Trapping; 
+  }
   /// Create a binary op that checks for overflow.
   /// Currently only supports +, - and *.
   Value *EmitOverflowCheckedBinOp(const BinOpInfo &Ops);
+  // Emit the overflow BB when -ftrapv option is activated. 
+  void EmitOverflowBB(llvm::BasicBlock *overflowBB) {
+    Builder.SetInsertPoint(overflowBB);
+    llvm::Function *Trap = CGF.CGM.getIntrinsic(llvm::Intrinsic::trap);
+    Builder.CreateCall(Trap);
+    Builder.CreateUnreachable();
+  }
+  // Check for undefined division and modulus behaviors.
+  void EmitUndefinedBehaviorIntegerDivAndRemCheck(const BinOpInfo &Ops, 
+                                                  llvm::Value *Zero,bool isDiv);
   Value *EmitDiv(const BinOpInfo &Ops);
   Value *EmitRem(const BinOpInfo &Ops);
   Value *EmitAdd(const BinOpInfo &Ops);
@@ -391,7 +492,7 @@ public:
 
   // Other Operators.
   Value *VisitBlockExpr(const BlockExpr *BE);
-  Value *VisitConditionalOperator(const ConditionalOperator *CO);
+  Value *VisitAbstractConditionalOperator(const AbstractConditionalOperator *);
   Value *VisitChooseExpr(ChooseExpr *CE);
   Value *VisitVAArgExpr(VAArgExpr *VE);
   Value *VisitObjCStringLiteral(const ObjCStringLiteral *E) {
@@ -409,11 +510,8 @@ public:
 Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
   assert(SrcType.isCanonical() && "EmitScalarConversion strips typedefs");
 
-  if (SrcType->isRealFloatingType()) {
-    // Compare against 0.0 for fp scalars.
-    llvm::Value *Zero = llvm::Constant::getNullValue(Src->getType());
-    return Builder.CreateFCmpUNE(Src, Zero, "tobool");
-  }
+  if (SrcType->isRealFloatingType())
+    return EmitFloatToBoolConversion(Src);
 
   if (const MemberPointerType *MPT = dyn_cast<MemberPointerType>(SrcType))
     return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, Src, MPT);
@@ -421,25 +519,11 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
   assert((SrcType->isIntegerType() || isa<llvm::PointerType>(Src->getType())) &&
          "Unknown scalar type to convert");
 
-  // Because of the type rules of C, we often end up computing a logical value,
-  // then zero extending it to int, then wanting it as a logical value again.
-  // Optimize this common case.
-  if (llvm::ZExtInst *ZI = dyn_cast<llvm::ZExtInst>(Src)) {
-    if (ZI->getOperand(0)->getType() ==
-        llvm::Type::getInt1Ty(CGF.getLLVMContext())) {
-      Value *Result = ZI->getOperand(0);
-      // If there aren't any more uses, zap the instruction to save space.
-      // Note that there can be more uses, for example if this
-      // is the result of an assignment.
-      if (ZI->use_empty())
-        ZI->eraseFromParent();
-      return Result;
-    }
-  }
+  if (isa<llvm::IntegerType>(Src->getType()))
+    return EmitIntToBoolConversion(Src);
 
-  // Compare against an integer or pointer null.
-  llvm::Value *Zero = llvm::Constant::getNullValue(Src->getType());
-  return Builder.CreateICmpNE(Src, Zero, "tobool");
+  assert(isa<llvm::PointerType>(Src->getType()));
+  return EmitPointerToBoolConversion(Src);
 }
 
 /// EmitScalarConversion - Emit a conversion from the specified type to the
@@ -501,10 +585,10 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
     // Splat the element across to all elements
     llvm::SmallVector<llvm::Constant*, 16> Args;
     unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
-    for (unsigned i = 0; i < NumElements; i++)
+    for (unsigned i = 0; i != NumElements; ++i)
       Args.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));
 
-    llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], NumElements);
+    llvm::Constant *Mask = llvm::ConstantVector::get(Args);
     llvm::Value *Yay = Builder.CreateShuffleVector(UnV, UnV, Mask, "splat");
     return Yay;
   }
@@ -603,7 +687,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
         concat.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 2*i+1));
       }
       
-      Value* CV = llvm::ConstantVector::get(concat.begin(), concat.size());
+      Value* CV = llvm::ConstantVector::get(concat);
       LHS = Builder.CreateShuffleVector(LHS, RHS, CV, "concat");
       LHSElts *= 2;
     } else {
@@ -629,7 +713,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
     for (unsigned i = 0, e = MTy->getNumElements(); i != e; ++i)
       MaskV.push_back(EltMask);
     
-    Value* MaskBits = llvm::ConstantVector::get(MaskV.begin(), MaskV.size());
+    Value* MaskBits = llvm::ConstantVector::get(MaskV);
     Mask = Builder.CreateAnd(Mask, MaskBits, "mask");
     
     // newv = undef
@@ -681,7 +765,7 @@ Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
     indices.push_back(C);
   }
 
-  Value* SV = llvm::ConstantVector::get(indices.begin(), indices.size());
+  Value *SV = llvm::ConstantVector::get(indices);
   return Builder.CreateShuffleVector(V1, V2, SV, "shuffle");
 }
 Value *ScalarExprEmitter::VisitMemberExpr(MemberExpr *E) {
@@ -693,6 +777,17 @@ Value *ScalarExprEmitter::VisitMemberExpr(MemberExpr *E) {
       EmitLValue(E->getBase());
     return llvm::ConstantInt::get(VMContext, Result.Val.getInt());
   }
+
+  // Emit debug info for aggregate now, if it was delayed to reduce 
+  // debug info size.
+  CGDebugInfo *DI = CGF.getDebugInfo();
+  if (DI && CGF.CGM.getCodeGenOpts().LimitDebugInfo) {
+    QualType PQTy = E->getBase()->IgnoreParenImpCasts()->getType();
+    if (const PointerType * PTy = dyn_cast<PointerType>(PQTy))
+      if (FieldDecl *M = dyn_cast<FieldDecl>(E->getMemberDecl()))
+        DI->getOrCreateRecordType(PTy->getPointeeType(), 
+                                  M->getParent()->getLocation());
+  }
   return EmitLoadOfLValue(E);
 }
 
@@ -790,7 +885,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
             VIsUndefShuffle = false;
           }
           if (!Args.empty()) {
-            llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], ResElts);
+            llvm::Constant *Mask = llvm::ConstantVector::get(Args);
             V = Builder.CreateShuffleVector(LHS, RHS, Mask);
             ++CurIdx;
             continue;
@@ -845,7 +940,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
         Args.push_back(llvm::ConstantInt::get(CGF.Int32Ty, j));
       for (unsigned j = InitElts; j != ResElts; ++j)
         Args.push_back(llvm::UndefValue::get(CGF.Int32Ty));
-      llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], ResElts);
+      llvm::Constant *Mask = llvm::ConstantVector::get(Args);
       Init = Builder.CreateShuffleVector(Init, llvm::UndefValue::get(VVT),
                                          Mask, "vext");
 
@@ -862,7 +957,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
     // merging subsequent shuffles into this one.
     if (CurIdx == 0)
       std::swap(V, Init);
-    llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], ResElts);
+    llvm::Constant *Mask = llvm::ConstantVector::get(Args);
     V = Builder.CreateShuffleVector(V, Init, Mask, "vecinit");
     VIsUndefShuffle = isa<llvm::UndefValue>(Init);
     CurIdx += InitElts;
@@ -916,11 +1011,8 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
   // a default case, so the compiler will warn on a missing case.  The cases
   // are in the same order as in the CastKind enum.
   switch (Kind) {
-  case CK_Unknown:
-    // FIXME: All casts should have a known kind!
-    //assert(0 && "Unknown cast kind!");
-    break;
-
+  case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
+      
   case CK_LValueBitCast: 
   case CK_ObjCObjectLValueCast: {
     Value *V = EmitLValue(E).getAddress();
@@ -963,9 +1055,6 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(CE);
     return CGF.EmitDynamicCast(V, DCE);
   }
-  case CK_ToUnion:
-    assert(0 && "Should be unreachable!");
-    break;
 
   case CK_ArrayToPointerDecay: {
     assert(E->getType()->isArrayType() &&
@@ -988,10 +1077,15 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
   case CK_FunctionToPointerDecay:
     return EmitLValue(E).getAddress();
 
+  case CK_NullToPointer:
+    if (MustVisitNullValue(E))
+      (void) Visit(E);
+
+    return llvm::ConstantPointerNull::get(
+                               cast<llvm::PointerType>(ConvertType(DestTy)));
+
   case CK_NullToMemberPointer: {
-    // If the subexpression's type is the C++0x nullptr_t, emit the
-    // subexpression, which may have side effects.
-    if (E->getType()->isNullPtrType())
+    if (MustVisitNullValue(E))
       (void) Visit(E);
 
     const MemberPointerType *MPT = CE->getType()->getAs<MemberPointerType>();
@@ -1011,11 +1105,30 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     return CGF.CGM.getCXXABI().EmitMemberPointerConversion(CGF, CE, Src);
   }
   
-
+  case CK_FloatingRealToComplex:
+  case CK_FloatingComplexCast:
+  case CK_IntegralRealToComplex:
+  case CK_IntegralComplexCast:
+  case CK_IntegralComplexToFloatingComplex:
+  case CK_FloatingComplexToIntegralComplex:
   case CK_ConstructorConversion:
-    assert(0 && "Should be unreachable!");
+  case CK_ToUnion:
+    llvm_unreachable("scalar cast to non-scalar value");
     break;
 
+  case CK_GetObjCProperty: {
+    assert(CGF.getContext().hasSameUnqualifiedType(E->getType(), DestTy));
+    assert(E->isGLValue() && E->getObjectKind() == OK_ObjCProperty &&
+           "CK_GetObjCProperty for non-lvalue or non-ObjCProperty");
+    RValue RV = CGF.EmitLoadOfLValue(CGF.EmitLValue(E), E->getType());
+    return RV.getScalarVal();
+  }
+      
+  case CK_LValueToRValue:
+    assert(CGF.getContext().hasSameUnqualifiedType(E->getType(), DestTy));
+    assert(E->isGLValue() && "lvalue-to-rvalue applied to r-value!");
+    return Visit(const_cast<Expr*>(E));
+
   case CK_IntegralToPointer: {
     Value *Src = Visit(const_cast<Expr*>(E));
 
@@ -1038,10 +1151,7 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     return Builder.CreatePtrToInt(Src, ConvertType(DestTy));
   }
   case CK_ToVoid: {
-    if (E->Classify(CGF.getContext()).isGLValue())
-      CGF.EmitLValue(E);
-    else
-      CGF.EmitAnyExpr(E, 0, false, true);
+    CGF.EmitIgnoredExpr(E);
     return 0;
   }
   case CK_VectorSplat: {
@@ -1056,62 +1166,55 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     // Splat the element across to all elements
     llvm::SmallVector<llvm::Constant*, 16> Args;
     unsigned NumElements = cast<llvm::VectorType>(DstTy)->getNumElements();
+    llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Int32Ty, 0);
     for (unsigned i = 0; i < NumElements; i++)
-      Args.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));
+      Args.push_back(Zero);
 
-    llvm::Constant *Mask = llvm::ConstantVector::get(&Args[0], NumElements);
+    llvm::Constant *Mask = llvm::ConstantVector::get(Args);
     llvm::Value *Yay = Builder.CreateShuffleVector(UnV, UnV, Mask, "splat");
     return Yay;
   }
+
   case CK_IntegralCast:
   case CK_IntegralToFloating:
   case CK_FloatingToIntegral:
   case CK_FloatingCast:
     return EmitScalarConversion(Visit(E), E->getType(), DestTy);
 
+  case CK_IntegralToBoolean:
+    return EmitIntToBoolConversion(Visit(E));
+  case CK_PointerToBoolean:
+    return EmitPointerToBoolConversion(Visit(E));
+  case CK_FloatingToBoolean:
+    return EmitFloatToBoolConversion(Visit(E));
   case CK_MemberPointerToBoolean: {
     llvm::Value *MemPtr = Visit(E);
     const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>();
     return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr, MPT);
   }
-  }
-  
-  // Handle cases where the source is an non-complex type.
 
-  if (!CGF.hasAggregateLLVMType(E->getType())) {
-    Value *Src = Visit(const_cast<Expr*>(E));
+  case CK_FloatingComplexToReal:
+  case CK_IntegralComplexToReal:
+    return CGF.EmitComplexExpr(E, false, true).first;
 
-    // Use EmitScalarConversion to perform the conversion.
-    return EmitScalarConversion(Src, E->getType(), DestTy);
-  }
+  case CK_FloatingComplexToBoolean:
+  case CK_IntegralComplexToBoolean: {
+    CodeGenFunction::ComplexPairTy V = CGF.EmitComplexExpr(E);
 
-  if (E->getType()->isAnyComplexType()) {
-    // Handle cases where the source is a complex type.
-    bool IgnoreImag = true;
-    bool IgnoreImagAssign = true;
-    bool IgnoreReal = IgnoreResultAssign;
-    bool IgnoreRealAssign = IgnoreResultAssign;
-    if (DestTy->isBooleanType())
-      IgnoreImagAssign = IgnoreImag = false;
-    else if (DestTy->isVoidType()) {
-      IgnoreReal = IgnoreImag = false;
-      IgnoreRealAssign = IgnoreImagAssign = true;
-    }
-    CodeGenFunction::ComplexPairTy V
-      = CGF.EmitComplexExpr(E, IgnoreReal, IgnoreImag, IgnoreRealAssign,
-                            IgnoreImagAssign);
+    // TODO: kill this function off, inline appropriate case here
     return EmitComplexToScalarConversion(V, E->getType(), DestTy);
   }
 
-  // Okay, this is a cast from an aggregate.  It must be a cast to void.  Just
-  // evaluate the result and return.
-  CGF.EmitAggExpr(E, 0, false, true);
+  }
+
+  llvm_unreachable("unknown scalar cast");
   return 0;
 }
 
 Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) {
-  return CGF.EmitCompoundStmt(*E->getSubStmt(),
-                              !E->getType()->isVoidType()).getScalarVal();
+  CodeGenFunction::StmtExprEvaluation eval(CGF);
+  return CGF.EmitCompoundStmt(*E->getSubStmt(), !E->getType()->isVoidType())
+    .getScalarVal();
 }
 
 Value *ScalarExprEmitter::VisitBlockDeclRefExpr(const BlockDeclRefExpr *E) {
@@ -1126,108 +1229,147 @@ Value *ScalarExprEmitter::VisitBlockDeclRefExpr(const BlockDeclRefExpr *E) {
 //===----------------------------------------------------------------------===//
 
 llvm::Value *ScalarExprEmitter::
-EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
-                        bool isInc, bool isPre) {
-  
-  QualType ValTy = E->getSubExpr()->getType();
-  llvm::Value *InVal = EmitLoadOfLValue(LV, ValTy);
-  
-  int AmountVal = isInc ? 1 : -1;
-  
-  if (ValTy->isPointerType() &&
-      ValTy->getAs<PointerType>()->isVariableArrayType()) {
-    // The amount of the addition/subtraction needs to account for the VLA size
-    CGF.ErrorUnsupported(E, "VLA pointer inc/dec");
+EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
+                                llvm::Value *InVal,
+                                llvm::Value *NextVal, bool IsInc) {
+  switch (CGF.getContext().getLangOptions().getSignedOverflowBehavior()) {
+  case LangOptions::SOB_Undefined:
+    return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+    break;
+  case LangOptions::SOB_Defined:
+    return Builder.CreateAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+    break;
+  case LangOptions::SOB_Trapping:
+    BinOpInfo BinOp;
+    BinOp.LHS = InVal;
+    BinOp.RHS = NextVal;
+    BinOp.Ty = E->getType();
+    BinOp.Opcode = BO_Add;
+    BinOp.E = E;
+    return EmitOverflowCheckedBinOp(BinOp);
+    break;
   }
+  assert(false && "Unknown SignedOverflowBehaviorTy");
+  return 0;
+}
+
+llvm::Value *
+ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
+                                           bool isInc, bool isPre) {
+  
+  QualType type = E->getSubExpr()->getType();
+  llvm::Value *value = EmitLoadOfLValue(LV, type);
+  llvm::Value *input = value;
+
+  int amount = (isInc ? 1 : -1);
+
+  // Special case of integer increment that we have to check first: bool++.
+  // Due to promotion rules, we get:
+  //   bool++ -> bool = bool + 1
+  //          -> bool = (int)bool + 1
+  //          -> bool = ((int)bool + 1 != 0)
+  // An interesting aspect of this is that increment is always true.
+  // Decrement does not have this property.
+  if (isInc && type->isBooleanType()) {
+    value = Builder.getTrue();
+
+  // Most common case by far: integer increment.
+  } else if (type->isIntegerType()) {
+
+    llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount);
+
+    if (type->isSignedIntegerType())
+      value = EmitAddConsiderOverflowBehavior(E, value, amt, isInc);
+
+    // Unsigned integer inc is always two's complement.
+    else
+      value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec");
   
-  llvm::Value *NextVal;
-  if (const llvm::PointerType *PT =
-      dyn_cast<llvm::PointerType>(InVal->getType())) {
-    llvm::Constant *Inc = llvm::ConstantInt::get(CGF.Int32Ty, AmountVal);
-    if (!isa<llvm::FunctionType>(PT->getElementType())) {
-      QualType PTEE = ValTy->getPointeeType();
-      if (const ObjCObjectType *OIT = PTEE->getAs<ObjCObjectType>()) {
-        // Handle interface types, which are not represented with a concrete
-        // type.
-        int size = CGF.getContext().getTypeSize(OIT) / 8;
-        if (!isInc)
-          size = -size;
-        Inc = llvm::ConstantInt::get(Inc->getType(), size);
-        const llvm::Type *i8Ty = llvm::Type::getInt8PtrTy(VMContext);
-        InVal = Builder.CreateBitCast(InVal, i8Ty);
-        NextVal = Builder.CreateGEP(InVal, Inc, "add.ptr");
-        llvm::Value *lhs = LV.getAddress();
-        lhs = Builder.CreateBitCast(lhs, llvm::PointerType::getUnqual(i8Ty));
-        LV = CGF.MakeAddrLValue(lhs, ValTy);
-      } else
-        NextVal = Builder.CreateInBoundsGEP(InVal, Inc, "ptrincdec");
+  // Next most common: pointer increment.
+  } else if (const PointerType *ptr = type->getAs<PointerType>()) {
+    QualType type = ptr->getPointeeType();
+
+    // VLA types don't have constant size.
+    if (type->isVariableArrayType()) {
+      llvm::Value *vlaSize =
+        CGF.GetVLASize(CGF.getContext().getAsVariableArrayType(type));
+      value = CGF.EmitCastToVoidPtr(value);
+      if (!isInc) vlaSize = Builder.CreateNSWNeg(vlaSize, "vla.negsize");
+      value = Builder.CreateInBoundsGEP(value, vlaSize, "vla.inc");
+      value = Builder.CreateBitCast(value, input->getType());
+    
+    // Arithmetic on function pointers (!) is just +-1.
+    } else if (type->isFunctionType()) {
+      llvm::Value *amt = llvm::ConstantInt::get(CGF.Int32Ty, amount);
+
+      value = CGF.EmitCastToVoidPtr(value);
+      value = Builder.CreateInBoundsGEP(value, amt, "incdec.funcptr");
+      value = Builder.CreateBitCast(value, input->getType());
+
+    // For everything else, we can just do a simple increment.
     } else {
-      const llvm::Type *i8Ty = llvm::Type::getInt8PtrTy(VMContext);
-      NextVal = Builder.CreateBitCast(InVal, i8Ty, "tmp");
-      NextVal = Builder.CreateGEP(NextVal, Inc, "ptrincdec");
-      NextVal = Builder.CreateBitCast(NextVal, InVal->getType());
+      llvm::Value *amt = llvm::ConstantInt::get(CGF.Int32Ty, amount);
+      value = Builder.CreateInBoundsGEP(value, amt, "incdec.ptr");
     }
-  } else if (InVal->getType()->isIntegerTy(1) && isInc) {
-    // Bool++ is an interesting case, due to promotion rules, we get:
-    // Bool++ -> Bool = Bool+1 -> Bool = (int)Bool+1 ->
-    // Bool = ((int)Bool+1) != 0
-    // An interesting aspect of this is that increment is always true.
-    // Decrement does not have this property.
-    NextVal = llvm::ConstantInt::getTrue(VMContext);
-  } else if (isa<llvm::IntegerType>(InVal->getType())) {
-    NextVal = llvm::ConstantInt::get(InVal->getType(), AmountVal);
-    
-    if (!ValTy->isSignedIntegerType())
-      // Unsigned integer inc is always two's complement.
-      NextVal = Builder.CreateAdd(InVal, NextVal, isInc ? "inc" : "dec");
-    else {
-      switch (CGF.getContext().getLangOptions().getSignedOverflowBehavior()) {
-      case LangOptions::SOB_Undefined:
-        NextVal = Builder.CreateNSWAdd(InVal, NextVal, isInc ? "inc" : "dec");
-        break;
-      case LangOptions::SOB_Defined:
-        NextVal = Builder.CreateAdd(InVal, NextVal, isInc ? "inc" : "dec");
-        break;
-      case LangOptions::SOB_Trapping:
-        BinOpInfo BinOp;
-        BinOp.LHS = InVal;
-        BinOp.RHS = NextVal;
-        BinOp.Ty = E->getType();
-        BinOp.Opcode = BO_Add;
-        BinOp.E = E;
-        NextVal = EmitOverflowCheckedBinOp(BinOp);
-        break;
-      }
+
+  // Vector increment/decrement.
+  } else if (type->isVectorType()) {
+    if (type->hasIntegerRepresentation()) {
+      llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount);
+
+      if (type->hasSignedIntegerRepresentation())
+        value = EmitAddConsiderOverflowBehavior(E, value, amt, isInc);
+      else
+        value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec");
+    } else {
+      value = Builder.CreateFAdd(
+                  value,
+                  llvm::ConstantFP::get(value->getType(), amount),
+                  isInc ? "inc" : "dec");
     }
-  } else {
+
+  // Floating point.
+  } else if (type->isRealFloatingType()) {
     // Add the inc/dec to the real part.
-    if (InVal->getType()->isFloatTy())
-      NextVal =
-      llvm::ConstantFP::get(VMContext,
-                            llvm::APFloat(static_cast<float>(AmountVal)));
-    else if (InVal->getType()->isDoubleTy())
-      NextVal =
-      llvm::ConstantFP::get(VMContext,
-                            llvm::APFloat(static_cast<double>(AmountVal)));
+    llvm::Value *amt;
+    if (value->getType()->isFloatTy())
+      amt = llvm::ConstantFP::get(VMContext,
+                                  llvm::APFloat(static_cast<float>(amount)));
+    else if (value->getType()->isDoubleTy())
+      amt = llvm::ConstantFP::get(VMContext,
+                                  llvm::APFloat(static_cast<double>(amount)));
     else {
-      llvm::APFloat F(static_cast<float>(AmountVal));
+      llvm::APFloat F(static_cast<float>(amount));
       bool ignored;
       F.convert(CGF.Target.getLongDoubleFormat(), llvm::APFloat::rmTowardZero,
                 &ignored);
-      NextVal = llvm::ConstantFP::get(VMContext, F);
+      amt = llvm::ConstantFP::get(VMContext, F);
     }
-    NextVal = Builder.CreateFAdd(InVal, NextVal, isInc ? "inc" : "dec");
+    value = Builder.CreateFAdd(value, amt, isInc ? "inc" : "dec");
+
+  // Objective-C pointer types.
+  } else {
+    const ObjCObjectPointerType *OPT = type->castAs<ObjCObjectPointerType>();
+    value = CGF.EmitCastToVoidPtr(value);
+
+    CharUnits size = CGF.getContext().getTypeSizeInChars(OPT->getObjectType());
+    if (!isInc) size = -size;
+    llvm::Value *sizeValue =
+      llvm::ConstantInt::get(CGF.SizeTy, size.getQuantity());
+
+    value = Builder.CreateInBoundsGEP(value, sizeValue, "incdec.objptr");
+    value = Builder.CreateBitCast(value, input->getType());
   }
   
   // Store the updated result through the lvalue.
   if (LV.isBitField())
-    CGF.EmitStoreThroughBitfieldLValue(RValue::get(NextVal), LV, ValTy, &NextVal);
+    CGF.EmitStoreThroughBitfieldLValue(RValue::get(value), LV, type, &value);
   else
-    CGF.EmitStoreThroughLValue(RValue::get(NextVal), LV, ValTy);
+    CGF.EmitStoreThroughLValue(RValue::get(value), LV, type);
   
   // If this is a postinc, return the value read from memory, otherwise use the
   // updated value.
-  return isPre ? NextVal : InVal;
+  return isPre ? value : input;
 }
 
 
@@ -1346,7 +1488,7 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
       // Compute the offset to the base.
       const RecordType *BaseRT = CurrentType->getAs<RecordType>();
       CXXRecordDecl *BaseRD = cast<CXXRecordDecl>(BaseRT->getDecl());
-      int64_t OffsetInt = RL.getBaseClassOffset(BaseRD) /
+      int64_t OffsetInt = RL.getBaseClassOffsetInBits(BaseRD) /
                           CGF.getContext().getCharWidth();
       Offset = llvm::ConstantInt::get(ResultType, OffsetInt);
       break;
@@ -1371,7 +1513,7 @@ ScalarExprEmitter::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E) {
       } else {
         // C99 6.5.3.4p2: If the argument is an expression of type
         // VLA, it is evaluated.
-        CGF.EmitAnyExpr(E->getArgumentExpr());
+        CGF.EmitIgnoredExpr(E->getArgumentExpr());
       }
 
       return CGF.GetVLASize(VAT);
@@ -1387,21 +1529,38 @@ ScalarExprEmitter::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E) {
 
 Value *ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *E) {
   Expr *Op = E->getSubExpr();
-  if (Op->getType()->isAnyComplexType())
-    return CGF.EmitComplexExpr(Op, false, true, false, true).first;
+  if (Op->getType()->isAnyComplexType()) {
+    // If it's an l-value, load through the appropriate subobject l-value.
+    // Note that we have to ask E because Op might be an l-value that
+    // this won't work for, e.g. an Obj-C property.
+    if (E->isGLValue())
+      return CGF.EmitLoadOfLValue(CGF.EmitLValue(E), E->getType())
+                .getScalarVal();
+
+    // Otherwise, calculate and project.
+    return CGF.EmitComplexExpr(Op, false, true).first;
+  }
+
   return Visit(Op);
 }
+
 Value *ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *E) {
   Expr *Op = E->getSubExpr();
-  if (Op->getType()->isAnyComplexType())
-    return CGF.EmitComplexExpr(Op, true, false, true, false).second;
+  if (Op->getType()->isAnyComplexType()) {
+    // If it's an l-value, load through the appropriate subobject l-value.
+    // Note that we have to ask E because Op might be an l-value that
+    // this won't work for, e.g. an Obj-C property.
+    if (Op->isGLValue())
+      return CGF.EmitLoadOfLValue(CGF.EmitLValue(E), E->getType())
+                .getScalarVal();
+
+    // Otherwise, calculate and project.
+    return CGF.EmitComplexExpr(Op, true, false).second;
+  }
 
   // __imag on a scalar returns zero.  Emit the subexpr to ensure side
   // effects are evaluated, but not the actual value.
-  if (E->isLvalue(CGF.getContext()) == Expr::LV_Valid)
-    CGF.EmitLValue(Op);
-  else
-    CGF.EmitScalarExpr(Op, true);
+  CGF.EmitScalarExpr(Op, true);
   return llvm::Constant::getNullValue(ConvertType(E->getType()));
 }
 
@@ -1478,8 +1637,12 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
   if (Ignore)
     return 0;
 
+  // The result of an assignment in C is the assigned r-value.
+  if (!CGF.getContext().getLangOptions().CPlusPlus)
+    return RHS;
+
   // Objective-C property assignment never reloads the value following a store.
-  if (LHS.isPropertyRef() || LHS.isKVCRef())
+  if (LHS.isPropertyRef())
     return RHS;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
@@ -1490,8 +1653,51 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
   return EmitLoadOfLValue(LHS, E->getType());
 }
 
+void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
+     					    const BinOpInfo &Ops, 
+				     	    llvm::Value *Zero, bool isDiv) {
+  llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
+  llvm::BasicBlock *contBB =
+    CGF.createBasicBlock(isDiv ? "div.cont" : "rem.cont", CGF.CurFn);
+
+  const llvm::IntegerType *Ty = cast<llvm::IntegerType>(Zero->getType());
+
+  if (Ops.Ty->hasSignedIntegerRepresentation()) {
+    llvm::Value *IntMin =
+      llvm::ConstantInt::get(VMContext,
+                             llvm::APInt::getSignedMinValue(Ty->getBitWidth()));
+    llvm::Value *NegOne = llvm::ConstantInt::get(Ty, -1ULL);
+
+    llvm::Value *Cond1 = Builder.CreateICmpEQ(Ops.RHS, Zero);
+    llvm::Value *LHSCmp = Builder.CreateICmpEQ(Ops.LHS, IntMin);
+    llvm::Value *RHSCmp = Builder.CreateICmpEQ(Ops.RHS, NegOne);
+    llvm::Value *Cond2 = Builder.CreateAnd(LHSCmp, RHSCmp, "and");
+    Builder.CreateCondBr(Builder.CreateOr(Cond1, Cond2, "or"), 
+                         overflowBB, contBB);
+  } else {
+    CGF.Builder.CreateCondBr(Builder.CreateICmpEQ(Ops.RHS, Zero), 
+                             overflowBB, contBB);
+  }
+  EmitOverflowBB(overflowBB);
+  Builder.SetInsertPoint(contBB);
+}
 
 Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
+  if (isTrapvOverflowBehavior()) { 
+    llvm::Value *Zero = llvm::Constant::getNullValue(ConvertType(Ops.Ty));
+
+    if (Ops.Ty->isIntegerType())
+      EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, true);
+    else if (Ops.Ty->isRealFloatingType()) {
+      llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow",
+                                                          CGF.CurFn);
+      llvm::BasicBlock *DivCont = CGF.createBasicBlock("div.cont", CGF.CurFn);
+      CGF.Builder.CreateCondBr(Builder.CreateFCmpOEQ(Ops.RHS, Zero), 
+                               overflowBB, DivCont);
+      EmitOverflowBB(overflowBB);
+      Builder.SetInsertPoint(DivCont);
+    }
+  }
   if (Ops.LHS->getType()->isFPOrFPVectorTy())
     return Builder.CreateFDiv(Ops.LHS, Ops.RHS, "div");
   else if (Ops.Ty->hasUnsignedIntegerRepresentation())
@@ -1502,6 +1708,13 @@ Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
 
 Value *ScalarExprEmitter::EmitRem(const BinOpInfo &Ops) {
   // Rem in C can't be a floating point type: C99 6.5.5p2.
+  if (isTrapvOverflowBehavior()) {
+    llvm::Value *Zero = llvm::Constant::getNullValue(ConvertType(Ops.Ty));
+
+    if (Ops.Ty->isIntegerType()) 
+      EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, false);
+  }
+
   if (Ops.Ty->isUnsignedIntegerType())
     return Builder.CreateURem(Ops.LHS, Ops.RHS, "rem");
   else
@@ -1544,21 +1757,56 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1);
 
   // Branch in case of overflow.
+  llvm::BasicBlock *initialBB = Builder.GetInsertBlock();
   llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
   llvm::BasicBlock *continueBB = CGF.createBasicBlock("nooverflow", CGF.CurFn);
 
   Builder.CreateCondBr(overflow, overflowBB, continueBB);
 
   // Handle overflow with llvm.trap.
-  // TODO: it would be better to generate one of these blocks per function.
+  const std::string *handlerName = 
+    &CGF.getContext().getLangOptions().OverflowHandler;
+  if (handlerName->empty()) {
+    EmitOverflowBB(overflowBB);
+    Builder.SetInsertPoint(continueBB);
+    return result;
+  }
+
+  // If an overflow handler is set, then we want to call it and then use its
+  // result, if it returns.
   Builder.SetInsertPoint(overflowBB);
-  llvm::Function *Trap = CGF.CGM.getIntrinsic(llvm::Intrinsic::trap);
-  Builder.CreateCall(Trap);
-  Builder.CreateUnreachable();
-  
-  // Continue on.
+
+  // Get the overflow handler.
+  const llvm::Type *Int8Ty = llvm::Type::getInt8Ty(VMContext);
+  std::vector<const llvm::Type*> argTypes;
+  argTypes.push_back(CGF.Int64Ty); argTypes.push_back(CGF.Int64Ty);
+  argTypes.push_back(Int8Ty); argTypes.push_back(Int8Ty);
+  llvm::FunctionType *handlerTy =
+      llvm::FunctionType::get(CGF.Int64Ty, argTypes, true);
+  llvm::Value *handler = CGF.CGM.CreateRuntimeFunction(handlerTy, *handlerName);
+
+  // Sign extend the args to 64-bit, so that we can use the same handler for
+  // all types of overflow.
+  llvm::Value *lhs = Builder.CreateSExt(Ops.LHS, CGF.Int64Ty);
+  llvm::Value *rhs = Builder.CreateSExt(Ops.RHS, CGF.Int64Ty);
+
+  // Call the handler with the two arguments, the operation, and the size of
+  // the result.
+  llvm::Value *handlerResult = Builder.CreateCall4(handler, lhs, rhs,
+      Builder.getInt8(OpID),
+      Builder.getInt8(cast<llvm::IntegerType>(opTy)->getBitWidth()));
+
+  // Truncate the result back to the desired size.
+  handlerResult = Builder.CreateTrunc(handlerResult, opTy);
+  Builder.CreateBr(continueBB);
+
   Builder.SetInsertPoint(continueBB);
-  return result;
+  llvm::PHINode *phi = Builder.CreatePHI(opTy);
+  phi->reserveOperandSpace(2);
+  phi->addIncoming(result, initialBB);
+  phi->addIncoming(handlerResult, overflowBB);
+
+  return phi;
 }
 
 Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &Ops) {
@@ -1609,7 +1857,7 @@ Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &Ops) {
   }
 
   unsigned Width = cast<llvm::IntegerType>(Idx->getType())->getBitWidth();
-  if (Width < CGF.LLVMPointerWidth) {
+  if (Width < CGF.PointerWidthInBits) {
     // Zero or sign extend the pointer value based on whether the index is
     // signed or not.
     const llvm::Type *IdxType = CGF.IntPtrTy;
@@ -1682,7 +1930,7 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &Ops) {
     // pointer - int
     Value *Idx = Ops.RHS;
     unsigned Width = cast<llvm::IntegerType>(Idx->getType())->getBitWidth();
-    if (Width < CGF.LLVMPointerWidth) {
+    if (Width < CGF.PointerWidthInBits) {
       // Zero or sign extend the pointer value based on whether the index is
       // signed or not.
       const llvm::Type *IdxType = CGF.IntPtrTy;
@@ -1792,6 +2040,48 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
   return Builder.CreateAShr(Ops.LHS, RHS, "shr");
 }
 
+enum IntrinsicType { VCMPEQ, VCMPGT };
+// return corresponding comparison intrinsic for given vector type
+static llvm::Intrinsic::ID GetIntrinsic(IntrinsicType IT,
+                                        BuiltinType::Kind ElemKind) {
+  switch (ElemKind) {
+  default: assert(0 && "unexpected element type");
+  case BuiltinType::Char_U:
+  case BuiltinType::UChar:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequb_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtub_p;
+    break;
+  case BuiltinType::Char_S:
+  case BuiltinType::SChar:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequb_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtsb_p;
+    break;
+  case BuiltinType::UShort:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequh_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtuh_p;
+    break;
+  case BuiltinType::Short:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequh_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtsh_p;
+    break;
+  case BuiltinType::UInt:
+  case BuiltinType::ULong:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequw_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtuw_p;
+    break;
+  case BuiltinType::Int:
+  case BuiltinType::Long:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequw_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtsw_p;
+    break;
+  case BuiltinType::Float:
+    return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpeqfp_p :
+                            llvm::Intrinsic::ppc_altivec_vcmpgtfp_p;
+    break;
+  }
+  return llvm::Intrinsic::not_intrinsic;
+}
+
 Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
                                       unsigned SICmpOpc, unsigned FCmpOpc) {
   TestAndClearIgnoreResultAssign();
@@ -1808,6 +2098,71 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
     Value *LHS = Visit(E->getLHS());
     Value *RHS = Visit(E->getRHS());
 
+    // If AltiVec, the comparison results in a numeric type, so we use
+    // intrinsics comparing vectors and giving 0 or 1 as a result
+    if (LHSTy->isVectorType() && CGF.getContext().getLangOptions().AltiVec) {
+      // constants for mapping CR6 register bits to predicate result
+      enum { CR6_EQ=0, CR6_EQ_REV, CR6_LT, CR6_LT_REV } CR6;
+
+      llvm::Intrinsic::ID ID = llvm::Intrinsic::not_intrinsic;
+
+      // in several cases vector arguments order will be reversed
+      Value *FirstVecArg = LHS,
+            *SecondVecArg = RHS;
+
+      QualType ElTy = LHSTy->getAs<VectorType>()->getElementType();
+      const BuiltinType *BTy = ElTy->getAs<BuiltinType>();
+      BuiltinType::Kind ElementKind = BTy->getKind();
+
+      switch(E->getOpcode()) {
+      default: assert(0 && "is not a comparison operation");
+      case BO_EQ:
+        CR6 = CR6_LT;
+        ID = GetIntrinsic(VCMPEQ, ElementKind);
+        break;
+      case BO_NE:
+        CR6 = CR6_EQ;
+        ID = GetIntrinsic(VCMPEQ, ElementKind);
+        break;
+      case BO_LT:
+        CR6 = CR6_LT;
+        ID = GetIntrinsic(VCMPGT, ElementKind);
+        std::swap(FirstVecArg, SecondVecArg);
+        break;
+      case BO_GT:
+        CR6 = CR6_LT;
+        ID = GetIntrinsic(VCMPGT, ElementKind);
+        break;
+      case BO_LE:
+        if (ElementKind == BuiltinType::Float) {
+          CR6 = CR6_LT;
+          ID = llvm::Intrinsic::ppc_altivec_vcmpgefp_p;
+          std::swap(FirstVecArg, SecondVecArg);
+        }
+        else {
+          CR6 = CR6_EQ;
+          ID = GetIntrinsic(VCMPGT, ElementKind);
+        }
+        break;
+      case BO_GE:
+        if (ElementKind == BuiltinType::Float) {
+          CR6 = CR6_LT;
+          ID = llvm::Intrinsic::ppc_altivec_vcmpgefp_p;
+        }
+        else {
+          CR6 = CR6_EQ;
+          ID = GetIntrinsic(VCMPGT, ElementKind);
+          std::swap(FirstVecArg, SecondVecArg);
+        }
+        break;
+      }
+
+      Value *CR6Param = llvm::ConstantInt::get(CGF.Int32Ty, CR6);
+      llvm::Function *F = CGF.CGM.getIntrinsic(ID);
+      Result = Builder.CreateCall3(F, CR6Param, FirstVecArg, SecondVecArg, "");
+      return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType());
+    }
+
     if (LHS->getType()->isFPOrFPVectorTy()) {
       Result = Builder.CreateFCmp((llvm::CmpInst::Predicate)FCmpOpc,
                                   LHS, RHS, "cmp");
@@ -1881,8 +2236,12 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   if (Ignore)
     return 0;
 
+  // The result of an assignment in C is the assigned r-value.
+  if (!CGF.getContext().getLangOptions().CPlusPlus)
+    return RHS;
+
   // Objective-C property assignment never reloads the value following a store.
-  if (LHS.isPropertyRef() || LHS.isKVCRef())
+  if (LHS.isPropertyRef())
     return RHS;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
@@ -1913,6 +2272,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("land.end");
   llvm::BasicBlock *RHSBlock  = CGF.createBasicBlock("land.rhs");
 
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
+
   // Branch on the LHS first.  If it is false, go to the failure (cont) block.
   CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock);
 
@@ -1926,10 +2287,10 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
        PI != PE; ++PI)
     PN->addIncoming(llvm::ConstantInt::getFalse(VMContext), *PI);
 
-  CGF.BeginConditionalBranch();
+  eval.begin(CGF);
   CGF.EmitBlock(RHSBlock);
   Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
-  CGF.EndConditionalBranch();
+  eval.end(CGF);
 
   // Reaquire the RHS block, as there may be subblocks inserted.
   RHSBlock = Builder.GetInsertBlock();
@@ -1963,6 +2324,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("lor.end");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("lor.rhs");
 
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
+
   // Branch on the LHS first.  If it is true, go to the success (cont) block.
   CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock);
 
@@ -1976,13 +2339,13 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
        PI != PE; ++PI)
     PN->addIncoming(llvm::ConstantInt::getTrue(VMContext), *PI);
 
-  CGF.BeginConditionalBranch();
+  eval.begin(CGF);
 
   // Emit the RHS condition as a bool value.
   CGF.EmitBlock(RHSBlock);
   Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
 
-  CGF.EndConditionalBranch();
+  eval.end(CGF);
 
   // Reaquire the RHS block, as there may be subblocks inserted.
   RHSBlock = Builder.GetInsertBlock();
@@ -1997,7 +2360,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
 }
 
 Value *ScalarExprEmitter::VisitBinComma(const BinaryOperator *E) {
-  CGF.EmitStmt(E->getLHS());
+  CGF.EmitIgnoredExpr(E->getLHS());
   CGF.EnsureInsertPoint();
   return Visit(E->getRHS());
 }
@@ -2034,95 +2397,107 @@ static bool isCheapEnoughToEvaluateUnconditionally(const Expr *E,
 
 
 Value *ScalarExprEmitter::
-VisitConditionalOperator(const ConditionalOperator *E) {
+VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   TestAndClearIgnoreResultAssign();
+
+  // Bind the common expression if necessary.
+  CodeGenFunction::OpaqueValueMapping binding(CGF, E);
+
+  Expr *condExpr = E->getCond();
+  Expr *lhsExpr = E->getTrueExpr();
+  Expr *rhsExpr = E->getFalseExpr();
+
   // If the condition constant folds and can be elided, try to avoid emitting
   // the condition and the dead arm.
-  if (int Cond = CGF.ConstantFoldsToSimpleInteger(E->getCond())){
-    Expr *Live = E->getLHS(), *Dead = E->getRHS();
-    if (Cond == -1)
-      std::swap(Live, Dead);
+  if (int Cond = CGF.ConstantFoldsToSimpleInteger(condExpr)){
+    Expr *live = lhsExpr, *dead = rhsExpr;
+    if (Cond == -1) std::swap(live, dead);
 
     // If the dead side doesn't have labels we need, and if the Live side isn't
     // the gnu missing ?: extension (which we could handle, but don't bother
     // to), just emit the Live part.
-    if ((!Dead || !CGF.ContainsLabel(Dead)) &&  // No labels in dead part
-        Live)                                   // Live part isn't missing.
-      return Visit(Live);
+    if (!CGF.ContainsLabel(dead))
+      return Visit(live);
   }
 
+  // OpenCL: If the condition is a vector, we can treat this condition like
+  // the select function.
+  if (CGF.getContext().getLangOptions().OpenCL 
+      && condExpr->getType()->isVectorType()) {
+    llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
+    llvm::Value *LHS = Visit(lhsExpr);
+    llvm::Value *RHS = Visit(rhsExpr);
+    
+    const llvm::Type *condType = ConvertType(condExpr->getType());
+    const llvm::VectorType *vecTy = cast<llvm::VectorType>(condType);
+    
+    unsigned numElem = vecTy->getNumElements();      
+    const llvm::Type *elemType = vecTy->getElementType();
+    
+    std::vector<llvm::Constant*> Zvals;
+    for (unsigned i = 0; i < numElem; ++i)
+      Zvals.push_back(llvm::ConstantInt::get(elemType,0));
+
+    llvm::Value *zeroVec = llvm::ConstantVector::get(Zvals);    
+    llvm::Value *TestMSB = Builder.CreateICmpSLT(CondV, zeroVec);
+    llvm::Value *tmp = Builder.CreateSExt(TestMSB, 
+                                          llvm::VectorType::get(elemType,
+                                                                numElem),         
+                                          "sext");
+    llvm::Value *tmp2 = Builder.CreateNot(tmp);
+    
+    // Cast float to int to perform ANDs if necessary.
+    llvm::Value *RHSTmp = RHS;
+    llvm::Value *LHSTmp = LHS;
+    bool wasCast = false;
+    const llvm::VectorType *rhsVTy = cast<llvm::VectorType>(RHS->getType());
+    if (rhsVTy->getElementType()->isFloatTy()) {
+      RHSTmp = Builder.CreateBitCast(RHS, tmp2->getType());
+      LHSTmp = Builder.CreateBitCast(LHS, tmp->getType());
+      wasCast = true;
+    }
+    
+    llvm::Value *tmp3 = Builder.CreateAnd(RHSTmp, tmp2);
+    llvm::Value *tmp4 = Builder.CreateAnd(LHSTmp, tmp);
+    llvm::Value *tmp5 = Builder.CreateOr(tmp3, tmp4, "cond");
+    if (wasCast)
+      tmp5 = Builder.CreateBitCast(tmp5, RHS->getType());
 
+    return tmp5;
+  }
+  
   // If this is a really simple expression (like x ? 4 : 5), emit this as a
   // select instead of as control flow.  We can only do this if it is cheap and
   // safe to evaluate the LHS and RHS unconditionally.
-  if (E->getLHS() && isCheapEnoughToEvaluateUnconditionally(E->getLHS(),
-                                                            CGF) &&
-      isCheapEnoughToEvaluateUnconditionally(E->getRHS(), CGF)) {
-    llvm::Value *CondV = CGF.EvaluateExprAsBool(E->getCond());
-    llvm::Value *LHS = Visit(E->getLHS());
-    llvm::Value *RHS = Visit(E->getRHS());
+  if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) &&
+      isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) {
+    llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr);
+    llvm::Value *LHS = Visit(lhsExpr);
+    llvm::Value *RHS = Visit(rhsExpr);
     return Builder.CreateSelect(CondV, LHS, RHS, "cond");
   }
 
-  if (!E->getLHS() && CGF.getContext().getLangOptions().CPlusPlus) {
-    // Does not support GNU missing condition extension in C++ yet (see #7726)
-    CGF.ErrorUnsupported(E, "conditional operator with missing LHS");
-    return llvm::UndefValue::get(ConvertType(E->getType()));
-  }
-  
   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
-  Value *CondVal = 0;
-
-  // If we don't have the GNU missing condition extension, emit a branch on bool
-  // the normal way.
-  if (E->getLHS()) {
-    // Otherwise, just use EmitBranchOnBoolExpr to get small and simple code for
-    // the branch on bool.
-    CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
-  } else {
-    // Otherwise, for the ?: extension, evaluate the conditional and then
-    // convert it to bool the hard way.  We do this explicitly because we need
-    // the unconverted value for the missing middle value of the ?:.
-    CondVal = CGF.EmitScalarExpr(E->getCond());
-
-    // In some cases, EmitScalarConversion will delete the "CondVal" expression
-    // if there are no extra uses (an optimization).  Inhibit this by making an
-    // extra dead use, because we're going to add a use of CondVal later.  We
-    // don't use the builder for this, because we don't want it to get optimized
-    // away.  This leaves dead code, but the ?: extension isn't common.
-    new llvm::BitCastInst(CondVal, CondVal->getType(), "dummy?:holder",
-                          Builder.GetInsertBlock());
-
-    Value *CondBoolVal =
-      CGF.EmitScalarConversion(CondVal, E->getCond()->getType(),
-                               CGF.getContext().BoolTy);
-    Builder.CreateCondBr(CondBoolVal, LHSBlock, RHSBlock);
-  }
-
-  CGF.BeginConditionalBranch();
-  CGF.EmitBlock(LHSBlock);
 
-  // Handle the GNU extension for missing LHS.
-  Value *LHS;
-  if (E->getLHS())
-    LHS = Visit(E->getLHS());
-  else    // Perform promotions, to handle cases like "short ?: int"
-    LHS = EmitScalarConversion(CondVal, E->getCond()->getType(), E->getType());
+  CodeGenFunction::ConditionalEvaluation eval(CGF);
+  CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock);
+
+  CGF.EmitBlock(LHSBlock);
+  eval.begin(CGF);
+  Value *LHS = Visit(lhsExpr);
+  eval.end(CGF);
 
-  CGF.EndConditionalBranch();
   LHSBlock = Builder.GetInsertBlock();
-  CGF.EmitBranch(ContBlock);
+  Builder.CreateBr(ContBlock);
 
-  CGF.BeginConditionalBranch();
   CGF.EmitBlock(RHSBlock);
+  eval.begin(CGF);
+  Value *RHS = Visit(rhsExpr);
+  eval.end(CGF);
 
-  Value *RHS = Visit(E->getRHS());
-  CGF.EndConditionalBranch();
   RHSBlock = Builder.GetInsertBlock();
-  CGF.EmitBranch(ContBlock);
-
   CGF.EmitBlock(ContBlock);
 
   // If the LHS or RHS is a throw expression, it will be legitimately null.
@@ -2155,8 +2530,8 @@ Value *ScalarExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
   return Builder.CreateLoad(ArgPtr);
 }
 
-Value *ScalarExprEmitter::VisitBlockExpr(const BlockExpr *BE) {
-  return CGF.BuildBlockLiteralTmp(BE);
+Value *ScalarExprEmitter::VisitBlockExpr(const BlockExpr *block) {
+  return CGF.EmitBlockLiteral(block);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2209,7 +2584,7 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
   const llvm::Type *ClassPtrTy = ConvertType(E->getType());
 
   Expr *BaseExpr = E->getBase();
-  if (BaseExpr->isLvalue(getContext()) != Expr::LV_Valid) {
+  if (BaseExpr->isRValue()) {
     V = CreateTempAlloca(ClassPtrTy, "resval");
     llvm::Value *Src = EmitScalarExpr(BaseExpr);
     Builder.CreateStore(Src, V);
@@ -2229,7 +2604,7 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
 }
 
 
-LValue CodeGenFunction::EmitCompoundAssignOperatorLValue(
+LValue CodeGenFunction::EmitCompoundAssignmentLValue(
                                             const CompoundAssignOperator *E) {
   ScalarExprEmitter Scalar(*this);
   Value *Result = 0;
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index 6a6d63df8f52..08c458bd52d3 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CGDebugInfo.h"
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
@@ -137,9 +138,49 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
        E = OMD->param_end(); PI != E; ++PI)
     Args.push_back(std::make_pair(*PI, (*PI)->getType()));
 
+  CurGD = OMD;
+
   StartFunction(OMD, OMD->getResultType(), Fn, Args, OMD->getLocStart());
 }
 
+void CodeGenFunction::GenerateObjCGetterBody(ObjCIvarDecl *Ivar, 
+                                             bool IsAtomic, bool IsStrong) {
+  LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
+                                Ivar, 0);
+  llvm::Value *GetCopyStructFn =
+  CGM.getObjCRuntime().GetGetStructFunction();
+  CodeGenTypes &Types = CGM.getTypes();
+  // objc_copyStruct (ReturnValue, &structIvar, 
+  //                  sizeof (Type of Ivar), isAtomic, false);
+  CallArgList Args;
+  RValue RV = RValue::get(Builder.CreateBitCast(ReturnValue,
+                                                Types.ConvertType(getContext().VoidPtrTy)));
+  Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
+  RV = RValue::get(Builder.CreateBitCast(LV.getAddress(),
+                                         Types.ConvertType(getContext().VoidPtrTy)));
+  Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
+  // sizeof (Type of Ivar)
+  CharUnits Size =  getContext().getTypeSizeInChars(Ivar->getType());
+  llvm::Value *SizeVal =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy), 
+                         Size.getQuantity());
+  Args.push_back(std::make_pair(RValue::get(SizeVal),
+                                getContext().LongTy));
+  llvm::Value *isAtomic =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
+                         IsAtomic ? 1 : 0);
+  Args.push_back(std::make_pair(RValue::get(isAtomic), 
+                                getContext().BoolTy));
+  llvm::Value *hasStrong =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
+                         IsStrong ? 1 : 0);
+  Args.push_back(std::make_pair(RValue::get(hasStrong), 
+                                getContext().BoolTy));
+  EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
+                                 FunctionType::ExtInfo()),
+           GetCopyStructFn, ReturnValueSlot(), Args);
+}
+
 /// Generate an Objective-C method.  An Objective-C method is a C function with
 /// its pointer, name, and types registered in the class struture.
 void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) {
@@ -211,51 +252,30 @@ void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
                                            Types.ConvertType(PD->getType())));
     EmitReturnOfRValue(RV, PD->getType());
   } else {
-    if (Ivar->getType()->isAnyComplexType()) {
+    const llvm::Triple &Triple = getContext().Target.getTriple();
+    QualType IVART = Ivar->getType();
+    if (IsAtomic &&
+        IVART->isScalarType() &&
+        (Triple.getArch() == llvm::Triple::arm ||
+         Triple.getArch() == llvm::Triple::thumb) &&
+        (getContext().getTypeSizeInChars(IVART) 
+         > CharUnits::fromQuantity(4)) &&
+        CGM.getObjCRuntime().GetGetStructFunction()) {
+      GenerateObjCGetterBody(Ivar, true, false);
+    }
+    else if (IVART->isAnyComplexType()) {
       LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
                                     Ivar, 0);
       ComplexPairTy Pair = LoadComplexFromAddr(LV.getAddress(),
                                                LV.isVolatileQualified());
       StoreComplexToAddr(Pair, ReturnValue, LV.isVolatileQualified());
     }
-    else if (hasAggregateLLVMType(Ivar->getType())) {
+    else if (hasAggregateLLVMType(IVART)) {
       bool IsStrong = false;
-      if ((IsAtomic || (IsStrong = IvarTypeWithAggrGCObjects(Ivar->getType())))
+      if ((IsAtomic || (IsStrong = IvarTypeWithAggrGCObjects(IVART)))
           && CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect
-          && CGM.getObjCRuntime().GetCopyStructFunction()) {
-        LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
-                                      Ivar, 0);
-        llvm::Value *GetCopyStructFn =
-          CGM.getObjCRuntime().GetCopyStructFunction();
-        CodeGenTypes &Types = CGM.getTypes();
-        // objc_copyStruct (ReturnValue, &structIvar, 
-        //                  sizeof (Type of Ivar), isAtomic, false);
-        CallArgList Args;
-        RValue RV = RValue::get(Builder.CreateBitCast(ReturnValue,
-                                    Types.ConvertType(getContext().VoidPtrTy)));
-        Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
-        RV = RValue::get(Builder.CreateBitCast(LV.getAddress(),
-                                    Types.ConvertType(getContext().VoidPtrTy)));
-        Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
-        // sizeof (Type of Ivar)
-        uint64_t Size =  getContext().getTypeSize(Ivar->getType()) / 8;
-        llvm::Value *SizeVal =
-          llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy), Size);
-        Args.push_back(std::make_pair(RValue::get(SizeVal),
-                                      getContext().LongTy));
-        llvm::Value *isAtomic =
-          llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
-                                 IsAtomic ? 1 : 0);
-        Args.push_back(std::make_pair(RValue::get(isAtomic), 
-                                      getContext().BoolTy));
-        llvm::Value *hasStrong =
-          llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 
-                                 IsStrong ? 1 : 0);
-        Args.push_back(std::make_pair(RValue::get(hasStrong), 
-                                      getContext().BoolTy));
-        EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
-                                       FunctionType::ExtInfo()),
-                 GetCopyStructFn, ReturnValueSlot(), Args);
+          && CGM.getObjCRuntime().GetGetStructFunction()) {
+        GenerateObjCGetterBody(Ivar, IsAtomic, IsStrong);
       }
       else {
         if (PID->getGetterCXXConstructor()) {
@@ -268,23 +288,61 @@ void CodeGenFunction::GenerateObjCGetter(ObjCImplementationDecl *IMP,
         else {
           LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
                                         Ivar, 0);
-          EmitAggregateCopy(ReturnValue, LV.getAddress(), Ivar->getType());
+          EmitAggregateCopy(ReturnValue, LV.getAddress(), IVART);
         }
       }
-    } else {
-      LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
+    } 
+    else {
+        LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), 
                                     Ivar, 0);
-      CodeGenTypes &Types = CGM.getTypes();
-      RValue RV = EmitLoadOfLValue(LV, Ivar->getType());
-      RV = RValue::get(Builder.CreateBitCast(RV.getScalarVal(),
-                       Types.ConvertType(PD->getType())));
-      EmitReturnOfRValue(RV, PD->getType());
+        CodeGenTypes &Types = CGM.getTypes();
+        RValue RV = EmitLoadOfLValue(LV, IVART);
+        RV = RValue::get(Builder.CreateBitCast(RV.getScalarVal(),
+                                               Types.ConvertType(PD->getType())));
+        EmitReturnOfRValue(RV, PD->getType());
     }
   }
 
   FinishFunction();
 }
 
+void CodeGenFunction::GenerateObjCAtomicSetterBody(ObjCMethodDecl *OMD,
+                                                   ObjCIvarDecl *Ivar) {
+  // objc_copyStruct (&structIvar, &Arg, 
+  //                  sizeof (struct something), true, false);
+  llvm::Value *GetCopyStructFn =
+  CGM.getObjCRuntime().GetSetStructFunction();
+  CodeGenTypes &Types = CGM.getTypes();
+  CallArgList Args;
+  LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), Ivar, 0);
+  RValue RV =
+    RValue::get(Builder.CreateBitCast(LV.getAddress(),
+                Types.ConvertType(getContext().VoidPtrTy)));
+  Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
+  llvm::Value *Arg = LocalDeclMap[*OMD->param_begin()];
+  llvm::Value *ArgAsPtrTy =
+  Builder.CreateBitCast(Arg,
+                      Types.ConvertType(getContext().VoidPtrTy));
+  RV = RValue::get(ArgAsPtrTy);
+  Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
+  // sizeof (Type of Ivar)
+  CharUnits Size =  getContext().getTypeSizeInChars(Ivar->getType());
+  llvm::Value *SizeVal =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy), 
+                         Size.getQuantity());
+  Args.push_back(std::make_pair(RValue::get(SizeVal),
+                                getContext().LongTy));
+  llvm::Value *True =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 1);
+  Args.push_back(std::make_pair(RValue::get(True), getContext().BoolTy));
+  llvm::Value *False =
+  llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 0);
+  Args.push_back(std::make_pair(RValue::get(False), getContext().BoolTy));
+  EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
+                                 FunctionType::ExtInfo()),
+           GetCopyStructFn, ReturnValueSlot(), Args);
+}
+
 /// GenerateObjCSetter - Generate an Objective-C property setter
 /// function. The given Decl must be an ObjCImplementationDecl. @synthesize
 /// is illegal within a category.
@@ -353,66 +411,49 @@ void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP,
   } else if (IsAtomic && hasAggregateLLVMType(Ivar->getType()) &&
              !Ivar->getType()->isAnyComplexType() &&
              IndirectObjCSetterArg(*CurFnInfo)
-             && CGM.getObjCRuntime().GetCopyStructFunction()) {
+             && CGM.getObjCRuntime().GetSetStructFunction()) {
     // objc_copyStruct (&structIvar, &Arg, 
     //                  sizeof (struct something), true, false);
-    llvm::Value *GetCopyStructFn =
-      CGM.getObjCRuntime().GetCopyStructFunction();
-    CodeGenTypes &Types = CGM.getTypes();
-    CallArgList Args;
-    LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), Ivar, 0);
-    RValue RV = RValue::get(Builder.CreateBitCast(LV.getAddress(),
-                                    Types.ConvertType(getContext().VoidPtrTy)));
-    Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
-    llvm::Value *Arg = LocalDeclMap[*OMD->param_begin()];
-    llvm::Value *ArgAsPtrTy =
-      Builder.CreateBitCast(Arg,
-                            Types.ConvertType(getContext().VoidPtrTy));
-    RV = RValue::get(ArgAsPtrTy);
-    Args.push_back(std::make_pair(RV, getContext().VoidPtrTy));
-    // sizeof (Type of Ivar)
-    uint64_t Size =  getContext().getTypeSize(Ivar->getType()) / 8;
-    llvm::Value *SizeVal =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().LongTy), Size);
-    Args.push_back(std::make_pair(RValue::get(SizeVal),
-                                  getContext().LongTy));
-    llvm::Value *True =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 1);
-    Args.push_back(std::make_pair(RValue::get(True), getContext().BoolTy));
-    llvm::Value *False =
-      llvm::ConstantInt::get(Types.ConvertType(getContext().BoolTy), 0);
-    Args.push_back(std::make_pair(RValue::get(False), getContext().BoolTy));
-    EmitCall(Types.getFunctionInfo(getContext().VoidTy, Args,
-                                   FunctionType::ExtInfo()),
-             GetCopyStructFn, ReturnValueSlot(), Args);
+    GenerateObjCAtomicSetterBody(OMD, Ivar);
   } else if (PID->getSetterCXXAssignment()) {
-    EmitAnyExpr(PID->getSetterCXXAssignment(), (llvm::Value *)0, false, true,
-                false);
-                
+    EmitIgnoredExpr(PID->getSetterCXXAssignment());
   } else {
-    // FIXME: Find a clean way to avoid AST node creation.
-    SourceLocation Loc = PD->getLocation();
-    ValueDecl *Self = OMD->getSelfDecl();
-    ObjCIvarDecl *Ivar = PID->getPropertyIvarDecl();
-    DeclRefExpr Base(Self, Self->getType(), Loc);
-    ParmVarDecl *ArgDecl = *OMD->param_begin();
-    DeclRefExpr Arg(ArgDecl, ArgDecl->getType(), Loc);
-    ObjCIvarRefExpr IvarRef(Ivar, Ivar->getType(), Loc, &Base, true, true);
+    const llvm::Triple &Triple = getContext().Target.getTriple();
+    QualType IVART = Ivar->getType();
+    if (IsAtomic &&
+        IVART->isScalarType() &&
+        (Triple.getArch() == llvm::Triple::arm ||
+         Triple.getArch() == llvm::Triple::thumb) &&
+        (getContext().getTypeSizeInChars(IVART)
+          > CharUnits::fromQuantity(4)) &&
+        CGM.getObjCRuntime().GetGetStructFunction()) {
+      GenerateObjCAtomicSetterBody(OMD, Ivar);
+    }
+    else {
+      // FIXME: Find a clean way to avoid AST node creation.
+      SourceLocation Loc = PD->getLocation();
+      ValueDecl *Self = OMD->getSelfDecl();
+      ObjCIvarDecl *Ivar = PID->getPropertyIvarDecl();
+      DeclRefExpr Base(Self, Self->getType(), VK_RValue, Loc);
+      ParmVarDecl *ArgDecl = *OMD->param_begin();
+      DeclRefExpr Arg(ArgDecl, ArgDecl->getType(), VK_LValue, Loc);
+      ObjCIvarRefExpr IvarRef(Ivar, Ivar->getType(), Loc, &Base, true, true);
     
-    // The property type can differ from the ivar type in some situations with
-    // Objective-C pointer types, we can always bit cast the RHS in these cases.
-    if (getContext().getCanonicalType(Ivar->getType()) !=
-        getContext().getCanonicalType(ArgDecl->getType())) {
-      ImplicitCastExpr ArgCasted(ImplicitCastExpr::OnStack,
-                                 Ivar->getType(), CK_BitCast, &Arg,
-                                 VK_RValue);
-      BinaryOperator Assign(&IvarRef, &ArgCasted, BO_Assign,
-                            Ivar->getType(), Loc);
-      EmitStmt(&Assign);
-    } else {
-      BinaryOperator Assign(&IvarRef, &Arg, BO_Assign,
-                            Ivar->getType(), Loc);
-      EmitStmt(&Assign);
+      // The property type can differ from the ivar type in some situations with
+      // Objective-C pointer types, we can always bit cast the RHS in these cases.
+      if (getContext().getCanonicalType(Ivar->getType()) !=
+          getContext().getCanonicalType(ArgDecl->getType())) {
+        ImplicitCastExpr ArgCasted(ImplicitCastExpr::OnStack,
+                                   Ivar->getType(), CK_BitCast, &Arg,
+                                   VK_RValue);
+        BinaryOperator Assign(&IvarRef, &ArgCasted, BO_Assign,
+                              Ivar->getType(), VK_RValue, OK_Ordinary, Loc);
+        EmitStmt(&Assign);
+      } else {
+        BinaryOperator Assign(&IvarRef, &Arg, BO_Assign,
+                              Ivar->getType(), VK_RValue, OK_Ordinary, Loc);
+        EmitStmt(&Assign);
+      }
     }
   }
 
@@ -422,24 +463,22 @@ void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP,
 void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
                                                  ObjCMethodDecl *MD,
                                                  bool ctor) {
-  llvm::SmallVector<CXXBaseOrMemberInitializer *, 8> IvarInitializers;
+  llvm::SmallVector<CXXCtorInitializer *, 8> IvarInitializers;
   MD->createImplicitParams(CGM.getContext(), IMP->getClassInterface());
   StartObjCMethod(MD, IMP->getClassInterface());
   for (ObjCImplementationDecl::init_const_iterator B = IMP->init_begin(),
        E = IMP->init_end(); B != E; ++B) {
-    CXXBaseOrMemberInitializer *Member = (*B);
+    CXXCtorInitializer *Member = (*B);
     IvarInitializers.push_back(Member);
   }
   if (ctor) {
     for (unsigned I = 0, E = IvarInitializers.size(); I != E; ++I) {
-      CXXBaseOrMemberInitializer *IvarInit = IvarInitializers[I];
-      FieldDecl *Field = IvarInit->getMember();
-      QualType FieldType = Field->getType();
+      CXXCtorInitializer *IvarInit = IvarInitializers[I];
+      FieldDecl *Field = IvarInit->getAnyMember();
       ObjCIvarDecl  *Ivar = cast<ObjCIvarDecl>(Field);
       LValue LV = EmitLValueForIvar(TypeOfSelfObject(), 
                                     LoadObjCSelf(), Ivar, 0);
-      EmitAggExpr(IvarInit->getInit(), LV.getAddress(),
-                  LV.isVolatileQualified(), false, true);
+      EmitAggExpr(IvarInit->getInit(), AggValueSlot::forLValue(LV, true));
     }
     // constructor returns 'self'.
     CodeGenTypes &Types = CGM.getTypes();
@@ -450,7 +489,7 @@ void CodeGenFunction::GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
   } else {
     // dtor
     for (size_t i = IvarInitializers.size(); i > 0; --i) {
-      FieldDecl *Field = IvarInitializers[i - 1]->getMember();
+      FieldDecl *Field = IvarInitializers[i - 1]->getAnyMember();
       QualType FieldType = Field->getType();
       const ConstantArrayType *Array = 
         getContext().getAsConstantArrayType(FieldType);
@@ -511,138 +550,128 @@ QualType CodeGenFunction::TypeOfSelfObject() {
   return PTy->getPointeeType();
 }
 
-RValue CodeGenFunction::EmitObjCSuperPropertyGet(const Expr *Exp,
-                                                 const Selector &S,
-                                                 ReturnValueSlot Return) {
-  llvm::Value *Receiver = LoadObjCSelf();
-  const ObjCMethodDecl *OMD = cast<ObjCMethodDecl>(CurFuncDecl);
+LValue
+CodeGenFunction::EmitObjCPropertyRefLValue(const ObjCPropertyRefExpr *E) {
+  // This is a special l-value that just issues sends when we load or
+  // store through it.
+
+  // For certain base kinds, we need to emit the base immediately.
+  llvm::Value *Base;
+  if (E->isSuperReceiver())
+    Base = LoadObjCSelf();
+  else if (E->isClassReceiver())
+    Base = CGM.getObjCRuntime().GetClass(Builder, E->getClassReceiver());
+  else
+    Base = EmitScalarExpr(E->getBase());
+  return LValue::MakePropertyRef(E, Base);
+}
+
+static RValue GenerateMessageSendSuper(CodeGenFunction &CGF,
+                                       ReturnValueSlot Return,
+                                       QualType ResultType,
+                                       Selector S,
+                                       llvm::Value *Receiver,
+                                       const CallArgList &CallArgs) {
+  const ObjCMethodDecl *OMD = cast<ObjCMethodDecl>(CGF.CurFuncDecl);
   bool isClassMessage = OMD->isClassMethod();
   bool isCategoryImpl = isa<ObjCCategoryImplDecl>(OMD->getDeclContext());
-  return CGM.getObjCRuntime().GenerateMessageSendSuper(*this,
-                                                       Return,
-                                                       Exp->getType(),
-                                                       S,
-                                                       OMD->getClassInterface(),
-                                                       isCategoryImpl,
-                                                       Receiver,
-                                                       isClassMessage,
-                                                       CallArgList());
-
+  return CGF.CGM.getObjCRuntime()
+                .GenerateMessageSendSuper(CGF, Return, ResultType,
+                                          S, OMD->getClassInterface(),
+                                          isCategoryImpl, Receiver,
+                                          isClassMessage, CallArgs);
 }
 
-RValue CodeGenFunction::EmitObjCPropertyGet(const Expr *Exp,
-                                            ReturnValueSlot Return) {
-  Exp = Exp->IgnoreParens();
-  // FIXME: Split it into two separate routines.
-  if (const ObjCPropertyRefExpr *E = dyn_cast<ObjCPropertyRefExpr>(Exp)) {
-    Selector S = E->getProperty()->getGetterName();
-    if (isa<ObjCSuperExpr>(E->getBase()))
-      return EmitObjCSuperPropertyGet(E, S, Return);
-    return CGM.getObjCRuntime().
-             GenerateMessageSend(*this, Return, Exp->getType(), S,
-                                 EmitScalarExpr(E->getBase()),
-                                 CallArgList());
+RValue CodeGenFunction::EmitLoadOfPropertyRefLValue(LValue LV,
+                                                    ReturnValueSlot Return) {
+  const ObjCPropertyRefExpr *E = LV.getPropertyRefExpr();
+  QualType ResultType;
+  Selector S;
+  if (E->isExplicitProperty()) {
+    const ObjCPropertyDecl *Property = E->getExplicitProperty();
+    S = Property->getGetterName();
+    ResultType = E->getType();
   } else {
-    const ObjCImplicitSetterGetterRefExpr *KE =
-      cast<ObjCImplicitSetterGetterRefExpr>(Exp);
-    Selector S = KE->getGetterMethod()->getSelector();
-    llvm::Value *Receiver;
-    if (KE->getInterfaceDecl()) {
-      const ObjCInterfaceDecl *OID = KE->getInterfaceDecl();
-      Receiver = CGM.getObjCRuntime().GetClass(Builder, OID);
-    } else if (isa<ObjCSuperExpr>(KE->getBase()))
-      return EmitObjCSuperPropertyGet(KE, S, Return);
-    else
-      Receiver = EmitScalarExpr(KE->getBase());
-    return CGM.getObjCRuntime().
-             GenerateMessageSend(*this, Return, Exp->getType(), S,
-                                 Receiver,
-                                 CallArgList(), KE->getInterfaceDecl());
+    const ObjCMethodDecl *Getter = E->getImplicitPropertyGetter();
+    S = Getter->getSelector();
+    ResultType = Getter->getResultType(); // with reference!
   }
+
+  llvm::Value *Receiver = LV.getPropertyRefBaseAddr();
+
+  // Accesses to 'super' follow a different code path.
+  if (E->isSuperReceiver())
+    return GenerateMessageSendSuper(*this, Return, ResultType,
+                                    S, Receiver, CallArgList());
+
+  const ObjCInterfaceDecl *ReceiverClass
+    = (E->isClassReceiver() ? E->getClassReceiver() : 0);
+  return CGM.getObjCRuntime().
+             GenerateMessageSend(*this, Return, ResultType, S,
+                                 Receiver, CallArgList(), ReceiverClass);
 }
 
-void CodeGenFunction::EmitObjCSuperPropertySet(const Expr *Exp,
-                                               const Selector &S,
-                                               RValue Src) {
+void CodeGenFunction::EmitStoreThroughPropertyRefLValue(RValue Src,
+                                                        LValue Dst) {
+  const ObjCPropertyRefExpr *E = Dst.getPropertyRefExpr();
+  Selector S = E->getSetterSelector();
+  QualType ArgType;
+  if (E->isImplicitProperty()) {
+    const ObjCMethodDecl *Setter = E->getImplicitPropertySetter();
+    ObjCMethodDecl::param_iterator P = Setter->param_begin(); 
+    ArgType = (*P)->getType();
+  } else {
+    ArgType = E->getType();
+  }
+  // FIXME. Other than scalars, AST is not adequate for setter and
+  // getter type mismatches which require conversion.
+  if (Src.isScalar()) {
+    llvm::Value *SrcVal = Src.getScalarVal();
+    QualType DstType = getContext().getCanonicalType(ArgType);
+    const llvm::Type *DstTy = ConvertType(DstType);
+    if (SrcVal->getType() != DstTy)
+      Src = 
+        RValue::get(EmitScalarConversion(SrcVal, E->getType(), DstType));
+  }
+  
   CallArgList Args;
-  llvm::Value *Receiver = LoadObjCSelf();
-  const ObjCMethodDecl *OMD = cast<ObjCMethodDecl>(CurFuncDecl);
-  bool isClassMessage = OMD->isClassMethod();
-  bool isCategoryImpl = isa<ObjCCategoryImplDecl>(OMD->getDeclContext());
-  Args.push_back(std::make_pair(Src, Exp->getType()));
-  CGM.getObjCRuntime().GenerateMessageSendSuper(*this,
-                                                ReturnValueSlot(),
-                                                getContext().VoidTy,
-                                                S,
-                                                OMD->getClassInterface(),
-                                                isCategoryImpl,
-                                                Receiver,
-                                                isClassMessage,
-                                                Args);
-  return;
-}
+  Args.push_back(std::make_pair(Src, ArgType));
 
-void CodeGenFunction::EmitObjCPropertySet(const Expr *Exp,
-                                          RValue Src) {
-  // FIXME: Split it into two separate routines.
-  if (const ObjCPropertyRefExpr *E = dyn_cast<ObjCPropertyRefExpr>(Exp)) {
-    Selector S = E->getProperty()->getSetterName();
-    if (isa<ObjCSuperExpr>(E->getBase())) {
-      EmitObjCSuperPropertySet(E, S, Src);
-      return;
-    }
-    CallArgList Args;
-    Args.push_back(std::make_pair(Src, E->getType()));
-    CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
-                                             getContext().VoidTy, S,
-                                             EmitScalarExpr(E->getBase()),
-                                             Args);
-  } else if (const ObjCImplicitSetterGetterRefExpr *E =
-               dyn_cast<ObjCImplicitSetterGetterRefExpr>(Exp)) {
-    const ObjCMethodDecl *SetterMD = E->getSetterMethod();
-    Selector S = SetterMD->getSelector();
-    CallArgList Args;
-    llvm::Value *Receiver;
-    if (E->getInterfaceDecl()) {
-      const ObjCInterfaceDecl *OID = E->getInterfaceDecl();
-      Receiver = CGM.getObjCRuntime().GetClass(Builder, OID);
-    } else if (isa<ObjCSuperExpr>(E->getBase())) {
-      EmitObjCSuperPropertySet(E, S, Src);
-      return;
-    } else
-      Receiver = EmitScalarExpr(E->getBase());
-    ObjCMethodDecl::param_iterator P = SetterMD->param_begin(); 
-    Args.push_back(std::make_pair(Src, (*P)->getType()));
-    CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
-                                             getContext().VoidTy, S,
-                                             Receiver,
-                                             Args, E->getInterfaceDecl());
-  } else
-    assert (0 && "bad expression node in EmitObjCPropertySet");
+  llvm::Value *Receiver = Dst.getPropertyRefBaseAddr();
+  QualType ResultType = getContext().VoidTy;
+
+  if (E->isSuperReceiver()) {
+    GenerateMessageSendSuper(*this, ReturnValueSlot(),
+                             ResultType, S, Receiver, Args);
+    return;
+  }
+
+  const ObjCInterfaceDecl *ReceiverClass
+    = (E->isClassReceiver() ? E->getClassReceiver() : 0);
+
+  CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
+                                           ResultType, S, Receiver, Args,
+                                           ReceiverClass);
 }
 
 void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   llvm::Constant *EnumerationMutationFn =
     CGM.getObjCRuntime().EnumerationMutationFunction();
-  llvm::Value *DeclAddress;
-  QualType ElementTy;
 
   if (!EnumerationMutationFn) {
     CGM.ErrorUnsupported(&S, "Obj-C fast enumeration for this runtime");
     return;
   }
 
-  if (const DeclStmt *SD = dyn_cast<DeclStmt>(S.getElement())) {
-    EmitStmt(SD);
-    assert(HaveInsertPoint() && "DeclStmt destroyed insert point!");
-    const Decl* D = SD->getSingleDecl();
-    ElementTy = cast<ValueDecl>(D)->getType();
-    DeclAddress = LocalDeclMap[D];
-  } else {
-    ElementTy = cast<Expr>(S.getElement())->getType();
-    DeclAddress = 0;
+  CGDebugInfo *DI = getDebugInfo();
+  if (DI) {
+    DI->setLocation(S.getSourceRange().getBegin());
+    DI->EmitRegionStart(Builder);
   }
 
+  JumpDest LoopEnd = getJumpDestInCurrentScope("forcoll.end");
+  JumpDest AfterBody = getJumpDestInCurrentScope("forcoll.next");
+
   // Fast enumeration state.
   QualType StateTy = getContext().getObjCFastEnumerationStateType();
   llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr");
@@ -651,7 +680,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   // Number of elements in the items array.
   static const unsigned NumItems = 16;
 
-  // Get selector
+  // Fetch the countByEnumeratingWithState:objects:count: selector.
   IdentifierInfo *II[] = {
     &CGM.getContext().Idents.get("countByEnumeratingWithState"),
     &CGM.getContext().Idents.get("objects"),
@@ -666,79 +695,92 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                                       ArrayType::Normal, 0);
   llvm::Value *ItemsPtr = CreateMemTemp(ItemsTy, "items.ptr");
 
+  // Emit the collection pointer.
   llvm::Value *Collection = EmitScalarExpr(S.getCollection());
 
+  // Send it our message:
   CallArgList Args;
+
+  // The first argument is a temporary of the enumeration-state type.
   Args.push_back(std::make_pair(RValue::get(StatePtr),
                                 getContext().getPointerType(StateTy)));
 
+  // The second argument is a temporary array with space for NumItems
+  // pointers.  We'll actually be loading elements from the array
+  // pointer written into the control state; this buffer is so that
+  // collections that *aren't* backed by arrays can still queue up
+  // batches of elements.
   Args.push_back(std::make_pair(RValue::get(ItemsPtr),
                                 getContext().getPointerType(ItemsTy)));
 
+  // The third argument is the capacity of that temporary array.
   const llvm::Type *UnsignedLongLTy = ConvertType(getContext().UnsignedLongTy);
   llvm::Constant *Count = llvm::ConstantInt::get(UnsignedLongLTy, NumItems);
   Args.push_back(std::make_pair(RValue::get(Count),
                                 getContext().UnsignedLongTy));
 
+  // Start the enumeration.
   RValue CountRV =
     CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
                                              getContext().UnsignedLongTy,
                                              FastEnumSel,
                                              Collection, Args);
 
-  llvm::Value *LimitPtr = CreateMemTemp(getContext().UnsignedLongTy,
-                                        "limit.ptr");
-  Builder.CreateStore(CountRV.getScalarVal(), LimitPtr);
+  // The initial number of objects that were returned in the buffer.
+  llvm::Value *initialBufferLimit = CountRV.getScalarVal();
 
-  llvm::BasicBlock *NoElements = createBasicBlock("noelements");
-  llvm::BasicBlock *SetStartMutations = createBasicBlock("setstartmutations");
+  llvm::BasicBlock *EmptyBB = createBasicBlock("forcoll.empty");
+  llvm::BasicBlock *LoopInitBB = createBasicBlock("forcoll.loopinit");
 
-  llvm::Value *Limit = Builder.CreateLoad(LimitPtr);
-  llvm::Value *Zero = llvm::Constant::getNullValue(UnsignedLongLTy);
+  llvm::Value *zero = llvm::Constant::getNullValue(UnsignedLongLTy);
 
-  llvm::Value *IsZero = Builder.CreateICmpEQ(Limit, Zero, "iszero");
-  Builder.CreateCondBr(IsZero, NoElements, SetStartMutations);
+  // If the limit pointer was zero to begin with, the collection is
+  // empty; skip all this.
+  Builder.CreateCondBr(Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"),
+                       EmptyBB, LoopInitBB);
 
-  EmitBlock(SetStartMutations);
-
-  llvm::Value *StartMutationsPtr = CreateMemTemp(getContext().UnsignedLongTy);
+  // Otherwise, initialize the loop.
+  EmitBlock(LoopInitBB);
 
+  // Save the initial mutations value.  This is the value at an
+  // address that was written into the state object by
+  // countByEnumeratingWithState:objects:count:.
   llvm::Value *StateMutationsPtrPtr =
     Builder.CreateStructGEP(StatePtr, 2, "mutationsptr.ptr");
   llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr,
                                                       "mutationsptr");
 
-  llvm::Value *StateMutations = Builder.CreateLoad(StateMutationsPtr,
-                                                   "mutations");
-
-  Builder.CreateStore(StateMutations, StartMutationsPtr);
+  llvm::Value *initialMutations =
+    Builder.CreateLoad(StateMutationsPtr, "forcoll.initial-mutations");
 
-  llvm::BasicBlock *LoopStart = createBasicBlock("loopstart");
-  EmitBlock(LoopStart);
+  // Start looping.  This is the point we return to whenever we have a
+  // fresh, non-empty batch of objects.
+  llvm::BasicBlock *LoopBodyBB = createBasicBlock("forcoll.loopbody");
+  EmitBlock(LoopBodyBB);
 
-  llvm::Value *CounterPtr = CreateMemTemp(getContext().UnsignedLongTy,
-                                       "counter.ptr");
-  Builder.CreateStore(Zero, CounterPtr);
+  // The current index into the buffer.
+  llvm::PHINode *index = Builder.CreatePHI(UnsignedLongLTy, "forcoll.index");
+  index->addIncoming(zero, LoopInitBB);
 
-  llvm::BasicBlock *LoopBody = createBasicBlock("loopbody");
-  EmitBlock(LoopBody);
+  // The current buffer size.
+  llvm::PHINode *count = Builder.CreatePHI(UnsignedLongLTy, "forcoll.count");
+  count->addIncoming(initialBufferLimit, LoopInitBB);
 
+  // Check whether the mutations value has changed from where it was
+  // at start.  StateMutationsPtr should actually be invariant between
+  // refreshes.
   StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr");
-  StateMutations = Builder.CreateLoad(StateMutationsPtr, "statemutations");
-
-  llvm::Value *StartMutations = Builder.CreateLoad(StartMutationsPtr,
-                                                   "mutations");
-  llvm::Value *MutationsEqual = Builder.CreateICmpEQ(StateMutations,
-                                                     StartMutations,
-                                                     "tobool");
+  llvm::Value *currentMutations
+    = Builder.CreateLoad(StateMutationsPtr, "statemutations");
 
+  llvm::BasicBlock *WasMutatedBB = createBasicBlock("forcoll.mutated");
+  llvm::BasicBlock *WasNotMutatedBB = createBasicBlock("forcool.notmutated");
 
-  llvm::BasicBlock *WasMutated = createBasicBlock("wasmutated");
-  llvm::BasicBlock *WasNotMutated = createBasicBlock("wasnotmutated");
+  Builder.CreateCondBr(Builder.CreateICmpEQ(currentMutations, initialMutations),
+                       WasNotMutatedBB, WasMutatedBB);
 
-  Builder.CreateCondBr(MutationsEqual, WasNotMutated, WasMutated);
-
-  EmitBlock(WasMutated);
+  // If so, call the enumeration-mutation function.
+  EmitBlock(WasMutatedBB);
   llvm::Value *V =
     Builder.CreateBitCast(Collection,
                           ConvertType(getContext().getObjCIdType()),
@@ -752,81 +794,114 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                                           FunctionType::ExtInfo()),
            EnumerationMutationFn, ReturnValueSlot(), Args2);
 
-  EmitBlock(WasNotMutated);
+  // Otherwise, or if the mutation function returns, just continue.
+  EmitBlock(WasNotMutatedBB);
 
-  llvm::Value *StateItemsPtr =
-    Builder.CreateStructGEP(StatePtr, 1, "stateitems.ptr");
+  // Initialize the element variable.
+  RunCleanupsScope elementVariableScope(*this);
+  bool elementIsDecl;
+  LValue elementLValue;
+  QualType elementType;
+  if (const DeclStmt *SD = dyn_cast<DeclStmt>(S.getElement())) {
+    EmitStmt(SD);
+    const VarDecl* D = cast<VarDecl>(SD->getSingleDecl());
 
-  llvm::Value *Counter = Builder.CreateLoad(CounterPtr, "counter");
+    DeclRefExpr tempDRE(const_cast<VarDecl*>(D), D->getType(),
+                        VK_LValue, SourceLocation());
+    elementLValue = EmitLValue(&tempDRE);
+    elementType = D->getType();
+    elementIsDecl = true;
+  } else {
+    elementLValue = LValue(); // suppress warning
+    elementType = cast<Expr>(S.getElement())->getType();
+    elementIsDecl = false;
+  }
+  const llvm::Type *convertedElementType = ConvertType(elementType);
 
-  llvm::Value *EnumStateItems = Builder.CreateLoad(StateItemsPtr,
-                                                   "stateitems");
+  // Fetch the buffer out of the enumeration state.
+  // TODO: this pointer should actually be invariant between
+  // refreshes, which would help us do certain loop optimizations.
+  llvm::Value *StateItemsPtr =
+    Builder.CreateStructGEP(StatePtr, 1, "stateitems.ptr");
+  llvm::Value *EnumStateItems =
+    Builder.CreateLoad(StateItemsPtr, "stateitems");
 
+  // Fetch the value at the current index from the buffer.
   llvm::Value *CurrentItemPtr =
-    Builder.CreateGEP(EnumStateItems, Counter, "currentitem.ptr");
-
-  llvm::Value *CurrentItem = Builder.CreateLoad(CurrentItemPtr, "currentitem");
-
-  // Cast the item to the right type.
-  CurrentItem = Builder.CreateBitCast(CurrentItem,
-                                      ConvertType(ElementTy), "tmp");
+    Builder.CreateGEP(EnumStateItems, index, "currentitem.ptr");
+  llvm::Value *CurrentItem = Builder.CreateLoad(CurrentItemPtr);
 
-  if (!DeclAddress) {
-    LValue LV = EmitLValue(cast<Expr>(S.getElement()));
+  // Cast that value to the right type.
+  CurrentItem = Builder.CreateBitCast(CurrentItem, convertedElementType,
+                                      "currentitem");
 
-    // Set the value to null.
-    Builder.CreateStore(CurrentItem, LV.getAddress());
-  } else
-    Builder.CreateStore(CurrentItem, DeclAddress);
+  // Make sure we have an l-value.  Yes, this gets evaluated every
+  // time through the loop.
+  if (!elementIsDecl)
+    elementLValue = EmitLValue(cast<Expr>(S.getElement()));
 
-  // Increment the counter.
-  Counter = Builder.CreateAdd(Counter,
-                              llvm::ConstantInt::get(UnsignedLongLTy, 1));
-  Builder.CreateStore(Counter, CounterPtr);
-
-  JumpDest LoopEnd = getJumpDestInCurrentScope("loopend");
-  JumpDest AfterBody = getJumpDestInCurrentScope("afterbody");
+  EmitStoreThroughLValue(RValue::get(CurrentItem), elementLValue, elementType);
 
+  // Perform the loop body, setting up break and continue labels.
   BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody));
-
-  EmitStmt(S.getBody());
-
+  {
+    RunCleanupsScope Scope(*this);
+    EmitStmt(S.getBody());
+  }
   BreakContinueStack.pop_back();
 
+  // Destroy the element variable now.
+  elementVariableScope.ForceCleanup();
+
+  // Check whether there are more elements.
   EmitBlock(AfterBody.getBlock());
 
-  llvm::BasicBlock *FetchMore = createBasicBlock("fetchmore");
+  llvm::BasicBlock *FetchMoreBB = createBasicBlock("forcoll.refetch");
+
+  // First we check in the local buffer.
+  llvm::Value *indexPlusOne
+    = Builder.CreateAdd(index, llvm::ConstantInt::get(UnsignedLongLTy, 1));
+
+  // If we haven't overrun the buffer yet, we can continue.
+  Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count),
+                       LoopBodyBB, FetchMoreBB);
 
-  Counter = Builder.CreateLoad(CounterPtr);
-  Limit = Builder.CreateLoad(LimitPtr);
-  llvm::Value *IsLess = Builder.CreateICmpULT(Counter, Limit, "isless");
-  Builder.CreateCondBr(IsLess, LoopBody, FetchMore);
+  index->addIncoming(indexPlusOne, AfterBody.getBlock());
+  count->addIncoming(count, AfterBody.getBlock());
 
-  // Fetch more elements.
-  EmitBlock(FetchMore);
+  // Otherwise, we have to fetch more elements.
+  EmitBlock(FetchMoreBB);
 
   CountRV =
     CGM.getObjCRuntime().GenerateMessageSend(*this, ReturnValueSlot(),
                                              getContext().UnsignedLongTy,
                                              FastEnumSel,
                                              Collection, Args);
-  Builder.CreateStore(CountRV.getScalarVal(), LimitPtr);
-  Limit = Builder.CreateLoad(LimitPtr);
 
-  IsZero = Builder.CreateICmpEQ(Limit, Zero, "iszero");
-  Builder.CreateCondBr(IsZero, NoElements, LoopStart);
+  // If we got a zero count, we're done.
+  llvm::Value *refetchCount = CountRV.getScalarVal();
+
+  // (note that the message send might split FetchMoreBB)
+  index->addIncoming(zero, Builder.GetInsertBlock());
+  count->addIncoming(refetchCount, Builder.GetInsertBlock());
+
+  Builder.CreateCondBr(Builder.CreateICmpEQ(refetchCount, zero),
+                       EmptyBB, LoopBodyBB);
 
   // No more elements.
-  EmitBlock(NoElements);
+  EmitBlock(EmptyBB);
 
-  if (!DeclAddress) {
+  if (!elementIsDecl) {
     // If the element was not a declaration, set it to be null.
 
-    LValue LV = EmitLValue(cast<Expr>(S.getElement()));
+    llvm::Value *null = llvm::Constant::getNullValue(convertedElementType);
+    elementLValue = EmitLValue(cast<Expr>(S.getElement()));
+    EmitStoreThroughLValue(RValue::get(null), elementLValue, elementType);
+  }
 
-    // Set the value to null.
-    Builder.CreateStore(llvm::Constant::getNullValue(ConvertType(ElementTy)),
-                        LV.getAddress());
+  if (DI) {
+    DI->setLocation(S.getSourceRange().getEnd());
+    DI->EmitRegionEnd(Builder);
   }
 
   EmitBlock(LoopEnd.getBlock());
@@ -846,5 +921,3 @@ void CodeGenFunction::EmitObjCAtSynchronizedStmt(
 }
 
 CGObjCRuntime::~CGObjCRuntime() {}
-
-
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index d7960beb1d1d..d481e7792674 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -17,7 +17,7 @@
 #include "CGObjCRuntime.h"
 #include "CodeGenModule.h"
 #include "CodeGenFunction.h"
-#include "CGException.h"
+#include "CGCleanup.h"
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
@@ -62,7 +62,10 @@ private:
   const llvm::IntegerType *IntTy;
   const llvm::PointerType *PtrTy;
   const llvm::IntegerType *LongTy;
+  const llvm::IntegerType *SizeTy;
+  const llvm::IntegerType *PtrDiffTy;
   const llvm::PointerType *PtrToIntTy;
+  const llvm::Type *BoolTy;
   llvm::GlobalAlias *ClassPtrAlias;
   llvm::GlobalAlias *MetaClassPtrAlias;
   std::vector<llvm::Constant*> Classes;
@@ -179,7 +182,8 @@ public:
   virtual llvm::Function *ModuleInitFunction();
   virtual llvm::Function *GetPropertyGetFunction();
   virtual llvm::Function *GetPropertySetFunction();
-  virtual llvm::Function *GetCopyStructFunction();
+  virtual llvm::Function *GetSetStructFunction();
+  virtual llvm::Function *GetGetStructFunction();
   virtual llvm::Constant *EnumerationMutationFunction();
 
   virtual void EmitTryStmt(CodeGen::CodeGenFunction &CGF,
@@ -212,8 +216,8 @@ public:
   virtual llvm::Value *EmitIvarOffset(CodeGen::CodeGenFunction &CGF,
                                       const ObjCInterfaceDecl *Interface,
                                       const ObjCIvarDecl *Ivar);
-  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
-              const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &) {
+  virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
+                                             const CGBlockInfo &blockInfo) {
     return NULLPtr;
   }
 };
@@ -273,6 +277,11 @@ CGObjCGNU::CGObjCGNU(CodeGen::CodeGenModule &cgm)
       CGM.getTypes().ConvertType(CGM.getContext().IntTy));
   LongTy = cast<llvm::IntegerType>(
       CGM.getTypes().ConvertType(CGM.getContext().LongTy));
+  SizeTy = cast<llvm::IntegerType>(
+      CGM.getTypes().ConvertType(CGM.getContext().getSizeType()));
+  PtrDiffTy = cast<llvm::IntegerType>(
+      CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()));
+  BoolTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
 
   Int8Ty = llvm::Type::getInt8Ty(VMContext);
   // C string type.  Used in lots of places.
@@ -318,8 +327,7 @@ CGObjCGNU::CGObjCGNU(CodeGen::CodeGenModule &cgm)
     // id objc_assign_ivar(id, id, ptrdiff_t);
     std::vector<const llvm::Type*> Args(1, IdTy);
     Args.push_back(PtrToIdTy);
-    // FIXME: ptrdiff_t
-    Args.push_back(LongTy);
+    Args.push_back(PtrDiffTy);
     llvm::FunctionType *FTy = llvm::FunctionType::get(IdTy, Args, false);
     IvarAssignFn = CGM.CreateRuntimeFunction(FTy, "objc_assign_ivar");
     // id objc_assign_strongCast (id, id*)
@@ -342,8 +350,7 @@ CGObjCGNU::CGObjCGNU(CodeGen::CodeGenModule &cgm)
     Args.clear();
     Args.push_back(PtrToInt8Ty);
     Args.push_back(PtrToInt8Ty);
-    // FIXME: size_t
-    Args.push_back(LongTy);
+    Args.push_back(SizeTy);
     FTy = llvm::FunctionType::get(IdTy, Args, false);
     MemMoveFn = CGM.CreateRuntimeFunction(FTy, "objc_memmove_collectable");
   }
@@ -435,7 +442,7 @@ llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::StructType *Ty,
     llvm::GlobalValue::LinkageTypes linkage) {
   llvm::Constant *C = llvm::ConstantStruct::get(Ty, V);
   return new llvm::GlobalVariable(TheModule, Ty, false,
-      llvm::GlobalValue::InternalLinkage, C, Name);
+      linkage, C, Name);
 }
 
 llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::ArrayType *Ty,
@@ -443,7 +450,7 @@ llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::ArrayType *Ty,
     llvm::GlobalValue::LinkageTypes linkage) {
   llvm::Constant *C = llvm::ConstantArray::get(Ty, V);
   return new llvm::GlobalVariable(TheModule, Ty, false,
-                                  llvm::GlobalValue::InternalLinkage, C, Name);
+                                  linkage, C, Name);
 }
 
 /// Generate an NSConstantString object.
@@ -995,7 +1002,7 @@ llvm::Constant *CGObjCGNU::GenerateProtocolList(
       Protocols.size());
   llvm::StructType *ProtocolListTy = llvm::StructType::get(VMContext,
       PtrTy, //Should be a recurisve pointer, but it's always NULL here.
-      LongTy,//FIXME: Should be size_t
+      SizeTy,
       ProtocolArrayTy,
       NULL);
   std::vector<llvm::Constant*> Elements;
@@ -1250,7 +1257,7 @@ void CGObjCGNU::GenerateProtocolHolderCategory(void) {
       ExistingProtocols.size());
   llvm::StructType *ProtocolListTy = llvm::StructType::get(VMContext,
       PtrTy, //Should be a recurisve pointer, but it's always NULL here.
-      LongTy,//FIXME: Should be size_t
+      SizeTy,
       ProtocolArrayTy,
       NULL);
   std::vector<llvm::Constant*> ProtocolElements;
@@ -1430,7 +1437,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   }
 
   // Get the size of instances.
-  int instanceSize = Context.getASTObjCImplementationLayout(OID).getSize() / 8;
+  int instanceSize = 
+    Context.getASTObjCImplementationLayout(OID).getSize().getQuantity();
 
   // Collect information about instance variables.
   llvm::SmallVector<llvm::Constant*, 16> IvarNames;
@@ -1440,7 +1448,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   std::vector<llvm::Constant*> IvarOffsetValues;
 
   int superInstanceSize = !SuperClassDecl ? 0 :
-    Context.getASTObjCInterfaceLayout(SuperClassDecl).getSize() / 8;
+    Context.getASTObjCInterfaceLayout(SuperClassDecl).getSize().getQuantity();
   // For non-fragile ivars, set the instance size to 0 - {the size of just this
   // class}.  The runtime will then set this to the correct value on load.
   if (CGM.getContext().getLangOptions().ObjCNonFragileABI) {
@@ -1469,7 +1477,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
           llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), Offset));
       IvarOffsetValues.push_back(new llvm::GlobalVariable(TheModule, IntTy,
           false, llvm::GlobalValue::ExternalLinkage,
-          llvm::ConstantInt::get(IntTy, BaseOffset),
+          llvm::ConstantInt::get(IntTy, Offset),
           "__objc_ivar_offset_value_" + ClassName +"." +
           IVD->getNameAsString()));
   }
@@ -1538,7 +1546,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   // allows code compiled for the non-Fragile ABI to inherit from code compiled
   // for the legacy ABI, without causing problems.  The converse is also
   // possible, but causes all ivar accesses to be fragile.
-  int i = 0;
+
   // Offset pointer for getting at the correct field in the ivar list when
   // setting up the alias.  These are: The base address for the global, the
   // ivar array (second field), the ivar in this list (set for each ivar), and
@@ -1548,15 +1556,16 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
       llvm::ConstantInt::get(IndexTy, 1), 0,
       llvm::ConstantInt::get(IndexTy, 2) };
 
-  for (ObjCInterfaceDecl::ivar_iterator iter = ClassDecl->ivar_begin(),
-      endIter = ClassDecl->ivar_end() ; iter != endIter ; iter++) {
+
+  for (unsigned i = 0, e = OIvars.size(); i != e; ++i) {
+      ObjCIvarDecl *IVD = OIvars[i];
       const std::string Name = "__objc_ivar_offset_" + ClassName + '.'
-          +(*iter)->getNameAsString();
-      offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, i++);
+          + IVD->getNameAsString();
+      offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, i);
       // Get the correct ivar field
       llvm::Constant *offsetValue = llvm::ConstantExpr::getGetElementPtr(
               IvarList, offsetPointerIndexes, 4);
-      // Get the existing alias, if one exists.
+      // Get the existing variable, if one exists.
       llvm::GlobalVariable *offset = TheModule.getNamedGlobal(Name);
       if (offset) {
           offset->setInitializer(offsetValue);
@@ -1585,13 +1594,15 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
 
   // Resolve the class aliases, if they exist.
   if (ClassPtrAlias) {
-    ClassPtrAlias->setAliasee(
+    ClassPtrAlias->replaceAllUsesWith(
         llvm::ConstantExpr::getBitCast(ClassStruct, IdTy));
+    ClassPtrAlias->eraseFromParent();
     ClassPtrAlias = 0;
   }
   if (MetaClassPtrAlias) {
-    MetaClassPtrAlias->setAliasee(
+    MetaClassPtrAlias->replaceAllUsesWith(
         llvm::ConstantExpr::getBitCast(MetaClassStruct, IdTy));
+    MetaClassPtrAlias->eraseFromParent();
     MetaClassPtrAlias = 0;
   }
 
@@ -1818,8 +1829,6 @@ llvm::Function *CGObjCGNU::GenerateMethod(const ObjCMethodDecl *OMD,
 
 llvm::Function *CGObjCGNU::GetPropertyGetFunction() {
   std::vector<const llvm::Type*> Params;
-  const llvm::Type *BoolTy =
-    CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
   Params.push_back(IdTy);
   Params.push_back(SelectorTy);
   Params.push_back(IntTy);
@@ -1833,8 +1842,6 @@ llvm::Function *CGObjCGNU::GetPropertyGetFunction() {
 
 llvm::Function *CGObjCGNU::GetPropertySetFunction() {
   std::vector<const llvm::Type*> Params;
-  const llvm::Type *BoolTy =
-    CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
   Params.push_back(IdTy);
   Params.push_back(SelectorTy);
   Params.push_back(IntTy);
@@ -1848,9 +1855,31 @@ llvm::Function *CGObjCGNU::GetPropertySetFunction() {
                                                         "objc_setProperty"));
 }
 
-// FIXME. Implement this.
-llvm::Function *CGObjCGNU::GetCopyStructFunction() {
-  return 0;
+llvm::Function *CGObjCGNU::GetGetStructFunction() {
+  std::vector<const llvm::Type*> Params;
+  Params.push_back(PtrTy);
+  Params.push_back(PtrTy);
+  Params.push_back(PtrDiffTy);
+  Params.push_back(BoolTy);
+  Params.push_back(BoolTy);
+  // objc_setPropertyStruct (void*, void*, ptrdiff_t, BOOL, BOOL)
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), Params, false);
+  return cast<llvm::Function>(CGM.CreateRuntimeFunction(FTy, 
+                                                    "objc_getPropertyStruct"));
+}
+llvm::Function *CGObjCGNU::GetSetStructFunction() {
+  std::vector<const llvm::Type*> Params;
+  Params.push_back(PtrTy);
+  Params.push_back(PtrTy);
+  Params.push_back(PtrDiffTy);
+  Params.push_back(BoolTy);
+  Params.push_back(BoolTy);
+  // objc_setPropertyStruct (void*, void*, ptrdiff_t, BOOL, BOOL)
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), Params, false);
+  return cast<llvm::Function>(CGM.CreateRuntimeFunction(FTy, 
+                                                    "objc_setPropertyStruct"));
 }
 
 llvm::Constant *CGObjCGNU::EnumerationMutationFunction() {
@@ -2008,7 +2037,7 @@ void CGObjCGNU::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
       const llvm::Type *CatchType = CGF.ConvertType(CatchParam->getType());
       Exn = CGF.Builder.CreateBitCast(Exn, CatchType);
 
-      CGF.EmitLocalBlockVarDecl(*CatchParam);
+      CGF.EmitAutoVarDecl(*CatchParam);
       CGF.Builder.CreateStore(Exn, CGF.GetAddrOfLocalVar(CatchParam));
     }
 
@@ -2142,12 +2171,18 @@ llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable(
   // when linked against code which isn't (most of the time).
   llvm::GlobalVariable *IvarOffsetPointer = TheModule.getNamedGlobal(Name);
   if (!IvarOffsetPointer) {
-    uint64_t Offset;
-    if (ObjCImplementationDecl *OID =
-            CGM.getContext().getObjCImplementation(
+    // This will cause a run-time crash if we accidentally use it.  A value of
+    // 0 would seem more sensible, but will silently overwrite the isa pointer
+    // causing a great deal of confusion.
+    uint64_t Offset = -1;
+    // We can't call ComputeIvarBaseOffset() here if we have the
+    // implementation, because it will create an invalid ASTRecordLayout object
+    // that we are then stuck with forever, so we only initialize the ivar
+    // offset variable with a guess if we only have the interface.  The
+    // initializer will be reset later anyway, when we are generating the class
+    // description.
+    if (!CGM.getContext().getObjCImplementation(
               const_cast<ObjCInterfaceDecl *>(ID)))
-      Offset = ComputeIvarBaseOffset(CGM, OID, Ivar);
-    else
       Offset = ComputeIvarBaseOffset(CGM, ID, Ivar);
 
     llvm::ConstantInt *OffsetGuess =
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index 54d0ff21ea44..7c679b90590c 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -16,7 +16,8 @@
 #include "CGRecordLayout.h"
 #include "CodeGenModule.h"
 #include "CodeGenFunction.h"
-#include "CGException.h"
+#include "CGBlocks.h"
+#include "CGCleanup.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
@@ -77,6 +78,7 @@ static uint64_t LookupFieldBitOffset(CodeGen::CodeGenModule &CGM,
     ++Index;
   }
   assert(Index != Ivars.size() && "Ivar is not inside container!");
+  assert(Index < RL->getFieldCount() && "Ivar is not inside record layout!");
 
   return RL->getFieldOffset(Index);
 }
@@ -129,7 +131,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   // a synthesized ivar can never be a bit-field, so this is safe.
   const ASTRecordLayout &RL =
     CGF.CGM.getContext().getASTObjCInterfaceLayout(OID);
-  uint64_t TypeSizeInBits = RL.getSize();
+  uint64_t TypeSizeInBits = CGF.CGM.getContext().toBits(RL.getSize());
   uint64_t FieldBitOffset = LookupFieldBitOffset(CGF.CGM, OID, 0, Ivar);
   uint64_t BitOffset = FieldBitOffset % 8;
   uint64_t ContainingTypeAlign = 8;
@@ -462,7 +464,7 @@ public:
     // void objc_exception_rethrow(void)
     std::vector<const llvm::Type*> Args;
     llvm::FunctionType *FTy =
-      llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), Args, true);
+      llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), Args, false);
     return CGM.CreateRuntimeFunction(FTy, "objc_exception_rethrow");
   }
   
@@ -999,8 +1001,8 @@ public:
   /// forward references will be filled in with empty bodies if no
   /// definition is seen. The return value has type ProtocolPtrTy.
   virtual llvm::Constant *GetOrEmitProtocolRef(const ObjCProtocolDecl *PD)=0;
-  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
-                      const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &);
+  virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
+                                             const CGBlockInfo &blockInfo);
   
 };
 
@@ -1156,7 +1158,8 @@ public:
 
   virtual llvm::Constant *GetPropertyGetFunction();
   virtual llvm::Constant *GetPropertySetFunction();
-  virtual llvm::Constant *GetCopyStructFunction();
+  virtual llvm::Constant *GetGetStructFunction();
+  virtual llvm::Constant *GetSetStructFunction();
   virtual llvm::Constant *EnumerationMutationFunction();
 
   virtual void EmitTryStmt(CodeGen::CodeGenFunction &CGF,
@@ -1401,7 +1404,10 @@ public:
     return ObjCTypes.getSetPropertyFn();
   }
   
-  virtual llvm::Constant *GetCopyStructFunction() {
+  virtual llvm::Constant *GetSetStructFunction() {
+    return ObjCTypes.getCopyStructFn();
+  }
+  virtual llvm::Constant *GetGetStructFunction() {
     return ObjCTypes.getCopyStructFn();
   }
   
@@ -1520,7 +1526,7 @@ llvm::Constant *CGObjCCommonMac::GenerateConstantString(
   const StringLiteral *SL) {
   return (CGM.getLangOptions().NoConstantCFStrings == 0 ? 
           CGM.GetAddrOfConstantCFString(SL) :
-          CGM.GetAddrOfConstantNSString(SL));
+          CGM.GetAddrOfConstantString(SL));
 }
 
 /// Generates a message send where the super is the receiver.  This is
@@ -1662,57 +1668,76 @@ static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) {
   return Qualifiers::GCNone;
 }
 
-llvm::Constant *CGObjCCommonMac::GCBlockLayout(CodeGen::CodeGenFunction &CGF,
-              const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &DeclRefs) {
-  llvm::Constant *NullPtr = 
+llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
+                                                const CGBlockInfo &blockInfo) {
+  llvm::Constant *nullPtr = 
     llvm::Constant::getNullValue(llvm::Type::getInt8PtrTy(VMContext));
-  if ((CGM.getLangOptions().getGCMode() == LangOptions::NonGC) ||
-      DeclRefs.empty())
-    return NullPtr;
+
+  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC)
+    return nullPtr;
+
   bool hasUnion = false;
   SkipIvars.clear();
   IvarsInfo.clear();
   unsigned WordSizeInBits = CGM.getContext().Target.getPointerWidth(0);
   unsigned ByteSizeInBits = CGM.getContext().Target.getCharWidth();
   
-  for (size_t i = 0; i < DeclRefs.size(); ++i) {
-    const BlockDeclRefExpr *BDRE = DeclRefs[i];
-    const ValueDecl *VD = BDRE->getDecl();
-    CharUnits Offset = CGF.BlockDecls[VD];
-    uint64_t FieldOffset = Offset.getQuantity();
-    QualType Ty = VD->getType();
-    assert(!Ty->isArrayType() && 
-           "Array block variable should have been caught");
-    if ((Ty->isRecordType() || Ty->isUnionType()) && !BDRE->isByRef()) {
-      BuildAggrIvarRecordLayout(Ty->getAs<RecordType>(),
-                                FieldOffset,
-                                true,
-                                hasUnion);
+  // __isa is the first field in block descriptor and must assume by runtime's
+  // convention that it is GC'able.
+  IvarsInfo.push_back(GC_IVAR(0, 1));
+
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
+
+  // Calculate the basic layout of the block structure.
+  const llvm::StructLayout *layout =
+    CGM.getTargetData().getStructLayout(blockInfo.StructureType);
+
+  // Ignore the optional 'this' capture: C++ objects are not assumed
+  // to be GC'ed.
+
+  // Walk the captured variables.
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+         ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    QualType type = variable->getType();
+
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+
+    // Ignore constant captures.
+    if (capture.isConstant()) continue;
+
+    uint64_t fieldOffset = layout->getElementOffset(capture.getIndex());
+
+    // __block variables are passed by their descriptor address.
+    if (ci->isByRef()) {
+      IvarsInfo.push_back(GC_IVAR(fieldOffset, /*size in words*/ 1));
+      continue;
+    }
+
+    assert(!type->isArrayType() && "array variable should not be caught");
+    if (const RecordType *record = type->getAs<RecordType>()) {
+      BuildAggrIvarRecordLayout(record, fieldOffset, true, hasUnion);
       continue;
     }
       
-    Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), Ty);
-    unsigned FieldSize = CGM.getContext().getTypeSize(Ty);
-    // __block variables are passed by their descriptior address. So, size
-    // must reflect this.
-    if (BDRE->isByRef())
-      FieldSize = WordSizeInBits;
-    if (GCAttr == Qualifiers::Strong || BDRE->isByRef())
-      IvarsInfo.push_back(GC_IVAR(FieldOffset,
-                                  FieldSize / WordSizeInBits));
+    Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), type);
+    unsigned fieldSize = CGM.getContext().getTypeSize(type);
+
+    if (GCAttr == Qualifiers::Strong)
+      IvarsInfo.push_back(GC_IVAR(fieldOffset,
+                                  fieldSize / WordSizeInBits));
     else if (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak)
-      SkipIvars.push_back(GC_IVAR(FieldOffset,
-                                  FieldSize / ByteSizeInBits));
+      SkipIvars.push_back(GC_IVAR(fieldOffset,
+                                  fieldSize / ByteSizeInBits));
   }
   
   if (IvarsInfo.empty())
-    return NullPtr;
-  // Sort on byte position in case we encounterred a union nested in
-  // block variable type's aggregate type.
-  if (hasUnion && !IvarsInfo.empty())
-    std::sort(IvarsInfo.begin(), IvarsInfo.end());
-  if (hasUnion && !SkipIvars.empty())
-    std::sort(SkipIvars.begin(), SkipIvars.end());
+    return nullPtr;
+
+  // Sort on byte position; captures might not be allocated in order,
+  // and unions can do funny things.
+  llvm::array_pod_sort(IvarsInfo.begin(), IvarsInfo.end());
+  llvm::array_pod_sort(SkipIvars.begin(), SkipIvars.end());
   
   std::string BitMap;
   llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
@@ -2189,10 +2214,10 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) {
   if (ID->getNumIvarInitializers())
     Flags |= eClassFlags_HasCXXStructors;
   unsigned Size =
-    CGM.getContext().getASTObjCImplementationLayout(ID).getSize() / 8;
+    CGM.getContext().getASTObjCImplementationLayout(ID).getSize().getQuantity();
 
   // FIXME: Set CXX-structors flag.
-  if (CGM.getDeclVisibilityMode(ID->getClassInterface()) == LangOptions::Hidden)
+  if (ID->getClassInterface()->getVisibility() == HiddenVisibility)
     Flags |= eClassFlags_Hidden;
 
   std::vector<llvm::Constant*> InstanceMethods, ClassMethods;
@@ -2277,7 +2302,7 @@ llvm::Constant *CGObjCMac::EmitMetaClass(const ObjCImplementationDecl *ID,
   unsigned Flags = eClassFlags_Meta;
   unsigned Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.ClassTy);
 
-  if (CGM.getDeclVisibilityMode(ID->getClassInterface()) == LangOptions::Hidden)
+  if (ID->getClassInterface()->getVisibility() == HiddenVisibility)
     Flags |= eClassFlags_Hidden;
 
   std::vector<llvm::Constant*> Values(12);
@@ -2578,7 +2603,10 @@ llvm::Constant *CGObjCMac::GetPropertySetFunction() {
   return ObjCTypes.getSetPropertyFn();
 }
 
-llvm::Constant *CGObjCMac::GetCopyStructFunction() {
+llvm::Constant *CGObjCMac::GetGetStructFunction() {
+  return ObjCTypes.getCopyStructFn();
+}
+llvm::Constant *CGObjCMac::GetSetStructFunction() {
   return ObjCTypes.getCopyStructFn();
 }
 
@@ -2968,6 +2996,10 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   llvm::Value *CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(),
                                                      "_call_try_exit");
 
+  // A slot containing the exception to rethrow.  Only needed when we
+  // have both a @catch and a @finally.
+  llvm::Value *PropagatingExnVar = 0;
+
   // Push a normal cleanup to leave the try scope.
   CGF.EHStack.pushCleanup<PerformFragileFinally>(NormalCleanup, &S,
                                                  SyncArgSlot,
@@ -3039,6 +3071,12 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
     llvm::BasicBlock *CatchBlock = 0;
     llvm::BasicBlock *CatchHandler = 0;
     if (HasFinally) {
+      // Save the currently-propagating exception before
+      // objc_exception_try_enter clears the exception slot.
+      PropagatingExnVar = CGF.CreateTempAlloca(Caught->getType(),
+                                               "propagating_exception");
+      CGF.Builder.CreateStore(Caught, PropagatingExnVar);
+
       // Enter a new exception try block (in case a @catch block
       // throws an exception).
       CGF.Builder.CreateCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData)
@@ -3090,7 +3128,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
         CodeGenFunction::RunCleanupsScope CatchVarCleanups(CGF);
 
         if (CatchParam) {
-          CGF.EmitLocalBlockVarDecl(*CatchParam);
+          CGF.EmitAutoVarDecl(*CatchParam);
           assert(CGF.HaveInsertPoint() && "DeclStmt destroyed insert point?");
 
           // These types work out because ConvertType(id) == i8*.
@@ -3134,7 +3172,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // the end of the catch body.
       CodeGenFunction::RunCleanupsScope CatchVarCleanups(CGF);
 
-      CGF.EmitLocalBlockVarDecl(*CatchParam);
+      CGF.EmitAutoVarDecl(*CatchParam);
       assert(CGF.HaveInsertPoint() && "DeclStmt destroyed insert point?");
 
       // Initialize the catch variable.
@@ -3173,6 +3211,15 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // the try's write hazard and here.
       //Hazards.emitWriteHazard();
 
+      // Extract the new exception and save it to the
+      // propagating-exception slot.
+      assert(PropagatingExnVar);
+      llvm::CallInst *NewCaught =
+        CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
+                               ExceptionData, "caught");
+      NewCaught->setDoesNotThrow();
+      CGF.Builder.CreateStore(NewCaught, PropagatingExnVar);
+
       // Don't pop the catch handler; the throw already did.
       CGF.Builder.CreateStore(CGF.Builder.getFalse(), CallTryExitVar);
       CGF.EmitBranchThroughCleanup(FinallyRethrow);
@@ -3193,13 +3240,21 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
   CGF.EmitBlock(FinallyRethrow.getBlock(), true);
   if (CGF.HaveInsertPoint()) {
-    // Just look in the buffer for the exception to throw.
-    llvm::CallInst *Caught =
-      CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
-                             ExceptionData);
-    Caught->setDoesNotThrow();
+    // If we have a propagating-exception variable, check it.
+    llvm::Value *PropagatingExn;
+    if (PropagatingExnVar) {
+      PropagatingExn = CGF.Builder.CreateLoad(PropagatingExnVar);
+
+    // Otherwise, just look in the buffer for the exception to throw.
+    } else {
+      llvm::CallInst *Caught =
+        CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
+                               ExceptionData);
+      Caught->setDoesNotThrow();
+      PropagatingExn = Caught;
+    }
 
-    CGF.Builder.CreateCall(ObjCTypes.getExceptionThrowFn(), Caught)
+    CGF.Builder.CreateCall(ObjCTypes.getExceptionThrowFn(), PropagatingExn)
       ->setDoesNotThrow();
     CGF.Builder.CreateUnreachable();
   }
@@ -3586,10 +3641,10 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
   uint64_t MaxSkippedUnionIvarSize = 0;
   FieldDecl *MaxField = 0;
   FieldDecl *MaxSkippedField = 0;
-  FieldDecl *LastFieldBitfield = 0;
+  FieldDecl *LastFieldBitfieldOrUnnamed = 0;
   uint64_t MaxFieldOffset = 0;
   uint64_t MaxSkippedFieldOffset = 0;
-  uint64_t LastBitfieldOffset = 0;
+  uint64_t LastBitfieldOrUnnamedOffset = 0;
 
   if (RecFields.empty())
     return;
@@ -3609,12 +3664,12 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
 
     // Skip over unnamed or bitfields
     if (!Field->getIdentifier() || Field->isBitField()) {
-      LastFieldBitfield = Field;
-      LastBitfieldOffset = FieldOffset;
+      LastFieldBitfieldOrUnnamed = Field;
+      LastBitfieldOrUnnamedOffset = FieldOffset;
       continue;
     }
 
-    LastFieldBitfield = 0;
+    LastFieldBitfieldOrUnnamed = 0;
     QualType FQT = Field->getType();
     if (FQT->isRecordType() || FQT->isUnionType()) {
       if (FQT->isUnionType())
@@ -3641,7 +3696,7 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
 
       assert(!FQT->isUnionType() &&
              "layout for array of unions not supported");
-      if (FQT->isRecordType()) {
+      if (FQT->isRecordType() && ElCount) {
         int OldIndex = IvarsInfo.size() - 1;
         int OldSkIndex = SkipIvars.size() -1;
 
@@ -3703,16 +3758,25 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
     }
   }
 
-  if (LastFieldBitfield) {
-    // Last field was a bitfield. Must update skip info.
-    Expr *BitWidth = LastFieldBitfield->getBitWidth();
-    uint64_t BitFieldSize =
-      BitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
-    GC_IVAR skivar;
-    skivar.ivar_bytepos = BytePos + LastBitfieldOffset;
-    skivar.ivar_size = (BitFieldSize / ByteSizeInBits)
-      + ((BitFieldSize % ByteSizeInBits) != 0);
-    SkipIvars.push_back(skivar);
+  if (LastFieldBitfieldOrUnnamed) {
+    if (LastFieldBitfieldOrUnnamed->isBitField()) {
+      // Last field was a bitfield. Must update skip info.
+      Expr *BitWidth = LastFieldBitfieldOrUnnamed->getBitWidth();
+      uint64_t BitFieldSize =
+        BitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
+      GC_IVAR skivar;
+      skivar.ivar_bytepos = BytePos + LastBitfieldOrUnnamedOffset;
+      skivar.ivar_size = (BitFieldSize / ByteSizeInBits)
+        + ((BitFieldSize % ByteSizeInBits) != 0);
+      SkipIvars.push_back(skivar);
+    } else {
+      assert(!LastFieldBitfieldOrUnnamed->getIdentifier() &&"Expected unnamed");
+      // Last field was unnamed. Must update skip info.
+      unsigned FieldSize
+          = CGM.getContext().getTypeSize(LastFieldBitfieldOrUnnamed->getType());
+      SkipIvars.push_back(GC_IVAR(BytePos + LastBitfieldOrUnnamedOffset,
+                                  FieldSize / ByteSizeInBits));
+    }
   }
 
   if (MaxField)
@@ -4886,7 +4950,7 @@ void CGObjCNonFragileABIMac::GetClassSizeInfo(const ObjCImplementationDecl *OID,
     CGM.getContext().getASTObjCImplementationLayout(OID);
 
   // InstanceSize is really instance end.
-  InstanceSize = llvm::RoundUpToAlignment(RL.getDataSize(), 8) / 8;
+  InstanceSize = RL.getDataSize().getQuantity();
 
   // If there are no fields, the start is the same as the end.
   if (!RL.getFieldCount())
@@ -4927,7 +4991,7 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
   llvm::GlobalVariable *SuperClassGV, *IsAGV;
 
   bool classIsHidden =
-    CGM.getDeclVisibilityMode(ID->getClassInterface()) == LangOptions::Hidden;
+    ID->getClassInterface()->getVisibility() == HiddenVisibility;
   if (classIsHidden)
     flags |= OBJC2_CLS_HIDDEN;
   if (ID->getNumIvarInitializers())
@@ -5222,7 +5286,7 @@ CGObjCNonFragileABIMac::EmitIvarOffsetVar(const ObjCInterfaceDecl *ID,
   // well (i.e., in ObjCIvarOffsetVariable).
   if (Ivar->getAccessControl() == ObjCIvarDecl::Private ||
       Ivar->getAccessControl() == ObjCIvarDecl::Package ||
-      CGM.getDeclVisibilityMode(ID) == LangOptions::Hidden)
+      ID->getVisibility() == HiddenVisibility)
     IvarOffsetGV->setVisibility(llvm::GlobalValue::HiddenVisibility);
   else
     IvarOffsetGV->setVisibility(llvm::GlobalValue::DefaultVisibility);
@@ -6096,7 +6160,7 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
       const llvm::Type *CatchType = CGF.ConvertType(CatchParam->getType());
       llvm::Value *CastExn = CGF.Builder.CreateBitCast(Exn, CatchType);
 
-      CGF.EmitLocalBlockVarDecl(*CatchParam);
+      CGF.EmitAutoVarDecl(*CatchParam);
       CGF.Builder.CreateStore(CastExn, CGF.GetAddrOfLocalVar(CatchParam));
     }
 
@@ -6124,32 +6188,20 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
 /// EmitThrowStmt - Generate code for a throw statement.
 void CGObjCNonFragileABIMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
                                            const ObjCAtThrowStmt &S) {
-  llvm::Value *Exception;
-  llvm::Constant *FunctionThrowOrRethrow;
   if (const Expr *ThrowExpr = S.getThrowExpr()) {
-    Exception = CGF.EmitScalarExpr(ThrowExpr);
-    FunctionThrowOrRethrow = ObjCTypes.getExceptionThrowFn();
-  } else {
-    assert((!CGF.ObjCEHValueStack.empty() && CGF.ObjCEHValueStack.back()) &&
-           "Unexpected rethrow outside @catch block.");
-    Exception = CGF.ObjCEHValueStack.back();
-    FunctionThrowOrRethrow = ObjCTypes.getExceptionRethrowFn();
-  }
-
-  llvm::Value *ExceptionAsObject =
-    CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy, "tmp");
-  llvm::BasicBlock *InvokeDest = CGF.getInvokeDest();
-  if (InvokeDest) {
-    CGF.Builder.CreateInvoke(FunctionThrowOrRethrow,
-                             CGF.getUnreachableBlock(), InvokeDest,
-                             &ExceptionAsObject, &ExceptionAsObject + 1);
+    llvm::Value *Exception = CGF.EmitScalarExpr(ThrowExpr);
+    Exception = CGF.Builder.CreateBitCast(Exception, ObjCTypes.ObjectPtrTy,
+                                          "tmp");
+    llvm::Value *Args[] = { Exception };
+    CGF.EmitCallOrInvoke(ObjCTypes.getExceptionThrowFn(),
+                         Args, Args+1)
+      .setDoesNotReturn();
   } else {
-    CGF.Builder.CreateCall(FunctionThrowOrRethrow, ExceptionAsObject)
-      ->setDoesNotReturn();
-    CGF.Builder.CreateUnreachable();
+    CGF.EmitCallOrInvoke(ObjCTypes.getExceptionRethrowFn(), 0, 0)
+      .setDoesNotReturn();
   }
 
-  // Clear the insertion point to indicate we are in unreachable code.
+  CGF.Builder.CreateUnreachable();
   CGF.Builder.ClearInsertionPoint();
 }
 
@@ -6208,7 +6260,7 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID,
                                       ID->getIdentifier()->getName()));
   }
 
-  if (CGM.getLangOptions().getVisibilityMode() == LangOptions::Hidden)
+  if (CGM.getLangOptions().getVisibilityMode() == HiddenVisibility)
     Entry->setVisibility(llvm::GlobalValue::HiddenVisibility);
   Entry->setAlignment(CGM.getTargetData().getABITypeAlignment(
       ObjCTypes.EHTypeTy));
diff --git a/lib/CodeGen/CGObjCRuntime.h b/lib/CodeGen/CGObjCRuntime.h
index 584760f6f343..5ad3a50adc3f 100644
--- a/lib/CodeGen/CGObjCRuntime.h
+++ b/lib/CodeGen/CGObjCRuntime.h
@@ -56,6 +56,7 @@ namespace CodeGen {
 
 namespace CodeGen {
   class CodeGenModule;
+  class CGBlockInfo;
 
 // FIXME: Several methods should be pure virtual but aren't to avoid the
 // partially-implemented subclass breaking.
@@ -176,8 +177,10 @@ public:
   /// Return the runtime function for setting properties.
   virtual llvm::Constant *GetPropertySetFunction() = 0;
 
-  // API for atomic copying of qualified aggregates in setter/getter.
-  virtual llvm::Constant *GetCopyStructFunction() = 0;
+  // API for atomic copying of qualified aggregates in getter.
+  virtual llvm::Constant *GetGetStructFunction() = 0;
+  // API for atomic copying of qualified aggregates in setter.
+  virtual llvm::Constant *GetSetStructFunction() = 0;
   
   /// GetClass - Return a reference to the class for the given
   /// interface decl.
@@ -219,9 +222,8 @@ public:
                                         llvm::Value *DestPtr,
                                         llvm::Value *SrcPtr,
                                         llvm::Value *Size) = 0;
-  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
-                  const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &) = 0;
-                                        
+  virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
+                                  const CodeGen::CGBlockInfo &blockInfo) = 0;
 };
 
 /// Creates an instance of an Objective-C runtime class.
diff --git a/lib/CodeGen/CGRTTI.cpp b/lib/CodeGen/CGRTTI.cpp
index 60df613f65e8..7ec0ee4b5cab 100644
--- a/lib/CodeGen/CGRTTI.cpp
+++ b/lib/CodeGen/CGRTTI.cpp
@@ -30,6 +30,10 @@ class RTTIBuilder {
   /// Fields - The fields of the RTTI descriptor currently being built.
   llvm::SmallVector<llvm::Constant *, 16> Fields;
 
+  /// GetAddrOfTypeName - Returns the mangled type name of the given type.
+  llvm::GlobalVariable *
+  GetAddrOfTypeName(QualType Ty, llvm::GlobalVariable::LinkageTypes Linkage);
+
   /// GetAddrOfExternalRTTIDescriptor - Returns the constant for the RTTI 
   /// descriptor of the given type.
   llvm::Constant *GetAddrOfExternalRTTIDescriptor(QualType Ty);
@@ -59,64 +63,10 @@ class RTTIBuilder {
   void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty);
   
 public:
-  RTTIBuilder(CodeGenModule &cgm)
-    : CGM(cgm), VMContext(cgm.getModule().getContext()),
-      Int8PtrTy(llvm::Type::getInt8PtrTy(VMContext)) { }
-
-  llvm::Constant *BuildName(QualType Ty, bool Hidden, 
-                            llvm::GlobalVariable::LinkageTypes Linkage) {
-    llvm::SmallString<256> OutName;
-    CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, OutName);
-    llvm::StringRef Name = OutName.str();
-
-    llvm::GlobalVariable *OGV = CGM.getModule().getNamedGlobal(Name);
-    if (OGV && !OGV->isDeclaration())
-      return llvm::ConstantExpr::getBitCast(OGV, Int8PtrTy);
-
-    llvm::Constant *C = llvm::ConstantArray::get(VMContext, Name.substr(4));
-
-    llvm::GlobalVariable *GV = 
-      new llvm::GlobalVariable(CGM.getModule(), C->getType(), true, Linkage,
-                               C, Name);
-    if (OGV) {
-      GV->takeName(OGV);
-      llvm::Constant *NewPtr = llvm::ConstantExpr::getBitCast(GV,
-                                                              OGV->getType());
-      OGV->replaceAllUsesWith(NewPtr);
-      OGV->eraseFromParent();
-    }
-    if (Hidden)
-      GV->setVisibility(llvm::GlobalVariable::HiddenVisibility);
-    return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
-  }
+  RTTIBuilder(CodeGenModule &CGM) : CGM(CGM), 
+    VMContext(CGM.getModule().getContext()),
+    Int8PtrTy(llvm::Type::getInt8PtrTy(VMContext)) { }
 
-  // FIXME: unify with DecideExtern
-  bool DecideHidden(QualType Ty) {
-    // For this type, see if all components are never hidden.
-    if (const MemberPointerType *MPT = Ty->getAs<MemberPointerType>())
-      return (DecideHidden(MPT->getPointeeType())
-              && DecideHidden(QualType(MPT->getClass(), 0)));
-    if (const PointerType *PT = Ty->getAs<PointerType>())
-      return DecideHidden(PT->getPointeeType());
-    if (const FunctionType *FT = Ty->getAs<FunctionType>()) {
-      if (DecideHidden(FT->getResultType()) == false)
-        return false;
-      if (const FunctionProtoType *FPT = Ty->getAs<FunctionProtoType>()) {
-        for (unsigned i = 0; i <FPT->getNumArgs(); ++i)
-          if (DecideHidden(FPT->getArgType(i)) == false)
-            return false;
-        for (unsigned i = 0; i <FPT->getNumExceptions(); ++i)
-          if (DecideHidden(FPT->getExceptionType(i)) == false)
-            return false;
-        return true;
-      }
-    }
-    if (const RecordType *RT = Ty->getAs<RecordType>())
-      if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-        return CGM.getDeclVisibilityMode(RD) == LangOptions::Hidden;
-    return false;
-  }
-  
   // Pointer type info flags.
   enum {
     /// PTI_Const - Type has const qualifier.
@@ -162,10 +112,34 @@ public:
 };
 }
 
+llvm::GlobalVariable *
+RTTIBuilder::GetAddrOfTypeName(QualType Ty, 
+                               llvm::GlobalVariable::LinkageTypes Linkage) {
+  llvm::SmallString<256> OutName;
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
+  Out.flush();
+  llvm::StringRef Name = OutName.str();
+
+  // We know that the mangled name of the type starts at index 4 of the
+  // mangled name of the typename, so we can just index into it in order to
+  // get the mangled name of the type.
+  llvm::Constant *Init = llvm::ConstantArray::get(VMContext, Name.substr(4));
+
+  llvm::GlobalVariable *GV = 
+    CGM.CreateOrReplaceCXXRuntimeVariable(Name, Init->getType(), Linkage);
+
+  GV->setInitializer(Init);
+
+  return GV;
+}
+
 llvm::Constant *RTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
   // Mangle the RTTI name.
   llvm::SmallString<256> OutName;
-  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, OutName);
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
+  Out.flush();
   llvm::StringRef Name = OutName.str();
 
   // Look for an existing global.
@@ -187,15 +161,17 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
   //   Basic type information (e.g. for "int", "bool", etc.) will be kept in
   //   the run-time support library. Specifically, the run-time support
   //   library should contain type_info objects for the types X, X* and 
-  //   X const*, for every X in: void, bool, wchar_t, char, unsigned char, 
-  //   signed char, short, unsigned short, int, unsigned int, long, 
-  //   unsigned long, long long, unsigned long long, float, double, long double, 
-  //   char16_t, char32_t, and the IEEE 754r decimal and half-precision 
-  //   floating point types.
+  //   X const*, for every X in: void, std::nullptr_t, bool, wchar_t, char,
+  //   unsigned char, signed char, short, unsigned short, int, unsigned int,
+  //   long, unsigned long, long long, unsigned long long, float, double,
+  //   long double, char16_t, char32_t, and the IEEE 754r decimal and 
+  //   half-precision floating point types.
   switch (Ty->getKind()) {
     case BuiltinType::Void:
+    case BuiltinType::NullPtr:
     case BuiltinType::Bool:
-    case BuiltinType::WChar:
+    case BuiltinType::WChar_S:
+    case BuiltinType::WChar_U:
     case BuiltinType::Char_U:
     case BuiltinType::Char_S:
     case BuiltinType::UChar:
@@ -219,12 +195,8 @@ static bool TypeInfoIsInStandardLibrary(const BuiltinType *Ty) {
       
     case BuiltinType::Overload:
     case BuiltinType::Dependent:
-    case BuiltinType::UndeducedAuto:
       assert(false && "Should not see this type here!");
       
-    case BuiltinType::NullPtr:
-      assert(false && "FIXME: nullptr_t is not handled!");
-
     case BuiltinType::ObjCId:
     case BuiltinType::ObjCClass:
     case BuiltinType::ObjCSel:
@@ -270,8 +242,9 @@ static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
 /// the given type exists somewhere else, and that we should not emit the type
 /// information in this translation unit.  Assumes that it is not a
 /// standard-library type.
-static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
-                                            QualType Ty) {
+static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM, QualType Ty) {
+  ASTContext &Context = CGM.getContext();
+
   // If RTTI is disabled, don't consider key functions.
   if (!Context.getLangOptions().RTTI) return false;
 
@@ -283,13 +256,7 @@ static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
     if (!RD->isDynamicClass())
       return false;
 
-    // Get the key function.
-    const CXXMethodDecl *KeyFunction = Context.getKeyFunction(RD);
-    if (KeyFunction && !KeyFunction->hasBody()) {
-      // The class has a key function, but it is not defined in this translation
-      // unit, so we should use the external descriptor for it.
-      return true;
-    }
+    return !CGM.getVTables().ShouldEmitVTableInThisTU(RD);
   }
   
   return false;
@@ -335,7 +302,8 @@ static bool ContainsIncompleteClassType(QualType Ty) {
 
 /// getTypeInfoLinkage - Return the linkage that the type info and type info
 /// name constants should have for the given type.
-static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(QualType Ty) {
+static llvm::GlobalVariable::LinkageTypes 
+getTypeInfoLinkage(CodeGenModule &CGM, QualType Ty) {
   // Itanium C++ ABI 2.9.5p7:
   //   In addition, it and all of the intermediate abi::__pointer_type_info 
   //   structs in the chain down to the abi::__class_type_info for the
@@ -355,16 +323,22 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(QualType Ty) {
     return llvm::GlobalValue::InternalLinkage;
 
   case ExternalLinkage:
+    if (!CGM.getLangOptions().RTTI) {
+      // RTTI is not enabled, which means that this type info struct is going
+      // to be used for exception handling. Give it linkonce_odr linkage.
+      return llvm::GlobalValue::LinkOnceODRLinkage;
+    }
+
     if (const RecordType *Record = dyn_cast<RecordType>(Ty)) {
       const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
       if (RD->isDynamicClass())
-        return CodeGenModule::getVTableLinkage(RD);
+        return CGM.getVTableLinkage(RD);
     }
 
-    return llvm::GlobalValue::WeakODRLinkage;
+    return llvm::GlobalValue::LinkOnceODRLinkage;
   }
 
-  return llvm::GlobalValue::WeakODRLinkage;
+  return llvm::GlobalValue::LinkOnceODRLinkage;
 }
 
 // CanUseSingleInheritance - Return whether the given record decl has a "single, 
@@ -513,23 +487,81 @@ void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
   Fields.push_back(VTable);
 }
 
+// maybeUpdateRTTILinkage - Will update the linkage of the RTTI data structures
+// from available_externally to the correct linkage if necessary. An example of
+// this is:
+//
+//   struct A {
+//     virtual void f();
+//   };
+//
+//   const std::type_info &g() {
+//     return typeid(A);
+//   }
+//
+//   void A::f() { }
+//
+// When we're generating the typeid(A) expression, we do not yet know that
+// A's key function is defined in this translation unit, so we will give the
+// typeinfo and typename structures available_externally linkage. When A::f
+// forces the vtable to be generated, we need to change the linkage of the
+// typeinfo and typename structs, otherwise we'll end up with undefined
+// externals when linking.
+static void 
+maybeUpdateRTTILinkage(CodeGenModule &CGM, llvm::GlobalVariable *GV,
+                       QualType Ty) {
+  // We're only interested in globals with available_externally linkage.
+  if (!GV->hasAvailableExternallyLinkage())
+    return;
+
+  // Get the real linkage for the type.
+  llvm::GlobalVariable::LinkageTypes Linkage = getTypeInfoLinkage(CGM, Ty);
+
+  // If variable is supposed to have available_externally linkage, we don't
+  // need to do anything.
+  if (Linkage == llvm::GlobalVariable::AvailableExternallyLinkage)
+    return;
+
+  // Update the typeinfo linkage.
+  GV->setLinkage(Linkage);
+
+  // Get the typename global.
+  llvm::SmallString<256> OutName;
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, Out);
+  Out.flush();
+  llvm::StringRef Name = OutName.str();
+
+  llvm::GlobalVariable *TypeNameGV = CGM.getModule().getNamedGlobal(Name);
+
+  assert(TypeNameGV->hasAvailableExternallyLinkage() &&
+         "Type name has different linkage from type info!");
+
+  // And update its linkage.
+  TypeNameGV->setLinkage(Linkage);
+}
+
 llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   // We want to operate on the canonical type.
   Ty = CGM.getContext().getCanonicalType(Ty);
 
   // Check if we've already emitted an RTTI descriptor for this type.
   llvm::SmallString<256> OutName;
-  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, OutName);
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, Out);
+  Out.flush();
   llvm::StringRef Name = OutName.str();
-  
+
   llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
-  if (OldGV && !OldGV->isDeclaration())
+  if (OldGV && !OldGV->isDeclaration()) {
+    maybeUpdateRTTILinkage(CGM, OldGV, Ty);
+
     return llvm::ConstantExpr::getBitCast(OldGV, Int8PtrTy);
+  }
 
   // Check if there is already an external RTTI descriptor for this type.
   bool IsStdLib = IsStandardLibraryRTTIDescriptor(Ty);
-  if (!Force &&
-      (IsStdLib || ShouldUseExternalRTTIDescriptor(CGM.getContext(), Ty)))
+  if (!Force && (IsStdLib || ShouldUseExternalRTTIDescriptor(CGM, Ty)))
     return GetAddrOfExternalRTTIDescriptor(Ty);
 
   // Emit the standard library with external linkage.
@@ -537,13 +569,16 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   if (IsStdLib)
     Linkage = llvm::GlobalValue::ExternalLinkage;
   else
-    Linkage = getTypeInfoLinkage(Ty);
+    Linkage = getTypeInfoLinkage(CGM, Ty);
 
   // Add the vtable pointer.
   BuildVTablePointer(cast<Type>(Ty));
   
   // And the name.
-  Fields.push_back(BuildName(Ty, DecideHidden(Ty), Linkage));
+  llvm::GlobalVariable *TypeName = GetAddrOfTypeName(Ty, Linkage);
+
+  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  Fields.push_back(llvm::ConstantExpr::getBitCast(TypeName, Int8PtrTy));
 
   switch (Ty->getTypeClass()) {
 #define TYPE(Class, Base)
@@ -641,13 +676,28 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   // type_infos themselves, so we can emit these as hidden symbols.
   // But don't do this if we're worried about strict visibility
   // compatibility.
-  if (const RecordType *RT = dyn_cast<RecordType>(Ty))
-    CGM.setTypeVisibility(GV, cast<CXXRecordDecl>(RT->getDecl()),
-                          /*ForRTTI*/ true);
-  else if (CGM.getCodeGenOpts().HiddenWeakVTables &&
-           Linkage == llvm::GlobalValue::WeakODRLinkage)
-    GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
-  
+  if (const RecordType *RT = dyn_cast<RecordType>(Ty)) {
+    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+
+    CGM.setTypeVisibility(GV, RD, CodeGenModule::TVK_ForRTTI);
+    CGM.setTypeVisibility(TypeName, RD, CodeGenModule::TVK_ForRTTIName);
+  } else {
+    Visibility TypeInfoVisibility = DefaultVisibility;
+    if (CGM.getCodeGenOpts().HiddenWeakVTables &&
+        Linkage == llvm::GlobalValue::LinkOnceODRLinkage)
+      TypeInfoVisibility = HiddenVisibility;
+
+    // The type name should have the same visibility as the type itself.
+    Visibility ExplicitVisibility = Ty->getVisibility();
+    TypeName->setVisibility(CodeGenModule::
+                            GetLLVMVisibility(ExplicitVisibility));
+  
+    TypeInfoVisibility = minVisibility(TypeInfoVisibility, Ty->getVisibility());
+    GV->setVisibility(CodeGenModule::GetLLVMVisibility(TypeInfoVisibility));
+  }
+
+  GV->setUnnamedAddr(true);
+
   return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
 }
 
@@ -701,12 +751,14 @@ void RTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
   Fields.push_back(BaseTypeInfo);
 }
 
-/// SeenBases - Contains virtual and non-virtual bases seen when traversing
-/// a class hierarchy.
-struct SeenBases {
-  llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
-  llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
-};
+namespace {
+  /// SeenBases - Contains virtual and non-virtual bases seen when traversing
+  /// a class hierarchy.
+  struct SeenBases {
+    llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
+    llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
+  };
+}
 
 /// ComputeVMIClassTypeInfoFlags - Compute the value of the flags member in
 /// abi::__vmi_class_type_info.
@@ -827,7 +879,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
       OffsetFlags = CGM.getVTables().getVirtualBaseOffsetOffset(RD, BaseDecl);
     else {
       const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
-      OffsetFlags = Layout.getBaseClassOffset(BaseDecl) / 8;
+      OffsetFlags = Layout.getBaseClassOffsetInBits(BaseDecl) / 8;
     };
     
     OffsetFlags <<= 8;
@@ -938,16 +990,16 @@ void CodeGenModule::EmitFundamentalRTTIDescriptor(QualType Type) {
 }
 
 void CodeGenModule::EmitFundamentalRTTIDescriptors() {
-  QualType FundamentalTypes[] = { Context.VoidTy, Context.Char32Ty,
-                                  Context.Char16Ty, Context.UnsignedLongLongTy,
-                                  Context.LongLongTy, Context.WCharTy,
-                                  Context.UnsignedShortTy, Context.ShortTy,
-                                  Context.UnsignedLongTy, Context.LongTy,
-                                  Context.UnsignedIntTy, Context.IntTy,
-                                  Context.UnsignedCharTy, Context.FloatTy,
-                                  Context.LongDoubleTy, Context.DoubleTy,
-                                  Context.CharTy, Context.BoolTy,
-                                  Context.SignedCharTy };
+  QualType FundamentalTypes[] = { Context.VoidTy, Context.NullPtrTy,
+                                  Context.BoolTy, Context.WCharTy,
+                                  Context.CharTy, Context.UnsignedCharTy,
+                                  Context.SignedCharTy, Context.ShortTy, 
+                                  Context.UnsignedShortTy, Context.IntTy,
+                                  Context.UnsignedIntTy, Context.LongTy, 
+                                  Context.UnsignedLongTy, Context.LongLongTy, 
+                                  Context.UnsignedLongLongTy, Context.FloatTy,
+                                  Context.DoubleTy, Context.LongDoubleTy,
+                                  Context.Char16Ty, Context.Char32Ty };
   for (unsigned i = 0; i < sizeof(FundamentalTypes)/sizeof(QualType); ++i)
     EmitFundamentalRTTIDescriptor(FundamentalTypes[i]);
 }
diff --git a/lib/CodeGen/CGRecordLayout.h b/lib/CodeGen/CGRecordLayout.h
index 9b4e9f86c6da..30da05f421fe 100644
--- a/lib/CodeGen/CGRecordLayout.h
+++ b/lib/CodeGen/CGRecordLayout.h
@@ -11,10 +11,11 @@
 #define CLANG_CODEGEN_CGRECORDLAYOUT_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/DerivedTypes.h"
 #include "clang/AST/Decl.h"
 namespace llvm {
   class raw_ostream;
-  class Type;
+  class StructType;
 }
 
 namespace clang {
@@ -172,8 +173,13 @@ class CGRecordLayout {
   void operator=(const CGRecordLayout&); // DO NOT IMPLEMENT
 
 private:
-  /// The LLVMType corresponding to this record layout.
-  const llvm::Type *LLVMType;
+  /// The LLVM type corresponding to this record layout; used when
+  /// laying it out as a complete object.
+  llvm::PATypeHolder CompleteObjectType;
+
+  /// The LLVM type for the non-virtual part of this record layout;
+  /// used when laying it out as a base subobject.
+  llvm::PATypeHolder BaseSubobjectType;
 
   /// Map from (non-bit-field) struct field to the corresponding llvm struct
   /// type field no. This info is populated by record builder.
@@ -185,19 +191,41 @@ private:
 
   // FIXME: Maybe we could use a CXXBaseSpecifier as the key and use a single
   // map for both virtual and non virtual bases.
-  llvm::DenseMap<const CXXRecordDecl *, unsigned> NonVirtualBaseFields;
+  llvm::DenseMap<const CXXRecordDecl *, unsigned> NonVirtualBases;
+
+  /// Map from virtual bases to their field index in the complete object.
+  llvm::DenseMap<const CXXRecordDecl *, unsigned> CompleteObjectVirtualBases;
 
-  /// Whether one of the fields in this record layout is a pointer to data
-  /// member, or a struct that contains pointer to data member.
+  /// False if any direct or indirect subobject of this class, when
+  /// considered as a complete object, requires a non-zero bitpattern
+  /// when zero-initialized.
   bool IsZeroInitializable : 1;
 
+  /// False if any direct or indirect subobject of this class, when
+  /// considered as a base subobject, requires a non-zero bitpattern
+  /// when zero-initialized.
+  bool IsZeroInitializableAsBase : 1;
+
 public:
-  CGRecordLayout(const llvm::Type *T, bool IsZeroInitializable)
-    : LLVMType(T), IsZeroInitializable(IsZeroInitializable) {}
+  CGRecordLayout(const llvm::StructType *CompleteObjectType,
+                 const llvm::StructType *BaseSubobjectType,
+                 bool IsZeroInitializable,
+                 bool IsZeroInitializableAsBase)
+    : CompleteObjectType(CompleteObjectType),
+      BaseSubobjectType(BaseSubobjectType),
+      IsZeroInitializable(IsZeroInitializable),
+      IsZeroInitializableAsBase(IsZeroInitializableAsBase) {}
+
+  /// \brief Return the "complete object" LLVM type associated with
+  /// this record.
+  const llvm::StructType *getLLVMType() const {
+    return cast<llvm::StructType>(CompleteObjectType.get());
+  }
 
-  /// \brief Return the LLVM type associated with this record.
-  const llvm::Type *getLLVMType() const {
-    return LLVMType;
+  /// \brief Return the "base subobject" LLVM type associated with
+  /// this record.
+  const llvm::StructType *getBaseSubobjectLLVMType() const {
+    return cast<llvm::StructType>(BaseSubobjectType.get());
   }
 
   /// \brief Check whether this struct can be C++ zero-initialized
@@ -206,6 +234,12 @@ public:
     return IsZeroInitializable;
   }
 
+  /// \brief Check whether this struct can be C++ zero-initialized
+  /// with a zeroinitializer when considered as a base subobject.
+  bool isZeroInitializableAsBase() const {
+    return IsZeroInitializableAsBase;
+  }
+
   /// \brief Return llvm::StructType element number that corresponds to the
   /// field FD.
   unsigned getLLVMFieldNo(const FieldDecl *FD) const {
@@ -215,8 +249,15 @@ public:
   }
 
   unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const {
-    assert(NonVirtualBaseFields.count(RD) && "Invalid non-virtual base!");
-    return NonVirtualBaseFields.lookup(RD);
+    assert(NonVirtualBases.count(RD) && "Invalid non-virtual base!");
+    return NonVirtualBases.lookup(RD);
+  }
+
+  /// \brief Return the LLVM field index corresponding to the given
+  /// virtual base.  Only valid when operating on the complete object.
+  unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const {
+    assert(CompleteObjectVirtualBases.count(base) && "Invalid virtual base!");
+    return CompleteObjectVirtualBases.lookup(base);
   }
 
   /// \brief Return the BitFieldInfo that corresponds to the field FD.
@@ -224,7 +265,7 @@ public:
     assert(FD->isBitField() && "Invalid call for non bit-field decl!");
     llvm::DenseMap<const FieldDecl *, CGBitFieldInfo>::const_iterator
       it = BitFields.find(FD);
-    assert(it != BitFields.end()  && "Unable to find bitfield info");
+    assert(it != BitFields.end() && "Unable to find bitfield info");
     return it->second;
   }
 
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 77a319fa3ab1..4b19aefcf9c4 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -14,6 +14,7 @@
 #include "CGRecordLayout.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
+#include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/RecordLayout.h"
@@ -27,28 +28,52 @@
 using namespace clang;
 using namespace CodeGen;
 
-namespace clang {
-namespace CodeGen {
+namespace {
 
 class CGRecordLayoutBuilder {
 public:
   /// FieldTypes - Holds the LLVM types that the struct is created from.
+  /// 
   std::vector<const llvm::Type *> FieldTypes;
 
-  /// LLVMFieldInfo - Holds a field and its corresponding LLVM field number.
-  typedef std::pair<const FieldDecl *, unsigned> LLVMFieldInfo;
-  llvm::SmallVector<LLVMFieldInfo, 16> LLVMFields;
-
-  /// LLVMBitFieldInfo - Holds location and size information about a bit field.
-  typedef std::pair<const FieldDecl *, CGBitFieldInfo> LLVMBitFieldInfo;
-  llvm::SmallVector<LLVMBitFieldInfo, 16> LLVMBitFields;
-
-  typedef std::pair<const CXXRecordDecl *, unsigned> LLVMBaseInfo;
-  llvm::SmallVector<LLVMBaseInfo, 16> LLVMNonVirtualBases;
+  /// BaseSubobjectType - Holds the LLVM type for the non-virtual part
+  /// of the struct. For example, consider:
+  ///
+  /// struct A { int i; };
+  /// struct B { void *v; };
+  /// struct C : virtual A, B { };
+  ///
+  /// The LLVM type of C will be
+  /// %struct.C = type { i32 (...)**, %struct.A, i32, %struct.B }
+  ///
+  /// And the LLVM type of the non-virtual base struct will be
+  /// %struct.C.base = type { i32 (...)**, %struct.A, i32 }
+  ///
+  /// This only gets initialized if the base subobject type is
+  /// different from the complete-object type.
+  const llvm::StructType *BaseSubobjectType;
+
+  /// FieldInfo - Holds a field and its corresponding LLVM field number.
+  llvm::DenseMap<const FieldDecl *, unsigned> Fields;
+
+  /// BitFieldInfo - Holds location and size information about a bit field.
+  llvm::DenseMap<const FieldDecl *, CGBitFieldInfo> BitFields;
+
+  llvm::DenseMap<const CXXRecordDecl *, unsigned> NonVirtualBases;
+  llvm::DenseMap<const CXXRecordDecl *, unsigned> VirtualBases;
+
+  /// IndirectPrimaryBases - Virtual base classes, direct or indirect, that are
+  /// primary base classes for some other direct or indirect base class.
+  CXXIndirectPrimaryBaseSet IndirectPrimaryBases;
+
+  /// LaidOutVirtualBases - A set of all laid out virtual bases, used to avoid
+  /// avoid laying out virtual bases more than once.
+  llvm::SmallPtrSet<const CXXRecordDecl *, 4> LaidOutVirtualBases;
   
   /// IsZeroInitializable - Whether this struct can be C++
   /// zero-initialized with an LLVM zeroinitializer.
   bool IsZeroInitializable;
+  bool IsZeroInitializableAsBase;
 
   /// Packed - Whether the resulting LLVM struct will be packed or not.
   bool Packed;
@@ -59,18 +84,14 @@ private:
   /// Alignment - Contains the alignment of the RecordDecl.
   //
   // FIXME: This is not needed and should be removed.
-  unsigned Alignment;
-
-  /// AlignmentAsLLVMStruct - Will contain the maximum alignment of all the
-  /// LLVM types.
-  unsigned AlignmentAsLLVMStruct;
+  CharUnits Alignment;
 
   /// BitsAvailableInLastField - If a bit field spans only part of a LLVM field,
   /// this will have the number of bits still available in the field.
   char BitsAvailableInLastField;
 
-  /// NextFieldOffsetInBytes - Holds the next field offset in bytes.
-  uint64_t NextFieldOffsetInBytes;
+  /// NextFieldOffset - Holds the next field offset.
+  CharUnits NextFieldOffset;
 
   /// LayoutUnionField - Will layout a field in an union and return the type
   /// that the field will have.
@@ -84,14 +105,30 @@ private:
   /// Returns false if the operation failed because the struct is not packed.
   bool LayoutFields(const RecordDecl *D);
 
+  /// Layout a single base, virtual or non-virtual
+  void LayoutBase(const CXXRecordDecl *base,
+                  const CGRecordLayout &baseLayout,
+                  CharUnits baseOffset);
+
+  /// LayoutVirtualBase - layout a single virtual base.
+  void LayoutVirtualBase(const CXXRecordDecl *base,
+                         CharUnits baseOffset);
+
+  /// LayoutVirtualBases - layout the virtual bases of a record decl.
+  void LayoutVirtualBases(const CXXRecordDecl *RD,
+                          const ASTRecordLayout &Layout);
+  
   /// LayoutNonVirtualBase - layout a single non-virtual base.
-  void LayoutNonVirtualBase(const CXXRecordDecl *BaseDecl,
-                            uint64_t BaseOffset);
+  void LayoutNonVirtualBase(const CXXRecordDecl *base,
+                            CharUnits baseOffset);
   
-  /// LayoutNonVirtualBases - layout the non-virtual bases of a record decl.
+  /// LayoutNonVirtualBases - layout the virtual bases of a record decl.
   void LayoutNonVirtualBases(const CXXRecordDecl *RD, 
                              const ASTRecordLayout &Layout);
 
+  /// ComputeNonVirtualBaseType - Compute the non-virtual base field types.
+  bool ComputeNonVirtualBaseType(const CXXRecordDecl *RD);
+  
   /// LayoutField - layout a single field. Returns false if the operation failed
   /// because the current struct is not packed.
   bool LayoutField(const FieldDecl *D, uint64_t FieldOffset);
@@ -100,41 +137,47 @@ private:
   void LayoutBitField(const FieldDecl *D, uint64_t FieldOffset);
 
   /// AppendField - Appends a field with the given offset and type.
-  void AppendField(uint64_t FieldOffsetInBytes, const llvm::Type *FieldTy);
+  void AppendField(CharUnits fieldOffset, const llvm::Type *FieldTy);
 
   /// AppendPadding - Appends enough padding bytes so that the total
   /// struct size is a multiple of the field alignment.
-  void AppendPadding(uint64_t FieldOffsetInBytes, unsigned FieldAlignment);
+  void AppendPadding(CharUnits fieldOffset, CharUnits fieldAlignment);
 
+  /// getByteArrayType - Returns a byte array type with the given number of
+  /// elements.
+  const llvm::Type *getByteArrayType(CharUnits NumBytes);
+  
   /// AppendBytes - Append a given number of bytes to the record.
-  void AppendBytes(uint64_t NumBytes);
+  void AppendBytes(CharUnits numBytes);
 
   /// AppendTailPadding - Append enough tail padding so that the type will have
   /// the passed size.
   void AppendTailPadding(uint64_t RecordSize);
 
-  unsigned getTypeAlignment(const llvm::Type *Ty) const;
+  CharUnits getTypeAlignment(const llvm::Type *Ty) const;
+
+  /// getAlignmentAsLLVMStruct - Returns the maximum alignment of all the
+  /// LLVM element types.
+  CharUnits getAlignmentAsLLVMStruct() const;
 
   /// CheckZeroInitializable - Check if the given type contains a pointer
   /// to data member.
   void CheckZeroInitializable(QualType T);
-  void CheckZeroInitializable(const CXXRecordDecl *RD);
 
 public:
   CGRecordLayoutBuilder(CodeGenTypes &Types)
-    : IsZeroInitializable(true), Packed(false), Types(Types),
-      Alignment(0), AlignmentAsLLVMStruct(1),
-      BitsAvailableInLastField(0), NextFieldOffsetInBytes(0) { }
+    : BaseSubobjectType(0),
+      IsZeroInitializable(true), IsZeroInitializableAsBase(true),
+      Packed(false), Types(Types), BitsAvailableInLastField(0) { }
 
   /// Layout - Will layout a RecordDecl.
   void Layout(const RecordDecl *D);
 };
 
 }
-}
 
 void CGRecordLayoutBuilder::Layout(const RecordDecl *D) {
-  Alignment = Types.getContext().getASTRecordLayout(D).getAlignment() / 8;
+  Alignment = Types.getContext().getASTRecordLayout(D).getAlignment();
   Packed = D->hasAttr<PackedAttr>();
 
   if (D->isUnion()) {
@@ -147,12 +190,12 @@ void CGRecordLayoutBuilder::Layout(const RecordDecl *D) {
 
   // We weren't able to layout the struct. Try again with a packed struct
   Packed = true;
-  AlignmentAsLLVMStruct = 1;
-  NextFieldOffsetInBytes = 0;
+  NextFieldOffset = CharUnits::Zero();
   FieldTypes.clear();
-  LLVMFields.clear();
-  LLVMBitFields.clear();
-  LLVMNonVirtualBases.clear();
+  Fields.clear();
+  BitFields.clear();
+  NonVirtualBases.clear();
+  VirtualBases.clear();
 
   LayoutFields(D);
 }
@@ -182,6 +225,12 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
     FieldSize = TypeSizeInBits;
   }
 
+  // in big-endian machines the first fields are in higher bit positions,
+  // so revert the offset. The byte offsets are reversed(back) later.
+  if (Types.getTargetData().isBigEndian()) {
+    FieldOffset = ((ContainingTypeSizeInBits)-FieldOffset-FieldSize);
+  }
+
   // Compute the access components. The policy we use is to start by attempting
   // to access using the width of the bit-field type itself and to always access
   // at aligned indices of that type. If such an access would fail because it
@@ -189,7 +238,6 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
   // power of two and retry. The current algorithm assumes pow2 sized types,
   // although this is easy to fix.
   //
-  // FIXME: This algorithm is wrong on big-endian systems, I think.
   assert(llvm::isPowerOf2_32(TypeSizeInBits) && "Unexpected type size!");
   CGBitFieldInfo::AccessInfo Components[3];
   unsigned NumComponents = 0;
@@ -237,7 +285,15 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
     // FIXME: We still follow the old access pattern of only using the field
     // byte offset. We should switch this once we fix the struct layout to be
     // pretty.
-    AI.FieldByteOffset = AccessStart / 8;
+
+    // on big-endian machines we reverted the bit offset because first fields are
+    // in higher bits. But this also reverts the bytes, so fix this here by reverting
+    // the byte offset on big-endian machines.
+    if (Types.getTargetData().isBigEndian()) {
+      AI.FieldByteOffset = (ContainingTypeSizeInBits - AccessStart - AccessWidth )/8;
+    } else {
+      AI.FieldByteOffset = AccessStart / 8;
+    }
     AI.FieldBitStart = AccessBitsInFieldStart - AccessStart;
     AI.AccessWidth = AccessWidth;
     AI.AccessAlignment = llvm::MinAlign(ContainingTypeAlign, AccessStart) / 8;
@@ -258,56 +314,54 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
                                         uint64_t FieldSize) {
   const RecordDecl *RD = FD->getParent();
   const ASTRecordLayout &RL = Types.getContext().getASTRecordLayout(RD);
-  uint64_t ContainingTypeSizeInBits = RL.getSize();
-  unsigned ContainingTypeAlign = RL.getAlignment();
+  uint64_t ContainingTypeSizeInBits = Types.getContext().toBits(RL.getSize());
+  unsigned ContainingTypeAlign = Types.getContext().toBits(RL.getAlignment());
 
   return MakeInfo(Types, FD, FieldOffset, FieldSize, ContainingTypeSizeInBits,
                   ContainingTypeAlign);
 }
 
 void CGRecordLayoutBuilder::LayoutBitField(const FieldDecl *D,
-                                           uint64_t FieldOffset) {
-  uint64_t FieldSize =
+                                           uint64_t fieldOffset) {
+  uint64_t fieldSize =
     D->getBitWidth()->EvaluateAsInt(Types.getContext()).getZExtValue();
 
-  if (FieldSize == 0)
+  if (fieldSize == 0)
     return;
 
-  uint64_t NextFieldOffset = NextFieldOffsetInBytes * 8;
-  unsigned NumBytesToAppend;
+  uint64_t nextFieldOffsetInBits = NextFieldOffset.getQuantity() * 8;
+  unsigned numBytesToAppend;
 
-  if (FieldOffset < NextFieldOffset) {
+  if (fieldOffset < nextFieldOffsetInBits) {
     assert(BitsAvailableInLastField && "Bitfield size mismatch!");
-    assert(NextFieldOffsetInBytes && "Must have laid out at least one byte!");
+    assert(!NextFieldOffset.isZero() && "Must have laid out at least one byte");
 
     // The bitfield begins in the previous bit-field.
-    NumBytesToAppend =
-      llvm::RoundUpToAlignment(FieldSize - BitsAvailableInLastField, 8) / 8;
+    numBytesToAppend =
+      llvm::RoundUpToAlignment(fieldSize - BitsAvailableInLastField, 8) / 8;
   } else {
-    assert(FieldOffset % 8 == 0 && "Field offset not aligned correctly");
+    assert(fieldOffset % 8 == 0 && "Field offset not aligned correctly");
 
     // Append padding if necessary.
-    AppendBytes((FieldOffset - NextFieldOffset) / 8);
+    AppendPadding(CharUnits::fromQuantity(fieldOffset / 8), CharUnits::One());
 
-    NumBytesToAppend =
-      llvm::RoundUpToAlignment(FieldSize, 8) / 8;
+    numBytesToAppend = llvm::RoundUpToAlignment(fieldSize, 8) / 8;
 
-    assert(NumBytesToAppend && "No bytes to append!");
+    assert(numBytesToAppend && "No bytes to append!");
   }
 
   // Add the bit field info.
-  LLVMBitFields.push_back(
-    LLVMBitFieldInfo(D, CGBitFieldInfo::MakeInfo(Types, D, FieldOffset,
-                                                 FieldSize)));
+  BitFields.insert(std::make_pair(D,
+                   CGBitFieldInfo::MakeInfo(Types, D, fieldOffset, fieldSize)));
 
-  AppendBytes(NumBytesToAppend);
+  AppendBytes(CharUnits::fromQuantity(numBytesToAppend));
 
   BitsAvailableInLastField =
-    NextFieldOffsetInBytes * 8 - (FieldOffset + FieldSize);
+    NextFieldOffset.getQuantity() * 8 - (fieldOffset + fieldSize);
 }
 
 bool CGRecordLayoutBuilder::LayoutField(const FieldDecl *D,
-                                        uint64_t FieldOffset) {
+                                        uint64_t fieldOffset) {
   // If the field is packed, then we need a packed struct.
   if (!Packed && D->hasAttr<PackedAttr>())
     return false;
@@ -318,54 +372,50 @@ bool CGRecordLayoutBuilder::LayoutField(const FieldDecl *D,
     if (!Packed && !D->getDeclName())
       return false;
 
-    LayoutBitField(D, FieldOffset);
+    LayoutBitField(D, fieldOffset);
     return true;
   }
 
   CheckZeroInitializable(D->getType());
 
-  assert(FieldOffset % 8 == 0 && "FieldOffset is not on a byte boundary!");
-  uint64_t FieldOffsetInBytes = FieldOffset / 8;
+  assert(fieldOffset % 8 == 0 && "FieldOffset is not on a byte boundary!");
+  CharUnits fieldOffsetInBytes = CharUnits::fromQuantity(fieldOffset / 8);
 
   const llvm::Type *Ty = Types.ConvertTypeForMemRecursive(D->getType());
-  unsigned TypeAlignment = getTypeAlignment(Ty);
+  CharUnits typeAlignment = getTypeAlignment(Ty);
 
   // If the type alignment is larger then the struct alignment, we must use
   // a packed struct.
-  if (TypeAlignment > Alignment) {
+  if (typeAlignment > Alignment) {
     assert(!Packed && "Alignment is wrong even with packed struct!");
     return false;
   }
 
-  if (const RecordType *RT = D->getType()->getAs<RecordType>()) {
-    const RecordDecl *RD = cast<RecordDecl>(RT->getDecl());
-    if (const MaxFieldAlignmentAttr *MFAA =
-          RD->getAttr<MaxFieldAlignmentAttr>()) {
-      if (MFAA->getAlignment() != TypeAlignment * 8 && !Packed)
-        return false;
+  if (!Packed) {
+    if (const RecordType *RT = D->getType()->getAs<RecordType>()) {
+      const RecordDecl *RD = cast<RecordDecl>(RT->getDecl());
+      if (const MaxFieldAlignmentAttr *MFAA =
+            RD->getAttr<MaxFieldAlignmentAttr>()) {
+        if (MFAA->getAlignment() != typeAlignment.getQuantity() * 8)
+          return false;
+      }
     }
   }
 
   // Round up the field offset to the alignment of the field type.
-  uint64_t AlignedNextFieldOffsetInBytes =
-    llvm::RoundUpToAlignment(NextFieldOffsetInBytes, TypeAlignment);
+  CharUnits alignedNextFieldOffsetInBytes =
+    NextFieldOffset.RoundUpToAlignment(typeAlignment);
 
-  if (FieldOffsetInBytes < AlignedNextFieldOffsetInBytes) {
+  if (fieldOffsetInBytes < alignedNextFieldOffsetInBytes) {
     assert(!Packed && "Could not place field even with packed struct!");
     return false;
   }
 
-  if (AlignedNextFieldOffsetInBytes < FieldOffsetInBytes) {
-    // Even with alignment, the field offset is not at the right place,
-    // insert padding.
-    uint64_t PaddingInBytes = FieldOffsetInBytes - NextFieldOffsetInBytes;
-
-    AppendBytes(PaddingInBytes);
-  }
+  AppendPadding(fieldOffsetInBytes, typeAlignment);
 
   // Now append the field.
-  LLVMFields.push_back(LLVMFieldInfo(D, FieldTypes.size()));
-  AppendField(FieldOffsetInBytes, Ty);
+  Fields[D] = FieldTypes.size();
+  AppendField(fieldOffsetInBytes, Ty);
 
   return true;
 }
@@ -389,95 +439,167 @@ CGRecordLayoutBuilder::LayoutUnionField(const FieldDecl *Field,
       FieldTy = llvm::ArrayType::get(FieldTy, NumBytesToAppend);
 
     // Add the bit field info.
-    LLVMBitFields.push_back(
-      LLVMBitFieldInfo(Field, CGBitFieldInfo::MakeInfo(Types, Field,
-                                                       0, FieldSize)));
+    BitFields.insert(std::make_pair(Field,
+                         CGBitFieldInfo::MakeInfo(Types, Field, 0, FieldSize)));
     return FieldTy;
   }
 
   // This is a regular union field.
-  LLVMFields.push_back(LLVMFieldInfo(Field, 0));
+  Fields[Field] = 0;
   return Types.ConvertTypeForMemRecursive(Field->getType());
 }
 
 void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
   assert(D->isUnion() && "Can't call LayoutUnion on a non-union record!");
 
-  const ASTRecordLayout &Layout = Types.getContext().getASTRecordLayout(D);
+  const ASTRecordLayout &layout = Types.getContext().getASTRecordLayout(D);
 
-  const llvm::Type *Ty = 0;
-  uint64_t Size = 0;
-  unsigned Align = 0;
+  const llvm::Type *unionType = 0;
+  CharUnits unionSize = CharUnits::Zero();
+  CharUnits unionAlign = CharUnits::Zero();
 
-  bool HasOnlyZeroSizedBitFields = true;
+  bool hasOnlyZeroSizedBitFields = true;
 
-  unsigned FieldNo = 0;
-  for (RecordDecl::field_iterator Field = D->field_begin(),
-       FieldEnd = D->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
-    assert(Layout.getFieldOffset(FieldNo) == 0 &&
+  unsigned fieldNo = 0;
+  for (RecordDecl::field_iterator field = D->field_begin(),
+       fieldEnd = D->field_end(); field != fieldEnd; ++field, ++fieldNo) {
+    assert(layout.getFieldOffset(fieldNo) == 0 &&
           "Union field offset did not start at the beginning of record!");
-    const llvm::Type *FieldTy = LayoutUnionField(*Field, Layout);
+    const llvm::Type *fieldType = LayoutUnionField(*field, layout);
 
-    if (!FieldTy)
+    if (!fieldType)
       continue;
 
-    HasOnlyZeroSizedBitFields = false;
+    hasOnlyZeroSizedBitFields = false;
 
-    unsigned FieldAlign = Types.getTargetData().getABITypeAlignment(FieldTy);
-    uint64_t FieldSize = Types.getTargetData().getTypeAllocSize(FieldTy);
+    CharUnits fieldAlign = CharUnits::fromQuantity(
+                          Types.getTargetData().getABITypeAlignment(fieldType));
+    CharUnits fieldSize = CharUnits::fromQuantity(
+                             Types.getTargetData().getTypeAllocSize(fieldType));
 
-    if (FieldAlign < Align)
+    if (fieldAlign < unionAlign)
       continue;
 
-    if (FieldAlign > Align || FieldSize > Size) {
-      Ty = FieldTy;
-      Align = FieldAlign;
-      Size = FieldSize;
+    if (fieldAlign > unionAlign || fieldSize > unionSize) {
+      unionType = fieldType;
+      unionAlign = fieldAlign;
+      unionSize = fieldSize;
     }
   }
 
   // Now add our field.
-  if (Ty) {
-    AppendField(0, Ty);
+  if (unionType) {
+    AppendField(CharUnits::Zero(), unionType);
 
-    if (getTypeAlignment(Ty) > Layout.getAlignment() / 8) {
+    if (getTypeAlignment(unionType) > layout.getAlignment()) {
       // We need a packed struct.
       Packed = true;
-      Align = 1;
+      unionAlign = CharUnits::One();
     }
   }
-  if (!Align) {
-    assert(HasOnlyZeroSizedBitFields &&
+  if (unionAlign.isZero()) {
+    assert(hasOnlyZeroSizedBitFields &&
            "0-align record did not have all zero-sized bit-fields!");
-    Align = 1;
+    unionAlign = CharUnits::One();
   }
 
   // Append tail padding.
-  if (Layout.getSize() / 8 > Size)
-    AppendPadding(Layout.getSize() / 8, Align);
+  CharUnits recordSize = layout.getSize();
+  if (recordSize > unionSize)
+    AppendPadding(recordSize, unionAlign);
+}
+
+void CGRecordLayoutBuilder::LayoutBase(const CXXRecordDecl *base,
+                                       const CGRecordLayout &baseLayout,
+                                       CharUnits baseOffset) {
+  AppendPadding(baseOffset, CharUnits::One());
+
+  const ASTRecordLayout &baseASTLayout
+    = Types.getContext().getASTRecordLayout(base);
+
+  // Fields and bases can be laid out in the tail padding of previous
+  // bases.  If this happens, we need to allocate the base as an i8
+  // array; otherwise, we can use the subobject type.  However,
+  // actually doing that would require knowledge of what immediately
+  // follows this base in the layout, so instead we do a conservative
+  // approximation, which is to use the base subobject type if it
+  // has the same LLVM storage size as the nvsize.
+
+  // The nvsize, i.e. the unpadded size of the base class.
+  CharUnits nvsize = baseASTLayout.getNonVirtualSize();
+
+#if 0
+  const llvm::StructType *subobjectType = baseLayout.getBaseSubobjectLLVMType();
+  const llvm::StructLayout *baseLLVMLayout =
+    Types.getTargetData().getStructLayout(subobjectType);
+  CharUnits stsize = CharUnits::fromQuantity(baseLLVMLayout->getSizeInBytes());
+
+  if (nvsize == stsize)
+    AppendField(baseOffset, subobjectType);
+  else 
+#endif
+    AppendBytes(nvsize);
 }
 
-void CGRecordLayoutBuilder::LayoutNonVirtualBase(const CXXRecordDecl *BaseDecl,
-                                                 uint64_t BaseOffset) {
-  const ASTRecordLayout &Layout = 
-    Types.getContext().getASTRecordLayout(BaseDecl);
+void CGRecordLayoutBuilder::LayoutNonVirtualBase(const CXXRecordDecl *base,
+                                                 CharUnits baseOffset) {
+  // Ignore empty bases.
+  if (base->isEmpty()) return;
 
-  uint64_t NonVirtualSize = Layout.getNonVirtualSize();
+  const CGRecordLayout &baseLayout = Types.getCGRecordLayout(base);
+  if (IsZeroInitializableAsBase) {
+    assert(IsZeroInitializable &&
+           "class zero-initializable as base but not as complete object");
 
-  if (BaseDecl->isEmpty()) {
-    // FIXME: Lay out empty bases.
-    return;
+    IsZeroInitializable = IsZeroInitializableAsBase =
+      baseLayout.isZeroInitializableAsBase();
   }
 
-  CheckZeroInitializable(BaseDecl);
+  LayoutBase(base, baseLayout, baseOffset);
+  NonVirtualBases[base] = (FieldTypes.size() - 1);
+}
 
-  // FIXME: Actually use a better type than [sizeof(BaseDecl) x i8] when we can.
-  AppendPadding(BaseOffset / 8, 1);
-  
-  // Append the base field.
-  LLVMNonVirtualBases.push_back(LLVMBaseInfo(BaseDecl, FieldTypes.size()));
+void
+CGRecordLayoutBuilder::LayoutVirtualBase(const CXXRecordDecl *base,
+                                         CharUnits baseOffset) {
+  // Ignore empty bases.
+  if (base->isEmpty()) return;
+
+  const CGRecordLayout &baseLayout = Types.getCGRecordLayout(base);
+  if (IsZeroInitializable)
+    IsZeroInitializable = baseLayout.isZeroInitializableAsBase();
+
+  LayoutBase(base, baseLayout, baseOffset);
+  VirtualBases[base] = (FieldTypes.size() - 1);
+}
 
-  AppendBytes(NonVirtualSize / 8);
+/// LayoutVirtualBases - layout the non-virtual bases of a record decl.
+void
+CGRecordLayoutBuilder::LayoutVirtualBases(const CXXRecordDecl *RD,
+                                          const ASTRecordLayout &Layout) {
+  for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+       E = RD->bases_end(); I != E; ++I) {
+    const CXXRecordDecl *BaseDecl = 
+      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+
+    // We only want to lay out virtual bases that aren't indirect primary bases
+    // of some other base.
+    if (I->isVirtual() && !IndirectPrimaryBases.count(BaseDecl)) {
+      // Only lay out the base once.
+      if (!LaidOutVirtualBases.insert(BaseDecl))
+        continue;
+
+      CharUnits vbaseOffset = Layout.getVBaseClassOffset(BaseDecl);
+      LayoutVirtualBase(BaseDecl, vbaseOffset);
+    }
+
+    if (!BaseDecl->getNumVBases()) {
+      // This base isn't interesting since it doesn't have any virtual bases.
+      continue;
+    }
+    
+    LayoutVirtualBases(BaseDecl, Layout);
+  }
 }
 
 void
@@ -493,13 +615,14 @@ CGRecordLayoutBuilder::LayoutNonVirtualBases(const CXXRecordDecl *RD,
                                 /*isVarArg=*/true);
       const llvm::Type *VTableTy = FunctionType->getPointerTo();
 
-      assert(NextFieldOffsetInBytes == 0 &&
+      assert(NextFieldOffset.isZero() &&
              "VTable pointer must come first!");
-      AppendField(NextFieldOffsetInBytes, VTableTy->getPointerTo());
+      AppendField(CharUnits::Zero(), VTableTy->getPointerTo());
     } else {
-      // FIXME: Handle a virtual primary base.
-      if (!Layout.getPrimaryBaseWasVirtual())
-        LayoutNonVirtualBase(PrimaryBase, 0);
+      if (!Layout.isPrimaryBaseVirtual())
+        LayoutNonVirtualBase(PrimaryBase, CharUnits::Zero());
+      else
+        LayoutVirtualBase(PrimaryBase, CharUnits::Zero());
     }
   }
 
@@ -513,20 +636,61 @@ CGRecordLayoutBuilder::LayoutNonVirtualBases(const CXXRecordDecl *RD,
       cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
 
     // We've already laid out the primary base.
-    if (BaseDecl == PrimaryBase && !Layout.getPrimaryBaseWasVirtual())
+    if (BaseDecl == PrimaryBase && !Layout.isPrimaryBaseVirtual())
       continue;
 
     LayoutNonVirtualBase(BaseDecl, Layout.getBaseClassOffset(BaseDecl));
   }
 }
 
+bool
+CGRecordLayoutBuilder::ComputeNonVirtualBaseType(const CXXRecordDecl *RD) {
+  const ASTRecordLayout &Layout = Types.getContext().getASTRecordLayout(RD);
+
+  CharUnits NonVirtualSize  = Layout.getNonVirtualSize();
+  CharUnits NonVirtualAlign = Layout.getNonVirtualAlign();
+  CharUnits AlignedNonVirtualTypeSize =
+    NonVirtualSize.RoundUpToAlignment(NonVirtualAlign);
+  
+  // First check if we can use the same fields as for the complete class.
+  CharUnits RecordSize = Layout.getSize();
+  if (AlignedNonVirtualTypeSize == RecordSize)
+    return true;
+
+  // Check if we need padding.
+  CharUnits AlignedNextFieldOffset =
+    NextFieldOffset.RoundUpToAlignment(getAlignmentAsLLVMStruct());
+
+  if (AlignedNextFieldOffset > AlignedNonVirtualTypeSize) {
+    assert(!Packed && "cannot layout even as packed struct");
+    return false; // Needs packing.
+  }
+
+  bool needsPadding = (AlignedNonVirtualTypeSize != AlignedNextFieldOffset);
+  if (needsPadding) {
+    CharUnits NumBytes = AlignedNonVirtualTypeSize - AlignedNextFieldOffset;
+    FieldTypes.push_back(getByteArrayType(NumBytes));
+  }
+
+  BaseSubobjectType = llvm::StructType::get(Types.getLLVMContext(),
+                                            FieldTypes, Packed);
+
+  if (needsPadding) {
+    // Pull the padding back off.
+    FieldTypes.pop_back();
+  }
+
+  return true;
+}
+
 bool CGRecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
   assert(!D->isUnion() && "Can't call LayoutFields on a union!");
-  assert(Alignment && "Did not set alignment!");
+  assert(!Alignment.isZero() && "Did not set alignment!");
 
   const ASTRecordLayout &Layout = Types.getContext().getASTRecordLayout(D);
 
-  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D))
+  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D);
+  if (RD)
     LayoutNonVirtualBases(RD, Layout);
 
   unsigned FieldNo = 0;
@@ -540,8 +704,23 @@ bool CGRecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
     }
   }
 
+  if (RD) {
+    // We've laid out the non-virtual bases and the fields, now compute the
+    // non-virtual base field types.
+    if (!ComputeNonVirtualBaseType(RD)) {
+      assert(!Packed && "Could not layout even with a packed LLVM struct!");
+      return false;
+    }
+
+    // And lay out the virtual bases.
+    RD->getIndirectPrimaryBases(IndirectPrimaryBases);
+    if (Layout.isPrimaryBaseVirtual())
+      IndirectPrimaryBases.insert(Layout.getPrimaryBase());
+    LayoutVirtualBases(RD, Layout);
+  }
+  
   // Append tail padding if necessary.
-  AppendTailPadding(Layout.getSize());
+  AppendTailPadding(Types.getContext().toBits(Layout.getSize()));
 
   return true;
 }
@@ -549,129 +728,137 @@ bool CGRecordLayoutBuilder::LayoutFields(const RecordDecl *D) {
 void CGRecordLayoutBuilder::AppendTailPadding(uint64_t RecordSize) {
   assert(RecordSize % 8 == 0 && "Invalid record size!");
 
-  uint64_t RecordSizeInBytes = RecordSize / 8;
-  assert(NextFieldOffsetInBytes <= RecordSizeInBytes && "Size mismatch!");
+  CharUnits RecordSizeInBytes = CharUnits::fromQuantity(RecordSize / 8);
+  assert(NextFieldOffset <= RecordSizeInBytes && "Size mismatch!");
 
-  uint64_t AlignedNextFieldOffset =
-    llvm::RoundUpToAlignment(NextFieldOffsetInBytes, AlignmentAsLLVMStruct);
+  CharUnits AlignedNextFieldOffset =
+    NextFieldOffset.RoundUpToAlignment(getAlignmentAsLLVMStruct());
 
   if (AlignedNextFieldOffset == RecordSizeInBytes) {
     // We don't need any padding.
     return;
   }
 
-  unsigned NumPadBytes = RecordSizeInBytes - NextFieldOffsetInBytes;
+  CharUnits NumPadBytes = RecordSizeInBytes - NextFieldOffset;
   AppendBytes(NumPadBytes);
 }
 
-void CGRecordLayoutBuilder::AppendField(uint64_t FieldOffsetInBytes,
-                                        const llvm::Type *FieldTy) {
-  AlignmentAsLLVMStruct = std::max(AlignmentAsLLVMStruct,
-                                   getTypeAlignment(FieldTy));
+void CGRecordLayoutBuilder::AppendField(CharUnits fieldOffset,
+                                        const llvm::Type *fieldType) {
+  CharUnits fieldSize =
+    CharUnits::fromQuantity(Types.getTargetData().getTypeAllocSize(fieldType));
 
-  uint64_t FieldSizeInBytes = Types.getTargetData().getTypeAllocSize(FieldTy);
+  FieldTypes.push_back(fieldType);
 
-  FieldTypes.push_back(FieldTy);
-
-  NextFieldOffsetInBytes = FieldOffsetInBytes + FieldSizeInBytes;
+  NextFieldOffset = fieldOffset + fieldSize;
   BitsAvailableInLastField = 0;
 }
 
-void CGRecordLayoutBuilder::AppendPadding(uint64_t FieldOffsetInBytes,
-                                          unsigned FieldAlignment) {
-  assert(NextFieldOffsetInBytes <= FieldOffsetInBytes &&
+void CGRecordLayoutBuilder::AppendPadding(CharUnits fieldOffset,
+                                          CharUnits fieldAlignment) {
+  assert(NextFieldOffset <= fieldOffset &&
          "Incorrect field layout!");
 
   // Round up the field offset to the alignment of the field type.
-  uint64_t AlignedNextFieldOffsetInBytes =
-    llvm::RoundUpToAlignment(NextFieldOffsetInBytes, FieldAlignment);
+  CharUnits alignedNextFieldOffset =
+    NextFieldOffset.RoundUpToAlignment(fieldAlignment);
 
-  if (AlignedNextFieldOffsetInBytes < FieldOffsetInBytes) {
+  if (alignedNextFieldOffset < fieldOffset) {
     // Even with alignment, the field offset is not at the right place,
     // insert padding.
-    uint64_t PaddingInBytes = FieldOffsetInBytes - NextFieldOffsetInBytes;
+    CharUnits padding = fieldOffset - NextFieldOffset;
 
-    AppendBytes(PaddingInBytes);
+    AppendBytes(padding);
   }
 }
 
-void CGRecordLayoutBuilder::AppendBytes(uint64_t NumBytes) {
-  if (NumBytes == 0)
-    return;
+const llvm::Type *CGRecordLayoutBuilder::getByteArrayType(CharUnits numBytes) {
+  assert(!numBytes.isZero() && "Empty byte arrays aren't allowed.");
 
   const llvm::Type *Ty = llvm::Type::getInt8Ty(Types.getLLVMContext());
-  if (NumBytes > 1)
-    Ty = llvm::ArrayType::get(Ty, NumBytes);
+  if (numBytes > CharUnits::One())
+    Ty = llvm::ArrayType::get(Ty, numBytes.getQuantity());
+
+  return Ty;
+}
+
+void CGRecordLayoutBuilder::AppendBytes(CharUnits numBytes) {
+  if (numBytes.isZero())
+    return;
 
   // Append the padding field
-  AppendField(NextFieldOffsetInBytes, Ty);
+  AppendField(NextFieldOffset, getByteArrayType(numBytes));
 }
 
-unsigned CGRecordLayoutBuilder::getTypeAlignment(const llvm::Type *Ty) const {
+CharUnits CGRecordLayoutBuilder::getTypeAlignment(const llvm::Type *Ty) const {
   if (Packed)
-    return 1;
+    return CharUnits::One();
 
-  return Types.getTargetData().getABITypeAlignment(Ty);
+  return CharUnits::fromQuantity(Types.getTargetData().getABITypeAlignment(Ty));
 }
 
+CharUnits CGRecordLayoutBuilder::getAlignmentAsLLVMStruct() const {
+  if (Packed)
+    return CharUnits::One();
+
+  CharUnits maxAlignment = CharUnits::One();
+  for (size_t i = 0; i != FieldTypes.size(); ++i)
+    maxAlignment = std::max(maxAlignment, getTypeAlignment(FieldTypes[i]));
+
+  return maxAlignment;
+}
+
+/// Merge in whether a field of the given type is zero-initializable.
 void CGRecordLayoutBuilder::CheckZeroInitializable(QualType T) {
   // This record already contains a member pointer.
-  if (!IsZeroInitializable)
+  if (!IsZeroInitializableAsBase)
     return;
 
   // Can only have member pointers if we're compiling C++.
   if (!Types.getContext().getLangOptions().CPlusPlus)
     return;
 
-  T = Types.getContext().getBaseElementType(T);
+  const Type *elementType = T->getBaseElementTypeUnsafe();
 
-  if (const MemberPointerType *MPT = T->getAs<MemberPointerType>()) {
+  if (const MemberPointerType *MPT = elementType->getAs<MemberPointerType>()) {
     if (!Types.getCXXABI().isZeroInitializable(MPT))
-      IsZeroInitializable = false;
-  } else if (const RecordType *RT = T->getAs<RecordType>()) {
+      IsZeroInitializable = IsZeroInitializableAsBase = false;
+  } else if (const RecordType *RT = elementType->getAs<RecordType>()) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-    CheckZeroInitializable(RD);
+    const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
+    if (!Layout.isZeroInitializable())
+      IsZeroInitializable = IsZeroInitializableAsBase = false;
   }
 }
 
-void CGRecordLayoutBuilder::CheckZeroInitializable(const CXXRecordDecl *RD) {
-  // This record already contains a member pointer.
-  if (!IsZeroInitializable)
-    return;
-
-  // FIXME: It would be better if there was a way to explicitly compute the
-  // record layout instead of converting to a type.
-  Types.ConvertTagDeclType(RD);
-  
-  const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
-  
-  if (!Layout.isZeroInitializable())
-    IsZeroInitializable = false;
-}
-
 CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
   CGRecordLayoutBuilder Builder(*this);
 
   Builder.Layout(D);
 
-  const llvm::Type *Ty = llvm::StructType::get(getLLVMContext(),
-                                               Builder.FieldTypes,
-                                               Builder.Packed);
+  const llvm::StructType *Ty = llvm::StructType::get(getLLVMContext(),
+                                                     Builder.FieldTypes,
+                                                     Builder.Packed);
+
+  // If we're in C++, compute the base subobject type.
+  const llvm::StructType *BaseTy = 0;
+  if (isa<CXXRecordDecl>(D)) {
+    BaseTy = Builder.BaseSubobjectType;
+    if (!BaseTy) BaseTy = Ty;
+  }
 
   CGRecordLayout *RL =
-    new CGRecordLayout(Ty, Builder.IsZeroInitializable);
+    new CGRecordLayout(Ty, BaseTy, Builder.IsZeroInitializable,
+                       Builder.IsZeroInitializableAsBase);
 
-  // Add all the non-virtual base field numbers.
-  RL->NonVirtualBaseFields.insert(Builder.LLVMNonVirtualBases.begin(),
-                                  Builder.LLVMNonVirtualBases.end());
+  RL->NonVirtualBases.swap(Builder.NonVirtualBases);
+  RL->CompleteObjectVirtualBases.swap(Builder.VirtualBases);
 
   // Add all the field numbers.
-  RL->FieldInfo.insert(Builder.LLVMFields.begin(),
-                       Builder.LLVMFields.end());
+  RL->FieldInfo.swap(Builder.Fields);
 
   // Add bitfield info.
-  RL->BitFields.insert(Builder.LLVMBitFields.begin(),
-                       Builder.LLVMBitFields.end());
+  RL->BitFields.swap(Builder.BitFields);
 
   // Dump the layout, if requested.
   if (getContext().getLangOptions().DumpRecordLayouts) {
@@ -684,10 +871,26 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
 
 #ifndef NDEBUG
   // Verify that the computed LLVM struct size matches the AST layout size.
-  uint64_t TypeSizeInBits = getContext().getASTRecordLayout(D).getSize();
+  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D);
+
+  uint64_t TypeSizeInBits = getContext().toBits(Layout.getSize());
   assert(TypeSizeInBits == getTargetData().getTypeAllocSizeInBits(Ty) &&
          "Type size mismatch!");
 
+  if (BaseTy) {
+    CharUnits NonVirtualSize  = Layout.getNonVirtualSize();
+    CharUnits NonVirtualAlign = Layout.getNonVirtualAlign();
+    CharUnits AlignedNonVirtualTypeSize = 
+      NonVirtualSize.RoundUpToAlignment(NonVirtualAlign);
+
+    uint64_t AlignedNonVirtualTypeSizeInBits = 
+      getContext().toBits(AlignedNonVirtualTypeSize);
+
+    assert(AlignedNonVirtualTypeSizeInBits == 
+           getTargetData().getTypeAllocSizeInBits(BaseTy) &&
+           "Type size mismatch!");
+  }
+                                     
   // Verify that the LLVM and AST field offsets agree.
   const llvm::StructType *ST =
     dyn_cast<llvm::StructType>(RL->getLLVMType());
@@ -729,7 +932,9 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
 
 void CGRecordLayout::print(llvm::raw_ostream &OS) const {
   OS << "<CGRecordLayout\n";
-  OS << "  LLVMType:" << *LLVMType << "\n";
+  OS << "  LLVMType:" << *CompleteObjectType << "\n";
+  if (BaseSubobjectType)
+    OS << "  NonVirtualBaseLLVMType:" << *BaseSubobjectType << "\n"; 
   OS << "  IsZeroInitializable:" << IsZeroInitializable << "\n";
   OS << "  BitFields:[\n";
 
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index 16145f766af2..cd238112ed1d 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -14,6 +14,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenModule.h"
 #include "CodeGenFunction.h"
+#include "TargetInfo.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Basic/TargetInfo.h"
@@ -69,24 +70,54 @@ void CodeGenFunction::EmitStmt(const Stmt *S) {
   EmitStopPoint(S);
 
   switch (S->getStmtClass()) {
-  default:
-    // Must be an expression in a stmt context.  Emit the value (to get
-    // side-effects) and ignore the result.
-    if (!isa<Expr>(S))
-      ErrorUnsupported(S, "statement");
-
-    EmitAnyExpr(cast<Expr>(S), 0, false, true);
-
-    // Expression emitters don't handle unreachable blocks yet, so look for one
-    // explicitly here. This handles the common case of a call to a noreturn
-    // function.
-    if (llvm::BasicBlock *CurBB = Builder.GetInsertBlock()) {
-      if (CurBB->empty() && CurBB->use_empty()) {
-        CurBB->eraseFromParent();
-        Builder.ClearInsertionPoint();
-      }
+  case Stmt::NoStmtClass:
+  case Stmt::CXXCatchStmtClass:
+    llvm_unreachable("invalid statement class to emit generically");
+  case Stmt::NullStmtClass:
+  case Stmt::CompoundStmtClass:
+  case Stmt::DeclStmtClass:
+  case Stmt::LabelStmtClass:
+  case Stmt::GotoStmtClass:
+  case Stmt::BreakStmtClass:
+  case Stmt::ContinueStmtClass:
+  case Stmt::DefaultStmtClass:
+  case Stmt::CaseStmtClass:
+    llvm_unreachable("should have emitted these statements as simple");
+
+#define STMT(Type, Base)
+#define ABSTRACT_STMT(Op)
+#define EXPR(Type, Base) \
+  case Stmt::Type##Class:
+#include "clang/AST/StmtNodes.inc"
+  {
+    // Remember the block we came in on.
+    llvm::BasicBlock *incoming = Builder.GetInsertBlock();
+    assert(incoming && "expression emission must have an insertion point");
+
+    EmitIgnoredExpr(cast<Expr>(S));
+
+    llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
+    assert(outgoing && "expression emission cleared block!");
+
+    // The expression emitters assume (reasonably!) that the insertion
+    // point is always set.  To maintain that, the call-emission code
+    // for noreturn functions has to enter a new block with no
+    // predecessors.  We want to kill that block and mark the current
+    // insertion point unreachable in the common case of a call like
+    // "exit();".  Since expression emission doesn't otherwise create
+    // blocks with no predecessors, we can just test for that.
+    // However, we must be careful not to do this to our incoming
+    // block, because *statement* emission does sometimes create
+    // reachable blocks which will have no predecessors until later in
+    // the function.  This occurs with, e.g., labels that are not
+    // reachable by fallthrough.
+    if (incoming != outgoing && outgoing->use_empty()) {
+      outgoing->eraseFromParent();
+      Builder.ClearInsertionPoint();
     }
     break;
+  }
+
   case Stmt::IndirectGotoStmtClass:
     EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;
 
@@ -146,7 +177,7 @@ bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) {
 /// this captures the expression result of the last sub-statement and returns it
 /// (for use by the statement expression extension).
 RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
-                                         llvm::Value *AggLoc, bool isAggVol) {
+                                         AggValueSlot AggSlot) {
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
                              "LLVM IR generation of compound statement ('{}')");
 
@@ -178,13 +209,13 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
     // emitting them before we evaluate the subexpr.
     const Stmt *LastStmt = S.body_back();
     while (const LabelStmt *LS = dyn_cast<LabelStmt>(LastStmt)) {
-      EmitLabel(*LS);
+      EmitLabel(LS->getDecl());
       LastStmt = LS->getSubStmt();
     }
 
     EnsureInsertPoint();
 
-    RV = EmitAnyExpr(cast<Expr>(LastStmt), AggLoc);
+    RV = EmitAnyExpr(cast<Expr>(LastStmt), AggSlot);
   }
 
   return RV;
@@ -246,24 +277,24 @@ void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
 }
 
 CodeGenFunction::JumpDest
-CodeGenFunction::getJumpDestForLabel(const LabelStmt *S) {
-  JumpDest &Dest = LabelMap[S];
+CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
+  JumpDest &Dest = LabelMap[D];
   if (Dest.isValid()) return Dest;
 
   // Create, but don't insert, the new block.
-  Dest = JumpDest(createBasicBlock(S->getName()),
+  Dest = JumpDest(createBasicBlock(D->getName()),
                   EHScopeStack::stable_iterator::invalid(),
                   NextCleanupDestIndex++);
   return Dest;
 }
 
-void CodeGenFunction::EmitLabel(const LabelStmt &S) {
-  JumpDest &Dest = LabelMap[&S];
+void CodeGenFunction::EmitLabel(const LabelDecl *D) {
+  JumpDest &Dest = LabelMap[D];
 
   // If we didn't need a forward reference to this label, just go
   // ahead and create a destination at the current scope.
   if (!Dest.isValid()) {
-    Dest = getJumpDestInCurrentScope(S.getName());
+    Dest = getJumpDestInCurrentScope(D->getName());
 
   // Otherwise, we need to give this label a target depth and remove
   // it from the branch-fixups list.
@@ -281,7 +312,7 @@ void CodeGenFunction::EmitLabel(const LabelStmt &S) {
 
 
 void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
-  EmitLabel(S);
+  EmitLabel(S.getDecl());
   EmitStmt(S.getSubStmt());
 }
 
@@ -297,10 +328,14 @@ void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
 
 
 void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
+  if (const LabelDecl *Target = S.getConstantTarget()) {
+    EmitBranchThroughCleanup(getJumpDestForLabel(Target));
+    return;
+  }
+
   // Ensure that we have an i8* for our PHI node.
   llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
-                                         llvm::Type::getInt8PtrTy(VMContext),
-                                          "addr");
+                                         Int8PtrTy, "addr");
   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
   
 
@@ -320,7 +355,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   RunCleanupsScope ConditionScope(*this);
 
   if (S.getConditionVariable())
-    EmitLocalBlockVarDecl(*S.getConditionVariable());
+    EmitAutoVarDecl(*S.getConditionVariable());
 
   // If the condition constant folds and can be elided, try to avoid emitting
   // the condition and the dead arm of the if/else.
@@ -395,7 +430,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
   RunCleanupsScope ConditionScope(*this);
 
   if (S.getConditionVariable())
-    EmitLocalBlockVarDecl(*S.getConditionVariable());
+    EmitAutoVarDecl(*S.getConditionVariable());
   
   // Evaluate the conditional in the while header.  C99 6.8.5.1: The
   // evaluation of the controlling expression takes place before each
@@ -527,7 +562,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
     // declaration.
     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
     if (S.getConditionVariable()) {
-      EmitLocalBlockVarDecl(*S.getConditionVariable());
+      EmitAutoVarDecl(*S.getConditionVariable());
     }
 
     // If there are any cleanups between here and the loop-exit scope,
@@ -621,11 +656,8 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
     
     // If there is an NRVO flag for this variable, set it to 1 into indicate
     // that the cleanup code should not destroy the variable.
-    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()]) {
-      const llvm::Type *BoolTy = llvm::Type::getInt1Ty(VMContext);
-      llvm::Value *One = llvm::ConstantInt::get(BoolTy, 1);
-      Builder.CreateStore(One, NRVOFlag);
-    }
+    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
+      Builder.CreateStore(Builder.getTrue(), NRVOFlag);
   } else if (!ReturnValue) {
     // Make sure not to return anything, but evaluate the expression
     // for side effects.
@@ -643,7 +675,7 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
   } else if (RV->getType()->isAnyComplexType()) {
     EmitComplexExprIntoAddr(RV, ReturnValue, false);
   } else {
-    EmitAggExpr(RV, ReturnValue, false);
+    EmitAggExpr(RV, AggValueSlot::forAddr(ReturnValue, false, true));
   }
 
   EmitBranchThroughCleanup(ReturnBlock);
@@ -713,7 +745,8 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
   if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
     // Range is small enough to add multiple switch instruction cases.
     for (unsigned i = 0, e = Range.getZExtValue() + 1; i != e; ++i) {
-      SwitchInsn->addCase(llvm::ConstantInt::get(VMContext, LHS), CaseDest);
+      SwitchInsn->addCase(llvm::ConstantInt::get(getLLVMContext(), LHS),
+                          CaseDest);
       LHS++;
     }
     return;
@@ -735,10 +768,10 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
   // Emit range check.
   llvm::Value *Diff =
     Builder.CreateSub(SwitchInsn->getCondition(),
-                      llvm::ConstantInt::get(VMContext, LHS),  "tmp");
+                      llvm::ConstantInt::get(getLLVMContext(), LHS),  "tmp");
   llvm::Value *Cond =
-    Builder.CreateICmpULE(Diff,
-                          llvm::ConstantInt::get(VMContext, Range), "tmp");
+    Builder.CreateICmpULE(Diff, llvm::ConstantInt::get(getLLVMContext(), Range),
+                          "inbounds");
   Builder.CreateCondBr(Cond, CaseDest, FalseDest);
 
   // Restore the appropriate insertion point.
@@ -757,7 +790,8 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
   EmitBlock(createBasicBlock("sw.bb"));
   llvm::BasicBlock *CaseDest = Builder.GetInsertBlock();
   llvm::APSInt CaseVal = S.getLHS()->EvaluateAsInt(getContext());
-  SwitchInsn->addCase(llvm::ConstantInt::get(VMContext, CaseVal), CaseDest);
+  SwitchInsn->addCase(llvm::ConstantInt::get(getLLVMContext(), CaseVal),
+                      CaseDest);
 
   // Recursively emitting the statement is acceptable, but is not wonderful for
   // code where we have many case statements nested together, i.e.:
@@ -775,7 +809,8 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
   while (NextCase && NextCase->getRHS() == 0) {
     CurCase = NextCase;
     CaseVal = CurCase->getLHS()->EvaluateAsInt(getContext());
-    SwitchInsn->addCase(llvm::ConstantInt::get(VMContext, CaseVal), CaseDest);
+    SwitchInsn->addCase(llvm::ConstantInt::get(getLLVMContext(), CaseVal),
+                        CaseDest);
 
     NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
   }
@@ -798,7 +833,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
   RunCleanupsScope ConditionScope(*this);
 
   if (S.getConditionVariable())
-    EmitLocalBlockVarDecl(*S.getConditionVariable());
+    EmitAutoVarDecl(*S.getConditionVariable());
 
   llvm::Value *CondV = EmitScalarExpr(S.getCond());
 
@@ -861,14 +896,11 @@ static std::string
 SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
                  llvm::SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
   std::string Result;
-  std::string tmp;
 
   while (*Constraint) {
     switch (*Constraint) {
     default:
-      tmp = Target.convertConstraint(*Constraint);
-      if (Result.find(tmp) == std::string::npos) // Combine unique constraints
-        Result += tmp;
+      Result += Target.convertConstraint(*Constraint);
       break;
     // Ignore these
     case '*':
@@ -877,8 +909,8 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
     case '=': // Will see this and the following in mult-alt constraints.
     case '+':
       break;
-    case ',':                 // FIXME - Until the back-end properly supports
-              return Result;  // multiple alternative constraints, we stop here.
+    case ',':
+      Result += "|";
       break;
     case 'g':
       Result += "imr";
@@ -890,7 +922,7 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
       bool result = Target.resolveSymbolicName(Constraint,
                                                &(*OutCons)[0],
                                                OutCons->size(), Index);
-      assert(result && "Could not resolve symbolic name"); result=result;
+      assert(result && "Could not resolve symbolic name"); (void)result;
       Result += llvm::utostr(Index);
       break;
     }
@@ -902,6 +934,33 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
   return Result;
 }
 
+/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
+/// as using a particular register add that as a constraint that will be used
+/// in this asm stmt.
+static std::string
+AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
+                       const TargetInfo &Target, CodeGenModule &CGM,
+                       const AsmStmt &Stmt) {
+  const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);
+  if (!AsmDeclRef)
+    return Constraint;
+  const ValueDecl &Value = *AsmDeclRef->getDecl();
+  const VarDecl *Variable = dyn_cast<VarDecl>(&Value);
+  if (!Variable)
+    return Constraint;
+  AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
+  if (!Attr)
+    return Constraint;
+  llvm::StringRef Register = Attr->getLabel();
+  assert(Target.isValidGCCRegisterName(Register));
+  // FIXME: We should check which registers are compatible with "r" or "x".
+  if (Constraint != "r" && Constraint != "x") {
+    CGM.ErrorUnsupported(&Stmt, "__asm__");
+    return Constraint;
+  }
+  return "{" + Register.str() + "}";
+}
+
 llvm::Value*
 CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
                                     const TargetInfo::ConstraintInfo &Info,
@@ -915,7 +974,7 @@ CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
       const llvm::Type *Ty = ConvertType(InputType);
       uint64_t Size = CGM.getTargetData().getTypeSizeInBits(Ty);
       if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
-        Ty = llvm::IntegerType::get(VMContext, Size);
+        Ty = llvm::IntegerType::get(getLLVMContext(), Size);
         Ty = llvm::PointerType::getUnqual(Ty);
 
         Arg = Builder.CreateLoad(Builder.CreateBitCast(InputValue.getAddress(),
@@ -946,6 +1005,35 @@ llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
   return EmitAsmInputLValue(S, Info, Dest, InputExpr->getType(), ConstraintStr);
 }
 
+/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
+/// asm call instruction.  The !srcloc MDNode contains a list of constant
+/// integers which are the source locations of the start of each line in the
+/// asm.
+static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
+                                      CodeGenFunction &CGF) {
+  llvm::SmallVector<llvm::Value *, 8> Locs;
+  // Add the location of the first line to the MDNode.
+  Locs.push_back(llvm::ConstantInt::get(CGF.Int32Ty,
+                                        Str->getLocStart().getRawEncoding()));
+  llvm::StringRef StrVal = Str->getString();
+  if (!StrVal.empty()) {
+    const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
+    const LangOptions &LangOpts = CGF.CGM.getLangOptions();
+    
+    // Add the location of the start of each subsequent line of the asm to the
+    // MDNode.
+    for (unsigned i = 0, e = StrVal.size()-1; i != e; ++i) {
+      if (StrVal[i] != '\n') continue;
+      SourceLocation LineLoc = Str->getLocationOfByte(i+1, SM, LangOpts,
+                                                      CGF.Target);
+      Locs.push_back(llvm::ConstantInt::get(CGF.Int32Ty,
+                                            LineLoc.getRawEncoding()));
+    }
+  }    
+  
+  return llvm::MDNode::get(CGF.getLLVMContext(), Locs.data(), Locs.size());
+}
+
 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   // Analyze the asm string to decompose it into its pieces.  We know that Sema
   // has already done this, so it is guaranteed to be successful.
@@ -1010,6 +1098,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     const Expr *OutExpr = S.getOutputExpr(i);
     OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());
 
+    OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr, Target,
+                                             CGM, S);
+
     LValue Dest = EmitLValue(OutExpr);
     if (!Constraints.empty())
       Constraints += ',';
@@ -1044,6 +1135,10 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
           ResultRegTypes.back() = ConvertType(InputTy);
         }
       }
+      if (const llvm::Type* AdjTy = 
+            getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
+                                                 ResultRegTypes.back()))
+        ResultRegTypes.back() = AdjTy;
     } else {
       ArgTypes.push_back(Dest.getAddress()->getType());
       Args.push_back(Dest.getAddress());
@@ -1083,6 +1178,11 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     InputConstraint = SimplifyConstraint(InputConstraint.c_str(), Target,
                                          &OutputConstraintInfos);
 
+    InputConstraint =
+      AddVariableConstraints(InputConstraint,
+                            *InputExpr->IgnoreParenNoopCasts(getContext()),
+                            Target, CGM, S);
+
     llvm::Value *Arg = EmitAsmInput(S, Info, InputExpr, Constraints);
 
     // If this input argument is tied to a larger output result, extend the
@@ -1107,7 +1207,10 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
           Arg = Builder.CreateFPExt(Arg, OutputTy);
       }
     }
-
+    if (const llvm::Type* AdjTy = 
+              getTargetHooks().adjustInlineAsmType(*this, InputConstraint,
+                                                   Arg->getType()))
+      Arg = Builder.CreateBitCast(Arg, AdjTy);
 
     ArgTypes.push_back(Arg->getType());
     Args.push_back(Arg);
@@ -1145,11 +1248,11 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
   const llvm::Type *ResultType;
   if (ResultRegTypes.empty())
-    ResultType = llvm::Type::getVoidTy(VMContext);
+    ResultType = llvm::Type::getVoidTy(getLLVMContext());
   else if (ResultRegTypes.size() == 1)
     ResultType = ResultRegTypes[0];
   else
-    ResultType = llvm::StructType::get(VMContext, ResultRegTypes);
+    ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
 
   const llvm::FunctionType *FTy =
     llvm::FunctionType::get(ResultType, ArgTypes, false);
@@ -1162,10 +1265,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
   // Slap the source location of the inline asm into a !srcloc metadata on the
   // call.
-  unsigned LocID = S.getAsmString()->getLocStart().getRawEncoding();
-  llvm::Value *LocIDC =
-    llvm::ConstantInt::get(Int32Ty, LocID);
-  Result->setMetadata("srcloc", llvm::MDNode::get(VMContext, &LocIDC, 1));
+  Result->setMetadata("srcloc", getAsmSrcLocInfo(S.getAsmString(), *this));
 
   // Extract all of the register value results from the asm.
   std::vector<llvm::Value*> RegResults;
@@ -1192,16 +1292,18 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
         Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
       else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
         uint64_t ResSize = CGM.getTargetData().getTypeSizeInBits(TruncTy);
-        Tmp = Builder.CreateTrunc(Tmp, llvm::IntegerType::get(VMContext,
-                                                            (unsigned)ResSize));
+        Tmp = Builder.CreateTrunc(Tmp,
+                   llvm::IntegerType::get(getLLVMContext(), (unsigned)ResSize));
         Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
       } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
         uint64_t TmpSize =CGM.getTargetData().getTypeSizeInBits(Tmp->getType());
-        Tmp = Builder.CreatePtrToInt(Tmp, llvm::IntegerType::get(VMContext,
-                                                            (unsigned)TmpSize));
+        Tmp = Builder.CreatePtrToInt(Tmp,
+                   llvm::IntegerType::get(getLLVMContext(), (unsigned)TmpSize));
         Tmp = Builder.CreateTrunc(Tmp, TruncTy);
       } else if (TruncTy->isIntegerTy()) {
         Tmp = Builder.CreateTrunc(Tmp, TruncTy);
+      } else if (TruncTy->isVectorTy()) {
+        Tmp = Builder.CreateBitCast(Tmp, TruncTy);
       }
     }
 
diff --git a/lib/CodeGen/CGTemporaries.cpp b/lib/CodeGen/CGTemporaries.cpp
index dfb8dc63c5c5..3b4c50910bf8 100644
--- a/lib/CodeGen/CGTemporaries.cpp
+++ b/lib/CodeGen/CGTemporaries.cpp
@@ -16,39 +16,11 @@ using namespace clang;
 using namespace CodeGen;
 
 namespace {
-  struct DestroyTemporary : EHScopeStack::Cleanup {
-    const CXXTemporary *Temporary;
-    llvm::Value *Addr;
-    llvm::Value *CondPtr;
-
-    DestroyTemporary(const CXXTemporary *Temporary, llvm::Value *Addr,
-                     llvm::Value *CondPtr)
-      : Temporary(Temporary), Addr(Addr), CondPtr(CondPtr) {}
-
-    void Emit(CodeGenFunction &CGF, bool IsForEH) {
-      llvm::BasicBlock *CondEnd = 0;
-    
-      // If this is a conditional temporary, we need to check the condition
-      // boolean and only call the destructor if it's true.
-      if (CondPtr) {
-        llvm::BasicBlock *CondBlock =
-          CGF.createBasicBlock("temp.cond-dtor.call");
-        CondEnd = CGF.createBasicBlock("temp.cond-dtor.cont");
-
-        llvm::Value *Cond = CGF.Builder.CreateLoad(CondPtr);
-        CGF.Builder.CreateCondBr(Cond, CondBlock, CondEnd);
-        CGF.EmitBlock(CondBlock);
-      }
-
-      CGF.EmitCXXDestructorCall(Temporary->getDestructor(),
-                                Dtor_Complete, /*ForVirtualBase=*/false,
-                                Addr);
-
-      if (CondPtr) {
-        // Reset the condition to false.
-        CGF.Builder.CreateStore(CGF.Builder.getFalse(), CondPtr);
-        CGF.EmitBlock(CondEnd);
-      }
+  struct DestroyTemporary {
+    static void Emit(CodeGenFunction &CGF, bool forEH,
+                     const CXXDestructorDecl *dtor, llvm::Value *addr) {
+      CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*ForVirtualBase=*/false,
+                                addr);
     }
   };
 }
@@ -56,37 +28,19 @@ namespace {
 /// Emits all the code to cause the given temporary to be cleaned up.
 void CodeGenFunction::EmitCXXTemporary(const CXXTemporary *Temporary,
                                        llvm::Value *Ptr) {
-  llvm::AllocaInst *CondPtr = 0;
-
-  // Check if temporaries need to be conditional. If so, we'll create a
-  // condition boolean, initialize it to 0 and
-  if (ConditionalBranchLevel != 0) {
-    CondPtr = CreateTempAlloca(llvm::Type::getInt1Ty(VMContext), "cond");
-
-    // Initialize it to false. This initialization takes place right after
-    // the alloca insert point.
-    InitTempAlloca(CondPtr, llvm::ConstantInt::getFalse(VMContext));
-
-    // Now set it to true.
-    Builder.CreateStore(Builder.getTrue(), CondPtr);
-  }
-
-  EHStack.pushCleanup<DestroyTemporary>(NormalAndEHCleanup,
-                                        Temporary, Ptr, CondPtr);
+  pushFullExprCleanup<DestroyTemporary>(NormalAndEHCleanup,
+                                        Temporary->getDestructor(),
+                                        Ptr);
 }
 
 RValue
-CodeGenFunction::EmitCXXExprWithTemporaries(const CXXExprWithTemporaries *E,
-                                            llvm::Value *AggLoc,
-                                            bool IsAggLocVolatile,
-                                            bool IsInitializer) {
+CodeGenFunction::EmitExprWithCleanups(const ExprWithCleanups *E,
+                                      AggValueSlot Slot) {
   RunCleanupsScope Scope(*this);
-  return EmitAnyExpr(E->getSubExpr(), AggLoc, IsAggLocVolatile,
-                     /*IgnoreResult=*/false, IsInitializer);
+  return EmitAnyExpr(E->getSubExpr(), Slot);
 }
 
-LValue CodeGenFunction::EmitCXXExprWithTemporariesLValue(
-                                              const CXXExprWithTemporaries *E) {
+LValue CodeGenFunction::EmitExprWithCleanupsLValue(const ExprWithCleanups *E) {
   RunCleanupsScope Scope(*this);
   return EmitLValue(E->getSubExpr());
 }
diff --git a/lib/CodeGen/CGVTT.cpp b/lib/CodeGen/CGVTT.cpp
index 56acfc84802e..78b2dbe2ba22 100644
--- a/lib/CodeGen/CGVTT.cpp
+++ b/lib/CodeGen/CGVTT.cpp
@@ -218,7 +218,7 @@ void VTTBuilder::LayoutSecondaryVTTs(BaseSubobject Base) {
 
     const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
     uint64_t BaseOffset = Base.getBaseOffset() + 
-      Layout.getBaseClassOffset(BaseDecl);
+      Layout.getBaseClassOffsetInBits(BaseDecl);
    
     // Layout the VTT for this base.
     LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/false);
@@ -262,14 +262,15 @@ VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base,
       if (!VBases.insert(BaseDecl))
         continue;
       
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffsetInBits(BaseDecl);
       BaseDeclIsMorallyVirtual = true;
     } else {
       const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
       
-      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
+      BaseOffset = 
+        Base.getBaseOffset() + Layout.getBaseClassOffsetInBits(BaseDecl);
       
-      if (!Layout.getPrimaryBaseWasVirtual() &&
+      if (!Layout.isPrimaryBaseVirtual() &&
           Layout.getPrimaryBase() == BaseDecl)
         BaseDeclIsNonVirtualPrimaryBase = true;
     }
@@ -316,7 +317,7 @@ void VTTBuilder::LayoutVirtualVTTs(const CXXRecordDecl *RD,
         continue;
     
       uint64_t BaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+        MostDerivedClassLayout.getVBaseClassOffsetInBits(BaseDecl);
       
       LayoutVTT(BaseSubobject(BaseDecl, BaseOffset), /*BaseIsVirtual=*/true);
     }
@@ -365,57 +366,52 @@ void VTTBuilder::LayoutVTT(BaseSubobject Base, bool BaseIsVirtual) {
   
 }
 
-llvm::GlobalVariable *
-CodeGenVTables::GenerateVTT(llvm::GlobalVariable::LinkageTypes Linkage,
-                            bool GenerateDefinition,
-                            const CXXRecordDecl *RD) {
-  // Only classes that have virtual bases need a VTT.
-  if (RD->getNumVBases() == 0)
-    return 0;
+void
+CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT,
+                                  llvm::GlobalVariable::LinkageTypes Linkage,
+                                  const CXXRecordDecl *RD) {
+  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/true);
 
-  llvm::SmallString<256> OutName;
-  CGM.getCXXABI().getMangleContext().mangleCXXVTT(RD, OutName);
-  llvm::StringRef Name = OutName.str();
+  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+  const llvm::ArrayType *ArrayType = 
+    llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size());
+  
+  llvm::Constant *Init = 
+    llvm::ConstantArray::get(ArrayType, Builder.getVTTComponents().data(),
+                             Builder.getVTTComponents().size());
+
+  VTT->setInitializer(Init);
+
+  // Set the correct linkage.
+  VTT->setLinkage(Linkage);
 
-  D1(printf("vtt %s\n", RD->getNameAsCString()));
+  // Set the right visibility.
+  CGM.setTypeVisibility(VTT, RD, CodeGenModule::TVK_ForVTT);
+}
 
-  llvm::GlobalVariable *GV = CGM.getModule().getGlobalVariable(Name, true);
-  if (GV == 0 || GV->isDeclaration()) {
-    const llvm::Type *Int8PtrTy = 
-      llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTT(const CXXRecordDecl *RD) {
+  assert(RD->getNumVBases() && "Only classes with virtual bases need a VTT");
 
-    VTTBuilder Builder(CGM, RD, GenerateDefinition);
+  llvm::SmallString<256> OutName;
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXVTT(RD, Out);
+  Out.flush();
+  llvm::StringRef Name = OutName.str();
 
-    const llvm::ArrayType *Type = 
-      llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size());
+  VTTBuilder Builder(CGM, RD, /*GenerateDefinition=*/false);
 
-    llvm::Constant *Init = 0;
-    if (GenerateDefinition)
-      Init = llvm::ConstantArray::get(Type, Builder.getVTTComponents().data(),
-                                      Builder.getVTTComponents().size());
+  const llvm::Type *Int8PtrTy = 
+    llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
+  const llvm::ArrayType *ArrayType = 
+    llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size());
 
-    llvm::GlobalVariable *OldGV = GV;
-    GV = new llvm::GlobalVariable(CGM.getModule(), Type, /*isConstant=*/true, 
-                                  Linkage, Init, Name);
-    CGM.setGlobalVisibility(GV, RD);
-    
-    if (OldGV) {
-      GV->takeName(OldGV);
-      llvm::Constant *NewPtr = 
-        llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
-      OldGV->replaceAllUsesWith(NewPtr);
-      OldGV->eraseFromParent();
-    }
-  }
-  
+  llvm::GlobalVariable *GV =
+    CGM.CreateOrReplaceCXXRuntimeVariable(Name, ArrayType, 
+                                          llvm::GlobalValue::ExternalLinkage);
+  GV->setUnnamedAddr(true);
   return GV;
 }
 
-llvm::GlobalVariable *CodeGenVTables::getVTT(const CXXRecordDecl *RD) {
-  return GenerateVTT(llvm::GlobalValue::ExternalLinkage, 
-                     /*GenerateDefinition=*/false, RD);
-}
-
 bool CodeGenVTables::needsVTTParameter(GlobalDecl GD) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   
diff --git a/lib/CodeGen/CGVTables.cpp b/lib/CodeGen/CGVTables.cpp
index bed4670f7f95..891697f4cd09 100644
--- a/lib/CodeGen/CGVTables.cpp
+++ b/lib/CodeGen/CGVTables.cpp
@@ -240,7 +240,7 @@ static BaseOffset ComputeBaseOffset(ASTContext &Context,
     const RecordType *BaseType = Element.Base->getType()->getAs<RecordType>();
     const CXXRecordDecl *Base = cast<CXXRecordDecl>(BaseType->getDecl());
 
-    NonVirtualOffset += Layout.getBaseClassOffset(Base);
+    NonVirtualOffset += Layout.getBaseClassOffsetInBits(Base);
   }
   
   // FIXME: This should probably use CharUnits or something. Maybe we should
@@ -358,12 +358,12 @@ FinalOverriders::ComputeBaseOffsets(BaseSubobject Base, bool IsVirtual,
       const ASTRecordLayout &LayoutClassLayout =
         Context.getASTRecordLayout(LayoutClass);
 
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffsetInBits(BaseDecl);
       BaseOffsetInLayoutClass = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+        LayoutClassLayout.getVBaseClassOffsetInBits(BaseDecl);
     } else {
       const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
-      uint64_t Offset = Layout.getBaseClassOffset(BaseDecl);
+      uint64_t Offset = Layout.getBaseClassOffsetInBits(BaseDecl);
     
       BaseOffset = Base.getBaseOffset() + Offset;
       BaseOffsetInLayoutClass = OffsetInLayoutClass + Offset;
@@ -396,9 +396,9 @@ void FinalOverriders::dump(llvm::raw_ostream &Out, BaseSubobject Base,
         continue;
       }
       
-      BaseOffset = MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseOffset = MostDerivedClassLayout.getVBaseClassOffsetInBits(BaseDecl);
     } else {
-      BaseOffset = Layout.getBaseClassOffset(BaseDecl) + 
+      BaseOffset = Layout.getBaseClassOffsetInBits(BaseDecl) + 
         Base.getBaseOffset();
     }
 
@@ -793,22 +793,22 @@ VCallAndVBaseOffsetBuilder::AddVCallAndVBaseOffsets(BaseSubobject Base,
   // (Since we're emitting the vcall and vbase offsets in reverse order, we'll
   // emit them for the primary base first).
   if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
-    bool PrimaryBaseIsVirtual = Layout.getPrimaryBaseWasVirtual();
+    bool PrimaryBaseIsVirtual = Layout.isPrimaryBaseVirtual();
 
     uint64_t PrimaryBaseOffset;
     
     // Get the base offset of the primary base.
     if (PrimaryBaseIsVirtual) {
-      assert(Layout.getVBaseClassOffset(PrimaryBase) == 0 &&
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
              "Primary vbase should have a zero offset!");
       
       const ASTRecordLayout &MostDerivedClassLayout =
         Context.getASTRecordLayout(MostDerivedClass);
       
       PrimaryBaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
+        MostDerivedClassLayout.getVBaseClassOffsetInBits(PrimaryBase);
     } else {
-      assert(Layout.getBaseClassOffset(PrimaryBase) == 0 &&
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
              "Primary base should have a zero offset!");
 
       PrimaryBaseOffset = Base.getBaseOffset();
@@ -849,9 +849,9 @@ void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base,
   // Handle the primary base first.
   // We only want to add vcall offsets if the base is non-virtual; a virtual
   // primary base will have its vcall and vbase offsets emitted already.
-  if (PrimaryBase && !Layout.getPrimaryBaseWasVirtual()) {
+  if (PrimaryBase && !Layout.isPrimaryBaseVirtual()) {
     // Get the base offset of the primary base.
-    assert(Layout.getBaseClassOffset(PrimaryBase) == 0 &&
+    assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
            "Primary base should have a zero offset!");
 
     AddVCallOffsets(BaseSubobject(PrimaryBase, Base.getBaseOffset()),
@@ -903,7 +903,7 @@ void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base,
 
     // Get the base offset of this base.
     uint64_t BaseOffset = Base.getBaseOffset() + 
-      Layout.getBaseClassOffset(BaseDecl);
+      Layout.getBaseClassOffsetInBits(BaseDecl);
     
     AddVCallOffsets(BaseSubobject(BaseDecl, BaseOffset), VBaseOffset);
   }
@@ -924,7 +924,7 @@ void VCallAndVBaseOffsetBuilder::AddVBaseOffsets(const CXXRecordDecl *RD,
     if (I->isVirtual() && VisitedVirtualBases.insert(BaseDecl)) {
       // FIXME: We shouldn't use / 8 here.
       int64_t Offset = 
-        (int64_t)(LayoutClassLayout.getVBaseClassOffset(BaseDecl) - 
+        (int64_t)(LayoutClassLayout.getVBaseClassOffsetInBits(BaseDecl) - 
                   OffsetInLayoutClass) / 8;
 
       // Add the vbase offset offset.
@@ -1372,7 +1372,7 @@ VTableBuilder::ComputeThisAdjustmentBaseOffset(BaseSubobject Base,
       /// Get the virtual base offset, relative to the most derived class 
       /// layout.
       OffsetToBaseSubobject += 
-        LayoutClassLayout.getVBaseClassOffset(Offset.VirtualBase);
+        LayoutClassLayout.getVBaseClassOffsetInBits(Offset.VirtualBase);
     } else {
       // Otherwise, the non-virtual offset is relative to the derived class 
       // offset.
@@ -1520,8 +1520,8 @@ VTableBuilder::IsOverriderUsed(const CXXMethodDecl *Overrider,
     if (!PrimaryBase)
       break;
     
-    if (Layout.getPrimaryBaseWasVirtual()) {
-      assert(Layout.getVBaseClassOffset(PrimaryBase) == 0 && 
+    if (Layout.isPrimaryBaseVirtual()) {
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 && 
              "Primary base should always be at offset 0!");
 
       const ASTRecordLayout &LayoutClassLayout =
@@ -1529,13 +1529,13 @@ VTableBuilder::IsOverriderUsed(const CXXMethodDecl *Overrider,
 
       // Now check if this is the primary base that is not a primary base in the
       // most derived class.
-      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
+      if (LayoutClassLayout.getVBaseClassOffsetInBits(PrimaryBase) !=
           FirstBaseOffsetInLayoutClass) {
         // We found it, stop walking the chain.
         break;
       }
     } else {
-      assert(Layout.getBaseClassOffset(PrimaryBase) == 0 && 
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 && 
              "Primary base should always be at offset 0!");
     }
     
@@ -1586,23 +1586,23 @@ VTableBuilder::AddMethods(BaseSubobject Base, uint64_t BaseOffsetInLayoutClass,
   if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
     uint64_t PrimaryBaseOffset;
     uint64_t PrimaryBaseOffsetInLayoutClass;
-    if (Layout.getPrimaryBaseWasVirtual()) {
-      assert(Layout.getVBaseClassOffset(PrimaryBase) == 0 &&
+    if (Layout.isPrimaryBaseVirtual()) {
+      assert(Layout.getVBaseClassOffsetInBits(PrimaryBase) == 0 &&
              "Primary vbase should have a zero offset!");
       
       const ASTRecordLayout &MostDerivedClassLayout =
         Context.getASTRecordLayout(MostDerivedClass);
       
       PrimaryBaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(PrimaryBase);
+        MostDerivedClassLayout.getVBaseClassOffsetInBits(PrimaryBase);
       
       const ASTRecordLayout &LayoutClassLayout =
         Context.getASTRecordLayout(LayoutClass);
 
       PrimaryBaseOffsetInLayoutClass =
-        LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
+        LayoutClassLayout.getVBaseClassOffsetInBits(PrimaryBase);
     } else {
-      assert(Layout.getBaseClassOffset(PrimaryBase) == 0 &&
+      assert(Layout.getBaseClassOffsetInBits(PrimaryBase) == 0 &&
              "Primary base should have a zero offset!");
 
       PrimaryBaseOffset = Base.getBaseOffset();
@@ -1664,9 +1664,18 @@ VTableBuilder::AddMethods(BaseSubobject Base, uint64_t BaseOffsetInLayoutClass,
 
           if (ThisAdjustment.VCallOffsetOffset &&
               Overrider.Method->getParent() == MostDerivedClass) {
+
+            // There's no return adjustment from OverriddenMD and MD,
+            // but that doesn't mean there isn't one between MD and
+            // the final overrider.
+            BaseOffset ReturnAdjustmentOffset =
+              ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
+            ReturnAdjustment ReturnAdjustment = 
+              ComputeReturnAdjustment(ReturnAdjustmentOffset);
+
             // This is a virtual thunk for the most derived class, add it.
             AddThunk(Overrider.Method, 
-                     ThunkInfo(ThisAdjustment, ReturnAdjustment()));
+                     ThunkInfo(ThisAdjustment, ReturnAdjustment));
           }
         }
 
@@ -1779,13 +1788,13 @@ VTableBuilder::LayoutPrimaryAndSecondaryVTables(BaseSubobject Base,
     if (!PrimaryBase)
       break;
     
-    if (Layout.getPrimaryBaseWasVirtual()) {
+    if (Layout.isPrimaryBaseVirtual()) {
       // Check if this virtual primary base is a primary base in the layout
       // class. If it's not, we don't want to add it.
       const ASTRecordLayout &LayoutClassLayout =
         Context.getASTRecordLayout(LayoutClass);
 
-      if (LayoutClassLayout.getVBaseClassOffset(PrimaryBase) !=
+      if (LayoutClassLayout.getVBaseClassOffsetInBits(PrimaryBase) !=
           OffsetInLayoutClass) {
         // We don't want to add this class (or any of its primary bases).
         break;
@@ -1835,7 +1844,7 @@ void VTableBuilder::LayoutSecondaryVTables(BaseSubobject Base,
     }
 
     // Get the base offset of this base.
-    uint64_t RelativeBaseOffset = Layout.getBaseClassOffset(BaseDecl);
+    uint64_t RelativeBaseOffset = Layout.getBaseClassOffsetInBits(BaseDecl);
     uint64_t BaseOffset = Base.getBaseOffset() + RelativeBaseOffset;
     
     uint64_t BaseOffsetInLayoutClass = OffsetInLayoutClass + RelativeBaseOffset;
@@ -1866,7 +1875,7 @@ VTableBuilder::DeterminePrimaryVirtualBases(const CXXRecordDecl *RD,
   if (const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase()) {
 
     // Check if it's virtual.
-    if (Layout.getPrimaryBaseWasVirtual()) {
+    if (Layout.isPrimaryBaseVirtual()) {
       bool IsPrimaryVirtualBase = true;
 
       if (isBuildingConstructorVTable()) {
@@ -1876,7 +1885,7 @@ VTableBuilder::DeterminePrimaryVirtualBases(const CXXRecordDecl *RD,
           Context.getASTRecordLayout(LayoutClass);
 
         uint64_t PrimaryBaseOffsetInLayoutClass =
-          LayoutClassLayout.getVBaseClassOffset(PrimaryBase);
+          LayoutClassLayout.getVBaseClassOffsetInBits(PrimaryBase);
         
         // We know that the base is not a primary base in the layout class if 
         // the base offsets are different.
@@ -1904,10 +1913,11 @@ VTableBuilder::DeterminePrimaryVirtualBases(const CXXRecordDecl *RD,
       const ASTRecordLayout &LayoutClassLayout =
         Context.getASTRecordLayout(LayoutClass);
 
-      BaseOffsetInLayoutClass = LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+      BaseOffsetInLayoutClass = 
+        LayoutClassLayout.getVBaseClassOffsetInBits(BaseDecl);
     } else {
       BaseOffsetInLayoutClass = 
-        OffsetInLayoutClass + Layout.getBaseClassOffset(BaseDecl);
+        OffsetInLayoutClass + Layout.getBaseClassOffsetInBits(BaseDecl);
     }
 
     DeterminePrimaryVirtualBases(BaseDecl, BaseOffsetInLayoutClass, VBases);
@@ -1933,12 +1943,12 @@ VTableBuilder::LayoutVTablesForVirtualBases(const CXXRecordDecl *RD,
       const ASTRecordLayout &MostDerivedClassLayout =
         Context.getASTRecordLayout(MostDerivedClass);
       uint64_t BaseOffset = 
-        MostDerivedClassLayout.getVBaseClassOffset(BaseDecl);
+        MostDerivedClassLayout.getVBaseClassOffsetInBits(BaseDecl);
       
       const ASTRecordLayout &LayoutClassLayout =
         Context.getASTRecordLayout(LayoutClass);
       uint64_t BaseOffsetInLayoutClass = 
-        LayoutClassLayout.getVBaseClassOffset(BaseDecl);
+        LayoutClassLayout.getVBaseClassOffsetInBits(BaseDecl);
 
       LayoutPrimaryAndSecondaryVTables(BaseSubobject(BaseDecl, BaseOffset),
                                        /*BaseIsMorallyVirtual=*/true,
@@ -2230,6 +2240,19 @@ void VTableBuilder::dumpLayout(llvm::raw_ostream& Out) {
   
 }
 
+static void 
+CollectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
+                    VTableBuilder::PrimaryBasesSetVectorTy &PrimaryBases) {
+  while (RD) {
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase();
+    if (PrimaryBase)
+      PrimaryBases.insert(PrimaryBase);
+
+    RD = PrimaryBase;
+  }
+}
+
 void CodeGenVTables::ComputeMethodVTableIndices(const CXXRecordDecl *RD) {
   
   // Itanium C++ ABI 2.5.2:
@@ -2258,10 +2281,7 @@ void CodeGenVTables::ComputeMethodVTableIndices(const CXXRecordDecl *RD) {
   // Collect all the primary bases, so we can check whether methods override
   // a method from the base.
   VTableBuilder::PrimaryBasesSetVectorTy PrimaryBases;
-  for (ASTRecordLayout::primary_base_info_iterator
-       I = Layout.primary_base_begin(), E = Layout.primary_base_end();
-       I != E; ++I)
-    PrimaryBases.insert((*I).getBase());
+  CollectPrimaryBases(RD, CGM.getContext(), PrimaryBases);
 
   const CXXDestructorDecl *ImplicitVirtualDtor = 0;
   
@@ -2336,6 +2356,33 @@ void CodeGenVTables::ComputeMethodVTableIndices(const CXXRecordDecl *RD) {
   NumVirtualFunctionPointers[RD] = CurrentIndex;
 }
 
+bool CodeGenVTables::ShouldEmitVTableInThisTU(const CXXRecordDecl *RD) {
+  assert(RD->isDynamicClass() && "Non dynamic classes have no VTable.");
+
+  TemplateSpecializationKind TSK = RD->getTemplateSpecializationKind();
+  if (TSK == TSK_ExplicitInstantiationDeclaration)
+    return false;
+
+  const CXXMethodDecl *KeyFunction = CGM.getContext().getKeyFunction(RD);
+  if (!KeyFunction)
+    return true;
+
+  // Itanium C++ ABI, 5.2.6 Instantiated Templates:
+  //    An instantiation of a class template requires:
+  //        - In the object where instantiated, the virtual table...
+  if (TSK == TSK_ImplicitInstantiation ||
+      TSK == TSK_ExplicitInstantiationDefinition)
+    return true;
+
+  // If we're building with optimization, we always emit VTables since that
+  // allows for virtual function calls to be devirtualized.
+  // (We don't want to do this in -fapple-kext mode however).
+  if (CGM.getCodeGenOpts().OptimizationLevel && !CGM.getLangOptions().AppleKext)
+    return true;
+
+  return KeyFunction->hasBody();
+}
+
 uint64_t CodeGenVTables::getNumVirtualFunctionPointers(const CXXRecordDecl *RD) {
   llvm::DenseMap<const CXXRecordDecl *, uint64_t>::iterator I = 
     NumVirtualFunctionPointers.find(RD);
@@ -2409,14 +2456,16 @@ llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD,
 
   // Compute the mangled name.
   llvm::SmallString<256> Name;
+  llvm::raw_svector_ostream Out(Name);
   if (const CXXDestructorDecl* DD = dyn_cast<CXXDestructorDecl>(MD))
     getCXXABI().getMangleContext().mangleCXXDtorThunk(DD, GD.getDtorType(),
-                                                      Thunk.This, Name);
+                                                      Thunk.This, Out);
   else
-    getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Name);
-  
+    getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Out);
+  Out.flush();
+
   const llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD);
-  return GetOrCreateLLVMFunction(Name, Ty, GD);
+  return GetOrCreateLLVMFunction(Name, Ty, GD, /*ForVTable=*/true);
 }
 
 static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
@@ -2563,7 +2612,7 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   const llvm::Type *Ty =
     CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(GD),
                                    FPT->isVariadic());
-  llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty);
+  llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true);
 
   const CGFunctionInfo &FnInfo = 
     CGM.getTypes().getFunctionInfo(ResultType, CallArgs,
@@ -2621,7 +2670,7 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   }
 
   if (!ResultType->isVoidType() && Slot.isNull())
-    CGM.getCXXABI().EmitReturnFromThunk(CGF, RV, ResultType);
+    CGM.getCXXABI().EmitReturnFromThunk(*this, RV, ResultType);
 
   FinishFunction();
 
@@ -2632,7 +2681,8 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   setThunkVisibility(CGM, MD, Thunk, Fn);
 }
 
-void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk)
+void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk, 
+                               bool UseAvailableExternallyLinkage)
 {
   llvm::Constant *Entry = CGM.GetAddrOfThunk(GD, Thunk);
   
@@ -2667,9 +2717,42 @@ void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk)
     OldThunkFn->eraseFromParent();
   }
 
-  // Actually generate the thunk body.
   llvm::Function *ThunkFn = cast<llvm::Function>(Entry);
+
+  if (!ThunkFn->isDeclaration()) {
+    if (UseAvailableExternallyLinkage) {
+      // There is already a thunk emitted for this function, do nothing.
+      return;
+    }
+
+    // If a function has a body, it should have available_externally linkage.
+    assert(ThunkFn->hasAvailableExternallyLinkage() &&
+           "Function should have available_externally linkage!");
+
+    // Change the linkage.
+    CGM.setFunctionLinkage(cast<CXXMethodDecl>(GD.getDecl()), ThunkFn);
+    return;
+  }
+
+  // Actually generate the thunk body.
   CodeGenFunction(CGM).GenerateThunk(ThunkFn, GD, Thunk);
+
+  if (UseAvailableExternallyLinkage)
+    ThunkFn->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
+}
+
+void CodeGenVTables::MaybeEmitThunkAvailableExternally(GlobalDecl GD,
+                                                       const ThunkInfo &Thunk) {
+  // We only want to do this when building with optimizations.
+  if (!CGM.getCodeGenOpts().OptimizationLevel)
+    return;
+
+  // We can't emit thunks for member functions with incomplete types.
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+  if (CGM.getTypes().VerifyFuncTypeComplete(MD->getType().getTypePtr()))
+    return;
+
+  EmitThunk(GD, Thunk, /*UseAvailableExternallyLinkage=*/true);
 }
 
 void CodeGenVTables::EmitThunks(GlobalDecl GD)
@@ -2694,7 +2777,7 @@ void CodeGenVTables::EmitThunks(GlobalDecl GD)
 
   const ThunkInfoVectorTy &ThunkInfoVector = I->second;
   for (unsigned I = 0, E = ThunkInfoVector.size(); I != E; ++I)
-    EmitThunk(GD, ThunkInfoVector[I]);
+    EmitThunk(GD, ThunkInfoVector[I], /*UseAvailableExternallyLinkage=*/false);
 }
 
 void CodeGenVTables::ComputeVTableRelatedInformation(const CXXRecordDecl *RD,
@@ -2703,9 +2786,7 @@ void CodeGenVTables::ComputeVTableRelatedInformation(const CXXRecordDecl *RD,
 
   // We may need to generate a definition for this vtable.
   if (RequireVTable && !Entry.getInt()) {
-    if (!isKeyFunctionInAnotherTU(CGM.getContext(), RD) &&
-        RD->getTemplateSpecializationKind()
-          != TSK_ExplicitInstantiationDeclaration)
+    if (ShouldEmitVTableInThisTU(RD))
       CGM.DeferredVTables.push_back(RD);
 
     Entry.setInt(true);
@@ -2719,7 +2800,14 @@ void CodeGenVTables::ComputeVTableRelatedInformation(const CXXRecordDecl *RD,
 
   // Add the VTable layout.
   uint64_t NumVTableComponents = Builder.getNumVTableComponents();
+  // -fapple-kext adds an extra entry at end of vtbl.
+  bool IsAppleKext = CGM.getContext().getLangOptions().AppleKext;
+  if (IsAppleKext)
+    NumVTableComponents += 1;
+
   uint64_t *LayoutData = new uint64_t[NumVTableComponents + 1];
+  if (IsAppleKext)
+    LayoutData[NumVTableComponents] = 0;
   Entry.setPointer(LayoutData);
 
   // Store the number of components.
@@ -2861,12 +2949,13 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
           const ThunkInfo &Thunk = VTableThunks[NextVTableThunkIndex].second;
         
           Init = CGM.GetAddrOfThunk(GD, Thunk);
-        
+          MaybeEmitThunkAvailableExternally(GD, Thunk);
+
           NextVTableThunkIndex++;
         } else {
           const llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(GD);
         
-          Init = CGM.GetAddrOfFunction(GD, Ty);
+          Init = CGM.GetAddrOfFunction(GD, Ty, /*ForVTable=*/true);
         }
 
         Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy);
@@ -2886,52 +2975,11 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
   return llvm::ConstantArray::get(ArrayType, Inits.data(), Inits.size());
 }
 
-/// GetGlobalVariable - Will return a global variable of the given type. 
-/// If a variable with a different type already exists then a new variable
-/// with the right type will be created.
-/// FIXME: We should move this to CodeGenModule and rename it to something 
-/// better and then use it in CGVTT and CGRTTI. 
-static llvm::GlobalVariable *
-GetGlobalVariable(llvm::Module &Module, llvm::StringRef Name,
-                  const llvm::Type *Ty,
-                  llvm::GlobalValue::LinkageTypes Linkage) {
-
-  llvm::GlobalVariable *GV = Module.getNamedGlobal(Name);
-  llvm::GlobalVariable *OldGV = 0;
-  
-  if (GV) {
-    // Check if the variable has the right type.
-    if (GV->getType()->getElementType() == Ty)
-      return GV;
-
-    assert(GV->isDeclaration() && "Declaration has wrong type!");
-    
-    OldGV = GV;
-  }
-  
-  // Create a new variable.
-  GV = new llvm::GlobalVariable(Module, Ty, /*isConstant=*/true,
-                                Linkage, 0, Name);
-  
-  if (OldGV) {
-    // Replace occurrences of the old variable if needed.
-    GV->takeName(OldGV);
-   
-    if (!OldGV->use_empty()) {
-      llvm::Constant *NewPtrForOldDecl =
-        llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
-      OldGV->replaceAllUsesWith(NewPtrForOldDecl);
-    }
-
-    OldGV->eraseFromParent();
-  }
-  
-  return GV;
-}
-
 llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTable(const CXXRecordDecl *RD) {
   llvm::SmallString<256> OutName;
-  CGM.getCXXABI().getMangleContext().mangleCXXVTable(RD, OutName);
+  llvm::raw_svector_ostream Out(OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXVTable(RD, Out);
+  Out.flush();
   llvm::StringRef Name = OutName.str();
 
   ComputeVTableRelatedInformation(RD, true);
@@ -2940,8 +2988,11 @@ llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTable(const CXXRecordDecl *RD) {
   llvm::ArrayType *ArrayType = 
     llvm::ArrayType::get(Int8PtrTy, getNumVTableComponents(RD));
 
-  return GetGlobalVariable(CGM.getModule(), Name, ArrayType, 
-                           llvm::GlobalValue::ExternalLinkage);
+  llvm::GlobalVariable *GV =
+    CGM.CreateOrReplaceCXXRuntimeVariable(Name, ArrayType, 
+                                          llvm::GlobalValue::ExternalLinkage);
+  GV->setUnnamedAddr(true);
+  return GV;
 }
 
 void
@@ -2970,7 +3021,7 @@ CodeGenVTables::EmitVTableDefinition(llvm::GlobalVariable *VTable,
   VTable->setLinkage(Linkage);
   
   // Set the right visibility.
-  CGM.setTypeVisibility(VTable, RD, /*ForRTTI*/ false);
+  CGM.setTypeVisibility(VTable, RD, CodeGenModule::TVK_ForVTable);
 }
 
 llvm::GlobalVariable *
@@ -2991,8 +3042,10 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
 
   // Get the mangled construction vtable name.
   llvm::SmallString<256> OutName;
+  llvm::raw_svector_ostream Out(OutName);
   CGM.getCXXABI().getMangleContext().
-    mangleCXXCtorVTable(RD, Base.getBaseOffset() / 8, Base.getBase(), OutName);
+    mangleCXXCtorVTable(RD, Base.getBaseOffset() / 8, Base.getBase(), Out);
+  Out.flush();
   llvm::StringRef Name = OutName.str();
 
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
@@ -3001,8 +3054,8 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
 
   // Create the variable that will hold the construction vtable.
   llvm::GlobalVariable *VTable = 
-    GetGlobalVariable(CGM.getModule(), Name, ArrayType, 
-                      llvm::GlobalValue::InternalLinkage);
+    CGM.CreateOrReplaceCXXRuntimeVariable(Name, ArrayType, 
+                                          llvm::GlobalValue::InternalLinkage);
 
   // Add the thunks.
   VTableThunksTy VTableThunks;
@@ -3034,7 +3087,10 @@ CodeGenVTables::GenerateClassData(llvm::GlobalVariable::LinkageTypes Linkage,
   VTable = GetAddrOfVTable(RD);
   EmitVTableDefinition(VTable, Linkage, RD);
 
-  GenerateVTT(Linkage, /*GenerateDefinition=*/true, RD);
+  if (RD->getNumVBases()) {
+    llvm::GlobalVariable *VTT = GetAddrOfVTT(RD);
+    EmitVTTDefinition(VTT, Linkage, RD);
+  }
 
   // If this is the magic class __cxxabiv1::__fundamental_type_info,
   // we will emit the typeinfo for the fundamental types. This is the
diff --git a/lib/CodeGen/CGVTables.h b/lib/CodeGen/CGVTables.h
index abcafd6c9c41..7c119fa42241 100644
--- a/lib/CodeGen/CGVTables.h
+++ b/lib/CodeGen/CGVTables.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/GlobalVariable.h"
+#include "clang/Basic/ABI.h"
 #include "GlobalDecl.h"
 
 namespace clang {
@@ -24,94 +25,6 @@ namespace clang {
 namespace CodeGen {
   class CodeGenModule;
 
-/// ReturnAdjustment - A return adjustment.
-struct ReturnAdjustment {
-  /// NonVirtual - The non-virtual adjustment from the derived object to its
-  /// nearest virtual base.
-  int64_t NonVirtual;
-  
-  /// VBaseOffsetOffset - The offset (in bytes), relative to the address point 
-  /// of the virtual base class offset.
-  int64_t VBaseOffsetOffset;
-  
-  ReturnAdjustment() : NonVirtual(0), VBaseOffsetOffset(0) { }
-  
-  bool isEmpty() const { return !NonVirtual && !VBaseOffsetOffset; }
-
-  friend bool operator==(const ReturnAdjustment &LHS, 
-                         const ReturnAdjustment &RHS) {
-    return LHS.NonVirtual == RHS.NonVirtual && 
-      LHS.VBaseOffsetOffset == RHS.VBaseOffsetOffset;
-  }
-
-  friend bool operator<(const ReturnAdjustment &LHS,
-                        const ReturnAdjustment &RHS) {
-    if (LHS.NonVirtual < RHS.NonVirtual)
-      return true;
-    
-    return LHS.NonVirtual == RHS.NonVirtual && 
-      LHS.VBaseOffsetOffset < RHS.VBaseOffsetOffset;
-  }
-};
-  
-/// ThisAdjustment - A 'this' pointer adjustment.
-struct ThisAdjustment {
-  /// NonVirtual - The non-virtual adjustment from the derived object to its
-  /// nearest virtual base.
-  int64_t NonVirtual;
-
-  /// VCallOffsetOffset - The offset (in bytes), relative to the address point,
-  /// of the virtual call offset.
-  int64_t VCallOffsetOffset;
-  
-  ThisAdjustment() : NonVirtual(0), VCallOffsetOffset(0) { }
-
-  bool isEmpty() const { return !NonVirtual && !VCallOffsetOffset; }
-
-  friend bool operator==(const ThisAdjustment &LHS, 
-                         const ThisAdjustment &RHS) {
-    return LHS.NonVirtual == RHS.NonVirtual && 
-      LHS.VCallOffsetOffset == RHS.VCallOffsetOffset;
-  }
-  
-  friend bool operator<(const ThisAdjustment &LHS,
-                        const ThisAdjustment &RHS) {
-    if (LHS.NonVirtual < RHS.NonVirtual)
-      return true;
-    
-    return LHS.NonVirtual == RHS.NonVirtual && 
-      LHS.VCallOffsetOffset < RHS.VCallOffsetOffset;
-  }
-};
-
-/// ThunkInfo - The 'this' pointer adjustment as well as an optional return
-/// adjustment for a thunk.
-struct ThunkInfo {
-  /// This - The 'this' pointer adjustment.
-  ThisAdjustment This;
-    
-  /// Return - The return adjustment.
-  ReturnAdjustment Return;
-
-  ThunkInfo() { }
-
-  ThunkInfo(const ThisAdjustment &This, const ReturnAdjustment &Return)
-    : This(This), Return(Return) { }
-
-  friend bool operator==(const ThunkInfo &LHS, const ThunkInfo &RHS) {
-    return LHS.This == RHS.This && LHS.Return == RHS.Return;
-  }
-
-  friend bool operator<(const ThunkInfo &LHS, const ThunkInfo &RHS) {
-    if (LHS.This < RHS.This)
-      return true;
-      
-    return LHS.This == RHS.This && LHS.Return < RHS.Return;
-  }
-
-  bool isEmpty() const { return This.isEmpty() && Return.isEmpty(); }
-};  
-
 // BaseSubobject - Uniquely identifies a direct or indirect base class. 
 // Stores both the base class decl and the offset from the most derived class to
 // the base class.
@@ -269,13 +182,16 @@ class CodeGenVTables {
   
   void ComputeMethodVTableIndices(const CXXRecordDecl *RD);
 
-  llvm::GlobalVariable *GenerateVTT(llvm::GlobalVariable::LinkageTypes Linkage,
-                                    bool GenerateDefinition,
-                                    const CXXRecordDecl *RD);
-
   /// EmitThunk - Emit a single thunk.
-  void EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk);
-  
+  void EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk, 
+                 bool UseAvailableExternallyLinkage);
+
+  /// MaybeEmitThunkAvailableExternally - Try to emit the given thunk with
+  /// available_externally linkage to allow for inlining of thunks.
+  /// This will be done iff optimizations are enabled and the member function
+  /// doesn't contain any incomplete types.
+  void MaybeEmitThunkAvailableExternally(GlobalDecl GD, const ThunkInfo &Thunk);
+
   /// ComputeVTableRelatedInformation - Compute and store all vtable related
   /// information (vtable layout, vbase offset offsets, thunks etc) for the
   /// given record decl.
@@ -295,14 +211,9 @@ public:
   CodeGenVTables(CodeGenModule &CGM)
     : CGM(CGM) { }
 
-  // isKeyFunctionInAnotherTU - True if this record has a key function and it is
-  // in another translation unit.
-  static bool isKeyFunctionInAnotherTU(ASTContext &Context,
-				       const CXXRecordDecl *RD) {
-    assert (RD->isDynamicClass() && "Non dynamic classes have no key.");
-    const CXXMethodDecl *KeyFunction = Context.getKeyFunction(RD);
-    return KeyFunction && !KeyFunction->hasBody();
-  }
+  /// \brief True if the VTable of this record must be emitted in the
+  /// translation unit.
+  bool ShouldEmitVTableInThisTU(const CXXRecordDecl *RD);
 
   /// needsVTTParameter - Return whether the given global decl needs a VTT
   /// parameter, which it does if it's a base constructor or destructor with
@@ -349,8 +260,15 @@ public:
   GenerateConstructionVTable(const CXXRecordDecl *RD, const BaseSubobject &Base, 
                              bool BaseIsVirtual, 
                              VTableAddressPointsMapTy& AddressPoints);
-  
-  llvm::GlobalVariable *getVTT(const CXXRecordDecl *RD);
+
+    
+  /// GetAddrOfVTable - Get the address of the VTT for the given record decl.
+  llvm::GlobalVariable *GetAddrOfVTT(const CXXRecordDecl *RD);
+
+  /// EmitVTTDefinition - Emit the definition of the given vtable.
+  void EmitVTTDefinition(llvm::GlobalVariable *VTT,
+                         llvm::GlobalVariable::LinkageTypes Linkage,
+                         const CXXRecordDecl *RD);
 
   /// EmitThunks - Emit the associated thunks for the given global decl.
   void EmitThunks(GlobalDecl GD);
diff --git a/lib/CodeGen/CGValue.h b/lib/CodeGen/CGValue.h
index f57ecd245f09..7f77d552032a 100644
--- a/lib/CodeGen/CGValue.h
+++ b/lib/CodeGen/CGValue.h
@@ -25,7 +25,6 @@ namespace llvm {
 
 namespace clang {
   class ObjCPropertyRefExpr;
-  class ObjCImplicitSetterGetterRefExpr;
 
 namespace CodeGen {
   class CGBitFieldInfo;
@@ -109,10 +108,8 @@ class LValue {
     VectorElt,    // This is a vector element l-value (V[i]), use getVector*
     BitField,     // This is a bitfield l-value, use getBitfield*.
     ExtVectorElt, // This is an extended vector subset, use getExtVectorComp
-    PropertyRef,  // This is an Objective-C property reference, use
+    PropertyRef   // This is an Objective-C property reference, use
                   // getPropertyRefExpr
-    KVCRef        // This is an objective-c 'implicit' property ref,
-                  // use getKVCRefExpr
   } LVType;
 
   llvm::Value *V;
@@ -129,9 +126,6 @@ class LValue {
 
     // Obj-C property reference expression
     const ObjCPropertyRefExpr *PropertyRefExpr;
-
-    // ObjC 'implicit' property reference expression
-    const ObjCImplicitSetterGetterRefExpr *KVCRefExpr;
   };
 
   // 'const' is unused here
@@ -157,8 +151,13 @@ class LValue {
   bool ThreadLocalRef : 1;
 
   Expr *BaseIvarExp;
+
+  /// TBAAInfo - TBAA information to attach to dereferences of this LValue.
+  llvm::MDNode *TBAAInfo;
+
 private:
-  void Initialize(Qualifiers Quals, unsigned Alignment = 0) {
+  void Initialize(Qualifiers Quals, unsigned Alignment = 0,
+                  llvm::MDNode *TBAAInfo = 0) {
     this->Quals = Quals;
     this->Alignment = Alignment;
     assert(this->Alignment == Alignment && "Alignment exceeds allowed max!");
@@ -167,6 +166,7 @@ private:
     this->Ivar = this->ObjIsArray = this->NonGC = this->GlobalObjCRef = false;
     this->ThreadLocalRef = false;
     this->BaseIvarExp = 0;
+    this->TBAAInfo = TBAAInfo;
   }
 
 public:
@@ -175,7 +175,6 @@ public:
   bool isBitField() const { return LVType == BitField; }
   bool isExtVectorElt() const { return LVType == ExtVectorElt; }
   bool isPropertyRef() const { return LVType == PropertyRef; }
-  bool isKVCRef() const { return LVType == KVCRef; }
 
   bool isVolatileQualified() const { return Quals.hasVolatile(); }
   bool isRestrictQualified() const { return Quals.hasRestrict(); }
@@ -208,6 +207,9 @@ public:
   Expr *getBaseIvarExp() const { return BaseIvarExp; }
   void setBaseIvarExp(Expr *V) { BaseIvarExp = V; }
 
+  llvm::MDNode *getTBAAInfo() const { return TBAAInfo; }
+  void setTBAAInfo(llvm::MDNode *N) { TBAAInfo = N; }
+
   const Qualifiers &getQuals() const { return Quals; }
   Qualifiers &getQuals() { return Quals; }
 
@@ -240,26 +242,25 @@ public:
   }
 
   // property ref lvalue
+  llvm::Value *getPropertyRefBaseAddr() const {
+    assert(isPropertyRef());
+    return V;
+  }
   const ObjCPropertyRefExpr *getPropertyRefExpr() const {
     assert(isPropertyRef());
     return PropertyRefExpr;
   }
 
-  // 'implicit' property ref lvalue
-  const ObjCImplicitSetterGetterRefExpr *getKVCRefExpr() const {
-    assert(isKVCRef());
-    return KVCRefExpr;
-  }
-
   static LValue MakeAddr(llvm::Value *V, QualType T, unsigned Alignment,
-                         ASTContext &Context) {
-    Qualifiers Quals = Context.getCanonicalType(T).getQualifiers();
+                         ASTContext &Context,
+                         llvm::MDNode *TBAAInfo = 0) {
+    Qualifiers Quals = T.getQualifiers();
     Quals.setObjCGCAttr(Context.getObjCGCAttrKind(T));
 
     LValue R;
     R.LVType = Simple;
     R.V = V;
-    R.Initialize(Quals, Alignment);
+    R.Initialize(Quals, Alignment, TBAAInfo);
     return R;
   }
 
@@ -303,21 +304,98 @@ public:
   // the lvalue. However, this complicates the code a bit, and I haven't figured
   // out how to make it go wrong yet.
   static LValue MakePropertyRef(const ObjCPropertyRefExpr *E,
-                                unsigned CVR) {
+                                llvm::Value *Base) {
     LValue R;
     R.LVType = PropertyRef;
+    R.V = Base;
     R.PropertyRefExpr = E;
-    R.Initialize(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers());
     return R;
   }
+};
 
-  static LValue MakeKVCRef(const ObjCImplicitSetterGetterRefExpr *E,
-                           unsigned CVR) {
-    LValue R;
-    R.LVType = KVCRef;
-    R.KVCRefExpr = E;
-    R.Initialize(Qualifiers::fromCVRMask(CVR));
-    return R;
+/// An aggregate value slot.
+class AggValueSlot {
+  /// The address.
+  llvm::Value *Addr;
+  
+  // Associated flags.
+  bool VolatileFlag : 1;
+  bool LifetimeFlag : 1;
+  bool RequiresGCollection : 1;
+  
+  /// IsZeroed - This is set to true if the destination is known to be zero
+  /// before the assignment into it.  This means that zero fields don't need to
+  /// be set.
+  bool IsZeroed : 1;
+
+public:
+  /// ignored - Returns an aggregate value slot indicating that the
+  /// aggregate value is being ignored.
+  static AggValueSlot ignored() {
+    AggValueSlot AV;
+    AV.Addr = 0;
+    AV.VolatileFlag = AV.LifetimeFlag = AV.RequiresGCollection = AV.IsZeroed =0;
+    return AV;
+  }
+
+  /// forAddr - Make a slot for an aggregate value.
+  ///
+  /// \param Volatile - true if the slot should be volatile-initialized
+  /// \param LifetimeExternallyManaged - true if the slot's lifetime
+  ///   is being externally managed; false if a destructor should be
+  ///   registered for any temporaries evaluated into the slot
+  /// \param RequiresGCollection - true if the slot is located
+  ///   somewhere that ObjC GC calls should be emitted for
+  static AggValueSlot forAddr(llvm::Value *Addr, bool Volatile,
+                              bool LifetimeExternallyManaged,
+                              bool RequiresGCollection = false,
+                              bool IsZeroed = false) {
+    AggValueSlot AV;
+    AV.Addr = Addr;
+    AV.VolatileFlag = Volatile;
+    AV.LifetimeFlag = LifetimeExternallyManaged;
+    AV.RequiresGCollection = RequiresGCollection;
+    AV.IsZeroed = IsZeroed;
+    return AV;
+  }
+
+  static AggValueSlot forLValue(LValue LV, bool LifetimeExternallyManaged,
+                                bool RequiresGCollection = false) {
+    return forAddr(LV.getAddress(), LV.isVolatileQualified(),
+                   LifetimeExternallyManaged, RequiresGCollection);
+  }
+
+  bool isLifetimeExternallyManaged() const {
+    return LifetimeFlag;
+  }
+  void setLifetimeExternallyManaged(bool Managed = true) {
+    LifetimeFlag = Managed;
+  }
+
+  bool isVolatile() const {
+    return VolatileFlag;
+  }
+
+  bool requiresGCollection() const {
+    return RequiresGCollection;
+  }
+  
+  llvm::Value *getAddr() const {
+    return Addr;
+  }
+
+  bool isIgnored() const {
+    return Addr == 0;
+  }
+
+  RValue asRValue() const {
+    return RValue::getAggregate(getAddr(), isVolatile());
+  }
+  
+  void setZeroed(bool V = true) { IsZeroed = V; }
+  bool isZeroed() const {
+    return IsZeroed;
   }
 };
 
diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt
index b5a23291f8a2..8c20f29903aa 100644
--- a/lib/CodeGen/CMakeLists.txt
+++ b/lib/CodeGen/CMakeLists.txt
@@ -1,4 +1,11 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_LINK_COMPONENTS
+  asmparser
+  bitreader
+  bitwriter
+  ipo
+  )
+
+set(LLVM_USED_LIBS clangBasic clangAST clangFrontend)
 
 add_clang_library(clangCodeGen
   BackendUtil.cpp
@@ -7,6 +14,8 @@ add_clang_library(clangCodeGen
   CGCall.cpp
   CGClass.cpp
   CGCXX.cpp
+  CGCXXABI.cpp
+  CGCleanup.cpp
   CGDebugInfo.cpp
   CGDecl.cpp
   CGDeclCXX.cpp
@@ -29,9 +38,9 @@ add_clang_library(clangCodeGen
   CodeGenAction.cpp
   CodeGenFunction.cpp
   CodeGenModule.cpp
+  CodeGenTBAA.cpp
   CodeGenTypes.cpp
   ItaniumCXXABI.cpp
-  Mangle.cpp
   MicrosoftCXXABI.cpp
   ModuleBuilder.cpp
   TargetInfo.cpp
diff --git a/lib/CodeGen/CodeGenAction.cpp b/lib/CodeGen/CodeGenAction.cpp
index 51c55a1a8390..a24bbc480c19 100644
--- a/lib/CodeGen/CodeGenAction.cpp
+++ b/lib/CodeGen/CodeGenAction.cpp
@@ -28,7 +28,7 @@
 using namespace clang;
 using namespace llvm;
 
-namespace {
+namespace clang {
   class BackendConsumer : public ASTConsumer {
     Diagnostic &Diags;
     BackendAction Action;
@@ -121,10 +121,10 @@ namespace {
       // Install an inline asm handler so that diagnostics get printed through
       // our diagnostics hooks.
       LLVMContext &Ctx = TheModule->getContext();
-      void *OldHandler = Ctx.getInlineAsmDiagnosticHandler();
+      LLVMContext::InlineAsmDiagHandlerTy OldHandler =
+        Ctx.getInlineAsmDiagnosticHandler();
       void *OldContext = Ctx.getInlineAsmDiagnosticContext();
-      Ctx.setInlineAsmDiagnosticHandler((void*)(intptr_t)InlineAsmDiagHandler,
-                                        this);
+      Ctx.setInlineAsmDiagnosticHandler(InlineAsmDiagHandler, this);
 
       EmitBackendOutput(Diags, CodeGenOpts, TargetOpts,
                         TheModule.get(), Action, AsmOutStream);
@@ -209,8 +209,7 @@ void BackendConsumer::InlineAsmDiagHandler2(const llvm::SMDiagnostic &D,
   // issue as being an error in the source with a note showing the instantiated
   // code.
   if (LocCookie.isValid()) {
-    Diags.Report(FullSourceLoc(LocCookie, Context->getSourceManager()),
-                 diag::err_fe_inline_asm).AddString(Message);
+    Diags.Report(LocCookie, diag::err_fe_inline_asm).AddString(Message);
     
     if (D.getLoc().isValid())
       Diags.Report(Loc, diag::note_fe_inline_asm_here);
@@ -225,9 +224,15 @@ void BackendConsumer::InlineAsmDiagHandler2(const llvm::SMDiagnostic &D,
 
 //
 
-CodeGenAction::CodeGenAction(unsigned _Act) : Act(_Act) {}
+CodeGenAction::CodeGenAction(unsigned _Act, LLVMContext *_VMContext)
+  : Act(_Act), VMContext(_VMContext ? _VMContext : new LLVMContext),
+    OwnsVMContext(!_VMContext) {}
 
-CodeGenAction::~CodeGenAction() {}
+CodeGenAction::~CodeGenAction() {
+  TheModule.reset();
+  if (OwnsVMContext)
+    delete VMContext;
+}
 
 bool CodeGenAction::hasIRSupport() const { return true; }
 
@@ -237,16 +242,18 @@ void CodeGenAction::EndSourceFileAction() {
     return;
 
   // Steal the module from the consumer.
-  BackendConsumer *Consumer = static_cast<BackendConsumer*>(
-    &getCompilerInstance().getASTConsumer());
-
-  TheModule.reset(Consumer->takeModule());
+  TheModule.reset(BEConsumer->takeModule());
 }
 
 llvm::Module *CodeGenAction::takeModule() {
   return TheModule.take();
 }
 
+llvm::LLVMContext *CodeGenAction::takeLLVMContext() {
+  OwnsVMContext = false;
+  return VMContext;
+}
+
 static raw_ostream *GetOutputStream(CompilerInstance &CI,
                                     llvm::StringRef InFile,
                                     BackendAction Action) {
@@ -275,10 +282,12 @@ ASTConsumer *CodeGenAction::CreateASTConsumer(CompilerInstance &CI,
   if (BA != Backend_EmitNothing && !OS)
     return 0;
 
-  return new BackendConsumer(BA, CI.getDiagnostics(),
-                             CI.getCodeGenOpts(), CI.getTargetOpts(),
-                             CI.getFrontendOpts().ShowTimers, InFile, OS.take(),
-                             CI.getLLVMContext());
+  BEConsumer = 
+      new BackendConsumer(BA, CI.getDiagnostics(),
+                          CI.getCodeGenOpts(), CI.getTargetOpts(),
+                          CI.getFrontendOpts().ShowTimers, InFile, OS.take(),
+                          *VMContext);
+  return BEConsumer;
 }
 
 void CodeGenAction::ExecuteAction() {
@@ -303,7 +312,7 @@ void CodeGenAction::ExecuteAction() {
                                            getCurrentFile().c_str());
 
     llvm::SMDiagnostic Err;
-    TheModule.reset(ParseIR(MainFileCopy, Err, CI.getLLVMContext()));
+    TheModule.reset(ParseIR(MainFileCopy, Err, *VMContext));
     if (!TheModule) {
       // Translate from the diagnostic info to the SourceManager location.
       SourceLocation Loc = SM.getLocation(
@@ -318,7 +327,7 @@ void CodeGenAction::ExecuteAction() {
       unsigned DiagID = CI.getDiagnostics().getCustomDiagID(Diagnostic::Error,
                                                             Msg);
 
-      CI.getDiagnostics().Report(FullSourceLoc(Loc, SM), DiagID);
+      CI.getDiagnostics().Report(Loc, DiagID);
       return;
     }
 
@@ -334,15 +343,20 @@ void CodeGenAction::ExecuteAction() {
 
 //
 
-EmitAssemblyAction::EmitAssemblyAction()
-  : CodeGenAction(Backend_EmitAssembly) {}
+EmitAssemblyAction::EmitAssemblyAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitAssembly, _VMContext) {}
 
-EmitBCAction::EmitBCAction() : CodeGenAction(Backend_EmitBC) {}
+EmitBCAction::EmitBCAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitBC, _VMContext) {}
 
-EmitLLVMAction::EmitLLVMAction() : CodeGenAction(Backend_EmitLL) {}
+EmitLLVMAction::EmitLLVMAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitLL, _VMContext) {}
 
-EmitLLVMOnlyAction::EmitLLVMOnlyAction() : CodeGenAction(Backend_EmitNothing) {}
+EmitLLVMOnlyAction::EmitLLVMOnlyAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitNothing, _VMContext) {}
 
-EmitCodeGenOnlyAction::EmitCodeGenOnlyAction() : CodeGenAction(Backend_EmitMCNull) {}
+EmitCodeGenOnlyAction::EmitCodeGenOnlyAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitMCNull, _VMContext) {}
 
-EmitObjAction::EmitObjAction() : CodeGenAction(Backend_EmitObj) {}
+EmitObjAction::EmitObjAction(llvm::LLVMContext *_VMContext)
+  : CodeGenAction(Backend_EmitObj, _VMContext) {}
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index 51d084e1d301..f1b72863caca 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -29,25 +29,17 @@ using namespace clang;
 using namespace CodeGen;
 
 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
-  : BlockFunction(cgm, *this, Builder), CGM(cgm),
-    Target(CGM.getContext().Target),
-    Builder(cgm.getModule().getContext()),
+  : CodeGenTypeCache(cgm), CGM(cgm),
+    Target(CGM.getContext().Target), Builder(cgm.getModule().getContext()),
+    BlockInfo(0), BlockPointer(0),
     NormalCleanupDest(0), EHCleanupDest(0), NextCleanupDestIndex(1),
     ExceptionSlot(0), DebugInfo(0), IndirectBranch(0),
     SwitchInsn(0), CaseRangeBlock(0),
     DidCallStackSave(false), UnreachableBlock(0),
     CXXThisDecl(0), CXXThisValue(0), CXXVTTDecl(0), CXXVTTValue(0),
-    ConditionalBranchLevel(0), TerminateLandingPad(0), TerminateHandler(0),
+    OutermostConditional(0), TerminateLandingPad(0), TerminateHandler(0),
     TrapBB(0) {
-      
-  // Get some frequently used types.
-  LLVMPointerWidth = Target.getPointerWidth(0);
-  llvm::LLVMContext &LLVMContext = CGM.getLLVMContext();
-  IntPtrTy = llvm::IntegerType::get(LLVMContext, LLVMPointerWidth);
-  Int32Ty  = llvm::Type::getInt32Ty(LLVMContext);
-  Int64Ty  = llvm::Type::getInt64Ty(LLVMContext);
-      
-  Exceptions = getContext().getLangOptions().Exceptions;
+
   CatchUndefined = getContext().getLangOptions().CatchUndefined;
   CGM.getCXXABI().getMangleContext().startNewFunction();
 }
@@ -125,7 +117,8 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   // Emit function epilog (to return).
   EmitReturnBlock();
 
-  EmitFunctionInstrumentation("__cyg_profile_func_exit");
+  if (ShouldInstrumentFunction())
+    EmitFunctionInstrumentation("__cyg_profile_func_exit");
 
   // Emit debug descriptor for function end.
   if (CGDebugInfo *DI = getDebugInfo()) {
@@ -184,20 +177,16 @@ bool CodeGenFunction::ShouldInstrumentFunction() {
 /// instrumentation function with the current function and the call site, if
 /// function instrumentation is enabled.
 void CodeGenFunction::EmitFunctionInstrumentation(const char *Fn) {
-  if (!ShouldInstrumentFunction())
-    return;
-
   const llvm::PointerType *PointerTy;
   const llvm::FunctionType *FunctionTy;
   std::vector<const llvm::Type*> ProfileFuncArgs;
 
   // void __cyg_profile_func_{enter,exit} (void *this_fn, void *call_site);
-  PointerTy = llvm::Type::getInt8PtrTy(VMContext);
+  PointerTy = Int8PtrTy;
   ProfileFuncArgs.push_back(PointerTy);
   ProfileFuncArgs.push_back(PointerTy);
-  FunctionTy = llvm::FunctionType::get(
-    llvm::Type::getVoidTy(VMContext),
-    ProfileFuncArgs, false);
+  FunctionTy = llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
+                                       ProfileFuncArgs, false);
 
   llvm::Constant *F = CGM.CreateRuntimeFunction(FunctionTy, Fn);
   llvm::CallInst *CallSite = Builder.CreateCall(
@@ -210,6 +199,15 @@ void CodeGenFunction::EmitFunctionInstrumentation(const char *Fn) {
                       CallSite);
 }
 
+void CodeGenFunction::EmitMCountInstrumentation() {
+  llvm::FunctionType *FTy =
+    llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()), false);
+
+  llvm::Constant *MCountFn = CGM.CreateRuntimeFunction(FTy,
+                                                       Target.getMCountName());
+  Builder.CreateCall(MCountFn);
+}
+
 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
                                     llvm::Function *Fn,
                                     const FunctionArgList &Args,
@@ -232,6 +230,19 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
         break;
       }
 
+  if (getContext().getLangOptions().OpenCL) {
+    // Add metadata for a kernel function.
+    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
+      if (FD->hasAttr<OpenCLKernelAttr>()) {
+        llvm::LLVMContext &Context = getLLVMContext();
+        llvm::NamedMDNode *OpenCLMetadata = 
+          CGM.getModule().getOrInsertNamedMetadata("opencl.kernels");
+          
+        llvm::Value *Op = Fn;
+        OpenCLMetadata->addOperand(llvm::MDNode::get(Context, &Op, 1));
+      }
+  }
+
   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
 
   // Create a marker to make it easy to insert allocas into the entryblock
@@ -246,18 +257,23 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
 
   Builder.SetInsertPoint(EntryBB);
 
-  QualType FnType = getContext().getFunctionType(RetTy, 0, 0, false, 0,
-                                                 false, false, 0, 0,
-                                                 /*FIXME?*/
-                                                 FunctionType::ExtInfo());
-
   // Emit subprogram debug descriptor.
   if (CGDebugInfo *DI = getDebugInfo()) {
+    // FIXME: what is going on here and why does it ignore all these
+    // interesting type properties?
+    QualType FnType =
+      getContext().getFunctionType(RetTy, 0, 0,
+                                   FunctionProtoType::ExtProtoInfo());
+
     DI->setLocation(StartLoc);
     DI->EmitFunctionStart(GD, FnType, CurFn, Builder);
   }
 
-  EmitFunctionInstrumentation("__cyg_profile_func_enter");
+  if (ShouldInstrumentFunction())
+    EmitFunctionInstrumentation("__cyg_profile_func_enter");
+
+  if (CGM.getCodeGenOpts().InstrumentForProfiling)
+    EmitMCountInstrumentation();
 
   // FIXME: Leaked.
   // CC info is ignored, hopefully?
@@ -384,8 +400,7 @@ bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
     IgnoreCaseStmts = true;
 
   // Scan subexpressions for verboten labels.
-  for (Stmt::const_child_iterator I = S->child_begin(), E = S->child_end();
-       I != E; ++I)
+  for (Stmt::const_child_range I = S->children(); I; ++I)
     if (ContainsLabel(*I, IgnoreCaseStmts))
       return true;
 
@@ -442,13 +457,15 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       // Emit the LHS as a conditional.  If the LHS conditional is false, we
       // want to jump to the FalseBlock.
       llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
+
+      ConditionalEvaluation eval(*this);
       EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock);
       EmitBlock(LHSTrue);
 
       // Any temporaries created here are conditional.
-      BeginConditionalBranch();
+      eval.begin(*this);
       EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
-      EndConditionalBranch();
+      eval.end(*this);
 
       return;
     } else if (CondBOp->getOpcode() == BO_LOr) {
@@ -469,13 +486,15 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       // Emit the LHS as a conditional.  If the LHS conditional is true, we
       // want to jump to the TrueBlock.
       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
+
+      ConditionalEvaluation eval(*this);
       EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse);
       EmitBlock(LHSFalse);
 
       // Any temporaries created here are conditional.
-      BeginConditionalBranch();
+      eval.begin(*this);
       EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
-      EndConditionalBranch();
+      eval.end(*this);
 
       return;
     }
@@ -495,11 +514,20 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
       llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
       llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
+
+      ConditionalEvaluation cond(*this);
       EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock);
+
+      cond.begin(*this);
       EmitBlock(LHSBlock);
       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock);
+      cond.end(*this);
+
+      cond.begin(*this);
       EmitBlock(RHSBlock);
       EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock);
+      cond.end(*this);
+
       return;
     }
   }
@@ -516,6 +544,57 @@ void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type,
   CGM.ErrorUnsupported(S, Type, OmitOnError);
 }
 
+/// emitNonZeroVLAInit - Emit the "zero" initialization of a
+/// variable-length array whose elements have a non-zero bit-pattern.
+///
+/// \param src - a char* pointing to the bit-pattern for a single
+/// base element of the array
+/// \param sizeInChars - the total size of the VLA, in chars
+/// \param align - the total alignment of the VLA
+static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
+                               llvm::Value *dest, llvm::Value *src, 
+                               llvm::Value *sizeInChars) {
+  std::pair<CharUnits,CharUnits> baseSizeAndAlign
+    = CGF.getContext().getTypeInfoInChars(baseType);
+
+  CGBuilderTy &Builder = CGF.Builder;
+
+  llvm::Value *baseSizeInChars
+    = llvm::ConstantInt::get(CGF.IntPtrTy, baseSizeAndAlign.first.getQuantity());
+
+  const llvm::Type *i8p = Builder.getInt8PtrTy();
+
+  llvm::Value *begin = Builder.CreateBitCast(dest, i8p, "vla.begin");
+  llvm::Value *end = Builder.CreateInBoundsGEP(dest, sizeInChars, "vla.end");
+
+  llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
+  llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
+  llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
+
+  // Make a loop over the VLA.  C99 guarantees that the VLA element
+  // count must be nonzero.
+  CGF.EmitBlock(loopBB);
+
+  llvm::PHINode *cur = Builder.CreatePHI(i8p, "vla.cur");
+  cur->reserveOperandSpace(2);
+  cur->addIncoming(begin, originBB);
+
+  // memcpy the individual element bit-pattern.
+  Builder.CreateMemCpy(cur, src, baseSizeInChars,
+                       baseSizeAndAlign.second.getQuantity(),
+                       /*volatile*/ false);
+
+  // Go to the next element.
+  llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(cur, 1, "vla.next");
+
+  // Leave if that's the end of the VLA.
+  llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
+  Builder.CreateCondBr(done, contBB, loopBB);
+  cur->addIncoming(next, loopBB);
+
+  CGF.EmitBlock(contBB);
+} 
+
 void
 CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
   // Ignore empty classes in C++.
@@ -529,26 +608,42 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
   // Cast the dest ptr to the appropriate i8 pointer type.
   unsigned DestAS =
     cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
-  const llvm::Type *BP =
-    llvm::Type::getInt8PtrTy(VMContext, DestAS);
+  const llvm::Type *BP = Builder.getInt8PtrTy(DestAS);
   if (DestPtr->getType() != BP)
     DestPtr = Builder.CreateBitCast(DestPtr, BP, "tmp");
 
   // Get size and alignment info for this aggregate.
   std::pair<uint64_t, unsigned> TypeInfo = getContext().getTypeInfo(Ty);
-  uint64_t Size = TypeInfo.first;
-  unsigned Align = TypeInfo.second;
+  uint64_t Size = TypeInfo.first / 8;
+  unsigned Align = TypeInfo.second / 8;
 
-  // Don't bother emitting a zero-byte memset.
-  if (Size == 0)
-    return;
+  llvm::Value *SizeVal;
+  const VariableArrayType *vla;
 
-  llvm::ConstantInt *SizeVal = llvm::ConstantInt::get(IntPtrTy, Size / 8);
-  llvm::ConstantInt *AlignVal = Builder.getInt32(Align / 8);
+  // Don't bother emitting a zero-byte memset.
+  if (Size == 0) {
+    // But note that getTypeInfo returns 0 for a VLA.
+    if (const VariableArrayType *vlaType =
+          dyn_cast_or_null<VariableArrayType>(
+                                          getContext().getAsArrayType(Ty))) {
+      SizeVal = GetVLASize(vlaType);
+      vla = vlaType;
+    } else {
+      return;
+    }
+  } else {
+    SizeVal = llvm::ConstantInt::get(IntPtrTy, Size);
+    vla = 0;
+  }
 
   // If the type contains a pointer to data member we can't memset it to zero.
   // Instead, create a null constant and copy it to the destination.
+  // TODO: there are other patterns besides zero that we can usefully memset,
+  // like -1, which happens to be the pattern used by member-pointers.
   if (!CGM.getTypes().isZeroInitializable(Ty)) {
+    // For a VLA, emit a single element, then splat that over the VLA.
+    if (vla) Ty = getContext().getBaseElementType(vla);
+
     llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
 
     llvm::GlobalVariable *NullVariable = 
@@ -559,27 +654,20 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
     llvm::Value *SrcPtr =
       Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy());
 
-    // FIXME: variable-size types?
+    if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
 
     // Get and call the appropriate llvm.memcpy overload.
-    llvm::Constant *Memcpy =
-      CGM.getMemCpyFn(DestPtr->getType(), SrcPtr->getType(), IntPtrTy);
-    Builder.CreateCall5(Memcpy, DestPtr, SrcPtr, SizeVal, AlignVal,
-                        /*volatile*/ Builder.getFalse());
+    Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, Align, false);
     return;
   } 
   
   // Otherwise, just memset the whole thing to zero.  This is legal
   // because in LLVM, all default initializers (other than the ones we just
   // handled above) are guaranteed to have a bit pattern of all zeros.
-
-  // FIXME: Handle variable sized types.
-  Builder.CreateCall5(CGM.getMemSetFn(BP, IntPtrTy), DestPtr,
-                      Builder.getInt8(0),
-                      SizeVal, AlignVal, /*volatile*/ Builder.getFalse());
+  Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, Align, false);
 }
 
-llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelStmt *L) {
+llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
   // Make sure that there is a block for the indirect goto.
   if (IndirectBranch == 0)
     GetIndirectGotoBlock();
@@ -597,8 +685,6 @@ llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
   
   CGBuilderTy TmpBuilder(createBasicBlock("indirectgoto"));
   
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-
   // Create the PHI node that indirect gotos will add entries to.
   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, "indirect.goto.dest");
   
@@ -621,6 +707,9 @@ llvm::Value *CodeGenFunction::EmitVLASize(QualType Ty) {
   EnsureInsertPoint();
 
   if (const VariableArrayType *VAT = getContext().getAsVariableArrayType(Ty)) {
+    // unknown size indication requires no size computation.
+    if (!VAT->getSizeExpr())
+      return 0;
     llvm::Value *&SizeEntry = VLASizeMap[VAT->getSizeExpr()];
 
     if (!SizeEntry) {
@@ -649,6 +738,11 @@ llvm::Value *CodeGenFunction::EmitVLASize(QualType Ty) {
     return 0;
   }
 
+  if (const ParenType *PT = dyn_cast<ParenType>(Ty)) {
+    EmitVLASize(PT->getInnerType());
+    return 0;
+  }
+
   const PointerType *PT = Ty->getAs<PointerType>();
   assert(PT && "unknown VM type!");
   EmitVLASize(PT->getPointeeType());
@@ -656,686 +750,41 @@ llvm::Value *CodeGenFunction::EmitVLASize(QualType Ty) {
 }
 
 llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) {
-  if (CGM.getContext().getBuiltinVaListType()->isArrayType())
+  if (getContext().getBuiltinVaListType()->isArrayType())
     return EmitScalarExpr(E);
   return EmitLValue(E).getAddress();
 }
 
-/// Pops cleanup blocks until the given savepoint is reached.
-void CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old) {
-  assert(Old.isValid());
-
-  while (EHStack.stable_begin() != Old) {
-    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
-
-    // As long as Old strictly encloses the scope's enclosing normal
-    // cleanup, we're going to emit another normal cleanup which
-    // fallthrough can propagate through.
-    bool FallThroughIsBranchThrough =
-      Old.strictlyEncloses(Scope.getEnclosingNormalCleanup());
-
-    PopCleanupBlock(FallThroughIsBranchThrough);
-  }
-}
-
-static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
-                                           EHCleanupScope &Scope) {
-  assert(Scope.isNormalCleanup());
-  llvm::BasicBlock *Entry = Scope.getNormalBlock();
-  if (!Entry) {
-    Entry = CGF.createBasicBlock("cleanup");
-    Scope.setNormalBlock(Entry);
-  }
-  return Entry;
-}
-
-static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
-                                       EHCleanupScope &Scope) {
-  assert(Scope.isEHCleanup());
-  llvm::BasicBlock *Entry = Scope.getEHBlock();
-  if (!Entry) {
-    Entry = CGF.createBasicBlock("eh.cleanup");
-    Scope.setEHBlock(Entry);
-  }
-  return Entry;
-}
-
-/// Transitions the terminator of the given exit-block of a cleanup to
-/// be a cleanup switch.
-static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
-                                                   llvm::BasicBlock *Block) {
-  // If it's a branch, turn it into a switch whose default
-  // destination is its original target.
-  llvm::TerminatorInst *Term = Block->getTerminator();
-  assert(Term && "can't transition block without terminator");
-
-  if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
-    assert(Br->isUnconditional());
-    llvm::LoadInst *Load =
-      new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term);
-    llvm::SwitchInst *Switch =
-      llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
-    Br->eraseFromParent();
-    return Switch;
-  } else {
-    return cast<llvm::SwitchInst>(Term);
-  }
-}
-
-/// Attempts to reduce a cleanup's entry block to a fallthrough.  This
-/// is basically llvm::MergeBlockIntoPredecessor, except
-/// simplified/optimized for the tighter constraints on cleanup blocks.
-///
-/// Returns the new block, whatever it is.
-static llvm::BasicBlock *SimplifyCleanupEntry(CodeGenFunction &CGF,
-                                              llvm::BasicBlock *Entry) {
-  llvm::BasicBlock *Pred = Entry->getSinglePredecessor();
-  if (!Pred) return Entry;
-
-  llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Pred->getTerminator());
-  if (!Br || Br->isConditional()) return Entry;
-  assert(Br->getSuccessor(0) == Entry);
-
-  // If we were previously inserting at the end of the cleanup entry
-  // block, we'll need to continue inserting at the end of the
-  // predecessor.
-  bool WasInsertBlock = CGF.Builder.GetInsertBlock() == Entry;
-  assert(!WasInsertBlock || CGF.Builder.GetInsertPoint() == Entry->end());
-
-  // Kill the branch.
-  Br->eraseFromParent();
-
-  // Merge the blocks.
-  Pred->getInstList().splice(Pred->end(), Entry->getInstList());
-
-  // Kill the entry block.
-  Entry->eraseFromParent();
-
-  if (WasInsertBlock)
-    CGF.Builder.SetInsertPoint(Pred);
-
-  return Pred;
-}
-
-static void EmitCleanup(CodeGenFunction &CGF,
-                        EHScopeStack::Cleanup *Fn,
-                        bool ForEH) {
-  if (ForEH) CGF.EHStack.pushTerminate();
-  Fn->Emit(CGF, ForEH);
-  if (ForEH) CGF.EHStack.popTerminate();
-  assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
-}
-
-/// Pops a cleanup block.  If the block includes a normal cleanup, the
-/// current insertion point is threaded through the cleanup, as are
-/// any branch fixups on the cleanup.
-void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
-  assert(!EHStack.empty() && "cleanup stack is empty!");
-  assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
-  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
-  assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
-  assert(Scope.isActive() && "cleanup was still inactive when popped!");
-
-  // Check whether we need an EH cleanup.  This is only true if we've
-  // generated a lazy EH cleanup block.
-  bool RequiresEHCleanup = Scope.hasEHBranches();
-
-  // Check the three conditions which might require a normal cleanup:
-
-  // - whether there are branch fix-ups through this cleanup
-  unsigned FixupDepth = Scope.getFixupDepth();
-  bool HasFixups = EHStack.getNumBranchFixups() != FixupDepth;
-
-  // - whether there are branch-throughs or branch-afters
-  bool HasExistingBranches = Scope.hasBranches();
-
-  // - whether there's a fallthrough
-  llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock();
-  bool HasFallthrough = (FallthroughSource != 0);
-
-  bool RequiresNormalCleanup = false;
-  if (Scope.isNormalCleanup() &&
-      (HasFixups || HasExistingBranches || HasFallthrough)) {
-    RequiresNormalCleanup = true;
-  }
-
-  // If we don't need the cleanup at all, we're done.
-  if (!RequiresNormalCleanup && !RequiresEHCleanup) {
-    EHStack.popCleanup(); // safe because there are no fixups
-    assert(EHStack.getNumBranchFixups() == 0 ||
-           EHStack.hasNormalCleanups());
-    return;
-  }
-
-  // Copy the cleanup emission data out.  Note that SmallVector
-  // guarantees maximal alignment for its buffer regardless of its
-  // type parameter.
-  llvm::SmallVector<char, 8*sizeof(void*)> CleanupBuffer;
-  CleanupBuffer.reserve(Scope.getCleanupSize());
-  memcpy(CleanupBuffer.data(),
-         Scope.getCleanupBuffer(), Scope.getCleanupSize());
-  CleanupBuffer.set_size(Scope.getCleanupSize());
-  EHScopeStack::Cleanup *Fn =
-    reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
-
-  // We want to emit the EH cleanup after the normal cleanup, but go
-  // ahead and do the setup for the EH cleanup while the scope is still
-  // alive.
-  llvm::BasicBlock *EHEntry = 0;
-  llvm::SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
-  if (RequiresEHCleanup) {
-    EHEntry = CreateEHEntry(*this, Scope);
-
-    // Figure out the branch-through dest if necessary.
-    llvm::BasicBlock *EHBranchThroughDest = 0;
-    if (Scope.hasEHBranchThroughs()) {
-      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
-      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
-      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
-    }
-
-    // If we have exactly one branch-after and no branch-throughs, we
-    // can dispatch it without a switch.
-    if (!Scope.hasEHBranchThroughs() &&
-        Scope.getNumEHBranchAfters() == 1) {
-      assert(!EHBranchThroughDest);
-
-      // TODO: remove the spurious eh.cleanup.dest stores if this edge
-      // never went through any switches.
-      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
-    
-    // Otherwise, if we have any branch-afters, we need a switch.
-    } else if (Scope.getNumEHBranchAfters()) {
-      // The default of the switch belongs to the branch-throughs if
-      // they exist.
-      llvm::BasicBlock *Default =
-        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
-
-      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
-
-      llvm::LoadInst *Load =
-        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
-      llvm::SwitchInst *Switch =
-        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
-
-      EHInstsToAppend.push_back(Load);
-      EHInstsToAppend.push_back(Switch);
-
-      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
-        Switch->addCase(Scope.getEHBranchAfterIndex(I),
-                        Scope.getEHBranchAfterBlock(I));
-
-    // Otherwise, we have only branch-throughs; jump to the next EH
-    // cleanup.
-    } else {
-      assert(EHBranchThroughDest);
-      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
-    }
-  }
-
-  if (!RequiresNormalCleanup) {
-    EHStack.popCleanup();
-  } else {
-    // As a kindof crazy internal case, branch-through fall-throughs
-    // leave the insertion point set to the end of the last cleanup.
-    bool HasPrebranchedFallthrough =
-      (HasFallthrough && FallthroughSource->getTerminator());
-    assert(!HasPrebranchedFallthrough ||
-           FallthroughSource->getTerminator()->getSuccessor(0)
-             == Scope.getNormalBlock());
-
-    // If we have a fallthrough and no other need for the cleanup,
-    // emit it directly.
-    if (HasFallthrough && !HasPrebranchedFallthrough &&
-        !HasFixups && !HasExistingBranches) {
-
-      // Fixups can cause us to optimistically create a normal block,
-      // only to later have no real uses for it.  Just delete it in
-      // this case.
-      // TODO: we can potentially simplify all the uses after this.
-      if (Scope.getNormalBlock()) {
-        Scope.getNormalBlock()->replaceAllUsesWith(getUnreachableBlock());
-        delete Scope.getNormalBlock();
-      }
-
-      EHStack.popCleanup();
-
-      EmitCleanup(*this, Fn, /*ForEH*/ false);
-
-    // Otherwise, the best approach is to thread everything through
-    // the cleanup block and then try to clean up after ourselves.
-    } else {
-      // Force the entry block to exist.
-      llvm::BasicBlock *NormalEntry = CreateNormalEntry(*this, Scope);
-
-      // If there's a fallthrough, we need to store the cleanup
-      // destination index.  For fall-throughs this is always zero.
-      if (HasFallthrough && !HasPrebranchedFallthrough)
-        Builder.CreateStore(Builder.getInt32(0), getNormalCleanupDestSlot());
-
-      // Emit the entry block.  This implicitly branches to it if we
-      // have fallthrough.  All the fixups and existing branches should
-      // already be branched to it.
-      EmitBlock(NormalEntry);
-
-      bool HasEnclosingCleanups =
-        (Scope.getEnclosingNormalCleanup() != EHStack.stable_end());
-
-      // Compute the branch-through dest if we need it:
-      //   - if there are branch-throughs threaded through the scope
-      //   - if fall-through is a branch-through
-      //   - if there are fixups that will be optimistically forwarded
-      //     to the enclosing cleanup
-      llvm::BasicBlock *BranchThroughDest = 0;
-      if (Scope.hasBranchThroughs() ||
-          (HasFallthrough && FallthroughIsBranchThrough) ||
-          (HasFixups && HasEnclosingCleanups)) {
-        assert(HasEnclosingCleanups);
-        EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
-        BranchThroughDest = CreateNormalEntry(*this, cast<EHCleanupScope>(S));
-      }
-
-      llvm::BasicBlock *FallthroughDest = 0;
-      llvm::SmallVector<llvm::Instruction*, 2> InstsToAppend;
-
-      // If there's exactly one branch-after and no other threads,
-      // we can route it without a switch.
-      if (!Scope.hasBranchThroughs() && !HasFixups && !HasFallthrough &&
-          Scope.getNumBranchAfters() == 1) {
-        assert(!BranchThroughDest);
-
-        // TODO: clean up the possibly dead stores to the cleanup dest slot.
-        llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
-        InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter));
-
-      // Build a switch-out if we need it:
-      //   - if there are branch-afters threaded through the scope
-      //   - if fall-through is a branch-after
-      //   - if there are fixups that have nowhere left to go and
-      //     so must be immediately resolved
-      } else if (Scope.getNumBranchAfters() ||
-                 (HasFallthrough && !FallthroughIsBranchThrough) ||
-                 (HasFixups && !HasEnclosingCleanups)) {
-
-        llvm::BasicBlock *Default =
-          (BranchThroughDest ? BranchThroughDest : getUnreachableBlock());
-
-        // TODO: base this on the number of branch-afters and fixups
-        const unsigned SwitchCapacity = 10;
-
-        llvm::LoadInst *Load =
-          new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest");
-        llvm::SwitchInst *Switch =
-          llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
-
-        InstsToAppend.push_back(Load);
-        InstsToAppend.push_back(Switch);
-
-        // Branch-after fallthrough.
-        if (HasFallthrough && !FallthroughIsBranchThrough) {
-          FallthroughDest = createBasicBlock("cleanup.cont");
-          Switch->addCase(Builder.getInt32(0), FallthroughDest);
-        }
-
-        for (unsigned I = 0, E = Scope.getNumBranchAfters(); I != E; ++I) {
-          Switch->addCase(Scope.getBranchAfterIndex(I),
-                          Scope.getBranchAfterBlock(I));
-        }
-
-        if (HasFixups && !HasEnclosingCleanups)
-          ResolveAllBranchFixups(Switch);
-      } else {
-        // We should always have a branch-through destination in this case.
-        assert(BranchThroughDest);
-        InstsToAppend.push_back(llvm::BranchInst::Create(BranchThroughDest));
-      }
-
-      // We're finally ready to pop the cleanup.
-      EHStack.popCleanup();
-      assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups);
-
-      EmitCleanup(*this, Fn, /*ForEH*/ false);
-
-      // Append the prepared cleanup prologue from above.
-      llvm::BasicBlock *NormalExit = Builder.GetInsertBlock();
-      for (unsigned I = 0, E = InstsToAppend.size(); I != E; ++I)
-        NormalExit->getInstList().push_back(InstsToAppend[I]);
-
-      // Optimistically hope that any fixups will continue falling through.
-      for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
-           I < E; ++I) {
-        BranchFixup &Fixup = CGF.EHStack.getBranchFixup(I);
-        if (!Fixup.Destination) continue;
-        if (!Fixup.OptimisticBranchBlock) {
-          new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex),
-                              getNormalCleanupDestSlot(),
-                              Fixup.InitialBranch);
-          Fixup.InitialBranch->setSuccessor(0, NormalEntry);
-        }
-        Fixup.OptimisticBranchBlock = NormalExit;
-      }
-      
-      if (FallthroughDest)
-        EmitBlock(FallthroughDest);
-      else if (!HasFallthrough)
-        Builder.ClearInsertionPoint();
-
-      // Check whether we can merge NormalEntry into a single predecessor.
-      // This might invalidate (non-IR) pointers to NormalEntry.
-      llvm::BasicBlock *NewNormalEntry =
-        SimplifyCleanupEntry(*this, NormalEntry);
-
-      // If it did invalidate those pointers, and NormalEntry was the same
-      // as NormalExit, go back and patch up the fixups.
-      if (NewNormalEntry != NormalEntry && NormalEntry == NormalExit)
-        for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
-               I < E; ++I)
-          CGF.EHStack.getBranchFixup(I).OptimisticBranchBlock = NewNormalEntry;
-    }
-  }
-
-  assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
-
-  // Emit the EH cleanup if required.
-  if (RequiresEHCleanup) {
-    CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
-
-    EmitBlock(EHEntry);
-    EmitCleanup(*this, Fn, /*ForEH*/ true);
-
-    // Append the prepared cleanup prologue from above.
-    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
-    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
-      EHExit->getInstList().push_back(EHInstsToAppend[I]);
-
-    Builder.restoreIP(SavedIP);
-
-    SimplifyCleanupEntry(*this, EHEntry);
-  }
-}
-
-/// Terminate the current block by emitting a branch which might leave
-/// the current cleanup-protected scope.  The target scope may not yet
-/// be known, in which case this will require a fixup.
-///
-/// As a side-effect, this method clears the insertion point.
-void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
-  assert(Dest.getScopeDepth().encloses(EHStack.getInnermostNormalCleanup())
-         && "stale jump destination");
-
-  if (!HaveInsertPoint())
-    return;
-
-  // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
-
-  // Calculate the innermost active normal cleanup.
-  EHScopeStack::stable_iterator
-    TopCleanup = EHStack.getInnermostActiveNormalCleanup();
-
-  // If we're not in an active normal cleanup scope, or if the
-  // destination scope is within the innermost active normal cleanup
-  // scope, we don't need to worry about fixups.
-  if (TopCleanup == EHStack.stable_end() ||
-      TopCleanup.encloses(Dest.getScopeDepth())) { // works for invalid
-    Builder.ClearInsertionPoint();
-    return;
-  }
-
-  // If we can't resolve the destination cleanup scope, just add this
-  // to the current cleanup scope as a branch fixup.
-  if (!Dest.getScopeDepth().isValid()) {
-    BranchFixup &Fixup = EHStack.addBranchFixup();
-    Fixup.Destination = Dest.getBlock();
-    Fixup.DestinationIndex = Dest.getDestIndex();
-    Fixup.InitialBranch = BI;
-    Fixup.OptimisticBranchBlock = 0;
-
-    Builder.ClearInsertionPoint();
-    return;
-  }
-
-  // Otherwise, thread through all the normal cleanups in scope.
-
-  // Store the index at the start.
-  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI);
-
-  // Adjust BI to point to the first cleanup block.
-  {
-    EHCleanupScope &Scope =
-      cast<EHCleanupScope>(*EHStack.find(TopCleanup));
-    BI->setSuccessor(0, CreateNormalEntry(*this, Scope));
-  }
-
-  // Add this destination to all the scopes involved.
-  EHScopeStack::stable_iterator I = TopCleanup;
-  EHScopeStack::stable_iterator E = Dest.getScopeDepth();
-  if (E.strictlyEncloses(I)) {
-    while (true) {
-      EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
-      assert(Scope.isNormalCleanup());
-      I = Scope.getEnclosingNormalCleanup();
-
-      // If this is the last cleanup we're propagating through, tell it
-      // that there's a resolved jump moving through it.
-      if (!E.strictlyEncloses(I)) {
-        Scope.addBranchAfter(Index, Dest.getBlock());
-        break;
-      }
-
-      // Otherwise, tell the scope that there's a jump propoagating
-      // through it.  If this isn't new information, all the rest of
-      // the work has been done before.
-      if (!Scope.addBranchThrough(Dest.getBlock()))
-        break;
-    }
-  }
-  
-  Builder.ClearInsertionPoint();
-}
-
-void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
-  // We should never get invalid scope depths for an UnwindDest; that
-  // implies that the destination wasn't set up correctly.
-  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
-
-  if (!HaveInsertPoint())
-    return;
-
-  // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
-
-  // Calculate the innermost active cleanup.
-  EHScopeStack::stable_iterator
-    InnermostCleanup = EHStack.getInnermostActiveEHCleanup();
-
-  // If the destination is in the same EH cleanup scope as us, we
-  // don't need to thread through anything.
-  if (InnermostCleanup.encloses(Dest.getScopeDepth())) {
-    Builder.ClearInsertionPoint();
-    return;
-  }
-  assert(InnermostCleanup != EHStack.stable_end());
-
-  // Store the index at the start.
-  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
-
-  // Adjust BI to point to the first cleanup block.
-  {
-    EHCleanupScope &Scope =
-      cast<EHCleanupScope>(*EHStack.find(InnermostCleanup));
-    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
-  }
-  
-  // Add this destination to all the scopes involved.
-  for (EHScopeStack::stable_iterator
-         I = InnermostCleanup, E = Dest.getScopeDepth(); ; ) {
-    assert(E.strictlyEncloses(I));
-    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
-    assert(Scope.isEHCleanup());
-    I = Scope.getEnclosingEHCleanup();
-
-    // If this is the last cleanup we're propagating through, add this
-    // as a branch-after.
-    if (I == E) {
-      Scope.addEHBranchAfter(Index, Dest.getBlock());
-      break;
-    }
-
-    // Otherwise, add it as a branch-through.  If this isn't new
-    // information, all the rest of the work has been done before.
-    if (!Scope.addEHBranchThrough(Dest.getBlock()))
-      break;
-  }
-  
-  Builder.ClearInsertionPoint();
-}
-
-/// All the branch fixups on the EH stack have propagated out past the
-/// outermost normal cleanup; resolve them all by adding cases to the
-/// given switch instruction.
-void CodeGenFunction::ResolveAllBranchFixups(llvm::SwitchInst *Switch) {
-  llvm::SmallPtrSet<llvm::BasicBlock*, 4> CasesAdded;
-
-  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
-    // Skip this fixup if its destination isn't set or if we've
-    // already treated it.
-    BranchFixup &Fixup = EHStack.getBranchFixup(I);
-    if (Fixup.Destination == 0) continue;
-    if (!CasesAdded.insert(Fixup.Destination)) continue;
-
-    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex),
-                    Fixup.Destination);
-  }
-
-  EHStack.clearFixups();
-}
-
-void CodeGenFunction::ResolveBranchFixups(llvm::BasicBlock *Block) {
-  assert(Block && "resolving a null target block");
-  if (!EHStack.getNumBranchFixups()) return;
-
-  assert(EHStack.hasNormalCleanups() &&
-         "branch fixups exist with no normal cleanups on stack");
-
-  llvm::SmallPtrSet<llvm::BasicBlock*, 4> ModifiedOptimisticBlocks;
-  bool ResolvedAny = false;
-
-  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
-    // Skip this fixup if its destination doesn't match.
-    BranchFixup &Fixup = EHStack.getBranchFixup(I);
-    if (Fixup.Destination != Block) continue;
-
-    Fixup.Destination = 0;
-    ResolvedAny = true;
-
-    // If it doesn't have an optimistic branch block, LatestBranch is
-    // already pointing to the right place.
-    llvm::BasicBlock *BranchBB = Fixup.OptimisticBranchBlock;
-    if (!BranchBB)
-      continue;
-
-    // Don't process the same optimistic branch block twice.
-    if (!ModifiedOptimisticBlocks.insert(BranchBB))
-      continue;
-
-    llvm::SwitchInst *Switch = TransitionToCleanupSwitch(*this, BranchBB);
-
-    // Add a case to the switch.
-    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex), Block);
-  }
-
-  if (ResolvedAny)
-    EHStack.popNullFixups();
+void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E, 
+                                              llvm::Constant *Init) {
+  assert (Init && "Invalid DeclRefExpr initializer!");
+  if (CGDebugInfo *Dbg = getDebugInfo())
+    Dbg->EmitGlobalVariable(E->getDecl(), Init);
 }
 
-/// Activate a cleanup that was created in an inactivated state.
-void CodeGenFunction::ActivateCleanup(EHScopeStack::stable_iterator C) {
-  assert(C != EHStack.stable_end() && "activating bottom of stack?");
-  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(C));
-  assert(!Scope.isActive() && "double activation");
-
-  // Calculate whether the cleanup was used:
-  bool Used = false;
+CodeGenFunction::PeepholeProtection
+CodeGenFunction::protectFromPeepholes(RValue rvalue) {
+  // At the moment, the only aggressive peephole we do in IR gen
+  // is trunc(zext) folding, but if we add more, we can easily
+  // extend this protection.
 
-  //   - as a normal cleanup
-  if (Scope.isNormalCleanup()) {
-    bool NormalUsed = false;
-    if (Scope.getNormalBlock()) {
-      NormalUsed = true;
-    } else {
-      // Check whether any enclosed cleanups were needed.
-      for (EHScopeStack::stable_iterator
-             I = EHStack.getInnermostNormalCleanup(); I != C; ) {
-        assert(C.strictlyEncloses(I));
-        EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
-        if (S.getNormalBlock()) {
-          NormalUsed = true;
-          break;
-        }
-        I = S.getEnclosingNormalCleanup();
-      }
-    }
+  if (!rvalue.isScalar()) return PeepholeProtection();
+  llvm::Value *value = rvalue.getScalarVal();
+  if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
 
-    if (NormalUsed)
-      Used = true;
-    else
-      Scope.setActivatedBeforeNormalUse(true);
-  }
+  // Just make an extra bitcast.
+  assert(HaveInsertPoint());
+  llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
+                                                  Builder.GetInsertBlock());
 
-  //  - as an EH cleanup
-  if (Scope.isEHCleanup()) {
-    bool EHUsed = false;
-    if (Scope.getEHBlock()) {
-      EHUsed = true;
-    } else {
-      // Check whether any enclosed cleanups were needed.
-      for (EHScopeStack::stable_iterator
-             I = EHStack.getInnermostEHCleanup(); I != C; ) {
-        assert(C.strictlyEncloses(I));
-        EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
-        if (S.getEHBlock()) {
-          EHUsed = true;
-          break;
-        }
-        I = S.getEnclosingEHCleanup();
-      }
-    }
-
-    if (EHUsed)
-      Used = true;
-    else
-      Scope.setActivatedBeforeEHUse(true);
-  }
-  
-  llvm::AllocaInst *Var = EHCleanupScope::activeSentinel();
-  if (Used) {
-    Var = CreateTempAlloca(Builder.getInt1Ty());
-    InitTempAlloca(Var, Builder.getFalse());
-  }
-  Scope.setActiveVar(Var);
+  PeepholeProtection protection;
+  protection.Inst = inst;
+  return protection;
 }
 
-llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
-  if (!NormalCleanupDest)
-    NormalCleanupDest =
-      CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
-  return NormalCleanupDest;
-}
+void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
+  if (!protection.Inst) return;
 
-llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
-  if (!EHCleanupDest)
-    EHCleanupDest =
-      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
-  return EHCleanupDest;
-}
-
-void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E, 
-                                              llvm::ConstantInt *Init) {
-  assert (Init && "Invalid DeclRefExpr initializer!");
-  if (CGDebugInfo *Dbg = getDebugInfo())
-    Dbg->EmitGlobalVariable(E->getDecl(), Init, Builder);
+  // In theory, we could try to duplicate the peepholes now, but whatever.
+  protection.Inst->eraseFromParent();
 }
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 4f0420536ad2..67ef41448e8d 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -18,15 +18,14 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/CharUnits.h"
+#include "clang/Basic/ABI.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ValueHandle.h"
 #include "CodeGenModule.h"
-#include "CGBlocks.h"
 #include "CGBuilder.h"
 #include "CGCall.h"
-#include "CGCXX.h"
 #include "CGValue.h"
 
 namespace llvm {
@@ -46,6 +45,7 @@ namespace clang {
   class CXXDestructorDecl;
   class CXXTryStmt;
   class Decl;
+  class LabelDecl;
   class EnumConstantDecl;
   class FunctionDecl;
   class FunctionProtoType;
@@ -71,6 +71,8 @@ namespace CodeGen {
   class CGRecordLayout;
   class CGBlockInfo;
   class CGCXXABI;
+  class BlockFlags;
+  class BlockFieldFlags;
 
 /// A branch fixup.  These are required when emitting a goto to a
 /// label which hasn't been emitted yet.  The goto is optimistically
@@ -97,6 +99,28 @@ struct BranchFixup {
   llvm::BranchInst *InitialBranch;
 };
 
+template <class T> struct InvariantValue {
+  typedef T type;
+  typedef T saved_type;
+  static bool needsSaving(type value) { return false; }
+  static saved_type save(CodeGenFunction &CGF, type value) { return value; }
+  static type restore(CodeGenFunction &CGF, saved_type value) { return value; }
+};
+
+/// A metaprogramming class for ensuring that a value will dominate an
+/// arbitrary position in a function.
+template <class T> struct DominatingValue : InvariantValue<T> {};
+
+template <class T, bool mightBeInstruction =
+            llvm::is_base_of<llvm::Value, T>::value &&
+            !llvm::is_base_of<llvm::Constant, T>::value &&
+            !llvm::is_base_of<llvm::BasicBlock, T>::value>
+struct DominatingPointer;
+template <class T> struct DominatingPointer<T,false> : InvariantValue<T*> {};
+// template <class T> struct DominatingPointer<T,true> at end of file
+
+template <class T> struct DominatingValue<T*> : DominatingPointer<T> {};
+
 enum CleanupKind {
   EHCleanup = 0x1,
   NormalCleanup = 0x2,
@@ -175,6 +199,63 @@ public:
     virtual void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) = 0;
   };
 
+  /// UnconditionalCleanupN stores its N parameters and just passes
+  /// them to the real cleanup function.
+  template <class T, class A0>
+  class UnconditionalCleanup1 : public Cleanup {
+    A0 a0;
+  public:
+    UnconditionalCleanup1(A0 a0) : a0(a0) {}
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      T::Emit(CGF, IsForEHCleanup, a0);
+    }
+  };
+
+  template <class T, class A0, class A1>
+  class UnconditionalCleanup2 : public Cleanup {
+    A0 a0; A1 a1;
+  public:
+    UnconditionalCleanup2(A0 a0, A1 a1) : a0(a0), a1(a1) {}
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      T::Emit(CGF, IsForEHCleanup, a0, a1);
+    }
+  };
+
+  /// ConditionalCleanupN stores the saved form of its N parameters,
+  /// then restores them and performs the cleanup.
+  template <class T, class A0>
+  class ConditionalCleanup1 : public Cleanup {
+    typedef typename DominatingValue<A0>::saved_type A0_saved;
+    A0_saved a0_saved;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved);
+      T::Emit(CGF, IsForEHCleanup, a0);
+    }
+
+  public:
+    ConditionalCleanup1(A0_saved a0)
+      : a0_saved(a0) {}
+  };
+
+  template <class T, class A0, class A1>
+  class ConditionalCleanup2 : public Cleanup {
+    typedef typename DominatingValue<A0>::saved_type A0_saved;
+    typedef typename DominatingValue<A1>::saved_type A1_saved;
+    A0_saved a0_saved;
+    A1_saved a1_saved;
+
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved);
+      A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved);
+      T::Emit(CGF, IsForEHCleanup, a0, a1);
+    }
+
+  public:
+    ConditionalCleanup2(A0_saved a0, A1_saved a1)
+      : a0_saved(a0), a1_saved(a1) {}
+  };
+
 private:
   // The implementation for this class is in CGException.h and
   // CGException.cpp; the definition is here because it's used as a
@@ -285,6 +366,25 @@ public:
     (void) Obj;
   }
 
+  // Feel free to add more variants of the following:
+
+  /// Push a cleanup with non-constant storage requirements on the
+  /// stack.  The cleanup type must provide an additional static method:
+  ///   static size_t getExtraSize(size_t);
+  /// The argument to this method will be the value N, which will also
+  /// be passed as the first argument to the constructor.
+  ///
+  /// The data stored in the extra storage must obey the same
+  /// restrictions as normal cleanup member data.
+  ///
+  /// The pointer returned from this method is valid until the cleanup
+  /// stack is modified.
+  template <class T, class A0, class A1, class A2>
+  T *pushCleanupWithExtra(CleanupKind Kind, size_t N, A0 a0, A1 a1, A2 a2) {
+    void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N));
+    return new (Buffer) T(N, a0, a1, a2);
+  }
+
   /// Pops a cleanup scope off the stack.  This should only be called
   /// by CodeGenFunction::PopCleanupBlock.
   void popCleanup();
@@ -395,7 +495,7 @@ public:
   void popNullFixups();
 
   /// Clears the branch-fixups list.  This should only be called by
-  /// CodeGenFunction::ResolveAllBranchFixups.
+  /// ResolveAllBranchFixups.
   void clearFixups() { BranchFixups.clear(); }
 
   /// Gets the next EH destination index.
@@ -404,7 +504,7 @@ public:
 
 /// CodeGenFunction - This class organizes the per-function state that is used
 /// while generating LLVM code.
-class CodeGenFunction : public BlockFunction {
+class CodeGenFunction : public CodeGenTypeCache {
   CodeGenFunction(const CodeGenFunction&); // DO NOT IMPLEMENT
   void operator=(const CodeGenFunction&);  // DO NOT IMPLEMENT
 
@@ -423,7 +523,7 @@ public:
     llvm::BasicBlock *getBlock() const { return Block; }
     EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
     unsigned getDestIndex() const { return Index; }
-    
+
   private:
     llvm::BasicBlock *Block;
     EHScopeStack::stable_iterator ScopeDepth;
@@ -482,13 +582,11 @@ public:
   /// we prefer to insert allocas.
   llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
 
-  // intptr_t, i32, i64
-  const llvm::IntegerType *IntPtrTy, *Int32Ty, *Int64Ty;
-  uint32_t LLVMPointerWidth;
-
-  bool Exceptions;
   bool CatchUndefined;
-  
+
+  const CodeGen::CGBlockInfo *BlockInfo;
+  llvm::Value *BlockPointer;
+
   /// \brief A mapping from NRVO variables to the flags used to indicate
   /// when the NRVO has been applied to this variable.
   llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;
@@ -510,6 +608,15 @@ public:
 
   llvm::BasicBlock *getInvokeDestImpl();
 
+  /// Set up the last cleaup that was pushed as a conditional
+  /// full-expression cleanup.
+  void initFullExprCleanup();
+
+  template <class T>
+  typename DominatingValue<T>::saved_type saveValueInCond(T value) {
+    return DominatingValue<T>::save(*this, value);
+  }
+
 public:
   /// ObjCEHValueStack - Stack of Objective-C exception values, used for
   /// rethrows.
@@ -526,6 +633,45 @@ public:
                                 llvm::Constant *RethrowFn);
   void ExitFinallyBlock(FinallyInfo &FinallyInfo);
 
+  /// pushFullExprCleanup - Push a cleanup to be run at the end of the
+  /// current full-expression.  Safe against the possibility that
+  /// we're currently inside a conditionally-evaluated expression.
+  template <class T, class A0>
+  void pushFullExprCleanup(CleanupKind kind, A0 a0) {
+    // If we're not in a conditional branch, or if none of the
+    // arguments requires saving, then use the unconditional cleanup.
+    if (!isInConditionalBranch()) {
+      typedef EHScopeStack::UnconditionalCleanup1<T, A0> CleanupType;
+      return EHStack.pushCleanup<CleanupType>(kind, a0);
+    }
+
+    typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0);
+
+    typedef EHScopeStack::ConditionalCleanup1<T, A0> CleanupType;
+    EHStack.pushCleanup<CleanupType>(kind, a0_saved);
+    initFullExprCleanup();
+  }
+
+  /// pushFullExprCleanup - Push a cleanup to be run at the end of the
+  /// current full-expression.  Safe against the possibility that
+  /// we're currently inside a conditionally-evaluated expression.
+  template <class T, class A0, class A1>
+  void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1) {
+    // If we're not in a conditional branch, or if none of the
+    // arguments requires saving, then use the unconditional cleanup.
+    if (!isInConditionalBranch()) {
+      typedef EHScopeStack::UnconditionalCleanup2<T, A0, A1> CleanupType;
+      return EHStack.pushCleanup<CleanupType>(kind, a0, a1);
+    }
+
+    typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0);
+    typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1);
+
+    typedef EHScopeStack::ConditionalCleanup2<T, A0, A1> CleanupType;
+    EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved);
+    initFullExprCleanup();
+  }
+
   /// PushDestructorCleanup - Push a cleanup to call the
   /// complete-object destructor of an object of the given type at the
   /// given address.  Does nothing if T is not a C++ class type with a
@@ -542,7 +688,14 @@ public:
   /// process all branch fixups.
   void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
 
-  void ActivateCleanup(EHScopeStack::stable_iterator Cleanup);
+  /// DeactivateCleanupBlock - Deactivates the given cleanup block.
+  /// The block cannot be reactivated.  Pops it if it's the top of the
+  /// stack.
+  void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup);
+
+  /// ActivateCleanupBlock - Activates an initially-inactive cleanup.
+  /// Cannot be used to resurrect a deactivated cleanup.
+  void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup);
 
   /// \brief Enters a new scope for capturing cleanups, all of which
   /// will be executed once the scope is exited.
@@ -557,11 +710,12 @@ public:
 
   public:
     /// \brief Enter a new cleanup scope.
-    explicit RunCleanupsScope(CodeGenFunction &CGF) 
-      : CGF(CGF), PerformCleanup(true) 
+    explicit RunCleanupsScope(CodeGenFunction &CGF)
+      : CGF(CGF), PerformCleanup(true)
     {
       CleanupStackDepth = CGF.EHStack.stable_begin();
       OldDidCallStackSave = CGF.DidCallStackSave;
+      CGF.DidCallStackSave = false;
     }
 
     /// \brief Exit this cleanup scope, emitting any accumulated
@@ -593,7 +747,6 @@ public:
   /// the cleanup blocks that have been added.
   void PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize);
 
-  void ResolveAllBranchFixups(llvm::SwitchInst *Switch);
   void ResolveBranchFixups(llvm::BasicBlock *Target);
 
   /// The given basic block lies in the current EH scope, but may be a
@@ -608,7 +761,7 @@ public:
   /// The given basic block lies in the current EH scope, but may be a
   /// target of a potentially scope-crossing jump; get a stable handle
   /// to which we can perform this jump later.
-  JumpDest getJumpDestInCurrentScope(const char *Name = 0) {
+  JumpDest getJumpDestInCurrentScope(llvm::StringRef Name = llvm::StringRef()) {
     return getJumpDestInCurrentScope(createBasicBlock(Name));
   }
 
@@ -626,27 +779,169 @@ public:
   /// destination.
   UnwindDest getRethrowDest();
 
-  /// BeginConditionalBranch - Should be called before a conditional part of an
-  /// expression is emitted. For example, before the RHS of the expression below
-  /// is emitted:
-  ///
-  /// b && f(T());
-  ///
-  /// This is used to make sure that any temporaries created in the conditional
-  /// branch are only destroyed if the branch is taken.
-  void BeginConditionalBranch() {
-    ++ConditionalBranchLevel;
-  }
+  /// An object to manage conditionally-evaluated expressions.
+  class ConditionalEvaluation {
+    llvm::BasicBlock *StartBB;
 
-  /// EndConditionalBranch - Should be called after a conditional part of an
-  /// expression has been emitted.
-  void EndConditionalBranch() {
-    assert(ConditionalBranchLevel != 0 &&
-           "Conditional branch mismatch!");
-    
-    --ConditionalBranchLevel;
-  }
+  public:
+    ConditionalEvaluation(CodeGenFunction &CGF)
+      : StartBB(CGF.Builder.GetInsertBlock()) {}
+
+    void begin(CodeGenFunction &CGF) {
+      assert(CGF.OutermostConditional != this);
+      if (!CGF.OutermostConditional)
+        CGF.OutermostConditional = this;
+    }
+
+    void end(CodeGenFunction &CGF) {
+      assert(CGF.OutermostConditional != 0);
+      if (CGF.OutermostConditional == this)
+        CGF.OutermostConditional = 0;
+    }
+
+    /// Returns a block which will be executed prior to each
+    /// evaluation of the conditional code.
+    llvm::BasicBlock *getStartingBlock() const {
+      return StartBB;
+    }
+  };
+
+  /// isInConditionalBranch - Return true if we're currently emitting
+  /// one branch or the other of a conditional expression.
+  bool isInConditionalBranch() const { return OutermostConditional != 0; }
+
+  /// An RAII object to record that we're evaluating a statement
+  /// expression.
+  class StmtExprEvaluation {
+    CodeGenFunction &CGF;
+
+    /// We have to save the outermost conditional: cleanups in a
+    /// statement expression aren't conditional just because the
+    /// StmtExpr is.
+    ConditionalEvaluation *SavedOutermostConditional;
+
+  public:
+    StmtExprEvaluation(CodeGenFunction &CGF)
+      : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {
+      CGF.OutermostConditional = 0;
+    }
+
+    ~StmtExprEvaluation() {
+      CGF.OutermostConditional = SavedOutermostConditional;
+      CGF.EnsureInsertPoint();
+    }
+  };
+
+  /// An object which temporarily prevents a value from being
+  /// destroyed by aggressive peephole optimizations that assume that
+  /// all uses of a value have been realized in the IR.
+  class PeepholeProtection {
+    llvm::Instruction *Inst;
+    friend class CodeGenFunction;
+
+  public:
+    PeepholeProtection() : Inst(0) {}
+  };  
+
+  /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
+  class OpaqueValueMapping {
+    CodeGenFunction &CGF;
+    const OpaqueValueExpr *OpaqueValue;
+    bool BoundLValue;
+    CodeGenFunction::PeepholeProtection Protection;
+
+  public:
+    static bool shouldBindAsLValue(const Expr *expr) {
+      return expr->isGLValue() || expr->getType()->isRecordType();
+    }
+
+    /// Build the opaque value mapping for the given conditional
+    /// operator if it's the GNU ?: extension.  This is a common
+    /// enough pattern that the convenience operator is really
+    /// helpful.
+    ///
+    OpaqueValueMapping(CodeGenFunction &CGF,
+                       const AbstractConditionalOperator *op) : CGF(CGF) {
+      if (isa<ConditionalOperator>(op)) {
+        OpaqueValue = 0;
+        BoundLValue = false;
+        return;
+      }
+
+      const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(op);
+      init(e->getOpaqueValue(), e->getCommon());
+    }
+
+    OpaqueValueMapping(CodeGenFunction &CGF,
+                       const OpaqueValueExpr *opaqueValue,
+                       LValue lvalue)
+      : CGF(CGF), OpaqueValue(opaqueValue), BoundLValue(true) {
+      assert(opaqueValue && "no opaque value expression!");
+      assert(shouldBindAsLValue(opaqueValue));
+      initLValue(lvalue);
+    }
+
+    OpaqueValueMapping(CodeGenFunction &CGF,
+                       const OpaqueValueExpr *opaqueValue,
+                       RValue rvalue)
+      : CGF(CGF), OpaqueValue(opaqueValue), BoundLValue(false) {
+      assert(opaqueValue && "no opaque value expression!");
+      assert(!shouldBindAsLValue(opaqueValue));
+      initRValue(rvalue);
+    }
+
+    void pop() {
+      assert(OpaqueValue && "mapping already popped!");
+      popImpl();
+      OpaqueValue = 0;
+    }
+
+    ~OpaqueValueMapping() {
+      if (OpaqueValue) popImpl();
+    }
+
+  private:
+    void popImpl() {
+      if (BoundLValue)
+        CGF.OpaqueLValues.erase(OpaqueValue);
+      else {
+        CGF.OpaqueRValues.erase(OpaqueValue);
+        CGF.unprotectFromPeepholes(Protection);
+      }
+    }
+
+    void init(const OpaqueValueExpr *ov, const Expr *e) {
+      OpaqueValue = ov;
+      BoundLValue = shouldBindAsLValue(ov);
+      assert(BoundLValue == shouldBindAsLValue(e)
+             && "inconsistent expression value kinds!");
+      if (BoundLValue)
+        initLValue(CGF.EmitLValue(e));
+      else
+        initRValue(CGF.EmitAnyExpr(e));
+    }
 
+    void initLValue(const LValue &lv) {
+      CGF.OpaqueLValues.insert(std::make_pair(OpaqueValue, lv));
+    }
+
+    void initRValue(const RValue &rv) {
+      // Work around an extremely aggressive peephole optimization in
+      // EmitScalarConversion which assumes that all other uses of a
+      // value are extant.
+      Protection = CGF.protectFromPeepholes(rv);
+      CGF.OpaqueRValues.insert(std::make_pair(OpaqueValue, rv));
+    }
+  };
+  
+  /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field
+  /// number that holds the value.
+  unsigned getByRefValueLLVMField(const ValueDecl *VD) const;
+
+  /// BuildBlockByrefAddress - Computes address location of the
+  /// variable which is declared as __block.
+  llvm::Value *BuildBlockByrefAddress(llvm::Value *BaseAddr,
+                                      const VarDecl *V);
 private:
   CGDebugInfo *DebugInfo;
 
@@ -658,10 +953,11 @@ private:
 
   /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C
   /// decls.
-  llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
+  typedef llvm::DenseMap<const Decl*, llvm::Value*> DeclMapTy;
+  DeclMapTy LocalDeclMap;
 
   /// LabelMap - This keeps track of the LLVM basic block for each C label.
-  llvm::DenseMap<const LabelStmt*, JumpDest> LabelMap;
+  llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap;
 
   // BreakContinueStack - This keeps track of where break and continue
   // statements should jump to.
@@ -682,6 +978,11 @@ private:
   /// statement range in current switch instruction.
   llvm::BasicBlock *CaseRangeBlock;
 
+  /// OpaqueLValues - Keeps track of the current set of opaque value
+  /// expressions.
+  llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;
+  llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;
+
   // VLASizeMap - This keeps track of the associated size for each VLA type.
   // We track this by the size expression rather than the type itself because
   // in certain situations, like a const qualifier applied to an VLA typedef,
@@ -708,21 +1009,17 @@ private:
   /// VTT parameter.
   ImplicitParamDecl *CXXVTTDecl;
   llvm::Value *CXXVTTValue;
-  
-  /// ConditionalBranchLevel - Contains the nesting level of the current
-  /// conditional branch. This is used so that we know if a temporary should be
-  /// destroyed conditionally.
-  unsigned ConditionalBranchLevel;
+
+  /// OutermostConditional - Points to the outermost active
+  /// conditional control.  This is used so that we know if a
+  /// temporary should be destroyed conditionally.
+  ConditionalEvaluation *OutermostConditional;
 
 
   /// ByrefValueInfoMap - For each __block variable, contains a pair of the LLVM
   /// type as well as the field number that contains the actual data.
-  llvm::DenseMap<const ValueDecl *, std::pair<const llvm::Type *, 
+  llvm::DenseMap<const ValueDecl *, std::pair<const llvm::Type *,
                                               unsigned> > ByRefValueInfo;
-  
-  /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field
-  /// number that holds the value.
-  unsigned getByRefValueLLVMField(const ValueDecl *VD) const;
 
   llvm::BasicBlock *TerminateLandingPad;
   llvm::BasicBlock *TerminateHandler;
@@ -735,6 +1032,8 @@ public:
   ASTContext &getContext() const;
   CGDebugInfo *getDebugInfo() { return DebugInfo; }
 
+  const LangOptions &getLangOptions() const { return CGM.getLangOptions(); }
+
   /// Returns a pointer to the function's exception object slot, which
   /// is assigned in every landing pad.
   llvm::Value *getExceptionSlot();
@@ -755,7 +1054,7 @@ public:
     return getInvokeDestImpl();
   }
 
-  llvm::LLVMContext &getLLVMContext() { return VMContext; }
+  llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }
 
   //===--------------------------------------------------------------------===//
   //                                  Objective-C
@@ -769,6 +1068,10 @@ public:
   /// GenerateObjCGetter - Synthesize an Objective-C property getter function.
   void GenerateObjCGetter(ObjCImplementationDecl *IMP,
                           const ObjCPropertyImplDecl *PID);
+  void GenerateObjCGetterBody(ObjCIvarDecl *Ivar, bool IsAtomic, bool IsStrong);
+  void GenerateObjCAtomicSetterBody(ObjCMethodDecl *OMD,
+                                    ObjCIvarDecl *Ivar);
+
   void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,
                                   ObjCMethodDecl *MD, bool ctor);
 
@@ -783,29 +1086,41 @@ public:
   //                                  Block Bits
   //===--------------------------------------------------------------------===//
 
-  llvm::Value *BuildBlockLiteralTmp(const BlockExpr *);
+  llvm::Value *EmitBlockLiteral(const BlockExpr *);
   llvm::Constant *BuildDescriptorBlockDecl(const BlockExpr *,
                                            const CGBlockInfo &Info,
                                            const llvm::StructType *,
-                                           llvm::Constant *BlockVarLayout,
-                                           std::vector<HelperInfo> *);
+                                           llvm::Constant *BlockVarLayout);
 
   llvm::Function *GenerateBlockFunction(GlobalDecl GD,
-                                        const BlockExpr *BExpr,
-                                        CGBlockInfo &Info,
+                                        const CGBlockInfo &Info,
                                         const Decl *OuterFuncDecl,
-                                        llvm::Constant *& BlockVarLayout,
-                                  llvm::DenseMap<const Decl*, llvm::Value*> ldm);
+                                        const DeclMapTy &ldm);
+
+  llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
+  llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
+
+  llvm::Constant *GeneratebyrefCopyHelperFunction(const llvm::Type *,
+                                                  BlockFieldFlags flags,
+                                                  const VarDecl *BD);
+  llvm::Constant *GeneratebyrefDestroyHelperFunction(const llvm::Type *T, 
+                                                     BlockFieldFlags flags, 
+                                                     const VarDecl *BD);
 
-  llvm::Value *LoadBlockStruct();
+  void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags);
+
+  llvm::Value *LoadBlockStruct() {
+    assert(BlockPointer && "no block pointer set!");
+    return BlockPointer;
+  }
 
   void AllocateBlockCXXThisPointer(const CXXThisExpr *E);
   void AllocateBlockDecl(const BlockDeclRefExpr *E);
   llvm::Value *GetAddrOfBlockDecl(const BlockDeclRefExpr *E) {
     return GetAddrOfBlockDecl(E->getDecl(), E->isByRef());
   }
-  llvm::Value *GetAddrOfBlockDecl(const ValueDecl *D, bool ByRef);
-  const llvm::Type *BuildByRefType(const ValueDecl *D);
+  llvm::Value *GetAddrOfBlockDecl(const VarDecl *var, bool ByRef);
+  const llvm::Type *BuildByRefType(const VarDecl *var);
 
   void GenerateCode(GlobalDecl GD, llvm::Function *Fn);
   void StartFunction(GlobalDecl GD, QualType RetTy,
@@ -827,21 +1142,21 @@ public:
 
   /// GenerateThunk - Generate a thunk for the given method.
   void GenerateThunk(llvm::Function *Fn, GlobalDecl GD, const ThunkInfo &Thunk);
-  
+
   void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,
                         FunctionArgList &Args);
 
   /// InitializeVTablePointer - Initialize the vtable pointer of the given
   /// subobject.
   ///
-  void InitializeVTablePointer(BaseSubobject Base, 
+  void InitializeVTablePointer(BaseSubobject Base,
                                const CXXRecordDecl *NearestVBase,
                                uint64_t OffsetFromNearestVBase,
                                llvm::Constant *VTable,
                                const CXXRecordDecl *VTableClass);
 
   typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;
-  void InitializeVTablePointers(BaseSubobject Base, 
+  void InitializeVTablePointers(BaseSubobject Base,
                                 const CXXRecordDecl *NearestVBase,
                                 uint64_t OffsetFromNearestVBase,
                                 bool BaseIsNonVirtualPrimaryBase,
@@ -851,6 +1166,9 @@ public:
 
   void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
 
+  /// GetVTablePtr - Return the Value of the vtable pointer member pointed
+  /// to by This.
+  llvm::Value *GetVTablePtr(llvm::Value *This, const llvm::Type *Ty);
 
   /// EnterDtorCleanups - Enter the cleanups necessary to complete the
   /// given phase of destruction for a destructor.  The end result
@@ -867,6 +1185,9 @@ public:
   /// function instrumentation is enabled.
   void EmitFunctionInstrumentation(const char *Fn);
 
+  /// EmitMCountInstrumentation - Emit call to .mcount.
+  void EmitMCountInstrumentation();
+
   /// EmitFunctionProlog - Emit the target specific LLVM code to load the
   /// arguments for the given function. This is also responsible for naming the
   /// LLVM function arguments.
@@ -910,19 +1231,19 @@ public:
   static bool hasAggregateLLVMType(QualType T);
 
   /// createBasicBlock - Create an LLVM basic block.
-  llvm::BasicBlock *createBasicBlock(const char *Name="",
-                                     llvm::Function *Parent=0,
-                                     llvm::BasicBlock *InsertBefore=0) {
+  llvm::BasicBlock *createBasicBlock(llvm::StringRef name = "",
+                                     llvm::Function *parent = 0,
+                                     llvm::BasicBlock *before = 0) {
 #ifdef NDEBUG
-    return llvm::BasicBlock::Create(VMContext, "", Parent, InsertBefore);
+    return llvm::BasicBlock::Create(getLLVMContext(), "", parent, before);
 #else
-    return llvm::BasicBlock::Create(VMContext, Name, Parent, InsertBefore);
+    return llvm::BasicBlock::Create(getLLVMContext(), name, parent, before);
 #endif
   }
 
   /// getBasicBlockForLabel - Return the LLVM basicblock that the specified
   /// label maps to.
-  JumpDest getJumpDestForLabel(const LabelStmt *S);
+  JumpDest getJumpDestForLabel(const LabelDecl *S);
 
   /// SimplifyForwardingBlocks - If the given basic block is only a branch to
   /// another basic block, simplify it. This assumes that no other code could
@@ -974,7 +1295,8 @@ public:
   //===--------------------------------------------------------------------===//
 
   LValue MakeAddrLValue(llvm::Value *V, QualType T, unsigned Alignment = 0) {
-    return LValue::MakeAddr(V, T, Alignment, getContext());
+    return LValue::MakeAddr(V, T, Alignment, getContext(),
+                            CGM.getTBAAInfo(T));
   }
 
   /// CreateTempAlloca - This creates a alloca and inserts it into the entry
@@ -997,19 +1319,31 @@ public:
   /// appropriate alignment.
   llvm::AllocaInst *CreateMemTemp(QualType T, const llvm::Twine &Name = "tmp");
 
+  /// CreateAggTemp - Create a temporary memory object for the given
+  /// aggregate type.
+  AggValueSlot CreateAggTemp(QualType T, const llvm::Twine &Name = "tmp") {
+    return AggValueSlot::forAddr(CreateMemTemp(T, Name), false, false);
+  }
+
+  /// Emit a cast to void* in the appropriate address space.
+  llvm::Value *EmitCastToVoidPtr(llvm::Value *value);
+
   /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
   /// expression and compare the result against zero, returning an Int1Ty value.
   llvm::Value *EvaluateExprAsBool(const Expr *E);
 
+  /// EmitIgnoredExpr - Emit an expression in a context which ignores the result.
+  void EmitIgnoredExpr(const Expr *E);
+
   /// EmitAnyExpr - Emit code to compute the specified expression which can have
   /// any type.  The result is returned as an RValue struct.  If this is an
   /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
   /// the result should be returned.
   ///
   /// \param IgnoreResult - True if the resulting value isn't used.
-  RValue EmitAnyExpr(const Expr *E, llvm::Value *AggLoc = 0,
-                     bool IsAggLocVolatile = false, bool IgnoreResult = false,
-                     bool IsInitializer = false);
+  RValue EmitAnyExpr(const Expr *E,
+                     AggValueSlot AggSlot = AggValueSlot::ignored(),
+                     bool IgnoreResult = false);
 
   // EmitVAListRef - Emit a "reference" to a va_list; this is either the address
   // or the value of the expression, depending on how va_list is defined.
@@ -1017,14 +1351,13 @@ public:
 
   /// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
   /// always be accessible even if no aggregate location is provided.
-  RValue EmitAnyExprToTemp(const Expr *E, bool IsAggLocVolatile = false,
-                           bool IsInitializer = false);
+  RValue EmitAnyExprToTemp(const Expr *E);
 
   /// EmitsAnyExprToMem - Emits the code necessary to evaluate an
   /// arbitrary expression into the given memory location.
   void EmitAnyExprToMem(const Expr *E, llvm::Value *Location,
-                        bool IsLocationVolatile = false,
-                        bool IsInitializer = false);
+                        bool IsLocationVolatile,
+                        bool IsInitializer);
 
   /// EmitAggregateCopy - Emit an aggrate copy.
   ///
@@ -1049,11 +1382,33 @@ public:
     return Res;
   }
 
+  /// getOpaqueLValueMapping - Given an opaque value expression (which
+  /// must be mapped to an l-value), return its mapping.
+  const LValue &getOpaqueLValueMapping(const OpaqueValueExpr *e) {
+    assert(OpaqueValueMapping::shouldBindAsLValue(e));
+
+    llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
+      it = OpaqueLValues.find(e);
+    assert(it != OpaqueLValues.end() && "no mapping for opaque value!");
+    return it->second;
+  }
+
+  /// getOpaqueRValueMapping - Given an opaque value expression (which
+  /// must be mapped to an r-value), return its mapping.
+  const RValue &getOpaqueRValueMapping(const OpaqueValueExpr *e) {
+    assert(!OpaqueValueMapping::shouldBindAsLValue(e));
+
+    llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
+      it = OpaqueRValues.find(e);
+    assert(it != OpaqueRValues.end() && "no mapping for opaque value!");
+    return it->second;
+  }
+
   /// getAccessedFieldNo - Given an encoded value and a result number, return
   /// the input field number being accessed.
   static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);
 
-  llvm::BlockAddress *GetAddrOfLabel(const LabelStmt *L);
+  llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);
   llvm::BasicBlock *GetIndirectGotoBlock();
 
   /// EmitNullInitialization - Generate code to set a value of the given type to
@@ -1102,7 +1457,7 @@ public:
 
   /// GetAddressOfBaseClass - This function will add the necessary delta to the
   /// load of 'this' and returns address of the base class.
-  llvm::Value *GetAddressOfBaseClass(llvm::Value *Value, 
+  llvm::Value *GetAddressOfBaseClass(llvm::Value *Value,
                                      const CXXRecordDecl *Derived,
                                      CastExpr::path_const_iterator PathBegin,
                                      CastExpr::path_const_iterator PathEnd,
@@ -1117,7 +1472,7 @@ public:
   llvm::Value *GetVirtualBaseClassOffset(llvm::Value *This,
                                          const CXXRecordDecl *ClassDecl,
                                          const CXXRecordDecl *BaseClassDecl);
-    
+
   void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
                                       CXXCtorType CtorType,
                                       const FunctionArgList &Args);
@@ -1125,6 +1480,11 @@ public:
                               bool ForVirtualBase, llvm::Value *This,
                               CallExpr::const_arg_iterator ArgBeg,
                               CallExpr::const_arg_iterator ArgEnd);
+  
+  void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
+                              llvm::Value *This, llvm::Value *Src,
+                              CallExpr::const_arg_iterator ArgBeg,
+                              CallExpr::const_arg_iterator ArgEnd);
 
   void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
                                   const ConstantArrayType *ArrayTy,
@@ -1132,7 +1492,7 @@ public:
                                   CallExpr::const_arg_iterator ArgBeg,
                                   CallExpr::const_arg_iterator ArgEnd,
                                   bool ZeroInitialization = false);
-  
+
   void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
                                   llvm::Value *NumElements,
                                   llvm::Value *ArrayPtr,
@@ -1154,7 +1514,7 @@ public:
 
   void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,
                              bool ForVirtualBase, llvm::Value *This);
-  
+
   void EmitNewArrayInitializer(const CXXNewExpr *E, llvm::Value *NewPtr,
                                llvm::Value *NumElements);
 
@@ -1184,25 +1544,37 @@ public:
   /// This function can be called with a null (unreachable) insert point.
   void EmitDecl(const Decl &D);
 
-  /// EmitBlockVarDecl - Emit a block variable declaration.
+  /// EmitVarDecl - Emit a local variable declaration.
   ///
   /// This function can be called with a null (unreachable) insert point.
-  void EmitBlockVarDecl(const VarDecl &D);
+  void EmitVarDecl(const VarDecl &D);
 
   typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
                              llvm::Value *Address);
 
-  /// EmitLocalBlockVarDecl - Emit a local block variable declaration.
+  /// EmitAutoVarDecl - Emit an auto variable declaration.
   ///
   /// This function can be called with a null (unreachable) insert point.
-  void EmitLocalBlockVarDecl(const VarDecl &D, SpecialInitFn *SpecialInit = 0);
+  void EmitAutoVarDecl(const VarDecl &D, SpecialInitFn *SpecialInit = 0);
 
-  void EmitStaticBlockVarDecl(const VarDecl &D,
-                              llvm::GlobalValue::LinkageTypes Linkage);
+  void EmitStaticVarDecl(const VarDecl &D,
+                         llvm::GlobalValue::LinkageTypes Linkage);
 
   /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.
   void EmitParmDecl(const VarDecl &D, llvm::Value *Arg);
 
+  /// protectFromPeepholes - Protect a value that we're intending to
+  /// store to the side, but which will probably be used later, from
+  /// aggressive peepholing optimizations that might delete it.
+  ///
+  /// Pass the result to unprotectFromPeepholes to declare that
+  /// protection is no longer required.
+  ///
+  /// There's no particular reason why this shouldn't apply to
+  /// l-values, it's just that no existing peepholes work on pointers.
+  PeepholeProtection protectFromPeepholes(RValue rvalue);
+  void unprotectFromPeepholes(PeepholeProtection protection);
+
   //===--------------------------------------------------------------------===//
   //                             Statement Emission
   //===--------------------------------------------------------------------===//
@@ -1227,11 +1599,11 @@ public:
   bool EmitSimpleStmt(const Stmt *S);
 
   RValue EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,
-                          llvm::Value *AggLoc = 0, bool isAggVol = false);
+                          AggValueSlot AVS = AggValueSlot::ignored());
 
   /// EmitLabel - Emit the block for the given label. It is legal to call this
   /// function even if there is no current insertion point.
-  void EmitLabel(const LabelStmt &S); // helper for EmitLabelStmt.
+  void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.
 
   void EmitLabelStmt(const LabelStmt &S);
   void EmitGotoStmt(const GotoStmt &S);
@@ -1260,7 +1632,7 @@ public:
   void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);
 
   void EmitCXXTryStmt(const CXXTryStmt &S);
-  
+
   //===--------------------------------------------------------------------===//
   //                         LValue Expression Emission
   //===--------------------------------------------------------------------===//
@@ -1303,17 +1675,27 @@ public:
   /// object.
   LValue EmitCheckedLValue(const Expr *E);
 
+  /// EmitToMemory - Change a scalar value from its value
+  /// representation to its in-memory representation.
+  llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
+
+  /// EmitFromMemory - Change a scalar value from its memory
+  /// representation to its value representation.
+  llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);
+
   /// EmitLoadOfScalar - Load a scalar value from an address, taking
   /// care to appropriately convert from the memory representation to
   /// the LLVM value representation.
   llvm::Value *EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
-                                unsigned Alignment, QualType Ty);
+                                unsigned Alignment, QualType Ty,
+                                llvm::MDNode *TBAAInfo = 0);
 
   /// EmitStoreOfScalar - Store a scalar value to an address, taking
   /// care to appropriately convert from the memory representation to
   /// the LLVM value representation.
   void EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
-                         bool Volatile, unsigned Alignment, QualType Ty);
+                         bool Volatile, unsigned Alignment, QualType Ty,
+                         llvm::MDNode *TBAAInfo = 0);
 
   /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
   /// this method emits the address of the lvalue, then loads the result as an
@@ -1321,9 +1703,8 @@ public:
   RValue EmitLoadOfLValue(LValue V, QualType LVType);
   RValue EmitLoadOfExtVectorElementLValue(LValue V, QualType LVType);
   RValue EmitLoadOfBitfieldLValue(LValue LV, QualType ExprType);
-  RValue EmitLoadOfPropertyRefLValue(LValue LV, QualType ExprType);
-  RValue EmitLoadOfKVCRefLValue(LValue LV, QualType ExprType);
-
+  RValue EmitLoadOfPropertyRefLValue(LValue LV,
+                                 ReturnValueSlot Return = ReturnValueSlot());
 
   /// EmitStoreThroughLValue - Store the specified rvalue into the specified
   /// lvalue, where both are guaranteed to the have the same type, and that type
@@ -1331,8 +1712,7 @@ public:
   void EmitStoreThroughLValue(RValue Src, LValue Dst, QualType Ty);
   void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst,
                                                 QualType Ty);
-  void EmitStoreThroughPropertyRefLValue(RValue Src, LValue Dst, QualType Ty);
-  void EmitStoreThroughKVCRefLValue(RValue Src, LValue Dst, QualType Ty);
+  void EmitStoreThroughPropertyRefLValue(RValue Src, LValue Dst);
 
   /// EmitStoreThroughLValue - Store Src into Dst with same constraints as
   /// EmitStoreThroughLValue.
@@ -1343,9 +1723,13 @@ public:
   void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, QualType Ty,
                                       llvm::Value **Result=0);
 
+  /// Emit an l-value for an assignment (simple or compound) of complex type.
+  LValue EmitComplexAssignmentLValue(const BinaryOperator *E);
+  LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);
+
   // Note: only availabe for agg return types
   LValue EmitBinaryOperatorLValue(const BinaryOperator *E);
-  LValue EmitCompoundAssignOperatorLValue(const CompoundAssignOperator *E);
+  LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);
   // Note: only available for agg return types
   LValue EmitCallExprLValue(const CallExpr *E);
   // Note: only available for agg return types
@@ -1360,25 +1744,26 @@ public:
   LValue EmitMemberExpr(const MemberExpr *E);
   LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);
   LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);
-  LValue EmitConditionalOperatorLValue(const ConditionalOperator *E);
+  LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);
   LValue EmitCastLValue(const CastExpr *E);
   LValue EmitNullInitializationLValue(const CXXScalarValueInitExpr *E);
-  
+  LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);
+
   llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,
                               const ObjCIvarDecl *Ivar);
   LValue EmitLValueForAnonRecordField(llvm::Value* Base,
-                                      const FieldDecl* Field,
+                                      const IndirectFieldDecl* Field,
                                       unsigned CVRQualifiers);
   LValue EmitLValueForField(llvm::Value* Base, const FieldDecl* Field,
                             unsigned CVRQualifiers);
-  
+
   /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that
   /// if the Field is a reference, this will return the address of the reference
   /// and not the address of the value stored in the reference.
-  LValue EmitLValueForFieldInitialization(llvm::Value* Base, 
+  LValue EmitLValueForFieldInitialization(llvm::Value* Base,
                                           const FieldDecl* Field,
                                           unsigned CVRQualifiers);
-  
+
   LValue EmitLValueForIvar(QualType ObjectTy,
                            llvm::Value* Base, const ObjCIvarDecl *Ivar,
                            unsigned CVRQualifiers);
@@ -1390,18 +1775,17 @@ public:
 
   LValue EmitCXXConstructLValue(const CXXConstructExpr *E);
   LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
-  LValue EmitCXXExprWithTemporariesLValue(const CXXExprWithTemporaries *E);
+  LValue EmitExprWithCleanupsLValue(const ExprWithCleanups *E);
   LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
-  
+
   LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
   LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
   LValue EmitObjCPropertyRefLValue(const ObjCPropertyRefExpr *E);
-  LValue EmitObjCKVCRefLValue(const ObjCImplicitSetterGetterRefExpr *E);
-  LValue EmitObjCSuperExprLValue(const ObjCSuperExpr *E);
   LValue EmitStmtExprLValue(const StmtExpr *E);
   LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
   LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
-  void   EmitDeclRefExprDbgValue(const DeclRefExpr *E, llvm::ConstantInt *Init);
+  void   EmitDeclRefExprDbgValue(const DeclRefExpr *E, llvm::Constant *Init);
+
   //===--------------------------------------------------------------------===//
   //                         Scalar Expression Emission
   //===--------------------------------------------------------------------===//
@@ -1424,7 +1808,7 @@ public:
                   CallExpr::const_arg_iterator ArgBeg,
                   CallExpr::const_arg_iterator ArgEnd,
                   const Decl *TargetDecl = 0);
-  RValue EmitCallExpr(const CallExpr *E, 
+  RValue EmitCallExpr(const CallExpr *E,
                       ReturnValueSlot ReturnValue = ReturnValueSlot());
 
   llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee,
@@ -1434,8 +1818,15 @@ public:
 
   llvm::Value *BuildVirtualCall(const CXXMethodDecl *MD, llvm::Value *This,
                                 const llvm::Type *Ty);
-  llvm::Value *BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type, 
-                                llvm::Value *&This, const llvm::Type *Ty);
+  llvm::Value *BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type,
+                                llvm::Value *This, const llvm::Type *Ty);
+  llvm::Value *BuildAppleKextVirtualCall(const CXXMethodDecl *MD, 
+                                         NestedNameSpecifier *Qual,
+                                         const llvm::Type *Ty);
+  
+  llvm::Value *BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
+                                                   CXXDtorType Type, 
+                                                   const CXXRecordDecl *RD);
 
   RValue EmitCXXMemberCall(const CXXMethodDecl *MD,
                            llvm::Value *Callee,
@@ -1453,7 +1844,7 @@ public:
                                        const CXXMethodDecl *MD,
                                        ReturnValueSlot ReturnValue);
 
-  
+
   RValue EmitBuiltinExpr(const FunctionDecl *FD,
                          unsigned BuiltinID, const CallExpr *E);
 
@@ -1464,15 +1855,15 @@ public:
   llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
   llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
-  llvm::Value *EmitNeonCall(llvm::Function *F, 
+  llvm::Value *EmitNeonCall(llvm::Function *F,
                             llvm::SmallVectorImpl<llvm::Value*> &O,
-                            const char *name, bool splat = false,
+                            const char *name,
                             unsigned shift = 0, bool rightshift = false);
-  llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
-                             bool widen = false);
+  llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
   llvm::Value *EmitNeonShiftVector(llvm::Value *V, const llvm::Type *Ty,
                                    bool negateForRightShift);
-  
+
+  llvm::Value *BuildVector(const llvm::SmallVectorImpl<llvm::Value*> &Ops);
   llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
@@ -1481,17 +1872,10 @@ public:
   llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);
   RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,
                              ReturnValueSlot Return = ReturnValueSlot());
-  RValue EmitObjCPropertyGet(const Expr *E,
-                             ReturnValueSlot Return = ReturnValueSlot());
-  RValue EmitObjCSuperPropertyGet(const Expr *Exp, const Selector &S,
-                                  ReturnValueSlot Return = ReturnValueSlot());
-  void EmitObjCPropertySet(const Expr *E, RValue Src);
-  void EmitObjCSuperPropertySet(const Expr *E, const Selector &S, RValue Src);
-
 
   /// EmitReferenceBindingToExpr - Emits a reference binding to the passed in
   /// expression. Will emit a temporary variable if E is not an LValue.
-  RValue EmitReferenceBindingToExpr(const Expr* E, 
+  RValue EmitReferenceBindingToExpr(const Expr* E,
                                     const NamedDecl *InitializedDecl);
 
   //===--------------------------------------------------------------------===//
@@ -1516,12 +1900,10 @@ public:
                                              QualType DstTy);
 
 
-  /// EmitAggExpr - Emit the computation of the specified expression of
-  /// aggregate type.  The result is computed into DestPtr.  Note that if
-  /// DestPtr is null, the value of the aggregate expression is not needed.
-  void EmitAggExpr(const Expr *E, llvm::Value *DestPtr, bool VolatileDest,
-                   bool IgnoreResult = false, bool IsInitializer = false,
-                   bool RequiresGCollection = false);
+  /// EmitAggExpr - Emit the computation of the specified expression
+  /// of aggregate type.  The result is computed into the given slot,
+  /// which may be null to indicate that the value is not needed.
+  void EmitAggExpr(const Expr *E, AggValueSlot AS, bool IgnoreResult = false);
 
   /// EmitAggExprToLValue - Emit the computation of the specified expression of
   /// aggregate type into a temporary LValue.
@@ -1534,10 +1916,9 @@ public:
 
   /// EmitComplexExpr - Emit the computation of the specified expression of
   /// complex type, returning the result.
-  ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,
-                                bool IgnoreImag = false,
-                                bool IgnoreRealAssign = false,
-                                bool IgnoreImagAssign = false);
+  ComplexPairTy EmitComplexExpr(const Expr *E,
+                                bool IgnoreReal = false,
+                                bool IgnoreImag = false);
 
   /// EmitComplexExprIntoAddr - Emit the computation of the specified expression
   /// of complex type, storing into the specified Value*.
@@ -1550,25 +1931,20 @@ public:
   /// LoadComplexFromAddr - Load a complex number from the specified address.
   ComplexPairTy LoadComplexFromAddr(llvm::Value *SrcAddr, bool SrcIsVolatile);
 
-  /// CreateStaticBlockVarDecl - Create a zero-initialized LLVM global for a
-  /// static block var decl.
-  llvm::GlobalVariable *CreateStaticBlockVarDecl(const VarDecl &D,
-                                                 const char *Separator,
+  /// CreateStaticVarDecl - Create a zero-initialized LLVM global for
+  /// a static local variable.
+  llvm::GlobalVariable *CreateStaticVarDecl(const VarDecl &D,
+                                            const char *Separator,
                                        llvm::GlobalValue::LinkageTypes Linkage);
-  
-  /// AddInitializerToGlobalBlockVarDecl - Add the initializer for 'D' to the
+
+  /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
   /// global variable that has already been created for it.  If the initializer
   /// has a different type than GV does, this may free GV and return a different
   /// one.  Otherwise it just returns GV.
   llvm::GlobalVariable *
-  AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
-                                     llvm::GlobalVariable *GV);
-  
+  AddInitializerToStaticVarDecl(const VarDecl &D,
+                                llvm::GlobalVariable *GV);
 
-  /// EmitStaticCXXBlockVarDeclInit - Create the initializer for a C++ runtime
-  /// initialized static block var decl.
-  void EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
-                                     llvm::GlobalVariable *GV);
 
   /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
   /// variable with global storage.
@@ -1579,6 +1955,13 @@ public:
   void EmitCXXGlobalDtorRegistration(llvm::Constant *DtorFn,
                                      llvm::Constant *DeclPtr);
 
+  /// Emit code in this function to perform a guarded variable
+  /// initialization.  Guarded initializations are used when it's not
+  /// possible to prove that an initialization will be done exactly
+  /// once, e.g. with a static local variable or a static data member
+  /// of a class template.
+  void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr);
+
   /// GenerateCXXGlobalInitFunc - Generates code for initializing global
   /// variables.
   void GenerateCXXGlobalInitFunc(llvm::Function *Fn,
@@ -1591,14 +1974,16 @@ public:
                                  const std::vector<std::pair<llvm::WeakVH,
                                    llvm::Constant*> > &DtorsAndObjects);
 
-  void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D);
+  void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
+                                        llvm::GlobalVariable *Addr);
 
-  void EmitCXXConstructExpr(llvm::Value *Dest, const CXXConstructExpr *E);
+  void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
+  
+  void EmitSynthesizedCXXCopyCtor(llvm::Value *Dest, llvm::Value *Src,
+                                  const Expr *Exp);
 
-  RValue EmitCXXExprWithTemporaries(const CXXExprWithTemporaries *E,
-                                    llvm::Value *AggLoc = 0,
-                                    bool IsAggLocVolatile = false,
-                                    bool IsInitializer = false);
+  RValue EmitExprWithCleanups(const ExprWithCleanups *E,
+                              AggValueSlot Slot =AggValueSlot::ignored());
 
   void EmitCXXThrowExpr(const CXXThrowExpr *E);
 
@@ -1626,7 +2011,7 @@ public:
   /// getTrapBB - Create a basic block that will call the trap intrinsic.  We'll
   /// generate a branch around the created basic block as necessary.
   llvm::BasicBlock *getTrapBB();
-  
+
   /// EmitCallArg - Emit a single call argument.
   RValue EmitCallArg(const Expr *E, QualType ArgType);
 
@@ -1678,12 +2063,26 @@ private:
            E = CallArgTypeInfo->arg_type_end(); I != E; ++I, ++Arg) {
         assert(Arg != ArgEnd && "Running over edge of argument list!");
         QualType ArgType = *I;
-
+#ifndef NDEBUG
+        QualType ActualArgType = Arg->getType();
+        if (ArgType->isPointerType() && ActualArgType->isPointerType()) {
+          QualType ActualBaseType =
+            ActualArgType->getAs<PointerType>()->getPointeeType();
+          QualType ArgBaseType =
+            ArgType->getAs<PointerType>()->getPointeeType();
+          if (ArgBaseType->isVariableArrayType()) {
+            if (const VariableArrayType *VAT =
+                getContext().getAsVariableArrayType(ActualBaseType)) {
+              if (!VAT->getSizeExpr())
+                ActualArgType = ArgType;
+            }
+          }
+        }
         assert(getContext().getCanonicalType(ArgType.getNonReferenceType()).
                getTypePtr() ==
-               getContext().getCanonicalType(Arg->getType()).getTypePtr() &&
+               getContext().getCanonicalType(ActualArgType).getTypePtr() &&
                "type mismatch in call argument!");
-
+#endif
         Args.push_back(std::make_pair(EmitCallArg(*Arg, ArgType),
                                       ArgType));
       }
@@ -1710,36 +2109,78 @@ private:
   void EmitDeclMetadata();
 };
 
-/// CGBlockInfo - Information to generate a block literal.
-class CGBlockInfo {
-public:
-  /// Name - The name of the block, kindof.
-  const char *Name;
-    
-  /// DeclRefs - Variables from parent scopes that have been
-  /// imported into this block.
-  llvm::SmallVector<const BlockDeclRefExpr *, 8> DeclRefs;
-    
-  /// InnerBlocks - This block and the blocks it encloses.
-  llvm::SmallPtrSet<const DeclContext *, 4> InnerBlocks;
-    
-  /// CXXThisRef - Non-null if 'this' was required somewhere, in
-  /// which case this is that expression.
-  const CXXThisExpr *CXXThisRef;
-    
-  /// NeedsObjCSelf - True if something in this block has an implicit
-  /// reference to 'self'.
-  bool NeedsObjCSelf;
-    
-  /// These are initialized by GenerateBlockFunction.
-  bool BlockHasCopyDispose;
-  CharUnits BlockSize;
-  CharUnits BlockAlign;
-  llvm::SmallVector<const Expr*, 8> BlockLayout;
-    
-  CGBlockInfo(const char *Name);
+/// Helper class with most of the code for saving a value for a
+/// conditional expression cleanup.
+struct DominatingLLVMValue {
+  typedef llvm::PointerIntPair<llvm::Value*, 1, bool> saved_type;
+
+  /// Answer whether the given value needs extra work to be saved.
+  static bool needsSaving(llvm::Value *value) {
+    // If it's not an instruction, we don't need to save.
+    if (!isa<llvm::Instruction>(value)) return false;
+
+    // If it's an instruction in the entry block, we don't need to save.
+    llvm::BasicBlock *block = cast<llvm::Instruction>(value)->getParent();
+    return (block != &block->getParent()->getEntryBlock());
+  }
+
+  /// Try to save the given value.
+  static saved_type save(CodeGenFunction &CGF, llvm::Value *value) {
+    if (!needsSaving(value)) return saved_type(value, false);
+
+    // Otherwise we need an alloca.
+    llvm::Value *alloca =
+      CGF.CreateTempAlloca(value->getType(), "cond-cleanup.save");
+    CGF.Builder.CreateStore(value, alloca);
+
+    return saved_type(alloca, true);
+  }
+
+  static llvm::Value *restore(CodeGenFunction &CGF, saved_type value) {
+    if (!value.getInt()) return value.getPointer();
+    return CGF.Builder.CreateLoad(value.getPointer());
+  }
 };
-  
+
+/// A partial specialization of DominatingValue for llvm::Values that
+/// might be llvm::Instructions.
+template <class T> struct DominatingPointer<T,true> : DominatingLLVMValue {
+  typedef T *type;
+  static type restore(CodeGenFunction &CGF, saved_type value) {
+    return static_cast<T*>(DominatingLLVMValue::restore(CGF, value));
+  }
+};
+
+/// A specialization of DominatingValue for RValue.
+template <> struct DominatingValue<RValue> {
+  typedef RValue type;
+  class saved_type {
+    enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral,
+                AggregateAddress, ComplexAddress };
+
+    llvm::Value *Value;
+    Kind K;
+    saved_type(llvm::Value *v, Kind k) : Value(v), K(k) {}
+
+  public:
+    static bool needsSaving(RValue value);
+    static saved_type save(CodeGenFunction &CGF, RValue value);
+    RValue restore(CodeGenFunction &CGF);
+
+    // implementations in CGExprCXX.cpp
+  };
+
+  static bool needsSaving(type value) {
+    return saved_type::needsSaving(value);
+  }
+  static saved_type save(CodeGenFunction &CGF, type value) {
+    return saved_type::save(CGF, value);
+  }
+  static type restore(CodeGenFunction &CGF, saved_type value) {
+    return value.restore(CGF);
+  }
+};
+
 }  // end namespace CodeGen
 }  // end namespace clang
 
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index d125b370a07a..9e5d7cf11276 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -14,10 +14,10 @@
 #include "CodeGenModule.h"
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
+#include "CodeGenTBAA.h"
 #include "CGCall.h"
 #include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
-#include "Mangle.h"
 #include "TargetInfo.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "clang/AST/ASTContext.h"
@@ -25,6 +25,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/AST/Mangle.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/Diagnostic.h"
@@ -36,6 +37,7 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -57,19 +59,19 @@ static CGCXXABI &createCXXABI(CodeGenModule &CGM) {
 CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
                              llvm::Module &M, const llvm::TargetData &TD,
                              Diagnostic &diags)
-  : BlockModule(C, M, TD, Types, *this), Context(C),
-    Features(C.getLangOptions()), CodeGenOpts(CGO), TheModule(M),
+  : Context(C), Features(C.getLangOptions()), CodeGenOpts(CGO), TheModule(M),
     TheTargetData(TD), TheTargetCodeGenInfo(0), Diags(diags),
     ABI(createCXXABI(*this)), 
     Types(C, M, TD, getTargetCodeGenInfo().getABIInfo(), ABI),
+    TBAA(0),
     VTables(*this), Runtime(0),
-    CFConstantStringClassRef(0), NSConstantStringClassRef(0),
+    CFConstantStringClassRef(0), ConstantStringClassRef(0),
     VMContext(M.getContext()),
     NSConcreteGlobalBlockDecl(0), NSConcreteStackBlockDecl(0),
     NSConcreteGlobalBlock(0), NSConcreteStackBlock(0),
     BlockObjectAssignDecl(0), BlockObjectDisposeDecl(0),
-    BlockObjectAssign(0), BlockObjectDispose(0){
-
+    BlockObjectAssign(0), BlockObjectDispose(0),
+    BlockDescriptorType(0), GenericBlockLiteralType(0) {
   if (!Features.ObjC1)
     Runtime = 0;
   else if (!Features.NeXTRuntime)
@@ -79,13 +81,32 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
   else
     Runtime = CreateMacObjCRuntime(*this);
 
+  // Enable TBAA unless it's suppressed.
+  if (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0)
+    TBAA = new CodeGenTBAA(Context, VMContext, getLangOptions(),
+                           ABI.getMangleContext());
+
   // If debug info generation is enabled, create the CGDebugInfo object.
   DebugInfo = CodeGenOpts.DebugInfo ? new CGDebugInfo(*this) : 0;
+
+  Block.GlobalUniqueCount = 0;
+
+  // Initialize the type cache.
+  llvm::LLVMContext &LLVMContext = M.getContext();
+  Int8Ty  = llvm::Type::getInt8Ty(LLVMContext);
+  Int32Ty  = llvm::Type::getInt32Ty(LLVMContext);
+  Int64Ty  = llvm::Type::getInt64Ty(LLVMContext);
+  PointerWidthInBits = C.Target.getPointerWidth(0);
+  IntTy = llvm::IntegerType::get(LLVMContext, C.Target.getIntWidth());
+  IntPtrTy = llvm::IntegerType::get(LLVMContext, PointerWidthInBits);
+  Int8PtrTy = Int8Ty->getPointerTo(0);
+  Int8PtrPtrTy = Int8PtrTy->getPointerTo(0);
 }
 
 CodeGenModule::~CodeGenModule() {
   delete Runtime;
   delete &ABI;
+  delete TBAA;
   delete DebugInfo;
 }
 
@@ -110,10 +131,23 @@ void CodeGenModule::Release() {
   EmitAnnotations();
   EmitLLVMUsed();
 
+  SimplifyPersonality();
+
   if (getCodeGenOpts().EmitDeclMetadata)
     EmitDeclMetadata();
 }
 
+llvm::MDNode *CodeGenModule::getTBAAInfo(QualType QTy) {
+  if (!TBAA)
+    return 0;
+  return TBAA->getTBAAInfo(QTy);
+}
+
+void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
+                                        llvm::MDNode *TBAAInfo) {
+  Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo);
+}
+
 bool CodeGenModule::isTargetDarwin() const {
   return getContext().Target.getTriple().getOS() == llvm::Triple::Darwin;
 }
@@ -143,93 +177,34 @@ void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type,
   getDiags().Report(Context.getFullLoc(D->getLocation()), DiagID) << Msg;
 }
 
-LangOptions::VisibilityMode
-CodeGenModule::getDeclVisibilityMode(const Decl *D) const {
-  if (const VarDecl *VD = dyn_cast<VarDecl>(D))
-    if (VD->getStorageClass() == SC_PrivateExtern)
-      return LangOptions::Hidden;
-
-  if (const VisibilityAttr *attr = D->getAttr<VisibilityAttr>()) {
-    switch (attr->getVisibility()) {
-    default: assert(0 && "Unknown visibility!");
-    case VisibilityAttr::Default:
-      return LangOptions::Default;
-    case VisibilityAttr::Hidden:
-      return LangOptions::Hidden;
-    case VisibilityAttr::Protected:
-      return LangOptions::Protected;
-    }
-  }
-  
-  if (getLangOptions().CPlusPlus) {
-    // Entities subject to an explicit instantiation declaration get default
-    // visibility.
-    if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
-      if (Function->getTemplateSpecializationKind()
-                                        == TSK_ExplicitInstantiationDeclaration)
-        return LangOptions::Default;
-    } else if (const ClassTemplateSpecializationDecl *ClassSpec
-                              = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
-      if (ClassSpec->getSpecializationKind()
-                                        == TSK_ExplicitInstantiationDeclaration)
-        return LangOptions::Default;
-    } else if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(D)) {
-      if (Record->getTemplateSpecializationKind()
-                                        == TSK_ExplicitInstantiationDeclaration)
-        return LangOptions::Default;
-    } else if (const VarDecl *Var = dyn_cast<VarDecl>(D)) {
-      if (Var->isStaticDataMember() &&
-          (Var->getTemplateSpecializationKind()
-                                      == TSK_ExplicitInstantiationDeclaration))
-        return LangOptions::Default;
-    }
-
-    // If -fvisibility-inlines-hidden was provided, then inline C++ member
-    // functions get "hidden" visibility by default.
-    if (getLangOptions().InlineVisibilityHidden)
-      if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
-        if (Method->isInlined())
-          return LangOptions::Hidden;
-  }
-           
-  // If this decl is contained in a class, it should have the same visibility
-  // as the parent class.
-  if (const DeclContext *DC = D->getDeclContext()) 
-    if (DC->isRecord())
-      return getDeclVisibilityMode(cast<Decl>(DC));
-
-  return getLangOptions().getVisibilityMode();
-}
-
 void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
-                                        const Decl *D) const {
+                                        const NamedDecl *D) const {
   // Internal definitions always have default visibility.
   if (GV->hasLocalLinkage()) {
     GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
     return;
   }
 
-  switch (getDeclVisibilityMode(D)) {
-  default: assert(0 && "Unknown visibility!");
-  case LangOptions::Default:
-    return GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
-  case LangOptions::Hidden:
-    return GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
-  case LangOptions::Protected:
-    return GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
-  }
+  // Set visibility for definitions.
+  NamedDecl::LinkageInfo LV = D->getLinkageAndVisibility();
+  if (LV.visibilityExplicit() || !GV->hasAvailableExternallyLinkage())
+    GV->setVisibility(GetLLVMVisibility(LV.visibility()));
 }
 
 /// Set the symbol visibility of type information (vtable and RTTI)
 /// associated with the given type.
 void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
                                       const CXXRecordDecl *RD,
-                                      bool IsForRTTI) const {
+                                      TypeVisibilityKind TVK) const {
   setGlobalVisibility(GV, RD);
 
   if (!CodeGenOpts.HiddenWeakVTables)
     return;
 
+  // We never want to drop the visibility for RTTI names.
+  if (TVK == TVK_ForRTTIName)
+    return;
+
   // We want to drop the visibility to hidden for weak type symbols.
   // This isn't possible if there might be unresolved references
   // elsewhere that rely on this symbol being visible.
@@ -238,7 +213,7 @@ void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
   // in CGVTables.cpp.
 
   // Preconditions.
-  if (GV->getLinkage() != llvm::GlobalVariable::WeakODRLinkage ||
+  if (GV->getLinkage() != llvm::GlobalVariable::LinkOnceODRLinkage ||
       GV->getVisibility() != llvm::GlobalVariable::DefaultVisibility)
     return;
 
@@ -273,12 +248,14 @@ void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
   // If there's a key function, there may be translation units
   // that don't have the key function's definition.  But ignore
   // this if we're emitting RTTI under -fno-rtti.
-  if (!IsForRTTI || Features.RTTI)
+  if (!(TVK != TVK_ForRTTI) || Features.RTTI) {
     if (Context.getKeyFunction(RD))
       return;
+  }
 
   // Otherwise, drop the visibility to hidden.
   GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
+  GV->setUnnamedAddr(true);
 }
 
 llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
@@ -297,16 +274,18 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   }
   
   llvm::SmallString<256> Buffer;
+  llvm::raw_svector_ostream Out(Buffer);
   if (const CXXConstructorDecl *D = dyn_cast<CXXConstructorDecl>(ND))
-    getCXXABI().getMangleContext().mangleCXXCtor(D, GD.getCtorType(), Buffer);
+    getCXXABI().getMangleContext().mangleCXXCtor(D, GD.getCtorType(), Out);
   else if (const CXXDestructorDecl *D = dyn_cast<CXXDestructorDecl>(ND))
-    getCXXABI().getMangleContext().mangleCXXDtor(D, GD.getDtorType(), Buffer);
+    getCXXABI().getMangleContext().mangleCXXDtor(D, GD.getDtorType(), Out);
   else if (const BlockDecl *BD = dyn_cast<BlockDecl>(ND))
-    getCXXABI().getMangleContext().mangleBlock(GD, BD, Buffer);
+    getCXXABI().getMangleContext().mangleBlock(BD, Out);
   else
-    getCXXABI().getMangleContext().mangleName(ND, Buffer);
+    getCXXABI().getMangleContext().mangleName(ND, Out);
 
   // Allocate space for the mangled name.
+  Out.flush();
   size_t Length = Buffer.size();
   char *Name = MangledNamesAllocator.Allocate<char>(Length);
   std::copy(Buffer.begin(), Buffer.end(), Name);
@@ -316,9 +295,19 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   return Str;
 }
 
-void CodeGenModule::getMangledName(GlobalDecl GD, MangleBuffer &Buffer,
-                                   const BlockDecl *BD) {
-  getCXXABI().getMangleContext().mangleBlock(GD, BD, Buffer.getBuffer());
+void CodeGenModule::getBlockMangledName(GlobalDecl GD, MangleBuffer &Buffer,
+                                        const BlockDecl *BD) {
+  MangleContext &MangleCtx = getCXXABI().getMangleContext();
+  const Decl *D = GD.getDecl();
+  llvm::raw_svector_ostream Out(Buffer.getBuffer());
+  if (D == 0)
+    MangleCtx.mangleGlobalBlock(BD, Out);
+  else if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(D))
+    MangleCtx.mangleCtorBlock(CD, GD.getCtorType(), BD, Out);
+  else if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(D))
+    MangleCtx.mangleDtorBlock(DD, GD.getDtorType(), BD, Out);
+  else
+    MangleCtx.mangleBlock(cast<DeclContext>(D), BD, Out);
 }
 
 llvm::GlobalValue *CodeGenModule::GetGlobalValue(llvm::StringRef Name) {
@@ -342,9 +331,7 @@ void CodeGenModule::AddGlobalDtor(llvm::Function * Dtor, int Priority) {
 void CodeGenModule::EmitCtorList(const CtorList &Fns, const char *GlobalName) {
   // Ctor function type is void()*.
   llvm::FunctionType* CtorFTy =
-    llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
-                            std::vector<const llvm::Type*>(),
-                            false);
+    llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), false);
   llvm::Type *CtorPFTy = llvm::PointerType::getUnqual(CtorFTy);
 
   // Get the type of a ctor entry, { i32, void ()* }.
@@ -412,14 +399,18 @@ CodeGenModule::getFunctionLinkage(const FunctionDecl *D) {
   // merged with other definitions. c) C++ has the ODR, so we know the
   // definition is dependable.
   if (Linkage == GVA_CXXInline || Linkage == GVA_TemplateInstantiation)
-    return llvm::Function::LinkOnceODRLinkage;
+    return !Context.getLangOptions().AppleKext 
+             ? llvm::Function::LinkOnceODRLinkage 
+             : llvm::Function::InternalLinkage;
   
   // An explicit instantiation of a template has weak linkage, since
   // explicit instantiations can occur in multiple translation units
   // and must all be equivalent. However, we are not allowed to
   // throw away these explicit instantiations.
   if (Linkage == GVA_ExplicitTemplateInstantiation)
-    return llvm::Function::WeakODRLinkage;
+    return !Context.getLangOptions().AppleKext
+             ? llvm::Function::WeakODRLinkage
+             : llvm::Function::InternalLinkage;
   
   // Otherwise, we have strong external linkage.
   assert(Linkage == GVA_StrongExternal);
@@ -455,9 +446,15 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
   if (D->hasAttr<AlwaysInlineAttr>())
     F->addFnAttr(llvm::Attribute::AlwaysInline);
 
+  if (D->hasAttr<NakedAttr>())
+    F->addFnAttr(llvm::Attribute::Naked);
+
   if (D->hasAttr<NoInlineAttr>())
     F->addFnAttr(llvm::Attribute::NoInline);
 
+  if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
+    F->setUnnamedAddr(true);
+
   if (Features.getStackProtectorMode() == LangOptions::SSPOn)
     F->addFnAttr(llvm::Attribute::StackProtect);
   else if (Features.getStackProtectorMode() == LangOptions::SSPReq)
@@ -474,7 +471,10 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
 
 void CodeGenModule::SetCommonAttributes(const Decl *D,
                                         llvm::GlobalValue *GV) {
-  setGlobalVisibility(GV, D);
+  if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
+    setGlobalVisibility(GV, ND);
+  else
+    GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
 
   if (D->hasAttr<UsedAttr>())
     AddUsedGlobal(GV);
@@ -516,6 +516,11 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD,
     F->setLinkage(llvm::Function::ExternalWeakLinkage);
   } else {
     F->setLinkage(llvm::Function::ExternalLinkage);
+
+    NamedDecl::LinkageInfo LV = FD->getLinkageAndVisibility();
+    if (LV.linkage() == ExternalLinkage && LV.visibilityExplicit()) {
+      F->setVisibility(GetLLVMVisibility(LV.visibility()));
+    }
   }
 
   if (const SectionAttr *SA = FD->getAttr<SectionAttr>())
@@ -634,6 +639,7 @@ llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
     new llvm::GlobalVariable(*M, unit->getType(), false,
                              llvm::GlobalValue::PrivateLinkage, unit,
                              ".str");
+  unitGV->setUnnamedAddr(true);
 
   // Create the ConstantStruct for the global annotation.
   llvm::Constant *Fields[4] = {
@@ -664,7 +670,8 @@ llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
 
   llvm::Constant *Aliasee;
   if (isa<llvm::FunctionType>(DeclTy))
-    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl());
+    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl(),
+                                      /*ForVTable=*/false);
   else
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(),
                                     llvm::PointerType::getUnqual(DeclTy), 0);
@@ -796,7 +803,7 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD) {
 llvm::Constant *
 CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
                                        const llvm::Type *Ty,
-                                       GlobalDecl D) {
+                                       GlobalDecl D, bool ForVTable) {
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
@@ -825,8 +832,7 @@ CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
   if (isa<llvm::FunctionType>(Ty)) {
     FTy = cast<llvm::FunctionType>(Ty);
   } else {
-    FTy = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
-                                  std::vector<const llvm::Type*>(), false);
+    FTy = llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext), false);
     IsIncompleteFunction = true;
   }
   
@@ -846,30 +852,34 @@ CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
     // list, and remove it from DeferredDecls (since we don't need it anymore).
     DeferredDeclsToEmit.push_back(DDI->second);
     DeferredDecls.erase(DDI);
-  } else if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D.getDecl())) {
-    // If this the first reference to a C++ inline function in a class, queue up
-    // the deferred function body for emission.  These are not seen as
-    // top-level declarations.
-    if (FD->isThisDeclarationADefinition() && MayDeferGeneration(FD))
-      DeferredDeclsToEmit.push_back(D);
-    // A called constructor which has no definition or declaration need be
-    // synthesized.
-    else if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD)) {
-      if (CD->isImplicit()) {
-        assert(CD->isUsed() && "Sema doesn't consider constructor as used.");
-        DeferredDeclsToEmit.push_back(D);
-      }
-    } else if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD)) {
-      if (DD->isImplicit()) {
-        assert(DD->isUsed() && "Sema doesn't consider destructor as used.");
-        DeferredDeclsToEmit.push_back(D);
-      }
-    } else if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
-      if (MD->isCopyAssignment() && MD->isImplicit()) {
-        assert(MD->isUsed() && "Sema doesn't consider CopyAssignment as used.");
-        DeferredDeclsToEmit.push_back(D);
+
+  // Otherwise, there are cases we have to worry about where we're
+  // using a declaration for which we must emit a definition but where
+  // we might not find a top-level definition:
+  //   - member functions defined inline in their classes
+  //   - friend functions defined inline in some class
+  //   - special member functions with implicit definitions
+  // If we ever change our AST traversal to walk into class methods,
+  // this will be unnecessary.
+  //
+  // We also don't emit a definition for a function if it's going to be an entry
+  // in a vtable, unless it's already marked as used.
+  } else if (getLangOptions().CPlusPlus && D.getDecl()) {
+    // Look for a declaration that's lexically in a record.
+    const FunctionDecl *FD = cast<FunctionDecl>(D.getDecl());
+    do {
+      if (isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
+        if (FD->isImplicit() && !ForVTable) {
+          assert(FD->isUsed() && "Sema didn't mark implicit function as used!");
+          DeferredDeclsToEmit.push_back(D.getWithDecl(FD));
+          break;
+        } else if (FD->isThisDeclarationADefinition()) {
+          DeferredDeclsToEmit.push_back(D.getWithDecl(FD));
+          break;
+        }
       }
-    }
+      FD = FD->getPreviousDeclaration();
+    } while (FD);
   }
 
   // Make sure the result is of the requested type.
@@ -886,13 +896,14 @@ CodeGenModule::GetOrCreateLLVMFunction(llvm::StringRef MangledName,
 /// non-null, then this function will use the specified type if it has to
 /// create it (this occurs when we see a definition of the function).
 llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
-                                                 const llvm::Type *Ty) {
+                                                 const llvm::Type *Ty,
+                                                 bool ForVTable) {
   // If there was no specific requested type, just convert it now.
   if (!Ty)
     Ty = getTypes().ConvertType(cast<ValueDecl>(GD.getDecl())->getType());
   
   llvm::StringRef MangledName = getMangledName(GD);
-  return GetOrCreateLLVMFunction(MangledName, Ty, GD);
+  return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable);
 }
 
 /// CreateRuntimeFunction - Create a new runtime function with the specified
@@ -900,7 +911,7 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
 llvm::Constant *
 CodeGenModule::CreateRuntimeFunction(const llvm::FunctionType *FTy,
                                      llvm::StringRef Name) {
-  return GetOrCreateLLVMFunction(Name, FTy, GlobalDecl());
+  return GetOrCreateLLVMFunction(Name, FTy, GlobalDecl(), /*ForVTable=*/false);
 }
 
 static bool DeclIsConstantGlobal(ASTContext &Context, const VarDecl *D) {
@@ -924,7 +935,8 @@ static bool DeclIsConstantGlobal(ASTContext &Context, const VarDecl *D) {
 llvm::Constant *
 CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
                                      const llvm::PointerType *Ty,
-                                     const VarDecl *D) {
+                                     const VarDecl *D,
+                                     bool UnnamedAddr) {
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
@@ -935,6 +947,9 @@ CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
       WeakRefReferences.erase(Entry);
     }
 
+    if (UnnamedAddr)
+      Entry->setUnnamedAddr(true);
+
     if (Entry->getType() == Ty)
       return Entry;
 
@@ -965,13 +980,20 @@ CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
     // handling.
     GV->setConstant(DeclIsConstantGlobal(Context, D));
 
-    // FIXME: Merge with other attribute handling code.
-    if (D->getStorageClass() == SC_PrivateExtern)
-      GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
-
-    if (D->hasAttr<WeakAttr>() ||
-        D->hasAttr<WeakImportAttr>())
-      GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
+    // Set linkage and visibility in case we never see a definition.
+    NamedDecl::LinkageInfo LV = D->getLinkageAndVisibility();
+    if (LV.linkage() != ExternalLinkage) {
+      // Don't set internal linkage on declarations.
+    } else {
+      if (D->hasAttr<DLLImportAttr>())
+        GV->setLinkage(llvm::GlobalValue::DLLImportLinkage);
+      else if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakImportAttr>())
+        GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
+
+      // Set visibility on a declaration only if it's explicit.
+      if (LV.visibilityExplicit())
+        GV->setVisibility(GetLLVMVisibility(LV.visibility()));
+    }
 
     GV->setThreadLocal(D->isThreadSpecified());
   }
@@ -980,6 +1002,45 @@ CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
 }
 
 
+llvm::GlobalVariable *
+CodeGenModule::CreateOrReplaceCXXRuntimeVariable(llvm::StringRef Name, 
+                                      const llvm::Type *Ty,
+                                      llvm::GlobalValue::LinkageTypes Linkage) {
+  llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
+  llvm::GlobalVariable *OldGV = 0;
+
+  
+  if (GV) {
+    // Check if the variable has the right type.
+    if (GV->getType()->getElementType() == Ty)
+      return GV;
+
+    // Because C++ name mangling, the only way we can end up with an already
+    // existing global with the same name is if it has been declared extern "C".
+      assert(GV->isDeclaration() && "Declaration has wrong type!");
+    OldGV = GV;
+  }
+  
+  // Create a new variable.
+  GV = new llvm::GlobalVariable(getModule(), Ty, /*isConstant=*/true,
+                                Linkage, 0, Name);
+  
+  if (OldGV) {
+    // Replace occurrences of the old variable if needed.
+    GV->takeName(OldGV);
+    
+    if (!OldGV->use_empty()) {
+      llvm::Constant *NewPtrForOldDecl =
+      llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
+      OldGV->replaceAllUsesWith(NewPtrForOldDecl);
+    }
+    
+    OldGV->eraseFromParent();
+  }
+  
+  return GV;
+}
+
 /// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
 /// given global variable.  If Ty is non-null and if the global doesn't exist,
 /// then it will be greated with the specified type instead of whatever the
@@ -1003,7 +1064,8 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
 llvm::Constant *
 CodeGenModule::CreateRuntimeVariable(const llvm::Type *Ty,
                                      llvm::StringRef Name) {
-  return GetOrCreateLLVMGlobal(Name, llvm::PointerType::getUnqual(Ty), 0);
+  return GetOrCreateLLVMGlobal(Name,  llvm::PointerType::getUnqual(Ty), 0,
+                               true);
 }
 
 void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
@@ -1045,44 +1107,63 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) {
     switch (KeyFunction->getTemplateSpecializationKind()) {
       case TSK_Undeclared:
       case TSK_ExplicitSpecialization:
+        // When compiling with optimizations turned on, we emit all vtables,
+        // even if the key function is not defined in the current translation
+        // unit. If this is the case, use available_externally linkage.
+        if (!Def && CodeGenOpts.OptimizationLevel)
+          return llvm::GlobalVariable::AvailableExternallyLinkage;
+
         if (KeyFunction->isInlined())
-          return llvm::GlobalVariable::WeakODRLinkage;
+          return !Context.getLangOptions().AppleKext ?
+                   llvm::GlobalVariable::LinkOnceODRLinkage :
+                   llvm::Function::InternalLinkage;
         
         return llvm::GlobalVariable::ExternalLinkage;
         
       case TSK_ImplicitInstantiation:
+        return !Context.getLangOptions().AppleKext ?
+                 llvm::GlobalVariable::LinkOnceODRLinkage :
+                 llvm::Function::InternalLinkage;
+
       case TSK_ExplicitInstantiationDefinition:
-        return llvm::GlobalVariable::WeakODRLinkage;
-        
+        return !Context.getLangOptions().AppleKext ?
+                 llvm::GlobalVariable::WeakODRLinkage :
+                 llvm::Function::InternalLinkage;
+  
       case TSK_ExplicitInstantiationDeclaration:
         // FIXME: Use available_externally linkage. However, this currently
         // breaks LLVM's build due to undefined symbols.
         //      return llvm::GlobalVariable::AvailableExternallyLinkage;
-        return llvm::GlobalVariable::WeakODRLinkage;
+        return !Context.getLangOptions().AppleKext ?
+                 llvm::GlobalVariable::LinkOnceODRLinkage :
+                 llvm::Function::InternalLinkage;
     }
   }
   
+  if (Context.getLangOptions().AppleKext)
+    return llvm::Function::InternalLinkage;
+  
   switch (RD->getTemplateSpecializationKind()) {
   case TSK_Undeclared:
   case TSK_ExplicitSpecialization:
   case TSK_ImplicitInstantiation:
-  case TSK_ExplicitInstantiationDefinition:
-    return llvm::GlobalVariable::WeakODRLinkage;
-    
-  case TSK_ExplicitInstantiationDeclaration:
     // FIXME: Use available_externally linkage. However, this currently
     // breaks LLVM's build due to undefined symbols.
     //   return llvm::GlobalVariable::AvailableExternallyLinkage;
-    return llvm::GlobalVariable::WeakODRLinkage;
+  case TSK_ExplicitInstantiationDeclaration:
+    return llvm::GlobalVariable::LinkOnceODRLinkage;
+
+  case TSK_ExplicitInstantiationDefinition:
+      return llvm::GlobalVariable::WeakODRLinkage;
   }
   
   // Silence GCC warning.
-  return llvm::GlobalVariable::WeakODRLinkage;
+  return llvm::GlobalVariable::LinkOnceODRLinkage;
 }
 
 CharUnits CodeGenModule::GetTargetTypeStoreSize(const llvm::Type *Ty) const {
-    return CharUnits::fromQuantity(
-      TheTargetData.getTypeStoreSizeInBits(Ty) / Context.getCharWidth());
+    return Context.toCharUnitsFromBits(
+      TheTargetData.getTypeStoreSizeInBits(Ty));
 }
 
 void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
@@ -1112,7 +1193,6 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
         T = D->getType();
       
       if (getLangOptions().CPlusPlus) {
-        EmitCXXGlobalVarDeclInitFunc(D);
         Init = EmitNullConstant(T);
         NonConstInit = true;
       } else {
@@ -1183,42 +1263,57 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     GV->setConstant(true);
 
   GV->setAlignment(getContext().getDeclAlign(D).getQuantity());
-
+  
   // Set the llvm linkage type as appropriate.
+  llvm::GlobalValue::LinkageTypes Linkage = 
+    GetLLVMLinkageVarDefinition(D, GV);
+  GV->setLinkage(Linkage);
+  if (Linkage == llvm::GlobalVariable::CommonLinkage)
+    // common vars aren't constant even if declared const.
+    GV->setConstant(false);
+
+  SetCommonAttributes(D, GV);
+
+  // Emit the initializer function if necessary.
+  if (NonConstInit)
+    EmitCXXGlobalVarDeclInitFunc(D, GV);
+
+  // Emit global variable debug information.
+  if (CGDebugInfo *DI = getDebugInfo()) {
+    DI->setLocation(D->getLocation());
+    DI->EmitGlobalVariable(GV, D);
+  }
+}
+
+llvm::GlobalValue::LinkageTypes
+CodeGenModule::GetLLVMLinkageVarDefinition(const VarDecl *D,
+                                           llvm::GlobalVariable *GV) {
   GVALinkage Linkage = getContext().GetGVALinkageForVariable(D);
   if (Linkage == GVA_Internal)
-    GV->setLinkage(llvm::Function::InternalLinkage);
+    return llvm::Function::InternalLinkage;
   else if (D->hasAttr<DLLImportAttr>())
-    GV->setLinkage(llvm::Function::DLLImportLinkage);
+    return llvm::Function::DLLImportLinkage;
   else if (D->hasAttr<DLLExportAttr>())
-    GV->setLinkage(llvm::Function::DLLExportLinkage);
+    return llvm::Function::DLLExportLinkage;
   else if (D->hasAttr<WeakAttr>()) {
     if (GV->isConstant())
-      GV->setLinkage(llvm::GlobalVariable::WeakODRLinkage);
+      return llvm::GlobalVariable::WeakODRLinkage;
     else
-      GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
+      return llvm::GlobalVariable::WeakAnyLinkage;
   } else if (Linkage == GVA_TemplateInstantiation ||
              Linkage == GVA_ExplicitTemplateInstantiation)
     // FIXME: It seems like we can provide more specific linkage here
     // (LinkOnceODR, WeakODR).
-    GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);   
-  else if (!getLangOptions().CPlusPlus && !CodeGenOpts.NoCommon &&
+    return llvm::GlobalVariable::WeakAnyLinkage;
+  else if (!getLangOptions().CPlusPlus && 
+           ((!CodeGenOpts.NoCommon && !D->getAttr<NoCommonAttr>()) ||
+             D->getAttr<CommonAttr>()) &&
            !D->hasExternalStorage() && !D->getInit() &&
            !D->getAttr<SectionAttr>() && !D->isThreadSpecified()) {
     // Thread local vars aren't considered common linkage.
-    GV->setLinkage(llvm::GlobalVariable::CommonLinkage);
-    // common vars aren't constant even if declared const.
-    GV->setConstant(false);
-  } else
-    GV->setLinkage(llvm::GlobalVariable::ExternalLinkage);
-
-  SetCommonAttributes(D, GV);
-
-  // Emit global variable debug information.
-  if (CGDebugInfo *DI = getDebugInfo()) {
-    DI->setLocation(D->getLocation());
-    DI->EmitGlobalVariable(GV, D);
+    return llvm::GlobalVariable::CommonLinkage;
   }
+  return llvm::GlobalVariable::ExternalLinkage;
 }
 
 /// ReplaceUsesOfNonProtoTypeWithRealFunction - This function is called when we
@@ -1295,7 +1390,6 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
 void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
   const FunctionDecl *D = cast<FunctionDecl>(GD.getDecl());
   const llvm::FunctionType *Ty = getTypes().GetFunctionType(GD);
-  getCXXABI().getMangleContext().mangleInitDiscriminator();
   // Get or create the prototype for the function.
   llvm::Constant *Entry = GetAddrOfFunction(GD, Ty);
 
@@ -1345,9 +1439,16 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
     Entry = NewFn;
   }
 
+  // We need to set linkage and visibility on the function before
+  // generating code for it because various parts of IR generation
+  // want to propagate this information down (e.g. to local static
+  // declarations).
   llvm::Function *Fn = cast<llvm::Function>(Entry);
   setFunctionLinkage(D, Fn);
 
+  // FIXME: this is redundant with part of SetFunctionDefinitionAttributes
+  setGlobalVisibility(Fn, D);
+
   CodeGenFunction(*this).GenerateCode(D, Fn);
 
   SetFunctionDefinitionAttributes(D, Fn);
@@ -1378,7 +1479,8 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   // if a deferred decl.
   llvm::Constant *Aliasee;
   if (isa<llvm::FunctionType>(DeclTy))
-    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl());
+    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl(),
+                                      /*ForVTable=*/false);
   else
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(),
                                     llvm::PointerType::getUnqual(DeclTy), 0);
@@ -1444,7 +1546,7 @@ llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
   const llvm::FunctionType *Ty =
     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
 
-  return GetOrCreateLLVMFunction(Name, Ty, GlobalDecl(FD));
+  return GetOrCreateLLVMFunction(Name, Ty, GlobalDecl(FD), /*ForVTable=*/false);
 }
 
 llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,const llvm::Type **Tys,
@@ -1453,27 +1555,6 @@ llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,const llvm::Type **Tys,
                                          (llvm::Intrinsic::ID)IID, Tys, NumTys);
 }
 
-
-llvm::Function *CodeGenModule::getMemCpyFn(const llvm::Type *DestType,
-                                           const llvm::Type *SrcType,
-                                           const llvm::Type *SizeType) {
-  const llvm::Type *ArgTypes[3] = {DestType, SrcType, SizeType };
-  return getIntrinsic(llvm::Intrinsic::memcpy, ArgTypes, 3);
-}
-
-llvm::Function *CodeGenModule::getMemMoveFn(const llvm::Type *DestType,
-                                            const llvm::Type *SrcType,
-                                            const llvm::Type *SizeType) {
-  const llvm::Type *ArgTypes[3] = {DestType, SrcType, SizeType };
-  return getIntrinsic(llvm::Intrinsic::memmove, ArgTypes, 3);
-}
-
-llvm::Function *CodeGenModule::getMemSetFn(const llvm::Type *DestType,
-                                           const llvm::Type *SizeType) {
-  const llvm::Type *ArgTypes[2] = { DestType, SizeType };
-  return getIntrinsic(llvm::Intrinsic::memset, ArgTypes, 2);
-}
-
 static llvm::StringMapEntry<llvm::Constant*> &
 GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
                          const StringLiteral *Literal,
@@ -1494,18 +1575,9 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
   const UTF8 *FromPtr = (UTF8 *)String.data();
   UTF16 *ToPtr = &ToBuf[0];
 
-  ConversionResult Result = ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
-                                               &ToPtr, ToPtr + NumBytes,
-                                               strictConversion);
-
-  // Check for conversion failure.
-  if (Result != conversionOK) {
-    // FIXME: Have Sema::CheckObjCString() validate the UTF-8 string and remove
-    // this duplicate code.
-    assert(Result == sourceIllegal && "UTF-8 to UTF-16 conversion failed");
-    StringLength = NumBytes;
-    return Map.GetOrCreateValue(String);
-  }
+  (void)ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
+                           &ToPtr, ToPtr + NumBytes,
+                           strictConversion);
 
   // ConvertUTF8toUTF16 returns the length in ToPtr.
   StringLength = ToPtr - &ToBuf[0];
@@ -1595,6 +1667,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(getModule(), C->getType(), isConstant, Linkage, C,
                              ".str");
+  GV->setUnnamedAddr(true);
   if (isUTF16) {
     CharUnits Align = getContext().getTypeAlignInChars(getContext().ShortTy);
     GV->setAlignment(Align.getQuantity());
@@ -1618,7 +1691,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
 }
 
 llvm::Constant *
-CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
+CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) {
   unsigned StringLength = 0;
   bool isUTF16 = false;
   llvm::StringMapEntry<llvm::Constant*> &Entry =
@@ -1634,16 +1707,27 @@ CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
   llvm::Constant *Zeros[] = { Zero, Zero };
   
   // If we don't already have it, get _NSConstantStringClassReference.
-  if (!NSConstantStringClassRef) {
+  if (!ConstantStringClassRef) {
+    std::string StringClass(getLangOptions().ObjCConstantStringClass);
     const llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
     Ty = llvm::ArrayType::get(Ty, 0);
-    llvm::Constant *GV = CreateRuntimeVariable(Ty, 
-                                        Features.ObjCNonFragileABI ?
-                                        "OBJC_CLASS_$_NSConstantString" :
-                                        "_NSConstantStringClassReference");
+    llvm::Constant *GV;
+    if (StringClass.empty())
+      GV = CreateRuntimeVariable(Ty, 
+                                 Features.ObjCNonFragileABI ?
+                                 "OBJC_CLASS_$_NSConstantString" :
+                                 "_NSConstantStringClassReference");
+    else {
+      std::string str;
+      if (Features.ObjCNonFragileABI)
+        str = "OBJC_CLASS_$_" + StringClass;
+      else
+        str = "_" + StringClass + "ClassReference";
+      GV = CreateRuntimeVariable(Ty, str);
+    }
     // Decay array -> ptr
-    NSConstantStringClassRef = 
-      llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
+    ConstantStringClassRef = 
+    llvm::ConstantExpr::getGetElementPtr(GV, Zeros, 2);
   }
   
   QualType NSTy = getContext().getNSConstantStringType();
@@ -1654,7 +1738,7 @@ CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
   std::vector<llvm::Constant*> Fields(3);
   
   // Class pointer.
-  Fields[0] = NSConstantStringClassRef;
+  Fields[0] = ConstantStringClassRef;
   
   // String pointer.
   llvm::Constant *C = llvm::ConstantArray::get(VMContext, Entry.getKey().str());
@@ -1675,6 +1759,7 @@ CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
   llvm::GlobalVariable *GV =
   new llvm::GlobalVariable(getModule(), C->getType(), isConstant, Linkage, C,
                            ".str");
+  GV->setUnnamedAddr(true);
   if (isUTF16) {
     CharUnits Align = getContext().getTypeAlignInChars(getContext().ShortTy);
     GV->setAlignment(Align.getQuantity());
@@ -1704,15 +1789,16 @@ CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
 /// GetStringForStringLiteral - Return the appropriate bytes for a
 /// string literal, properly padded to match the literal type.
 std::string CodeGenModule::GetStringForStringLiteral(const StringLiteral *E) {
+  const ASTContext &Context = getContext();
   const ConstantArrayType *CAT =
-    getContext().getAsConstantArrayType(E->getType());
+    Context.getAsConstantArrayType(E->getType());
   assert(CAT && "String isn't pointer or array!");
 
   // Resize the string to the right size.
   uint64_t RealLen = CAT->getSize().getZExtValue();
 
   if (E->isWide())
-    RealLen *= getContext().Target.getWCharWidth()/8;
+    RealLen *= Context.Target.getWCharWidth() / Context.getCharWidth();
 
   std::string Str = E->getString().str();
   Str.resize(RealLen, '\0');
@@ -1756,9 +1842,12 @@ static llvm::Constant *GenerateStringLiteral(const std::string &str,
       llvm::ConstantArray::get(CGM.getLLVMContext(), str, false);
 
   // Create a global variable for this string
-  return new llvm::GlobalVariable(CGM.getModule(), C->getType(), constant,
-                                  llvm::GlobalValue::PrivateLinkage,
-                                  C, GlobalName);
+  llvm::GlobalVariable *GV =
+    new llvm::GlobalVariable(CGM.getModule(), C->getType(), constant,
+                             llvm::GlobalValue::PrivateLinkage,
+                             C, GlobalName);
+  GV->setUnnamedAddr(true);
+  return GV;
 }
 
 /// GetAddrOfConstantString - Returns a pointer to a character array
@@ -2097,7 +2186,7 @@ llvm::Constant *CodeGenModule::getBlockObjectDispose() {
   const llvm::FunctionType *FTy;
   std::vector<const llvm::Type*> ArgTys;
   const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
-  ArgTys.push_back(PtrToInt8Ty);
+  ArgTys.push_back(Int8PtrTy);
   ArgTys.push_back(llvm::Type::getInt32Ty(VMContext));
   FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
   return BlockObjectDispose =
@@ -2119,8 +2208,8 @@ llvm::Constant *CodeGenModule::getBlockObjectAssign() {
   const llvm::FunctionType *FTy;
   std::vector<const llvm::Type*> ArgTys;
   const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
-  ArgTys.push_back(PtrToInt8Ty);
-  ArgTys.push_back(PtrToInt8Ty);
+  ArgTys.push_back(Int8PtrTy);
+  ArgTys.push_back(Int8PtrTy);
   ArgTys.push_back(llvm::Type::getInt32Ty(VMContext));
   FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
   return BlockObjectAssign =
@@ -2139,8 +2228,8 @@ llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
   }
 
   // Otherwise construct the variable by hand.
-  return NSConcreteGlobalBlock = CreateRuntimeVariable(
-    PtrToInt8Ty, "_NSConcreteGlobalBlock");
+  return NSConcreteGlobalBlock =
+    CreateRuntimeVariable(Int8PtrTy, "_NSConcreteGlobalBlock");
 }
 
 llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
@@ -2155,8 +2244,8 @@ llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
   }
 
   // Otherwise construct the variable by hand.
-  return NSConcreteStackBlock = CreateRuntimeVariable(
-    PtrToInt8Ty, "_NSConcreteStackBlock");
+  return NSConcreteStackBlock =
+    CreateRuntimeVariable(Int8PtrTy, "_NSConcreteStackBlock");
 }
 
 ///@}
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index cabff9e1cad5..b6bd37c1c0d3 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -14,17 +14,16 @@
 #ifndef CLANG_CODEGEN_CODEGENMODULE_H
 #define CLANG_CODEGEN_CODEGENMODULE_H
 
+#include "clang/Basic/ABI.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
-#include "CGBlocks.h"
+#include "clang/AST/Mangle.h"
 #include "CGCall.h"
-#include "CGCXX.h"
 #include "CGVTables.h"
 #include "CodeGenTypes.h"
 #include "GlobalDecl.h"
-#include "Mangle.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
@@ -66,14 +65,16 @@ namespace clang {
   class Diagnostic;
   class AnnotateAttr;
   class CXXDestructorDecl;
+  class MangleBuffer;
 
 namespace CodeGen {
 
   class CodeGenFunction;
+  class CodeGenTBAA;
   class CGCXXABI;
   class CGDebugInfo;
   class CGObjCRuntime;
-  class MangleBuffer;
+  class BlockFieldFlags;
   
   struct OrderGlobalInits {
     unsigned int priority;
@@ -93,10 +94,39 @@ namespace CodeGen {
       return priority == RHS.priority && lex_order < RHS.lex_order;
     }
   };
+
+  struct CodeGenTypeCache {
+    /// i8, i32, and i64
+    const llvm::IntegerType *Int8Ty, *Int32Ty, *Int64Ty;
+
+    /// int
+    const llvm::IntegerType *IntTy;
+
+    /// intptr_t and size_t, which we assume are the same
+    union {
+      const llvm::IntegerType *IntPtrTy;
+      const llvm::IntegerType *SizeTy;
+    };
+
+    /// void* in address space 0
+    union {
+      const llvm::PointerType *VoidPtrTy;
+      const llvm::PointerType *Int8PtrTy;
+    };
+
+    /// void** in address space 0
+    union {
+      const llvm::PointerType *VoidPtrPtrTy;
+      const llvm::PointerType *Int8PtrPtrTy;
+    };
+
+    /// The width of an address-zero pointer.
+    unsigned char PointerWidthInBits;
+  };
   
 /// CodeGenModule - This class organizes the cross-function state that is used
 /// while generating LLVM code.
-class CodeGenModule : public BlockModule {
+class CodeGenModule : public CodeGenTypeCache {
   CodeGenModule(const CodeGenModule&);  // DO NOT IMPLEMENT
   void operator=(const CodeGenModule&); // DO NOT IMPLEMENT
 
@@ -111,6 +141,7 @@ class CodeGenModule : public BlockModule {
   Diagnostic &Diags;
   CGCXXABI &ABI;
   CodeGenTypes Types;
+  CodeGenTBAA *TBAA;
 
   /// VTables - Holds information about C++ vtables.
   CodeGenVTables VTables;
@@ -183,9 +214,9 @@ class CodeGenModule : public BlockModule {
   /// strings. This value has type int * but is actually an Obj-C class pointer.
   llvm::Constant *CFConstantStringClassRef;
 
-  /// NSConstantStringClassRef - Cached reference to the class for constant
+  /// ConstantStringClassRef - Cached reference to the class for constant
   /// strings. This value has type int * but is actually an Obj-C class pointer.
-  llvm::Constant *NSConstantStringClassRef;
+  llvm::Constant *ConstantStringClassRef;
 
   /// Lazily create the Objective-C runtime
   void createObjCRuntime();
@@ -205,6 +236,16 @@ class CodeGenModule : public BlockModule {
   llvm::Constant *BlockObjectAssign;
   llvm::Constant *BlockObjectDispose;
 
+  const llvm::Type *BlockDescriptorType;
+  const llvm::Type *GenericBlockLiteralType;
+
+  struct {
+    int GlobalUniqueCount;
+  } Block;
+
+  llvm::DenseMap<uint64_t, llvm::Constant *> AssignCache;
+  llvm::DenseMap<uint64_t, llvm::Constant *> DestroyCache;
+
   /// @}
 public:
   CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
@@ -246,21 +287,43 @@ public:
   CodeGenVTables &getVTables() { return VTables; }
   Diagnostic &getDiags() const { return Diags; }
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
+  const TargetInfo &getTarget() const { return Context.Target; }
   llvm::LLVMContext &getLLVMContext() { return VMContext; }
   const TargetCodeGenInfo &getTargetCodeGenInfo();
   bool isTargetDarwin() const;
 
-  /// getDeclVisibilityMode - Compute the visibility of the decl \arg D.
-  LangOptions::VisibilityMode getDeclVisibilityMode(const Decl *D) const;
+  llvm::MDNode *getTBAAInfo(QualType QTy);
+
+  static void DecorateInstruction(llvm::Instruction *Inst,
+                                  llvm::MDNode *TBAAInfo);
 
   /// setGlobalVisibility - Set the visibility for the given LLVM
   /// GlobalValue.
-  void setGlobalVisibility(llvm::GlobalValue *GV, const Decl *D) const;
+  void setGlobalVisibility(llvm::GlobalValue *GV, const NamedDecl *D) const;
+
+  /// TypeVisibilityKind - The kind of global variable that is passed to 
+  /// setTypeVisibility
+  enum TypeVisibilityKind {
+    TVK_ForVTT,
+    TVK_ForVTable,
+    TVK_ForRTTI,
+    TVK_ForRTTIName
+  };
 
   /// setTypeVisibility - Set the visibility for the given global
   /// value which holds information about a type.
   void setTypeVisibility(llvm::GlobalValue *GV, const CXXRecordDecl *D,
-                         bool IsForRTTI) const;
+                         TypeVisibilityKind TVK) const;
+
+  static llvm::GlobalValue::VisibilityTypes GetLLVMVisibility(Visibility V) {
+    switch (V) {
+    case DefaultVisibility:   return llvm::GlobalValue::DefaultVisibility;
+    case HiddenVisibility:    return llvm::GlobalValue::HiddenVisibility;
+    case ProtectedVisibility: return llvm::GlobalValue::ProtectedVisibility;
+    }
+    llvm_unreachable("unknown visibility!");
+    return llvm::GlobalValue::DefaultVisibility;
+  }
 
   llvm::Constant *GetAddrOfGlobal(GlobalDecl GD) {
     if (isa<CXXConstructorDecl>(GD.getDecl()))
@@ -275,6 +338,14 @@ public:
       return GetAddrOfGlobalVar(cast<VarDecl>(GD.getDecl()));
   }
 
+  /// CreateOrReplaceCXXRuntimeVariable - Will return a global variable of the given
+  /// type. If a variable with a different type already exists then a new 
+  /// variable with the right type will be created and all uses of the old
+  /// variable will be replaced with a bitcast to the new variable.
+  llvm::GlobalVariable *
+  CreateOrReplaceCXXRuntimeVariable(llvm::StringRef Name, const llvm::Type *Ty,
+                                    llvm::GlobalValue::LinkageTypes Linkage);
+
   /// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
   /// given global variable.  If Ty is non-null and if the global doesn't exist,
   /// then it will be greated with the specified type instead of whatever the
@@ -286,7 +357,8 @@ public:
   /// non-null, then this function will use the specified type if it has to
   /// create it.
   llvm::Constant *GetAddrOfFunction(GlobalDecl GD,
-                                    const llvm::Type *Ty = 0);
+                                    const llvm::Type *Ty = 0,
+                                    bool ForVTable = false);
 
   /// GetAddrOfRTTIDescriptor - Get the address of the RTTI descriptor 
   /// for the given type.
@@ -304,6 +376,29 @@ public:
   GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
                                CastExpr::path_const_iterator PathBegin,
                                CastExpr::path_const_iterator PathEnd);
+
+  llvm::Constant *BuildbyrefCopyHelper(const llvm::Type *T,
+                                       BlockFieldFlags flags,
+                                       unsigned Align,
+                                       const VarDecl *variable);
+  llvm::Constant *BuildbyrefDestroyHelper(const llvm::Type *T,
+                                          BlockFieldFlags flags,
+                                          unsigned Align,
+                                          const VarDecl *variable);
+
+  /// getGlobalUniqueCount - Fetches the global unique block count.
+  int getGlobalUniqueCount() { return ++Block.GlobalUniqueCount; }
+
+  /// getBlockDescriptorType - Fetches the type of a generic block
+  /// descriptor.
+  const llvm::Type *getBlockDescriptorType();
+
+  /// getGenericBlockLiteralType - The type of a generic block literal.
+  const llvm::Type *getGenericBlockLiteralType();
+
+  /// GetAddrOfGlobalBlock - Gets the address of a block which
+  /// requires no captures.
+  llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *);
   
   /// GetStringForStringLiteral - Return the appropriate bytes for a string
   /// literal, properly padded to match the literal type. If only the address of
@@ -314,9 +409,10 @@ public:
   /// for the given string.
   llvm::Constant *GetAddrOfConstantCFString(const StringLiteral *Literal);
   
-  /// GetAddrOfConstantNSString - Return a pointer to a constant NSString object
-  /// for the given string.
-  llvm::Constant *GetAddrOfConstantNSString(const StringLiteral *Literal);
+  /// GetAddrOfConstantString - Return a pointer to a constant NSString object
+  /// for the given string. Or a user defined String object as defined via
+  /// -fconstant-string-class=class_name option.
+  llvm::Constant *GetAddrOfConstantString(const StringLiteral *Literal);
 
   /// GetAddrOfConstantStringFromLiteral - Return a pointer to a constant array
   /// for the given string literal.
@@ -363,17 +459,6 @@ public:
   llvm::Value *getBuiltinLibFunction(const FunctionDecl *FD,
                                      unsigned BuiltinID);
 
-  llvm::Function *getMemCpyFn(const llvm::Type *DestType,
-                              const llvm::Type *SrcType,
-                              const llvm::Type *SizeType);
-
-  llvm::Function *getMemMoveFn(const llvm::Type *DestType,
-                               const llvm::Type *SrcType,
-                               const llvm::Type *SizeType);
-
-  llvm::Function *getMemSetFn(const llvm::Type *DestType,
-                              const llvm::Type *SizeType);
-
   llvm::Function *getIntrinsic(unsigned IID, const llvm::Type **Tys = 0,
                                unsigned NumTys = 0);
 
@@ -417,6 +502,8 @@ public:
     Types.UpdateCompletedType(TD);
   }
 
+  llvm::Constant *getMemberPointerConstant(const UnaryOperator *e);
+
   /// EmitConstantExpr - Try to emit the given expression as a
   /// constant; returns 0 if the expression cannot be emitted as a
   /// constant.
@@ -485,7 +572,8 @@ public:
                               unsigned &CallingConv);
 
   llvm::StringRef getMangledName(GlobalDecl GD);
-  void getMangledName(GlobalDecl GD, MangleBuffer &Buffer, const BlockDecl *BD);
+  void getBlockMangledName(GlobalDecl GD, MangleBuffer &Buffer,
+                           const BlockDecl *BD);
 
   void EmitTentativeDefinition(const VarDecl *D);
 
@@ -500,13 +588,18 @@ public:
 
   /// getVTableLinkage - Return the appropriate linkage for the vtable, VTT,
   /// and type information of the given class.
-  static llvm::GlobalVariable::LinkageTypes 
-  getVTableLinkage(const CXXRecordDecl *RD);
+  llvm::GlobalVariable::LinkageTypes getVTableLinkage(const CXXRecordDecl *RD);
 
   /// GetTargetTypeStoreSize - Return the store size, in character units, of
   /// the given LLVM type.
   CharUnits GetTargetTypeStoreSize(const llvm::Type *Ty) const;
-
+  
+  /// GetLLVMLinkageVarDefinition - Returns LLVM linkage for a global 
+  /// variable.
+  llvm::GlobalValue::LinkageTypes 
+  GetLLVMLinkageVarDefinition(const VarDecl *D,
+                              llvm::GlobalVariable *GV);
+  
   std::vector<const CXXRecordDecl*> DeferredVTables;
 
 private:
@@ -514,10 +607,12 @@ private:
 
   llvm::Constant *GetOrCreateLLVMFunction(llvm::StringRef MangledName,
                                           const llvm::Type *Ty,
-                                          GlobalDecl D);
+                                          GlobalDecl D,
+                                          bool ForVTable);
   llvm::Constant *GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
                                         const llvm::PointerType *PTy,
-                                        const VarDecl *D);
+                                        const VarDecl *D,
+                                        bool UnnamedAddr = false);
 
   /// SetCommonAttributes - Set attributes which are common to any
   /// form of a global definition (alias, Objective-C method,
@@ -547,7 +642,7 @@ private:
   void EmitAliasDefinition(GlobalDecl GD);
   void EmitObjCPropertyImplementations(const ObjCImplementationDecl *D);
   void EmitObjCIvarInitializations(ObjCImplementationDecl *D);
-
+  
   // C++ related functions.
 
   bool TryEmitDefinitionAsAlias(GlobalDecl Alias, GlobalDecl Target);
@@ -578,7 +673,8 @@ private:
   /// EmitCXXGlobalDtorFunc - Emit the function that destroys C++ globals.
   void EmitCXXGlobalDtorFunc();
 
-  void EmitCXXGlobalVarDeclInitFunc(const VarDecl *D);
+  void EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
+                                    llvm::GlobalVariable *Addr);
 
   // FIXME: Hardcoding priority here is gross.
   void AddGlobalCtor(llvm::Function *Ctor, int Priority=65535);
@@ -613,6 +709,10 @@ private:
   /// lazily; this is only relevant for definitions. The given decl
   /// must be either a function or var decl.
   bool MayDeferGeneration(const ValueDecl *D);
+
+  /// SimplifyPersonality - Check whether we can use a "simpler", more
+  /// core exceptions personality function.
+  void SimplifyPersonality();
 };
 }  // end namespace CodeGen
 }  // end namespace clang
diff --git a/lib/CodeGen/CodeGenTBAA.cpp b/lib/CodeGen/CodeGenTBAA.cpp
new file mode 100644
index 000000000000..3f2c6cabf2ae
--- /dev/null
+++ b/lib/CodeGen/CodeGenTBAA.cpp
@@ -0,0 +1,180 @@
+//===--- CodeGenTypes.cpp - TBAA information for LLVM CodeGen -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the code that manages TBAA information and defines the TBAA policy
+// for the optimizer to use. Relevant standards text includes:
+//
+//   C99 6.5p7
+//   C++ [basic.lval] (p10 in n3126, p15 in some earlier versions)
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenTBAA.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Mangle.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/Metadata.h"
+#include "llvm/Constants.h"
+#include "llvm/Type.h"
+using namespace clang;
+using namespace CodeGen;
+
+CodeGenTBAA::CodeGenTBAA(ASTContext &Ctx, llvm::LLVMContext& VMContext,
+                         const LangOptions &Features, MangleContext &MContext)
+  : Context(Ctx), VMContext(VMContext), Features(Features), MContext(MContext),
+    Root(0), Char(0) {
+}
+
+CodeGenTBAA::~CodeGenTBAA() {
+}
+
+llvm::MDNode *CodeGenTBAA::getRoot() {
+  // Define the root of the tree. This identifies the tree, so that
+  // if our LLVM IR is linked with LLVM IR from a different front-end
+  // (or a different version of this front-end), their TBAA trees will
+  // remain distinct, and the optimizer will treat them conservatively.
+  if (!Root)
+    Root = getTBAAInfoForNamedType("Simple C/C++ TBAA", 0);
+
+  return Root;
+}
+
+llvm::MDNode *CodeGenTBAA::getChar() {
+  // Define the root of the tree for user-accessible memory. C and C++
+  // give special powers to char and certain similar types. However,
+  // these special powers only cover user-accessible memory, and doesn't
+  // include things like vtables.
+  if (!Char)
+    Char = getTBAAInfoForNamedType("omnipotent char", getRoot());
+
+  return Char;
+}
+
+/// getTBAAInfoForNamedType - Create a TBAA tree node with the given string
+/// as its identifier, and the given Parent node as its tree parent.
+llvm::MDNode *CodeGenTBAA::getTBAAInfoForNamedType(llvm::StringRef NameStr,
+                                                   llvm::MDNode *Parent,
+                                                   bool Readonly) {
+  // Currently there is only one flag defined - the readonly flag.
+  llvm::Value *Flags = 0;
+  if (Readonly)
+    Flags = llvm::ConstantInt::get(llvm::Type::getInt64Ty(VMContext), true);
+
+  // Set up the mdnode operand list.
+  llvm::Value *Ops[] = {
+    llvm::MDString::get(VMContext, NameStr),
+    Parent,
+    Flags
+  };
+
+  // Create the mdnode.
+  return llvm::MDNode::get(VMContext, Ops, llvm::array_lengthof(Ops) - !Flags);
+}
+
+static bool TypeHasMayAlias(QualType QTy) {
+  // Tagged types have declarations, and therefore may have attributes.
+  if (const TagType *TTy = dyn_cast<TagType>(QTy))
+    return TTy->getDecl()->hasAttr<MayAliasAttr>();
+
+  // Typedef types have declarations, and therefore may have attributes.
+  if (const TypedefType *TTy = dyn_cast<TypedefType>(QTy)) {
+    if (TTy->getDecl()->hasAttr<MayAliasAttr>())
+      return true;
+    // Also, their underlying types may have relevant attributes.
+    return TypeHasMayAlias(TTy->desugar());
+  }
+
+  return false;
+}
+
+llvm::MDNode *
+CodeGenTBAA::getTBAAInfo(QualType QTy) {
+  // If the type has the may_alias attribute (even on a typedef), it is
+  // effectively in the general char alias class.
+  if (TypeHasMayAlias(QTy))
+    return getChar();
+
+  const Type *Ty = Context.getCanonicalType(QTy).getTypePtr();
+
+  if (llvm::MDNode *N = MetadataCache[Ty])
+    return N;
+
+  // Handle builtin types.
+  if (const BuiltinType *BTy = dyn_cast<BuiltinType>(Ty)) {
+    switch (BTy->getKind()) {
+    // Character types are special and can alias anything.
+    // In C++, this technically only includes "char" and "unsigned char",
+    // and not "signed char". In C, it includes all three. For now,
+    // the risk of exploiting this detail in C++ seems likely to outweigh
+    // the benefit.
+    case BuiltinType::Char_U:
+    case BuiltinType::Char_S:
+    case BuiltinType::UChar:
+    case BuiltinType::SChar:
+      return getChar();
+
+    // Unsigned types can alias their corresponding signed types.
+    case BuiltinType::UShort:
+      return getTBAAInfo(Context.ShortTy);
+    case BuiltinType::UInt:
+      return getTBAAInfo(Context.IntTy);
+    case BuiltinType::ULong:
+      return getTBAAInfo(Context.LongTy);
+    case BuiltinType::ULongLong:
+      return getTBAAInfo(Context.LongLongTy);
+    case BuiltinType::UInt128:
+      return getTBAAInfo(Context.Int128Ty);
+
+    // Treat all other builtin types as distinct types. This includes
+    // treating wchar_t, char16_t, and char32_t as distinct from their
+    // "underlying types".
+    default:
+      return MetadataCache[Ty] =
+               getTBAAInfoForNamedType(BTy->getName(Features), getChar());
+    }
+  }
+
+  // Handle pointers.
+  // TODO: Implement C++'s type "similarity" and consider dis-"similar"
+  // pointers distinct.
+  if (Ty->isPointerType())
+    return MetadataCache[Ty] = getTBAAInfoForNamedType("any pointer",
+                                                       getChar());
+
+  // Enum types are distinct types. In C++ they have "underlying types",
+  // however they aren't related for TBAA.
+  if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
+    // In C mode, two anonymous enums are compatible iff their members
+    // are the same -- see C99 6.2.7p1. For now, be conservative. We could
+    // theoretically implement this by combining information about all the
+    // members into a single identifying MDNode.
+    if (!Features.CPlusPlus &&
+        ETy->getDecl()->getTypedefForAnonDecl())
+      return MetadataCache[Ty] = getChar();
+
+    // In C++ mode, types have linkage, so we can rely on the ODR and
+    // on their mangled names, if they're external.
+    // TODO: Is there a way to get a program-wide unique name for a
+    // decl with local linkage or no linkage?
+    if (Features.CPlusPlus &&
+        ETy->getDecl()->getLinkage() != ExternalLinkage)
+      return MetadataCache[Ty] = getChar();
+
+    // TODO: This is using the RTTI name. Is there a better way to get
+    // a unique string for a type?
+    llvm::SmallString<256> OutName;
+    llvm::raw_svector_ostream Out(OutName);
+    MContext.mangleCXXRTTIName(QualType(ETy, 0), Out);
+    Out.flush();
+    return MetadataCache[Ty] = getTBAAInfoForNamedType(OutName, getChar());
+  }
+
+  // For now, handle any other kind of type conservatively.
+  return MetadataCache[Ty] = getChar();
+}
diff --git a/lib/CodeGen/CodeGenTBAA.h b/lib/CodeGen/CodeGenTBAA.h
new file mode 100644
index 000000000000..c4583473a0e0
--- /dev/null
+++ b/lib/CodeGen/CodeGenTBAA.h
@@ -0,0 +1,76 @@
+//===--- CodeGenTBAA.h - TBAA information for LLVM CodeGen ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the code that manages TBAA information and defines the TBAA policy
+// for the optimizer to use.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CODEGEN_CODEGENTBAA_H
+#define CLANG_CODEGEN_CODEGENTBAA_H
+
+#include "llvm/LLVMContext.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace llvm {
+  class LLVMContext;
+  class MDNode;
+}
+
+namespace clang {
+  class ASTContext;
+  class LangOptions;
+  class MangleContext;
+  class QualType;
+  class Type;
+
+namespace CodeGen {
+  class CGRecordLayout;
+
+/// CodeGenTBAA - This class organizes the cross-module state that is used
+/// while lowering AST types to LLVM types.
+class CodeGenTBAA {
+  ASTContext &Context;
+  llvm::LLVMContext& VMContext;
+  const LangOptions &Features;
+  MangleContext &MContext;
+
+  /// MetadataCache - This maps clang::Types to llvm::MDNodes describing them.
+  llvm::DenseMap<const Type *, llvm::MDNode *> MetadataCache;
+
+  llvm::MDNode *Root;
+  llvm::MDNode *Char;
+
+  /// getRoot - This is the mdnode for the root of the metadata type graph
+  /// for this translation unit.
+  llvm::MDNode *getRoot();
+
+  /// getChar - This is the mdnode for "char", which is special, and any types
+  /// considered to be equivalent to it.
+  llvm::MDNode *getChar();
+
+  llvm::MDNode *getTBAAInfoForNamedType(llvm::StringRef NameStr,
+                                        llvm::MDNode *Parent,
+                                        bool Readonly = false);
+
+public:
+  CodeGenTBAA(ASTContext &Ctx, llvm::LLVMContext &VMContext,
+              const LangOptions &Features,
+              MangleContext &MContext);
+  ~CodeGenTBAA();
+
+  /// getTBAAInfo - Get the TBAA MDNode to be used for a dereference
+  /// of the given type.
+  llvm::MDNode *getTBAAInfo(QualType QTy);
+};
+
+}  // end namespace CodeGen
+}  // end namespace clang
+
+#endif
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index 5ab65c5779b6..5254922f13a1 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -85,7 +85,7 @@ const llvm::Type *CodeGenTypes::ConvertTypeRecursive(QualType T) {
   T = Context.getCanonicalType(T);
 
   // See if type is already cached.
-  llvm::DenseMap<Type *, llvm::PATypeHolder>::iterator
+  llvm::DenseMap<const Type *, llvm::PATypeHolder>::iterator
     I = TypeCache.find(T.getTypePtr());
   // If type is found in map and this is not a definition for a opaque
   // place holder type then use it. Otherwise, convert type T.
@@ -228,7 +228,8 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
     case BuiltinType::ULong:
     case BuiltinType::LongLong:
     case BuiltinType::ULongLong:
-    case BuiltinType::WChar:
+    case BuiltinType::WChar_S:
+    case BuiltinType::WChar_U:
     case BuiltinType::Char16:
     case BuiltinType::Char32:
       return llvm::IntegerType::get(getLLVMContext(),
@@ -252,7 +253,6 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
     
     case BuiltinType::Overload:
     case BuiltinType::Dependent:
-    case BuiltinType::UndeducedAuto:
       assert(0 && "Unexpected builtin type!");
       break;
     }
@@ -371,26 +371,30 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
     const TagDecl *TD = cast<TagType>(Ty).getDecl();
     const llvm::Type *Res = ConvertTagDeclType(TD);
 
-    std::string TypeName(TD->getKindName());
-    TypeName += '.';
+    llvm::SmallString<256> TypeName;
+    llvm::raw_svector_ostream OS(TypeName);
+    OS << TD->getKindName() << '.';
 
     // Name the codegen type after the typedef name
     // if there is no tag type name available
-    if (TD->getIdentifier())
+    if (TD->getIdentifier()) {
       // FIXME: We should not have to check for a null decl context here.
       // Right now we do it because the implicit Obj-C decls don't have one.
-      TypeName += TD->getDeclContext() ? TD->getQualifiedNameAsString() :
-        TD->getNameAsString();
-    else if (const TypedefType *TdT = dyn_cast<TypedefType>(T))
+      if (TD->getDeclContext())
+        OS << TD->getQualifiedNameAsString();
+      else
+        TD->printName(OS);
+    } else if (const TypedefDecl *TDD = TD->getTypedefForAnonDecl()) {
       // FIXME: We should not have to check for a null decl context here.
       // Right now we do it because the implicit Obj-C decls don't have one.
-      TypeName += TdT->getDecl()->getDeclContext() ? 
-        TdT->getDecl()->getQualifiedNameAsString() :
-        TdT->getDecl()->getNameAsString();
-    else
-      TypeName += "anon";
-
-    TheModule.addTypeName(TypeName, Res);
+      if (TDD->getDeclContext())
+        OS << TDD->getQualifiedNameAsString();
+      else
+        TDD->printName(OS);
+    } else
+      OS << "anon";
+
+    TheModule.addTypeName(OS.str(), Res);
     return Res;
   }
 
@@ -424,9 +428,13 @@ const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
   if (TDTI != TagDeclTypes.end())
     return TDTI->second;
 
+  const EnumDecl *ED = dyn_cast<EnumDecl>(TD);
+
   // If this is still a forward declaration, just define an opaque
   // type to use for this tagged decl.
-  if (!TD->isDefinition()) {
+  // C++0x: If this is a enumeration type with fixed underlying type,
+  // consider it complete.
+  if (!TD->isDefinition() && !(ED && ED->isFixed())) {
     llvm::Type *ResultType = llvm::OpaqueType::get(getLLVMContext());
     TagDeclTypes.insert(std::make_pair(Key, ResultType));
     return ResultType;
@@ -434,8 +442,8 @@ const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
 
   // Okay, this is a definition of a type.  Compile the implementation now.
 
-  if (TD->isEnum())  // Don't bother storing enums in TagDeclTypes.
-    return ConvertTypeRecursive(cast<EnumDecl>(TD)->getIntegerType());
+  if (ED)  // Don't bother storing enums in TagDeclTypes.
+    return ConvertTypeRecursive(ED->getIntegerType());
 
   // This decl could well be recursive.  In this case, insert an opaque
   // definition of this type, which the recursive uses will get.  We will then
@@ -474,11 +482,20 @@ const llvm::Type *CodeGenTypes::ConvertTagDeclType(const TagDecl *TD) {
   return ResultHolder.get();
 }
 
-/// getCGRecordLayout - Return record layout info for the given llvm::Type.
+/// getCGRecordLayout - Return record layout info for the given record decl.
 const CGRecordLayout &
-CodeGenTypes::getCGRecordLayout(const RecordDecl *TD) const {
-  const Type *Key = Context.getTagDeclType(TD).getTypePtr();
+CodeGenTypes::getCGRecordLayout(const RecordDecl *RD) {
+  const Type *Key = Context.getTagDeclType(RD).getTypePtr();
+
   const CGRecordLayout *Layout = CGRecordLayouts.lookup(Key);
+  if (!Layout) {
+    // Compute the type information.
+    ConvertTagDeclType(RD);
+
+    // Now try again.
+    Layout = CGRecordLayouts.lookup(Key);
+  }
+
   assert(Layout && "Unable to find record layout information for type");
   return *Layout;
 }
@@ -506,11 +523,5 @@ bool CodeGenTypes::isZeroInitializable(QualType T) {
 }
 
 bool CodeGenTypes::isZeroInitializable(const CXXRecordDecl *RD) {
-  
-  // FIXME: It would be better if there was a way to explicitly compute the
-  // record layout instead of converting to a type.
-  ConvertTagDeclType(RD);
-  
-  const CGRecordLayout &Layout = getCGRecordLayout(RD);
-  return Layout.isZeroInitializable();
+  return getCGRecordLayout(RD).isZeroInitializable();
 }
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index 1fc2153fcadb..41513daf17ca 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -88,14 +88,14 @@ private:
   /// and maps llvm::Types to corresponding clang::Type. llvm::PATypeHolder is
   /// used instead of llvm::Type because it allows us to bypass potential
   /// dangling type pointers due to type refinement on llvm side.
-  llvm::DenseMap<Type *, llvm::PATypeHolder> TypeCache;
+  llvm::DenseMap<const Type *, llvm::PATypeHolder> TypeCache;
 
   /// ConvertNewType - Convert type T into a llvm::Type. Do not use this
   /// method directly because it does not do any type caching. This method
   /// is available only for ConvertType(). CovertType() is preferred
   /// interface to convert type T into a llvm::Type.
   const llvm::Type *ConvertNewType(QualType T);
-  
+
   /// HandleLateResolvedPointers - For top-level ConvertType calls, this handles
   /// pointers that are referenced but have not been converted yet.  This is
   /// used to handle cyclic structures properly.
@@ -139,11 +139,11 @@ public:
   static const TagType *VerifyFuncTypeComplete(const Type* T);
 
   /// GetFunctionTypeForVTable - Get the LLVM function type for use in a vtable,
-  /// given a CXXMethodDecl. If the method to has an incomplete return type, 
+  /// given a CXXMethodDecl. If the method to has an incomplete return type,
   /// and/or incomplete argument types, this will return the opaque type.
   const llvm::Type *GetFunctionTypeForVTable(GlobalDecl GD);
-                                                     
-  const CGRecordLayout &getCGRecordLayout(const RecordDecl*) const;
+
+  const CGRecordLayout &getCGRecordLayout(const RecordDecl*);
 
   /// UpdateCompletedType - When we find the full definition for a TagDecl,
   /// replace the 'opaque' type we previously made for it if applicable.
@@ -151,7 +151,7 @@ public:
 
   /// getFunctionInfo - Get the function info for the specified function decl.
   const CGFunctionInfo &getFunctionInfo(GlobalDecl GD);
-  
+
   const CGFunctionInfo &getFunctionInfo(const FunctionDecl *FD);
   const CGFunctionInfo &getFunctionInfo(const CXXMethodDecl *MD);
   const CGFunctionInfo &getFunctionInfo(const ObjCMethodDecl *MD);
@@ -176,7 +176,7 @@ public:
   /// pointers.
   const CGFunctionInfo &getFunctionInfo(const CXXRecordDecl *RD,
                                         const FunctionProtoType *FTP);
-  
+
   /// getFunctionInfo - Get the function info for a function described by a
   /// return type and argument types. If the calling convention is not
   /// specified, the "C" calling convention will be used.
@@ -188,7 +188,7 @@ public:
                                         const FunctionType::ExtInfo &Info);
 
   /// Retrieves the ABI information for the given function signature.
-  /// 
+  ///
   /// \param ArgTys - must all actually be canonical as params
   const CGFunctionInfo &getFunctionInfo(CanQualType RetTy,
                                const llvm::SmallVectorImpl<CanQualType> &ArgTys,
@@ -208,11 +208,11 @@ public:  // These are internal details of CGT that shouldn't be used externally.
   /// ArgTys. See ABIArgInfo::Expand.
   void GetExpandedTypes(QualType Ty, std::vector<const llvm::Type*> &ArgTys,
                         bool IsRecursive);
-  
+
   /// IsZeroInitializable - Return whether a type can be
   /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
   bool isZeroInitializable(QualType T);
-  
+
   /// IsZeroInitializable - Return whether a record type can be
   /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
   bool isZeroInitializable(const CXXRecordDecl *RD);
diff --git a/lib/CodeGen/GlobalDecl.h b/lib/CodeGen/GlobalDecl.h
index 26dea402f4b6..c2f36d210bfc 100644
--- a/lib/CodeGen/GlobalDecl.h
+++ b/lib/CodeGen/GlobalDecl.h
@@ -15,9 +15,9 @@
 #ifndef CLANG_CODEGEN_GLOBALDECL_H
 #define CLANG_CODEGEN_GLOBALDECL_H
 
-#include "CGCXX.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/Basic/ABI.h"
 
 namespace clang {
 
@@ -81,6 +81,12 @@ public:
     GD.Value.setFromOpaqueValue(P);
     return GD;
   }
+  
+  GlobalDecl getWithDecl(const Decl *D) {
+    GlobalDecl Result(*this);
+    Result.Value.setPointer(D);
+    return Result;
+  }
 };
 
 } // end namespace CodeGen
diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp
index eefc530ccf18..95654a33a125 100644
--- a/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/lib/CodeGen/ItaniumCXXABI.cpp
@@ -22,7 +22,7 @@
 #include "CGRecordLayout.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
-#include "Mangle.h"
+#include <clang/AST/Mangle.h>
 #include <clang/AST/Type.h>
 #include <llvm/Target/TargetData.h>
 #include <llvm/Value.h>
@@ -35,7 +35,6 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
 private:
   const llvm::IntegerType *PtrDiffTy;
 protected:
-  CodeGen::MangleContext MangleCtx;
   bool IsARM;
 
   // It's a little silly for us to cache this.
@@ -48,16 +47,13 @@ protected:
     return PtrDiffTy;
   }
 
-  bool NeedsArrayCookie(QualType ElementType);
+  bool NeedsArrayCookie(const CXXNewExpr *expr);
+  bool NeedsArrayCookie(const CXXDeleteExpr *expr,
+                        QualType elementType);
 
 public:
   ItaniumCXXABI(CodeGen::CodeGenModule &CGM, bool IsARM = false) :
-    CGCXXABI(CGM), PtrDiffTy(0), MangleCtx(getContext(), CGM.getDiags()),
-    IsARM(IsARM) { }
-
-  CodeGen::MangleContext &getMangleContext() {
-    return MangleCtx;
-  }
+    CGCXXABI(CGM), PtrDiffTy(0), IsARM(IsARM) { }
 
   bool isZeroInitializable(const MemberPointerType *MPT);
 
@@ -83,7 +79,8 @@ public:
   llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT);
 
   llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD);
-  llvm::Constant *EmitMemberPointer(const FieldDecl *FD);
+  llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
+                                        CharUnits offset);
 
   llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
                                            llvm::Value *L,
@@ -111,14 +108,19 @@ public:
 
   void EmitInstanceFunctionProlog(CodeGenFunction &CGF);
 
-  CharUnits GetArrayCookieSize(QualType ElementType);
+  CharUnits GetArrayCookieSize(const CXXNewExpr *expr);
   llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
                                      llvm::Value *NewPtr,
                                      llvm::Value *NumElements,
+                                     const CXXNewExpr *expr,
                                      QualType ElementType);
   void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                       const CXXDeleteExpr *expr,
                        QualType ElementType, llvm::Value *&NumElements,
                        llvm::Value *&AllocPtr, CharUnits &CookieSize);
+
+  void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
+                       llvm::GlobalVariable *DeclPtr);
 };
 
 class ARMCXXABI : public ItaniumCXXABI {
@@ -143,12 +145,14 @@ public:
 
   void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV, QualType ResTy);
 
-  CharUnits GetArrayCookieSize(QualType ElementType);
+  CharUnits GetArrayCookieSize(const CXXNewExpr *expr);
   llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
                                      llvm::Value *NewPtr,
                                      llvm::Value *NumElements,
+                                     const CXXNewExpr *expr,
                                      QualType ElementType);
   void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                       const CXXDeleteExpr *expr,
                        QualType ElementType, llvm::Value *&NumElements,
                        llvm::Value *&AllocPtr, CharUnits &CookieSize);
 
@@ -352,11 +356,11 @@ ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
   bool DerivedToBase =
     E->getCastKind() == CK_DerivedToBaseMemberPointer;
 
-  const CXXRecordDecl *BaseDecl, *DerivedDecl;
+  const CXXRecordDecl *DerivedDecl;
   if (DerivedToBase)
-    DerivedDecl = SrcDecl, BaseDecl = DestDecl;
+    DerivedDecl = SrcDecl;
   else
-    BaseDecl = SrcDecl, DerivedDecl = DestDecl;
+    DerivedDecl = DestDecl;
 
   llvm::Constant *Adj = 
     CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl,
@@ -490,21 +494,13 @@ ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
                                    /*Packed=*/false);
 }
 
-llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const FieldDecl *FD) {
+llvm::Constant *
+ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
+                                     CharUnits offset) {
   // Itanium C++ ABI 2.3:
   //   A pointer to data member is an offset from the base address of
   //   the class object containing it, represented as a ptrdiff_t
-
-  QualType ClassType = getContext().getTypeDeclType(FD->getParent());
-  const llvm::StructType *ClassLTy =
-    cast<llvm::StructType>(CGM.getTypes().ConvertType(ClassType));
-
-  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(FD->getParent());
-  unsigned FieldNo = RL.getLLVMFieldNo(FD);
-  uint64_t Offset = 
-    CGM.getTargetData().getStructLayout(ClassLTy)->getElementOffset(FieldNo);
-
-  return llvm::ConstantInt::get(getPtrDiffTy(), Offset);
+  return llvm::ConstantInt::get(getPtrDiffTy(), offset.getQuantity());
 }
 
 llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
@@ -799,67 +795,49 @@ void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
 
 /************************** Array allocation cookies **************************/
 
-bool ItaniumCXXABI::NeedsArrayCookie(QualType ElementType) {
-  ElementType = getContext().getBaseElementType(ElementType);
-  const RecordType *RT = ElementType->getAs<RecordType>();
-  if (!RT) return false;
-  
-  const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-
-  // If the class has a non-trivial destructor, it always needs a cookie.
-  if (!RD->hasTrivialDestructor()) return true;
+bool ItaniumCXXABI::NeedsArrayCookie(const CXXNewExpr *expr) {
+  // If the class's usual deallocation function takes two arguments,
+  // it needs a cookie.
+  if (expr->doesUsualArrayDeleteWantSize())
+    return true;
+
+  // Otherwise, if the class has a non-trivial destructor, it always
+  // needs a cookie.
+  const CXXRecordDecl *record =
+    expr->getAllocatedType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
+  return (record && !record->hasTrivialDestructor());
+}
 
+bool ItaniumCXXABI::NeedsArrayCookie(const CXXDeleteExpr *expr,
+                                     QualType elementType) {
   // If the class's usual deallocation function takes two arguments,
-  // it needs a cookie.  Otherwise we don't need a cookie.
-  const CXXMethodDecl *UsualDeallocationFunction = 0;
-
-  // Usual deallocation functions of this form are always found on the
-  // class.
-  //
-  // FIXME: what exactly is this code supposed to do if there's an
-  // ambiguity?  That's possible with using declarations.
-  DeclarationName OpName =
-    getContext().DeclarationNames.getCXXOperatorName(OO_Array_Delete);
-  DeclContext::lookup_const_iterator Op, OpEnd;
-  for (llvm::tie(Op, OpEnd) = RD->lookup(OpName); Op != OpEnd; ++Op) {
-    const CXXMethodDecl *Delete =
-      cast<CXXMethodDecl>((*Op)->getUnderlyingDecl());
-
-    if (Delete->isUsualDeallocationFunction()) {
-      UsualDeallocationFunction = Delete;
-      break;
-    }
-  }
-    
-  // No usual deallocation function, we don't need a cookie.
-  if (!UsualDeallocationFunction)
-    return false;
-    
-  // The usual deallocation function doesn't take a size_t argument,
-  // so we don't need a cookie.
-  if (UsualDeallocationFunction->getNumParams() == 1)
-    return false;
-        
-  assert(UsualDeallocationFunction->getNumParams() == 2 && 
-         "Unexpected deallocation function type!");
-  return true;
-}
-
-CharUnits ItaniumCXXABI::GetArrayCookieSize(QualType ElementType) {
-  if (!NeedsArrayCookie(ElementType))
+  // it needs a cookie.
+  if (expr->doesUsualArrayDeleteWantSize())
+    return true;
+
+  // Otherwise, if the class has a non-trivial destructor, it always
+  // needs a cookie.
+  const CXXRecordDecl *record =
+    elementType->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
+  return (record && !record->hasTrivialDestructor());
+}
+
+CharUnits ItaniumCXXABI::GetArrayCookieSize(const CXXNewExpr *expr) {
+  if (!NeedsArrayCookie(expr))
     return CharUnits::Zero();
   
-  // Padding is the maximum of sizeof(size_t) and alignof(ElementType)
+  // Padding is the maximum of sizeof(size_t) and alignof(elementType)
   ASTContext &Ctx = getContext();
   return std::max(Ctx.getTypeSizeInChars(Ctx.getSizeType()),
-                  Ctx.getTypeAlignInChars(ElementType));
+                  Ctx.getTypeAlignInChars(expr->getAllocatedType()));
 }
 
 llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
                                                   llvm::Value *NewPtr,
                                                   llvm::Value *NumElements,
+                                                  const CXXNewExpr *expr,
                                                   QualType ElementType) {
-  assert(NeedsArrayCookie(ElementType));
+  assert(NeedsArrayCookie(expr));
 
   unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
 
@@ -892,6 +870,7 @@ llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
 
 void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
                                     llvm::Value *Ptr,
+                                    const CXXDeleteExpr *expr,
                                     QualType ElementType,
                                     llvm::Value *&NumElements,
                                     llvm::Value *&AllocPtr,
@@ -901,7 +880,7 @@ void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
   const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
 
   // If we don't need an array cookie, bail out early.
-  if (!NeedsArrayCookie(ElementType)) {
+  if (!NeedsArrayCookie(expr, ElementType)) {
     AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
     NumElements = 0;
     CookieSize = CharUnits::Zero();
@@ -932,8 +911,8 @@ void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
   NumElements = CGF.Builder.CreateLoad(NumElementsPtr);
 }
 
-CharUnits ARMCXXABI::GetArrayCookieSize(QualType ElementType) {
-  if (!NeedsArrayCookie(ElementType))
+CharUnits ARMCXXABI::GetArrayCookieSize(const CXXNewExpr *expr) {
+  if (!NeedsArrayCookie(expr))
     return CharUnits::Zero();
 
   // On ARM, the cookie is always:
@@ -949,8 +928,9 @@ CharUnits ARMCXXABI::GetArrayCookieSize(QualType ElementType) {
 llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
                                               llvm::Value *NewPtr,
                                               llvm::Value *NumElements,
+                                              const CXXNewExpr *expr,
                                               QualType ElementType) {
-  assert(NeedsArrayCookie(ElementType));
+  assert(NeedsArrayCookie(expr));
 
   // NewPtr is a char*.
 
@@ -983,6 +963,7 @@ llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
 
 void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
                                 llvm::Value *Ptr,
+                                const CXXDeleteExpr *expr,
                                 QualType ElementType,
                                 llvm::Value *&NumElements,
                                 llvm::Value *&AllocPtr,
@@ -992,7 +973,7 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
   const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
 
   // If we don't need an array cookie, bail out early.
-  if (!NeedsArrayCookie(ElementType)) {
+  if (!NeedsArrayCookie(expr, ElementType)) {
     AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
     NumElements = 0;
     CookieSize = CharUnits::Zero();
@@ -1005,7 +986,6 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
   
   // The cookie size is always 2 * sizeof(size_t).
   CookieSize = 2 * SizeSize;
-  CharUnits NumElementsOffset = CookieSize - SizeSize;
 
   // The allocated pointer is the input ptr, minus that amount.
   AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
@@ -1021,3 +1001,168 @@ void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
   NumElements = CGF.Builder.CreateLoad(NumElementsPtr);
 }
 
+/*********************** Static local initialization **************************/
+
+static llvm::Constant *getGuardAcquireFn(CodeGenModule &CGM,
+                                         const llvm::PointerType *GuardPtrTy) {
+  // int __cxa_guard_acquire(__guard *guard_object);
+  
+  std::vector<const llvm::Type*> Args(1, GuardPtrTy);  
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(CGM.getTypes().ConvertType(CGM.getContext().IntTy),
+                            Args, /*isVarArg=*/false);
+  
+  return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire");
+}
+
+static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
+                                         const llvm::PointerType *GuardPtrTy) {
+  // void __cxa_guard_release(__guard *guard_object);
+  
+  std::vector<const llvm::Type*> Args(1, GuardPtrTy);
+  
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
+                            Args, /*isVarArg=*/false);
+  
+  return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release");
+}
+
+static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM,
+                                       const llvm::PointerType *GuardPtrTy) {
+  // void __cxa_guard_abort(__guard *guard_object);
+  
+  std::vector<const llvm::Type*> Args(1, GuardPtrTy);
+  
+  const llvm::FunctionType *FTy =
+    llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()),
+                            Args, /*isVarArg=*/false);
+  
+  return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort");
+}
+
+namespace {
+  struct CallGuardAbort : EHScopeStack::Cleanup {
+    llvm::GlobalVariable *Guard;
+    CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      CGF.Builder.CreateCall(getGuardAbortFn(CGF.CGM, Guard->getType()), Guard)
+        ->setDoesNotThrow();
+    }
+  };
+}
+
+/// The ARM code here follows the Itanium code closely enough that we
+/// just special-case it at particular places.
+void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
+                                    const VarDecl &D,
+                                    llvm::GlobalVariable *GV) {
+  CGBuilderTy &Builder = CGF.Builder;
+
+  // We only need to use thread-safe statics for local variables;
+  // global initialization is always single-threaded.
+  bool ThreadsafeStatics = (getContext().getLangOptions().ThreadsafeStatics &&
+                            D.isLocalVarDecl());
+  
+  // Guard variables are 64 bits in the generic ABI and 32 bits on ARM.
+  const llvm::IntegerType *GuardTy
+    = (IsARM ? Builder.getInt32Ty() : Builder.getInt64Ty());
+  const llvm::PointerType *GuardPtrTy = GuardTy->getPointerTo();
+
+  // Create the guard variable.
+  llvm::SmallString<256> GuardVName;
+  llvm::raw_svector_ostream Out(GuardVName);
+  getMangleContext().mangleItaniumGuardVariable(&D, Out);
+  Out.flush();
+
+  // Just absorb linkage and visibility from the variable.
+  llvm::GlobalVariable *GuardVariable =
+    new llvm::GlobalVariable(CGM.getModule(), GuardTy,
+                             false, GV->getLinkage(),
+                             llvm::ConstantInt::get(GuardTy, 0),
+                             GuardVName.str());
+  GuardVariable->setVisibility(GV->getVisibility());
+
+  // Test whether the variable has completed initialization.
+  llvm::Value *IsInitialized;
+
+  // ARM C++ ABI 3.2.3.1:
+  //   To support the potential use of initialization guard variables
+  //   as semaphores that are the target of ARM SWP and LDREX/STREX
+  //   synchronizing instructions we define a static initialization
+  //   guard variable to be a 4-byte aligned, 4- byte word with the
+  //   following inline access protocol.
+  //     #define INITIALIZED 1
+  //     if ((obj_guard & INITIALIZED) != INITIALIZED) {
+  //       if (__cxa_guard_acquire(&obj_guard))
+  //         ...
+  //     }
+  if (IsARM) {
+    llvm::Value *V = Builder.CreateLoad(GuardVariable);
+    V = Builder.CreateAnd(V, Builder.getInt32(1));
+    IsInitialized = Builder.CreateIsNull(V, "guard.uninitialized");
+
+  // Itanium C++ ABI 3.3.2:
+  //   The following is pseudo-code showing how these functions can be used:
+  //     if (obj_guard.first_byte == 0) {
+  //       if ( __cxa_guard_acquire (&obj_guard) ) {
+  //         try {
+  //           ... initialize the object ...;
+  //         } catch (...) {
+  //            __cxa_guard_abort (&obj_guard);
+  //            throw;
+  //         }
+  //         ... queue object destructor with __cxa_atexit() ...;
+  //         __cxa_guard_release (&obj_guard);
+  //       }
+  //     }
+  } else {
+    // Load the first byte of the guard variable.
+    const llvm::Type *PtrTy = Builder.getInt8PtrTy();
+    llvm::Value *V = 
+      Builder.CreateLoad(Builder.CreateBitCast(GuardVariable, PtrTy), "tmp");
+
+    IsInitialized = Builder.CreateIsNull(V, "guard.uninitialized");
+  }
+
+  llvm::BasicBlock *InitCheckBlock = CGF.createBasicBlock("init.check");
+  llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end");
+
+  // Check if the first byte of the guard variable is zero.
+  Builder.CreateCondBr(IsInitialized, InitCheckBlock, EndBlock);
+
+  CGF.EmitBlock(InitCheckBlock);
+
+  // Variables used when coping with thread-safe statics and exceptions.
+  if (ThreadsafeStatics) {    
+    // Call __cxa_guard_acquire.
+    llvm::Value *V
+      = Builder.CreateCall(getGuardAcquireFn(CGM, GuardPtrTy), GuardVariable);
+               
+    llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init");
+  
+    Builder.CreateCondBr(Builder.CreateIsNotNull(V, "tobool"),
+                         InitBlock, EndBlock);
+  
+    // Call __cxa_guard_abort along the exceptional edge.
+    CGF.EHStack.pushCleanup<CallGuardAbort>(EHCleanup, GuardVariable);
+    
+    CGF.EmitBlock(InitBlock);
+  }
+
+  // Emit the initializer and add a global destructor if appropriate.
+  CGF.EmitCXXGlobalVarDeclInit(D, GV);
+
+  if (ThreadsafeStatics) {
+    // Pop the guard-abort cleanup if we pushed one.
+    CGF.PopCleanupBlock();
+
+    // Call __cxa_guard_release.  This cannot throw.
+    Builder.CreateCall(getGuardReleaseFn(CGM, GuardPtrTy), GuardVariable);
+  } else {
+    Builder.CreateStore(llvm::ConstantInt::get(GuardTy, 1), GuardVariable);
+  }
+
+  CGF.EmitBlock(EndBlock);
+}
diff --git a/lib/CodeGen/Mangle.h b/lib/CodeGen/Mangle.h
deleted file mode 100644
index 139f6c0377ec..000000000000
--- a/lib/CodeGen/Mangle.h
+++ /dev/null
@@ -1,177 +0,0 @@
-//===--- Mangle.h - Mangle C++ Names ----------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements C++ name mangling according to the Itanium C++ ABI,
-// which is used in GCC 3.2 and newer (and many compilers that are
-// ABI-compatible with GCC):
-//
-//   http://www.codesourcery.com/public/cxx-abi/abi.html
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_CODEGEN_MANGLE_H
-#define LLVM_CLANG_CODEGEN_MANGLE_H
-
-#include "CGCXX.h"
-#include "GlobalDecl.h"
-#include "clang/AST/Type.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace clang {
-  class ASTContext;
-  class BlockDecl;
-  class CXXConstructorDecl;
-  class CXXDestructorDecl;
-  class CXXMethodDecl;
-  class FunctionDecl;
-  class NamedDecl;
-  class ObjCMethodDecl;
-  class VarDecl;
-
-namespace CodeGen {
-  struct ThisAdjustment;
-  struct ThunkInfo;
-
-/// MangleBuffer - a convenient class for storing a name which is
-/// either the result of a mangling or is a constant string with
-/// external memory ownership.
-class MangleBuffer {
-public:
-  void setString(llvm::StringRef Ref) {
-    String = Ref;
-  }
-
-  llvm::SmallVectorImpl<char> &getBuffer() {
-    return Buffer;
-  }
-
-  llvm::StringRef getString() const {
-    if (!String.empty()) return String;
-    return Buffer.str();
-  }
-
-  operator llvm::StringRef() const {
-    return getString();
-  }
-
-private:
-  llvm::StringRef String;
-  llvm::SmallString<256> Buffer;
-};
-
-/// MangleContext - Context for tracking state which persists across multiple
-/// calls to the C++ name mangler.
-class MangleContext {
-  ASTContext &Context;
-  Diagnostic &Diags;
-
-  llvm::DenseMap<const TagDecl *, uint64_t> AnonStructIds;
-  unsigned Discriminator;
-  llvm::DenseMap<const NamedDecl*, unsigned> Uniquifier;
-  llvm::DenseMap<const BlockDecl*, unsigned> GlobalBlockIds;
-  llvm::DenseMap<const BlockDecl*, unsigned> LocalBlockIds;
-  
-public:
-  explicit MangleContext(ASTContext &Context,
-                         Diagnostic &Diags)
-    : Context(Context), Diags(Diags) { }
-
-  virtual ~MangleContext() { }
-
-  ASTContext &getASTContext() const { return Context; }
-
-  Diagnostic &getDiags() const { return Diags; }
-
-  void startNewFunction() { LocalBlockIds.clear(); }
-  
-  uint64_t getAnonymousStructId(const TagDecl *TD) {
-    std::pair<llvm::DenseMap<const TagDecl *,
-      uint64_t>::iterator, bool> Result =
-      AnonStructIds.insert(std::make_pair(TD, AnonStructIds.size()));
-    return Result.first->second;
-  }
-
-  unsigned getBlockId(const BlockDecl *BD, bool Local) {
-    llvm::DenseMap<const BlockDecl *, unsigned> &BlockIds
-      = Local? LocalBlockIds : GlobalBlockIds;
-    std::pair<llvm::DenseMap<const BlockDecl *, unsigned>::iterator, bool>
-      Result = BlockIds.insert(std::make_pair(BD, BlockIds.size()));
-    return Result.first->second;
-  }
-  
-  /// @name Mangler Entry Points
-  /// @{
-
-  virtual bool shouldMangleDeclName(const NamedDecl *D);
-  virtual void mangleName(const NamedDecl *D, llvm::SmallVectorImpl<char> &);
-  virtual void mangleThunk(const CXXMethodDecl *MD,
-                          const ThunkInfo &Thunk,
-                          llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
-                                  const ThisAdjustment &ThisAdjustment,
-                                  llvm::SmallVectorImpl<char> &);
-  virtual void mangleGuardVariable(const VarDecl *D,
-                                   llvm::SmallVectorImpl<char> &);
-  virtual void mangleReferenceTemporary(const VarDecl *D,
-                                        llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXVTable(const CXXRecordDecl *RD,
-                               llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXVTT(const CXXRecordDecl *RD,
-                            llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
-                                   const CXXRecordDecl *Type,
-                                   llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXRTTI(QualType T, llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXRTTIName(QualType T, llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                             llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                             llvm::SmallVectorImpl<char> &);
-  void mangleBlock(GlobalDecl GD,
-                   const BlockDecl *BD, llvm::SmallVectorImpl<char> &);
-
-  void mangleInitDiscriminator() {
-    Discriminator = 0;
-  }
-
-  bool getNextDiscriminator(const NamedDecl *ND, unsigned &disc) {
-    unsigned &discriminator = Uniquifier[ND];
-    if (!discriminator)
-      discriminator = ++Discriminator;
-    if (discriminator == 1)
-      return false;
-    disc = discriminator-2;
-    return true;
-  }
-  /// @}
-};
-
-/// MiscNameMangler - Mangles Objective-C method names and blocks.
-class MiscNameMangler {
-  MangleContext &Context;
-  llvm::raw_svector_ostream Out;
-  
-  ASTContext &getASTContext() const { return Context.getASTContext(); }
-
-public:
-  MiscNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res);
-
-  llvm::raw_svector_ostream &getStream() { return Out; }
-  
-  void mangleBlock(GlobalDecl GD, const BlockDecl *BD);
-  void mangleObjCMethodName(const ObjCMethodDecl *MD);
-};
-
-}
-}
-
-#endif
diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp
index 9407335e3283..3a63eba39741 100644
--- a/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -16,107 +16,17 @@
 
 #include "CGCXXABI.h"
 #include "CodeGenModule.h"
-#include "Mangle.h"
-#include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
-#include "clang/AST/DeclTemplate.h"
-#include "clang/AST/ExprCXX.h"
-#include "CGVTables.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 namespace {
 
-/// MicrosoftCXXNameMangler - Manage the mangling of a single name for the
-/// Microsoft Visual C++ ABI.
-class MicrosoftCXXNameMangler {
-  MangleContext &Context;
-  llvm::raw_svector_ostream Out;
-
-  ASTContext &getASTContext() const { return Context.getASTContext(); }
-
-public:
-  MicrosoftCXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res)
-  : Context(C), Out(Res) { }
-
-  void mangle(const NamedDecl *D, llvm::StringRef Prefix = "?");
-  void mangleName(const NamedDecl *ND);
-  void mangleFunctionEncoding(const FunctionDecl *FD);
-  void mangleVariableEncoding(const VarDecl *VD);
-  void mangleNumber(int64_t Number);
-  void mangleType(QualType T);
-
-private:
-  void mangleUnqualifiedName(const NamedDecl *ND) {
-    mangleUnqualifiedName(ND, ND->getDeclName());
-  }
-  void mangleUnqualifiedName(const NamedDecl *ND, DeclarationName Name);
-  void mangleSourceName(const IdentifierInfo *II);
-  void manglePostfix(const DeclContext *DC, bool NoFunction=false);
-  void mangleOperatorName(OverloadedOperatorKind OO);
-  void mangleQualifiers(Qualifiers Quals, bool IsMember);
-
-  void mangleObjCMethodName(const ObjCMethodDecl *MD);
-
-  // Declare manglers for every type class.
-#define ABSTRACT_TYPE(CLASS, PARENT)
-#define NON_CANONICAL_TYPE(CLASS, PARENT)
-#define TYPE(CLASS, PARENT) void mangleType(const CLASS##Type *T);
-#include "clang/AST/TypeNodes.def"
-  
-  void mangleType(const TagType*);
-  void mangleType(const FunctionType *T, const FunctionDecl *D,
-                  bool IsStructor, bool IsInstMethod);
-  void mangleType(const ArrayType *T, bool IsGlobal);
-  void mangleExtraDimensions(QualType T);
-  void mangleFunctionClass(const FunctionDecl *FD);
-  void mangleCallingConvention(const FunctionType *T);
-  void mangleThrowSpecification(const FunctionProtoType *T);
-
-};
-
-/// MicrosoftMangleContext - Overrides the default MangleContext for the
-/// Microsoft Visual C++ ABI.
-class MicrosoftMangleContext : public MangleContext {
-public:
-  MicrosoftMangleContext(ASTContext &Context,
-                         Diagnostic &Diags) : MangleContext(Context, Diags) { }
-  virtual bool shouldMangleDeclName(const NamedDecl *D);
-  virtual void mangleName(const NamedDecl *D, llvm::SmallVectorImpl<char> &);
-  virtual void mangleThunk(const CXXMethodDecl *MD,
-                           const ThunkInfo &Thunk,
-                           llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
-                                  const ThisAdjustment &ThisAdjustment,
-                                  llvm::SmallVectorImpl<char> &);
-  virtual void mangleGuardVariable(const VarDecl *D,
-                                   llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXVTable(const CXXRecordDecl *RD,
-                               llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXVTT(const CXXRecordDecl *RD,
-                            llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
-                                   const CXXRecordDecl *Type,
-                                   llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXRTTI(QualType T, llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXRTTIName(QualType T, llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXCtor(const CXXConstructorDecl *D, CXXCtorType Type,
-                             llvm::SmallVectorImpl<char> &);
-  virtual void mangleCXXDtor(const CXXDestructorDecl *D, CXXDtorType Type,
-                             llvm::SmallVectorImpl<char> &);
-};
-
 class MicrosoftCXXABI : public CGCXXABI {
-  MicrosoftMangleContext MangleCtx;
 public:
-  MicrosoftCXXABI(CodeGenModule &CGM)
-    : CGCXXABI(CGM), MangleCtx(CGM.getContext(), CGM.getDiags()) {}
-
-  MicrosoftMangleContext &getMangleContext() {
-    return MangleCtx;
-  }
+  MicrosoftCXXABI(CodeGenModule &CGM) : CGCXXABI(CGM) {}
 
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType Type,
@@ -145,1071 +55,31 @@ public:
     EmitThisParam(CGF);
     // TODO: 'for base' flag
   }
-};
-
-}
-
-static bool isInCLinkageSpecification(const Decl *D) {
-  D = D->getCanonicalDecl();
-  for (const DeclContext *DC = D->getDeclContext();
-       !DC->isTranslationUnit(); DC = DC->getParent()) {
-    if (const LinkageSpecDecl *Linkage = dyn_cast<LinkageSpecDecl>(DC))
-      return Linkage->getLanguage() == LinkageSpecDecl::lang_c;
-  }
-
-  return false;
-}
-
-bool MicrosoftMangleContext::shouldMangleDeclName(const NamedDecl *D) {
-  // In C, functions with no attributes never need to be mangled. Fastpath them.
-  if (!getASTContext().getLangOptions().CPlusPlus && !D->hasAttrs())
-    return false;
-
-  // Any decl can be declared with __asm("foo") on it, and this takes precedence
-  // over all other naming in the .o file.
-  if (D->hasAttr<AsmLabelAttr>())
-    return true;
-
-  // Clang's "overloadable" attribute extension to C/C++ implies name mangling
-  // (always) as does passing a C++ member function and a function
-  // whose name is not a simple identifier.
-  const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
-  if (FD && (FD->hasAttr<OverloadableAttr>() || isa<CXXMethodDecl>(FD) ||
-             !FD->getDeclName().isIdentifier()))
-    return true;
-
-  // Otherwise, no mangling is done outside C++ mode.
-  if (!getASTContext().getLangOptions().CPlusPlus)
-    return false;
-
-  // Variables at global scope with internal linkage are not mangled.
-  if (!FD) {
-    const DeclContext *DC = D->getDeclContext();
-    if (DC->isTranslationUnit() && D->getLinkage() == InternalLinkage)
-      return false;
-  }
-
-  // C functions and "main" are not mangled.
-  if ((FD && FD->isMain()) || isInCLinkageSpecification(D))
-    return false;
-
-  return true;
-}
-
-void MicrosoftCXXNameMangler::mangle(const NamedDecl *D,
-                                     llvm::StringRef Prefix) {
-  // MSVC doesn't mangle C++ names the same way it mangles extern "C" names.
-  // Therefore it's really important that we don't decorate the
-  // name with leading underscores or leading/trailing at signs. So, emit a
-  // asm marker at the start so we get the name right.
-  Out << '\01';  // LLVM IR Marker for __asm("foo")
-
-  // Any decl can be declared with __asm("foo") on it, and this takes precedence
-  // over all other naming in the .o file.
-  if (const AsmLabelAttr *ALA = D->getAttr<AsmLabelAttr>()) {
-    // If we have an asm name, then we use it as the mangling.
-    Out << ALA->getLabel();
-    return;
-  }
-
-  // <mangled-name> ::= ? <name> <type-encoding>
-  Out << Prefix;
-  mangleName(D);
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
-    mangleFunctionEncoding(FD);
-  else if (const VarDecl *VD = dyn_cast<VarDecl>(D))
-    mangleVariableEncoding(VD);
-  // TODO: Fields? Can MSVC even mangle them?
-}
-
-void MicrosoftCXXNameMangler::mangleFunctionEncoding(const FunctionDecl *FD) {
-  // <type-encoding> ::= <function-class> <function-type>
-
-  // Don't mangle in the type if this isn't a decl we should typically mangle.
-  if (!Context.shouldMangleDeclName(FD))
-    return;
-  
-  // We should never ever see a FunctionNoProtoType at this point.
-  // We don't even know how to mangle their types anyway :).
-  const FunctionProtoType *FT = cast<FunctionProtoType>(FD->getType());
-
-  bool InStructor = false, InInstMethod = false;
-  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
-  if (MD) {
-    if (MD->isInstance())
-      InInstMethod = true;
-    if (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD))
-      InStructor = true;
-  }
-
-  // First, the function class.
-  mangleFunctionClass(FD);
-
-  mangleType(FT, FD, InStructor, InInstMethod);
-}
-
-void MicrosoftCXXNameMangler::mangleVariableEncoding(const VarDecl *VD) {
-  // <type-encoding> ::= <storage-class> <variable-type>
-  // <storage-class> ::= 0  # private static member
-  //                 ::= 1  # protected static member
-  //                 ::= 2  # public static member
-  //                 ::= 3  # global
-  //                 ::= 4  # static local
-  
-  // The first character in the encoding (after the name) is the storage class.
-  if (VD->isStaticDataMember()) {
-    // If it's a static member, it also encodes the access level.
-    switch (VD->getAccess()) {
-      default:
-      case AS_private: Out << '0'; break;
-      case AS_protected: Out << '1'; break;
-      case AS_public: Out << '2'; break;
-    }
-  }
-  else if (!VD->isStaticLocal())
-    Out << '3';
-  else
-    Out << '4';
-  // Now mangle the type.
-  // <variable-type> ::= <type> <cvr-qualifiers>
-  //                 ::= <type> A # pointers, references, arrays
-  // Pointers and references are odd. The type of 'int * const foo;' gets
-  // mangled as 'QAHA' instead of 'PAHB', for example.
-  QualType Ty = VD->getType();
-  if (Ty->isPointerType() || Ty->isReferenceType()) {
-    mangleType(Ty);
-    Out << 'A';
-  } else if (Ty->isArrayType()) {
-    // Global arrays are funny, too.
-    mangleType(static_cast<ArrayType *>(Ty.getTypePtr()), true);
-    Out << 'A';
-  } else {
-    mangleType(Ty.getLocalUnqualifiedType());
-    mangleQualifiers(Ty.getLocalQualifiers(), false);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleName(const NamedDecl *ND) {
-  // <name> ::= <unscoped-name> {[<named-scope>]+ | [<nested-name>]}? @
-  const DeclContext *DC = ND->getDeclContext();
-
-  // Always start with the unqualified name.
-  mangleUnqualifiedName(ND);    
-
-  // If this is an extern variable declared locally, the relevant DeclContext
-  // is that of the containing namespace, or the translation unit.
-  if (isa<FunctionDecl>(DC) && ND->hasLinkage())
-    while (!DC->isNamespace() && !DC->isTranslationUnit())
-      DC = DC->getParent();
-
-  manglePostfix(DC);
-
-  // Terminate the whole name with an '@'.
-  Out << '@';
-}
-
-void MicrosoftCXXNameMangler::mangleNumber(int64_t Number) {
-  // <number> ::= [?] <decimal digit> # <= 9
-  //          ::= [?] <hex digit>+ @ # > 9; A = 0, B = 1, etc...
-  if (Number < 0) {
-    Out << '?';
-    Number = -Number;
-  }
-  if (Number >= 1 && Number <= 10) {
-    Out << Number-1;
-  } else {
-    // We have to build up the encoding in reverse order, so it will come
-    // out right when we write it out.
-    char Encoding[16];
-    char *EndPtr = Encoding+sizeof(Encoding);
-    char *CurPtr = EndPtr;
-    while (Number) {
-      *--CurPtr = 'A' + (Number % 16);
-      Number /= 16;
-    }
-    Out.write(CurPtr, EndPtr-CurPtr);
-    Out << '@';
-  }
-}
-
-void
-MicrosoftCXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
-                                               DeclarationName Name) {
-  //  <unqualified-name> ::= <operator-name>
-  //                     ::= <ctor-dtor-name>
-  //                     ::= <source-name>
-  switch (Name.getNameKind()) {
-    case DeclarationName::Identifier: {
-      if (const IdentifierInfo *II = Name.getAsIdentifierInfo()) {
-        mangleSourceName(II);
-        break;
-      }
-      
-      // Otherwise, an anonymous entity.  We must have a declaration.
-      assert(ND && "mangling empty name without declaration");
-      
-      if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
-        if (NS->isAnonymousNamespace()) {
-          Out << "?A";
-          break;
-        }
-      }
-      
-      // We must have an anonymous struct.
-      const TagDecl *TD = cast<TagDecl>(ND);
-      if (const TypedefDecl *D = TD->getTypedefForAnonDecl()) {
-        assert(TD->getDeclContext() == D->getDeclContext() &&
-               "Typedef should not be in another decl context!");
-        assert(D->getDeclName().getAsIdentifierInfo() &&
-               "Typedef was not named!");
-        mangleSourceName(D->getDeclName().getAsIdentifierInfo());
-        break;
-      }
-
-      // When VC encounters an anonymous type with no tag and no typedef,
-      // it literally emits '<unnamed-tag>'.
-      Out << "<unnamed-tag>";
-      break;
-    }
-      
-    case DeclarationName::ObjCZeroArgSelector:
-    case DeclarationName::ObjCOneArgSelector:
-    case DeclarationName::ObjCMultiArgSelector:
-      assert(false && "Can't mangle Objective-C selector names here!");
-      break;
-      
-    case DeclarationName::CXXConstructorName:
-      assert(false && "Can't mangle constructors yet!");
-      break;
-      
-    case DeclarationName::CXXDestructorName:
-      assert(false && "Can't mangle destructors yet!");
-      break;
-      
-    case DeclarationName::CXXConversionFunctionName:
-      // <operator-name> ::= ?B # (cast)
-      // The target type is encoded as the return type.
-      Out << "?B";
-      break;
-      
-    case DeclarationName::CXXOperatorName:
-      mangleOperatorName(Name.getCXXOverloadedOperator());
-      break;
-      
-    case DeclarationName::CXXLiteralOperatorName:
-      // FIXME: Was this added in VS2010? Does MS even know how to mangle this?
-      assert(false && "Don't know how to mangle literal operators yet!");
-      break;
-      
-    case DeclarationName::CXXUsingDirective:
-      assert(false && "Can't mangle a using directive name!");
-      break;
-  }
-}
-
-void MicrosoftCXXNameMangler::manglePostfix(const DeclContext *DC,
-                                            bool NoFunction) {
-  // <postfix> ::= <unqualified-name> [<postfix>]
-  //           ::= <template-postfix> <template-args> [<postfix>]
-  //           ::= <template-param>
-  //           ::= <substitution> [<postfix>]
-
-  if (!DC) return;
-
-  while (isa<LinkageSpecDecl>(DC))
-    DC = DC->getParent();
-
-  if (DC->isTranslationUnit())
-    return;
-
-  if (const BlockDecl *BD = dyn_cast<BlockDecl>(DC)) {
-    llvm::SmallString<64> Name;
-    Context.mangleBlock(GlobalDecl(), BD, Name);
-    Out << Name << '@';
-    return manglePostfix(DC->getParent(), NoFunction);
-  }
-
-  if (NoFunction && (isa<FunctionDecl>(DC) || isa<ObjCMethodDecl>(DC)))
-    return;
-  else if (const ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(DC))
-    mangleObjCMethodName(Method);
-  else {
-    mangleUnqualifiedName(cast<NamedDecl>(DC));
-    manglePostfix(DC->getParent(), NoFunction);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO) {
-  switch (OO) {
-  //                     ?0 # constructor
-  //                     ?1 # destructor
-  // <operator-name> ::= ?2 # new
-  case OO_New: Out << "?2"; break;
-  // <operator-name> ::= ?3 # delete
-  case OO_Delete: Out << "?3"; break;
-  // <operator-name> ::= ?4 # =
-  case OO_Equal: Out << "?4"; break;
-  // <operator-name> ::= ?5 # >>
-  case OO_GreaterGreater: Out << "?5"; break;
-  // <operator-name> ::= ?6 # <<
-  case OO_LessLess: Out << "?6"; break;
-  // <operator-name> ::= ?7 # !
-  case OO_Exclaim: Out << "?7"; break;
-  // <operator-name> ::= ?8 # ==
-  case OO_EqualEqual: Out << "?8"; break;
-  // <operator-name> ::= ?9 # !=
-  case OO_ExclaimEqual: Out << "?9"; break;
-  // <operator-name> ::= ?A # []
-  case OO_Subscript: Out << "?A"; break;
-  //                     ?B # conversion
-  // <operator-name> ::= ?C # ->
-  case OO_Arrow: Out << "?C"; break;
-  // <operator-name> ::= ?D # *
-  case OO_Star: Out << "?D"; break;
-  // <operator-name> ::= ?E # ++
-  case OO_PlusPlus: Out << "?E"; break;
-  // <operator-name> ::= ?F # --
-  case OO_MinusMinus: Out << "?F"; break;
-  // <operator-name> ::= ?G # -
-  case OO_Minus: Out << "?G"; break;
-  // <operator-name> ::= ?H # +
-  case OO_Plus: Out << "?H"; break;
-  // <operator-name> ::= ?I # &
-  case OO_Amp: Out << "?I"; break;
-  // <operator-name> ::= ?J # ->*
-  case OO_ArrowStar: Out << "?J"; break;
-  // <operator-name> ::= ?K # /
-  case OO_Slash: Out << "?K"; break;
-  // <operator-name> ::= ?L # %
-  case OO_Percent: Out << "?L"; break;
-  // <operator-name> ::= ?M # <
-  case OO_Less: Out << "?M"; break;
-  // <operator-name> ::= ?N # <=
-  case OO_LessEqual: Out << "?N"; break;
-  // <operator-name> ::= ?O # >
-  case OO_Greater: Out << "?O"; break;
-  // <operator-name> ::= ?P # >=
-  case OO_GreaterEqual: Out << "?P"; break;
-  // <operator-name> ::= ?Q # ,
-  case OO_Comma: Out << "?Q"; break;
-  // <operator-name> ::= ?R # ()
-  case OO_Call: Out << "?R"; break;
-  // <operator-name> ::= ?S # ~
-  case OO_Tilde: Out << "?S"; break;
-  // <operator-name> ::= ?T # ^
-  case OO_Caret: Out << "?T"; break;
-  // <operator-name> ::= ?U # |
-  case OO_Pipe: Out << "?U"; break;
-  // <operator-name> ::= ?V # &&
-  case OO_AmpAmp: Out << "?V"; break;
-  // <operator-name> ::= ?W # ||
-  case OO_PipePipe: Out << "?W"; break;
-  // <operator-name> ::= ?X # *=
-  case OO_StarEqual: Out << "?X"; break;
-  // <operator-name> ::= ?Y # +=
-  case OO_PlusEqual: Out << "?Y"; break;
-  // <operator-name> ::= ?Z # -=
-  case OO_MinusEqual: Out << "?Z"; break;
-  // <operator-name> ::= ?_0 # /=
-  case OO_SlashEqual: Out << "?_0"; break;
-  // <operator-name> ::= ?_1 # %=
-  case OO_PercentEqual: Out << "?_1"; break;
-  // <operator-name> ::= ?_2 # >>=
-  case OO_GreaterGreaterEqual: Out << "?_2"; break;
-  // <operator-name> ::= ?_3 # <<=
-  case OO_LessLessEqual: Out << "?_3"; break;
-  // <operator-name> ::= ?_4 # &=
-  case OO_AmpEqual: Out << "?_4"; break;
-  // <operator-name> ::= ?_5 # |=
-  case OO_PipeEqual: Out << "?_5"; break;
-  // <operator-name> ::= ?_6 # ^=
-  case OO_CaretEqual: Out << "?_6"; break;
-  //                     ?_7 # vftable
-  //                     ?_8 # vbtable
-  //                     ?_9 # vcall
-  //                     ?_A # typeof
-  //                     ?_B # local static guard
-  //                     ?_C # string
-  //                     ?_D # vbase destructor
-  //                     ?_E # vector deleting destructor
-  //                     ?_F # default constructor closure
-  //                     ?_G # scalar deleting destructor
-  //                     ?_H # vector constructor iterator
-  //                     ?_I # vector destructor iterator
-  //                     ?_J # vector vbase constructor iterator
-  //                     ?_K # virtual displacement map
-  //                     ?_L # eh vector constructor iterator
-  //                     ?_M # eh vector destructor iterator
-  //                     ?_N # eh vector vbase constructor iterator
-  //                     ?_O # copy constructor closure
-  //                     ?_P<name> # udt returning <name>
-  //                     ?_Q # <unknown>
-  //                     ?_R0 # RTTI Type Descriptor
-  //                     ?_R1 # RTTI Base Class Descriptor at (a,b,c,d)
-  //                     ?_R2 # RTTI Base Class Array
-  //                     ?_R3 # RTTI Class Hierarchy Descriptor
-  //                     ?_R4 # RTTI Complete Object Locator
-  //                     ?_S # local vftable
-  //                     ?_T # local vftable constructor closure
-  // <operator-name> ::= ?_U # new[]
-  case OO_Array_New: Out << "?_U"; break;
-  // <operator-name> ::= ?_V # delete[]
-  case OO_Array_Delete: Out << "?_V"; break;
-    
-  case OO_Conditional:
-    assert(false && "Don't know how to mangle ?:");
-    break;
-    
-  case OO_None:
-  case NUM_OVERLOADED_OPERATORS:
-    assert(false && "Not an overloaded operator");
-    break;
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleSourceName(const IdentifierInfo *II) {
-  // <source name> ::= <identifier> @
-  Out << II->getName() << '@';
-}
-
-void MicrosoftCXXNameMangler::mangleObjCMethodName(const ObjCMethodDecl *MD) {
-  llvm::SmallString<64> Buffer;
-  MiscNameMangler(Context, Buffer).mangleObjCMethodName(MD);
-  Out << Buffer;
-}
-
-void MicrosoftCXXNameMangler::mangleQualifiers(Qualifiers Quals,
-                                               bool IsMember) {
-  // <cvr-qualifiers> ::= [E] [F] [I] <base-cvr-qualifiers>
-  // 'E' means __ptr64 (32-bit only); 'F' means __unaligned (32/64-bit only);
-  // 'I' means __restrict (32/64-bit).
-  // Note that the MSVC __restrict keyword isn't the same as the C99 restrict
-  // keyword!
-  // <base-cvr-qualifiers> ::= A  # near
-  //                       ::= B  # near const
-  //                       ::= C  # near volatile
-  //                       ::= D  # near const volatile
-  //                       ::= E  # far (16-bit)
-  //                       ::= F  # far const (16-bit)
-  //                       ::= G  # far volatile (16-bit)
-  //                       ::= H  # far const volatile (16-bit)
-  //                       ::= I  # huge (16-bit)
-  //                       ::= J  # huge const (16-bit)
-  //                       ::= K  # huge volatile (16-bit)
-  //                       ::= L  # huge const volatile (16-bit)
-  //                       ::= M <basis> # based
-  //                       ::= N <basis> # based const
-  //                       ::= O <basis> # based volatile
-  //                       ::= P <basis> # based const volatile
-  //                       ::= Q  # near member
-  //                       ::= R  # near const member
-  //                       ::= S  # near volatile member
-  //                       ::= T  # near const volatile member
-  //                       ::= U  # far member (16-bit)
-  //                       ::= V  # far const member (16-bit)
-  //                       ::= W  # far volatile member (16-bit)
-  //                       ::= X  # far const volatile member (16-bit)
-  //                       ::= Y  # huge member (16-bit)
-  //                       ::= Z  # huge const member (16-bit)
-  //                       ::= 0  # huge volatile member (16-bit)
-  //                       ::= 1  # huge const volatile member (16-bit)
-  //                       ::= 2 <basis> # based member
-  //                       ::= 3 <basis> # based const member
-  //                       ::= 4 <basis> # based volatile member
-  //                       ::= 5 <basis> # based const volatile member
-  //                       ::= 6  # near function (pointers only)
-  //                       ::= 7  # far function (pointers only)
-  //                       ::= 8  # near method (pointers only)
-  //                       ::= 9  # far method (pointers only)
-  //                       ::= _A <basis> # based function (pointers only)
-  //                       ::= _B <basis> # based function (far?) (pointers only)
-  //                       ::= _C <basis> # based method (pointers only)
-  //                       ::= _D <basis> # based method (far?) (pointers only)
-  //                       ::= _E # block (Clang)
-  // <basis> ::= 0 # __based(void)
-  //         ::= 1 # __based(segment)?
-  //         ::= 2 <name> # __based(name)
-  //         ::= 3 # ?
-  //         ::= 4 # ?
-  //         ::= 5 # not really based
-  if (!IsMember) {
-    if (!Quals.hasVolatile()) {
-      if (!Quals.hasConst())
-        Out << 'A';
-      else
-        Out << 'B';
-    } else {
-      if (!Quals.hasConst())
-        Out << 'C';
-      else
-        Out << 'D';
-    }
-  } else {
-    if (!Quals.hasVolatile()) {
-      if (!Quals.hasConst())
-        Out << 'Q';
-      else
-        Out << 'R';
-    } else {
-      if (!Quals.hasConst())
-        Out << 'S';
-      else
-        Out << 'T';
-    }
-  }
-
-  // FIXME: For now, just drop all extension qualifiers on the floor.
-}
-
-void MicrosoftCXXNameMangler::mangleType(QualType T) {
-  // Only operate on the canonical type!
-  T = getASTContext().getCanonicalType(T);
-  
-  Qualifiers Quals = T.getLocalQualifiers();
-  if (Quals) {
-    // We have to mangle these now, while we still have enough information.
-    // <pointer-cvr-qualifiers> ::= P  # pointer
-    //                          ::= Q  # const pointer
-    //                          ::= R  # volatile pointer
-    //                          ::= S  # const volatile pointer
-    if (T->isAnyPointerType() || T->isMemberPointerType() ||
-        T->isBlockPointerType()) {
-      if (!Quals.hasVolatile())
-        Out << 'Q';
-      else {
-        if (!Quals.hasConst())
-          Out << 'R';
-        else
-          Out << 'S';
-      }
-    } else
-      // Just emit qualifiers like normal.
-      // NB: When we mangle a pointer/reference type, and the pointee
-      // type has no qualifiers, the lack of qualifier gets mangled
-      // in there.
-      mangleQualifiers(Quals, false);
-  } else if (T->isAnyPointerType() || T->isMemberPointerType() ||
-             T->isBlockPointerType()) {
-    Out << 'P';
-  }
-  switch (T->getTypeClass()) {
-#define ABSTRACT_TYPE(CLASS, PARENT)
-#define NON_CANONICAL_TYPE(CLASS, PARENT) \
-case Type::CLASS: \
-llvm_unreachable("can't mangle non-canonical type " #CLASS "Type"); \
-return;
-#define TYPE(CLASS, PARENT) \
-case Type::CLASS: \
-mangleType(static_cast<const CLASS##Type*>(T.getTypePtr())); \
-break;
-#include "clang/AST/TypeNodes.def"
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleType(const BuiltinType *T) {
-  //  <type>         ::= <builtin-type>
-  //  <builtin-type> ::= X  # void
-  //                 ::= C  # signed char
-  //                 ::= D  # char
-  //                 ::= E  # unsigned char
-  //                 ::= F  # short
-  //                 ::= G  # unsigned short (or wchar_t if it's not a builtin)
-  //                 ::= H  # int
-  //                 ::= I  # unsigned int
-  //                 ::= J  # long
-  //                 ::= K  # unsigned long
-  //                     L  # <none>
-  //                 ::= M  # float
-  //                 ::= N  # double
-  //                 ::= O  # long double (__float80 is mangled differently)
-  //                 ::= _D # __int8 (yup, it's a distinct type in MSVC)
-  //                 ::= _E # unsigned __int8
-  //                 ::= _F # __int16
-  //                 ::= _G # unsigned __int16
-  //                 ::= _H # __int32
-  //                 ::= _I # unsigned __int32
-  //                 ::= _J # long long, __int64
-  //                 ::= _K # unsigned long long, __int64
-  //                 ::= _L # __int128
-  //                 ::= _M # unsigned __int128
-  //                 ::= _N # bool
-  //                     _O # <array in parameter>
-  //                 ::= _T # __float80 (Intel)
-  //                 ::= _W # wchar_t
-  //                 ::= _Z # __float80 (Digital Mars)
-  switch (T->getKind()) {
-  case BuiltinType::Void: Out << 'X'; break;
-  case BuiltinType::SChar: Out << 'C'; break;
-  case BuiltinType::Char_U: case BuiltinType::Char_S: Out << 'D'; break;
-  case BuiltinType::UChar: Out << 'E'; break;
-  case BuiltinType::Short: Out << 'F'; break;
-  case BuiltinType::UShort: Out << 'G'; break;
-  case BuiltinType::Int: Out << 'H'; break;
-  case BuiltinType::UInt: Out << 'I'; break;
-  case BuiltinType::Long: Out << 'J'; break;
-  case BuiltinType::ULong: Out << 'K'; break;
-  case BuiltinType::Float: Out << 'M'; break;
-  case BuiltinType::Double: Out << 'N'; break;
-  // TODO: Determine size and mangle accordingly
-  case BuiltinType::LongDouble: Out << 'O'; break;
-  // TODO: __int8 and friends
-  case BuiltinType::LongLong: Out << "_J"; break;
-  case BuiltinType::ULongLong: Out << "_K"; break;
-  case BuiltinType::Int128: Out << "_L"; break;
-  case BuiltinType::UInt128: Out << "_M"; break;
-  case BuiltinType::Bool: Out << "_N"; break;
-  case BuiltinType::WChar: Out << "_W"; break;
-
-  case BuiltinType::Overload:
-  case BuiltinType::Dependent:
-    assert(false &&
-           "Overloaded and dependent types shouldn't get to name mangling");
-    break;
-  case BuiltinType::UndeducedAuto:
-    assert(0 && "Should not see undeduced auto here");
-    break;
-  case BuiltinType::ObjCId: Out << "PAUobjc_object@@"; break;
-  case BuiltinType::ObjCClass: Out << "PAUobjc_class@@"; break;
-  case BuiltinType::ObjCSel: Out << "PAUobjc_selector@@"; break;
-
-  case BuiltinType::Char16:
-  case BuiltinType::Char32:
-  case BuiltinType::NullPtr:
-    assert(false && "Don't know how to mangle this type");
-    break;
-  }
-}
-
-// <type>          ::= <function-type>
-void MicrosoftCXXNameMangler::mangleType(const FunctionProtoType *T) {
-  // Structors only appear in decls, so at this point we know it's not a
-  // structor type.
-  // I'll probably have mangleType(MemberPointerType) call the mangleType()
-  // method directly.
-  mangleType(T, NULL, false, false);
-}
-void MicrosoftCXXNameMangler::mangleType(const FunctionNoProtoType *T) {
-  llvm_unreachable("Can't mangle K&R function prototypes");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const FunctionType *T,
-                                         const FunctionDecl *D,
-                                         bool IsStructor,
-                                         bool IsInstMethod) {
-  // <function-type> ::= <this-cvr-qualifiers> <calling-convention>
-  //                     <return-type> <argument-list> <throw-spec>
-  const FunctionProtoType *Proto = cast<FunctionProtoType>(T);
-
-  // If this is a C++ instance method, mangle the CVR qualifiers for the
-  // this pointer.
-  if (IsInstMethod)
-    mangleQualifiers(Qualifiers::fromCVRMask(Proto->getTypeQuals()), false);
-
-  mangleCallingConvention(T);
-
-  // <return-type> ::= <type>
-  //               ::= @ # structors (they have no declared return type)
-  if (IsStructor)
-    Out << '@';
-  else
-    mangleType(Proto->getResultType());
-
-  // <argument-list> ::= X # void
-  //                 ::= <type>+ @
-  //                 ::= <type>* Z # varargs
-  if (Proto->getNumArgs() == 0 && !Proto->isVariadic()) {
-    Out << 'X';
-  } else {
-    if (D) {
-      // If we got a decl, use the "types-as-written" to make sure arrays
-      // get mangled right.
-      for (FunctionDecl::param_const_iterator Parm = D->param_begin(),
-           ParmEnd = D->param_end();
-           Parm != ParmEnd; ++Parm)
-        mangleType((*Parm)->getTypeSourceInfo()->getType());
-    } else {
-      for (FunctionProtoType::arg_type_iterator Arg = Proto->arg_type_begin(),
-           ArgEnd = Proto->arg_type_end();
-           Arg != ArgEnd; ++Arg)
-        mangleType(*Arg);
-    }
-    // <builtin-type>      ::= Z  # ellipsis
-    if (Proto->isVariadic())
-      Out << 'Z';
-    else
-      Out << '@';
-  }
-
-  mangleThrowSpecification(Proto);
-}
-
-void MicrosoftCXXNameMangler::mangleFunctionClass(const FunctionDecl *FD) {
-  // <function-class> ::= A # private: near
-  //                  ::= B # private: far
-  //                  ::= C # private: static near
-  //                  ::= D # private: static far
-  //                  ::= E # private: virtual near
-  //                  ::= F # private: virtual far
-  //                  ::= G # private: thunk near
-  //                  ::= H # private: thunk far
-  //                  ::= I # protected: near
-  //                  ::= J # protected: far
-  //                  ::= K # protected: static near
-  //                  ::= L # protected: static far
-  //                  ::= M # protected: virtual near
-  //                  ::= N # protected: virtual far
-  //                  ::= O # protected: thunk near
-  //                  ::= P # protected: thunk far
-  //                  ::= Q # public: near
-  //                  ::= R # public: far
-  //                  ::= S # public: static near
-  //                  ::= T # public: static far
-  //                  ::= U # public: virtual near
-  //                  ::= V # public: virtual far
-  //                  ::= W # public: thunk near
-  //                  ::= X # public: thunk far
-  //                  ::= Y # global near
-  //                  ::= Z # global far
-  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
-    switch (MD->getAccess()) {
-      default:
-      case AS_private:
-        if (MD->isStatic())
-          Out << 'C';
-        else if (MD->isVirtual())
-          Out << 'E';
-        else
-          Out << 'A';
-        break;
-      case AS_protected:
-        if (MD->isStatic())
-          Out << 'K';
-        else if (MD->isVirtual())
-          Out << 'M';
-        else
-          Out << 'I';
-        break;
-      case AS_public:
-        if (MD->isStatic())
-          Out << 'S';
-        else if (MD->isVirtual())
-          Out << 'U';
-        else
-          Out << 'Q';
-    }
-  } else
-    Out << 'Y';
-}
-void MicrosoftCXXNameMangler::mangleCallingConvention(const FunctionType *T) {
-  // <calling-convention> ::= A # __cdecl
-  //                      ::= B # __export __cdecl
-  //                      ::= C # __pascal
-  //                      ::= D # __export __pascal
-  //                      ::= E # __thiscall
-  //                      ::= F # __export __thiscall
-  //                      ::= G # __stdcall
-  //                      ::= H # __export __stdcall
-  //                      ::= I # __fastcall
-  //                      ::= J # __export __fastcall
-  // The 'export' calling conventions are from a bygone era
-  // (*cough*Win16*cough*) when functions were declared for export with
-  // that keyword. (It didn't actually export them, it just made them so
-  // that they could be in a DLL and somebody from another module could call
-  // them.)
-  switch (T->getCallConv()) {
-    case CC_Default:
-    case CC_C: Out << 'A'; break;
-    case CC_X86Pascal: Out << 'C'; break;
-    case CC_X86ThisCall: Out << 'E'; break;
-    case CC_X86StdCall: Out << 'G'; break;
-    case CC_X86FastCall: Out << 'I'; break;
-  }
-}
-void MicrosoftCXXNameMangler::mangleThrowSpecification(
-                                                const FunctionProtoType *FT) {
-  // <throw-spec> ::= Z # throw(...) (default)
-  //              ::= @ # throw() or __declspec/__attribute__((nothrow))
-  //              ::= <type>+
-  // NOTE: Since the Microsoft compiler ignores throw specifications, they are
-  // all actually mangled as 'Z'. (They're ignored because their associated
-  // functionality isn't implemented, and probably never will be.)
-  Out << 'Z';
-}
-
-void MicrosoftCXXNameMangler::mangleType(const UnresolvedUsingType *T) {
-  assert(false && "Don't know how to mangle UnresolvedUsingTypes yet!");
-}
-
-// <type>        ::= <union-type> | <struct-type> | <class-type> | <enum-type>
-// <union-type>  ::= T <name>
-// <struct-type> ::= U <name>
-// <class-type>  ::= V <name>
-// <enum-type>   ::= W <size> <name>
-void MicrosoftCXXNameMangler::mangleType(const EnumType *T) {
-  mangleType(static_cast<const TagType*>(T));
-}
-void MicrosoftCXXNameMangler::mangleType(const RecordType *T) {
-  mangleType(static_cast<const TagType*>(T));
-}
-void MicrosoftCXXNameMangler::mangleType(const TagType *T) {
-  switch (T->getDecl()->getTagKind()) {
-    case TTK_Union:
-      Out << 'T';
-      break;
-    case TTK_Struct:
-      Out << 'U';
-      break;
-    case TTK_Class:
-      Out << 'V';
-      break;
-    case TTK_Enum:
-      Out << 'W';
-      Out << getASTContext().getTypeSizeInChars(
-                cast<EnumDecl>(T->getDecl())->getIntegerType()).getQuantity();
-      break;
-  }
-  mangleName(T->getDecl());
-}
-
-// <type>       ::= <array-type>
-// <array-type> ::= P <cvr-qualifiers> [Y <dimension-count> <dimension>+]
-//                                                  <element-type> # as global
-//              ::= Q <cvr-qualifiers> [Y <dimension-count> <dimension>+]
-//                                                  <element-type> # as param
-// It's supposed to be the other way around, but for some strange reason, it
-// isn't. Today this behavior is retained for the sole purpose of backwards
-// compatibility.
-void MicrosoftCXXNameMangler::mangleType(const ArrayType *T, bool IsGlobal) {
-  // This isn't a recursive mangling, so now we have to do it all in this
-  // one call.
-  if (IsGlobal)
-    Out << 'P';
-  else
-    Out << 'Q';
-  mangleExtraDimensions(T->getElementType());
-}
-void MicrosoftCXXNameMangler::mangleType(const ConstantArrayType *T) {
-  mangleType(static_cast<const ArrayType *>(T), false);
-}
-void MicrosoftCXXNameMangler::mangleType(const VariableArrayType *T) {
-  mangleType(static_cast<const ArrayType *>(T), false);
-}
-void MicrosoftCXXNameMangler::mangleType(const DependentSizedArrayType *T) {
-  mangleType(static_cast<const ArrayType *>(T), false);
-}
-void MicrosoftCXXNameMangler::mangleType(const IncompleteArrayType *T) {
-  mangleType(static_cast<const ArrayType *>(T), false);
-}
-void MicrosoftCXXNameMangler::mangleExtraDimensions(QualType ElementTy) {
-  llvm::SmallVector<llvm::APInt, 3> Dimensions;
-  for (;;) {
-    if (ElementTy->isConstantArrayType()) {
-      const ConstantArrayType *CAT =
-      static_cast<const ConstantArrayType *>(ElementTy.getTypePtr());
-      Dimensions.push_back(CAT->getSize());
-      ElementTy = CAT->getElementType();
-    } else if (ElementTy->isVariableArrayType()) {
-      assert(false && "Don't know how to mangle VLAs!");
-    } else if (ElementTy->isDependentSizedArrayType()) {
-      // The dependent expression has to be folded into a constant (TODO).
-      assert(false && "Don't know how to mangle dependent-sized arrays!");
-    } else if (ElementTy->isIncompleteArrayType()) continue;
-    else break;
-  }
-  mangleQualifiers(ElementTy.getQualifiers(), false);
-  // If there are any additional dimensions, mangle them now.
-  if (Dimensions.size() > 0) {
-    Out << 'Y';
-    // <dimension-count> ::= <number> # number of extra dimensions
-    mangleNumber(Dimensions.size());
-    for (unsigned Dim = 0; Dim < Dimensions.size(); ++Dim) {
-      mangleNumber(Dimensions[Dim].getLimitedValue());
-    }
-  }
-  mangleType(ElementTy.getLocalUnqualifiedType());
-}
-
-// <type>                   ::= <pointer-to-member-type>
-// <pointer-to-member-type> ::= <pointer-cvr-qualifiers> <cvr-qualifiers>
-//                                                          <class name> <type>
-void MicrosoftCXXNameMangler::mangleType(const MemberPointerType *T) {
-  QualType PointeeType = T->getPointeeType();
-  if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(PointeeType)) {
-    Out << '8';
-    mangleName(cast<RecordType>(T->getClass())->getDecl());
-    mangleType(FPT, NULL, false, true);
-  } else {
-    mangleQualifiers(PointeeType.getQualifiers(), true);
-    mangleName(cast<RecordType>(T->getClass())->getDecl());
-    mangleType(PointeeType.getLocalUnqualifiedType());
-  }
-}
 
-void MicrosoftCXXNameMangler::mangleType(const TemplateTypeParmType *T) {
-  assert(false && "Don't know how to mangle TemplateTypeParmTypes yet!");
-}
-
-// <type> ::= <pointer-type>
-// <pointer-type> ::= <pointer-cvr-qualifiers> <cvr-qualifiers> <type>
-void MicrosoftCXXNameMangler::mangleType(const PointerType *T) {
-  QualType PointeeTy = T->getPointeeType();
-  if (PointeeTy->isArrayType()) {
-    // Pointers to arrays are mangled like arrays.
-    mangleExtraDimensions(T->getPointeeType());
-  } else if (PointeeTy->isFunctionType()) {
-    // Function pointers are special.
-    Out << '6';
-    mangleType(static_cast<const FunctionType *>(PointeeTy.getTypePtr()),
-               NULL, false, false);
-  } else {
-    if (!PointeeTy.hasQualifiers())
-      // Lack of qualifiers is mangled as 'A'.
-      Out << 'A';
-    mangleType(PointeeTy);
-  }
-}
-void MicrosoftCXXNameMangler::mangleType(const ObjCObjectPointerType *T) {
-  // Object pointers never have qualifiers.
-  Out << 'A';
-  mangleType(T->getPointeeType());
-}
-
-// <type> ::= <reference-type>
-// <reference-type> ::= A <cvr-qualifiers> <type>
-void MicrosoftCXXNameMangler::mangleType(const LValueReferenceType *T) {
-  Out << 'A';
-  QualType PointeeTy = T->getPointeeType();
-  if (!PointeeTy.hasQualifiers())
-    // Lack of qualifiers is mangled as 'A'.
-    Out << 'A';
-  mangleType(PointeeTy);
-}
-
-void MicrosoftCXXNameMangler::mangleType(const RValueReferenceType *T) {
-  assert(false && "Don't know how to mangle RValueReferenceTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const ComplexType *T) {
-  assert(false && "Don't know how to mangle ComplexTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const VectorType *T) {
-  assert(false && "Don't know how to mangle VectorTypes yet!");
-}
-void MicrosoftCXXNameMangler::mangleType(const ExtVectorType *T) {
-  assert(false && "Don't know how to mangle ExtVectorTypes yet!");
-}
-void MicrosoftCXXNameMangler::mangleType(const DependentSizedExtVectorType *T) {
-  assert(false && "Don't know how to mangle DependentSizedExtVectorTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const ObjCInterfaceType *T) {
-  // ObjC interfaces have structs underlying them.
-  Out << 'U';
-  mangleName(T->getDecl());
-}
-
-void MicrosoftCXXNameMangler::mangleType(const ObjCObjectType *T) {
-  // We don't allow overloading by different protocol qualification,
-  // so mangling them isn't necessary.
-  mangleType(T->getBaseType());
-}
-
-void MicrosoftCXXNameMangler::mangleType(const BlockPointerType *T) {
-  Out << "_E";
-  mangleType(T->getPointeeType());
-}
-
-void MicrosoftCXXNameMangler::mangleType(const InjectedClassNameType *T) {
-  assert(false && "Don't know how to mangle InjectedClassNameTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const TemplateSpecializationType *T) {
-  assert(false && "Don't know how to mangle TemplateSpecializationTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const DependentNameType *T) {
-  assert(false && "Don't know how to mangle DependentNameTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(
-                                 const DependentTemplateSpecializationType *T) {
-  assert(false &&
-         "Don't know how to mangle DependentTemplateSpecializationTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const TypeOfType *T) {
-  assert(false && "Don't know how to mangle TypeOfTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const TypeOfExprType *T) {
-  assert(false && "Don't know how to mangle TypeOfExprTypes yet!");
-}
-
-void MicrosoftCXXNameMangler::mangleType(const DecltypeType *T) {
-  assert(false && "Don't know how to mangle DecltypeTypes yet!");
-}
-
-void MicrosoftMangleContext::mangleName(const NamedDecl *D,
-                                        llvm::SmallVectorImpl<char> &Name) {
-  assert((isa<FunctionDecl>(D) || isa<VarDecl>(D)) &&
-         "Invalid mangleName() call, argument is not a variable or function!");
-  assert(!isa<CXXConstructorDecl>(D) && !isa<CXXDestructorDecl>(D) &&
-         "Invalid mangleName() call on 'structor decl!");
-
-  PrettyStackTraceDecl CrashInfo(D, SourceLocation(),
-                                 getASTContext().getSourceManager(),
-                                 "Mangling declaration");
+  // ==== Notes on array cookies =========
+  //
+  // MSVC seems to only use cookies when the class has a destructor; a
+  // two-argument usual array deallocation function isn't sufficient.
+  //
+  // For example, this code prints "100" and "1":
+  //   struct A {
+  //     char x;
+  //     void *operator new[](size_t sz) {
+  //       printf("%u\n", sz);
+  //       return malloc(sz);
+  //     }
+  //     void operator delete[](void *p, size_t sz) {
+  //       printf("%u\n", sz);
+  //       free(p);
+  //     }
+  //   };
+  //   int main() {
+  //     A *p = new A[100];
+  //     delete[] p;
+  //   }
+  // Whereas it prints "104" and "104" if you give A a destructor.
+};
 
-  MicrosoftCXXNameMangler Mangler(*this, Name);
-  return Mangler.mangle(D);
-}
-void MicrosoftMangleContext::mangleThunk(const CXXMethodDecl *MD,
-                                         const ThunkInfo &Thunk,
-                                         llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle thunks!");
-}
-void MicrosoftMangleContext::mangleCXXDtorThunk(const CXXDestructorDecl *DD,
-                                                CXXDtorType Type,
-                                                const ThisAdjustment &,
-                                                llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle destructor thunks!");
-}
-void MicrosoftMangleContext::mangleGuardVariable(const VarDecl *D,
-                                                 llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle guard variables!");
-}
-void MicrosoftMangleContext::mangleCXXVTable(const CXXRecordDecl *RD,
-                                             llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle virtual tables!");
-}
-void MicrosoftMangleContext::mangleCXXVTT(const CXXRecordDecl *RD,
-                                          llvm::SmallVectorImpl<char> &) {
-  llvm_unreachable("The MS C++ ABI does not have virtual table tables!");
-}
-void MicrosoftMangleContext::mangleCXXCtorVTable(const CXXRecordDecl *RD,
-                                                 int64_t Offset,
-                                                 const CXXRecordDecl *Type,
-                                                 llvm::SmallVectorImpl<char> &) {
-  llvm_unreachable("The MS C++ ABI does not have constructor vtables!");
-}
-void MicrosoftMangleContext::mangleCXXRTTI(QualType T,
-                                           llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle RTTI!");
-}
-void MicrosoftMangleContext::mangleCXXRTTIName(QualType T,
-                                               llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle RTTI names!");
-}
-void MicrosoftMangleContext::mangleCXXCtor(const CXXConstructorDecl *D,
-                                           CXXCtorType Type,
-                                           llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle constructors!");
-}
-void MicrosoftMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
-                                           CXXDtorType Type,
-                                           llvm::SmallVectorImpl<char> &) {
-  assert(false && "Can't yet mangle destructors!");
 }
 
 CGCXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) {
diff --git a/lib/CodeGen/ModuleBuilder.cpp b/lib/CodeGen/ModuleBuilder.cpp
index 6d9d2770c7c8..894502864464 100644
--- a/lib/CodeGen/ModuleBuilder.cpp
+++ b/lib/CodeGen/ModuleBuilder.cpp
@@ -71,6 +71,19 @@ namespace {
     /// (because these can be defined in declspecs).
     virtual void HandleTagDeclDefinition(TagDecl *D) {
       Builder->UpdateCompletedType(D);
+      
+      // In C++, we may have member functions that need to be emitted at this 
+      // point.
+      if (Ctx->getLangOptions().CPlusPlus && !D->isDependentContext()) {
+        for (DeclContext::decl_iterator M = D->decls_begin(), 
+                                     MEnd = D->decls_end();
+             M != MEnd; ++M)
+          if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(*M))
+            if (Method->isThisDeclarationADefinition() &&
+                (Method->hasAttr<UsedAttr>() || 
+                 Method->hasAttr<ConstructorAttr>()))
+              Builder->EmitTopLevelDecl(Method);
+      }
     }
 
     virtual void HandleTranslationUnit(ASTContext &Ctx) {
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index 4d221d4e657e..d74b3f32d954 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -18,7 +18,6 @@
 #include "clang/AST/RecordLayout.h"
 #include "llvm/Type.h"
 #include "llvm/Target/TargetData.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
@@ -75,7 +74,8 @@ void ABIArgInfo::dump() const {
     break;
   case Indirect:
     OS << "Indirect Align=" << getIndirectAlign()
-       << " Byal=" << getIndirectByVal();
+       << " Byal=" << getIndirectByVal()
+       << " Realign=" << getIndirectRealign();
     break;
   case Expand:
     OS << "Expand";
@@ -330,12 +330,47 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
           ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
+ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
+    return ABIArgInfo::getIgnore();
+
+  if (isAggregateTypeForABI(RetTy))
+    return ABIArgInfo::getIndirect(0);
+
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+    RetTy = EnumTy->getDecl()->getIntegerType();
+
+  return (RetTy->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
+/// UseX86_MMXType - Return true if this is an MMX type that should use the special
+/// x86_mmx type.
+bool UseX86_MMXType(const llvm::Type *IRType) {
+  // If the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>, use the
+  // special x86_mmx type.
+  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
+    cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
+    IRType->getScalarSizeInBits() != 64;
+}
+
+static const llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
+                                                llvm::StringRef Constraint,
+                                                const llvm::Type* Ty) {
+  if (Constraint=="y" && Ty->isVectorTy())
+    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
+  return Ty;
+}
+
 //===----------------------------------------------------------------------===//
 // X86-32 ABI Implementation
 //===----------------------------------------------------------------------===//
 
 /// X86_32ABIInfo - The X86-32 ABI information.
 class X86_32ABIInfo : public ABIInfo {
+  static const unsigned MinABIStackAlignInBytes = 4;
+
   bool IsDarwinVectorABI;
   bool IsSmallStructInRegABI;
 
@@ -349,6 +384,9 @@ class X86_32ABIInfo : public ABIInfo {
   /// such that the argument will be passed in memory.
   ABIArgInfo getIndirectResult(QualType Ty, bool ByVal = true) const;
 
+  /// \brief Return the alignment to use for the given type on the stack.
+  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
+
 public:
 
   ABIArgInfo classifyReturnType(QualType RetTy) const;
@@ -385,6 +423,13 @@ public:
 
   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
                                llvm::Value *Address) const;
+
+  const llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
+                                        llvm::StringRef Constraint,
+                                        const llvm::Type* Ty) const {
+    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
+  }
+
 };
 
 }
@@ -547,17 +592,70 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy) const {
           ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
+static bool isRecordWithSSEVectorType(ASTContext &Context, QualType Ty) {
+  const RecordType *RT = Ty->getAs<RecordType>();
+  if (!RT)
+    return 0;
+  const RecordDecl *RD = RT->getDecl();
+
+  // If this is a C++ record, check the bases first.
+  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
+    for (CXXRecordDecl::base_class_const_iterator i = CXXRD->bases_begin(),
+           e = CXXRD->bases_end(); i != e; ++i)
+      if (!isRecordWithSSEVectorType(Context, i->getType()))
+        return false;
+
+  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+       i != e; ++i) {
+    QualType FT = i->getType();
+
+    if (FT->getAs<VectorType>() && Context.getTypeSize(Ty) == 128)
+      return true;
+
+    if (isRecordWithSSEVectorType(Context, FT))
+      return true;
+  }
+
+  return false;
+}
+
+unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
+                                                 unsigned Align) const {
+  // Otherwise, if the alignment is less than or equal to the minimum ABI
+  // alignment, just use the default; the backend will handle this.
+  if (Align <= MinABIStackAlignInBytes)
+    return 0; // Use default alignment.
+
+  // On non-Darwin, the stack type alignment is always 4.
+  if (!IsDarwinVectorABI) {
+    // Set explicit alignment, since we may need to realign the top.
+    return MinABIStackAlignInBytes;
+  }
+
+  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
+  if (isRecordWithSSEVectorType(getContext(), Ty))
+    return 16;
+
+  return MinABIStackAlignInBytes;
+}
+
 ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal) const {
   if (!ByVal)
     return ABIArgInfo::getIndirect(0, false);
 
-  // Compute the byval alignment. We trust the back-end to honor the
-  // minimum ABI alignment for byval, to make cleaner IR.
-  const unsigned MinABIAlign = 4;
-  unsigned Align = getContext().getTypeAlign(Ty) / 8;
-  if (Align > MinABIAlign)
-    return ABIArgInfo::getIndirect(Align);
-  return ABIArgInfo::getIndirect(0);
+  // Compute the byval alignment.
+  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
+  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
+  if (StackAlign == 0)
+    return ABIArgInfo::getIndirect(0);
+
+  // If the stack alignment is less than the type alignment, realign the
+  // argument.
+  if (StackAlign < TypeAlign)
+    return ABIArgInfo::getIndirect(StackAlign, /*ByVal=*/true,
+                                   /*Realign=*/true);
+
+  return ABIArgInfo::getIndirect(StackAlign);
 }
 
 ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
@@ -599,11 +697,18 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
         return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
                                                             Size));
     }
-    
+
+    const llvm::Type *IRType = CGT.ConvertTypeRecursive(Ty);
+    if (UseX86_MMXType(IRType)) {
+      ABIArgInfo AAI = ABIArgInfo::getDirect(IRType);
+      AAI.setCoerceToType(llvm::Type::getX86_MMXTy(getVMContext()));
+      return AAI;
+    }
+
     return ABIArgInfo::getDirect();
   }
-  
-  
+
+
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
     Ty = EnumTy->getDecl()->getIntegerType();
 
@@ -755,7 +860,8 @@ class X86_64ABIInfo : public ABIInfo {
 
   ABIArgInfo classifyReturnType(QualType RetTy) const;
 
-  ABIArgInfo classifyArgumentType(QualType Ty, unsigned &neededInt,
+  ABIArgInfo classifyArgumentType(QualType Ty,
+                                  unsigned &neededInt,
                                   unsigned &neededSSE) const;
 
 public:
@@ -768,9 +874,14 @@ public:
 };
 
 /// WinX86_64ABIInfo - The Windows X86_64 ABI information.
-class WinX86_64ABIInfo : public X86_64ABIInfo {
+class WinX86_64ABIInfo : public ABIInfo {
+
+  ABIArgInfo classify(QualType Ty) const;
+
 public:
-  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : X86_64ABIInfo(CGT) {}
+  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  virtual void computeInfo(CGFunctionInfo &FI) const;
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                  CodeGenFunction &CGF) const;
@@ -799,6 +910,13 @@ public:
 
     return false;
   }
+
+  const llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
+                                        llvm::StringRef Constraint,
+                                        const llvm::Type* Ty) const {
+    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
+  }
+
 };
 
 class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
@@ -817,7 +935,7 @@ public:
 
     const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
     llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
-      
+
     // 0-15 are the 16 integer registers.
     // 16 is %rip.
     AssignToArrayRange(Builder, Address, Eight8, 0, 16);
@@ -1065,7 +1183,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
         // single eightbyte, each is classified separately. Each eightbyte gets
         // initialized to class NO_CLASS.
         Class FieldLo, FieldHi;
-        uint64_t Offset = OffsetBase + Layout.getBaseClassOffset(Base);
+        uint64_t Offset = OffsetBase + Layout.getBaseClassOffsetInBits(Base);
         classify(i->getType(), Offset, FieldLo, FieldHi);
         Lo = merge(Lo, FieldLo);
         Hi = merge(Hi, FieldHi);
@@ -1264,7 +1382,7 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
           cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
 
         // If the base is after the span we care about, ignore it.
-        unsigned BaseOffset = (unsigned)Layout.getBaseClassOffset(Base);
+        unsigned BaseOffset = (unsigned)Layout.getBaseClassOffsetInBits(Base);
         if (BaseOffset >= EndBit) continue;
 
         unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
@@ -1443,7 +1561,7 @@ GetX86_64ByValArgumentPair(const llvm::Type *Lo, const llvm::Type *Hi,
   unsigned HiAlign = TD.getABITypeAlignment(Hi);
   unsigned HiStart = llvm::TargetData::RoundUpAlignment(LoSize, HiAlign);
   assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
-  
+
   // To handle this, we have to increase the size of the low part so that the
   // second element will start at an 8 byte offset.  We can't increase the size
   // of the second element because it might make us access off the end of the
@@ -1459,11 +1577,11 @@ GetX86_64ByValArgumentPair(const llvm::Type *Lo, const llvm::Type *Hi,
       Lo = llvm::Type::getInt64Ty(Lo->getContext());
     }
   }
-  
-  const llvm::StructType *Result = 
+
+  const llvm::StructType *Result =
     llvm::StructType::get(Lo->getContext(), Lo, Hi, NULL);
-  
-  
+
+
   // Verify that the second element is at an 8-byte offset.
   assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
          "Invalid x86-64 argument pair!");
@@ -1592,7 +1710,7 @@ classifyReturnType(QualType RetTy) const {
     }
     break;
   }
-  
+
   // If a high part was specified, merge it together with the low part.  It is
   // known to pass in the high eightbyte of the result.  We do this by forming a
   // first class struct aggregate with the high and low part: {low, high}
@@ -1665,11 +1783,18 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned &neededInt,
     // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
     // available SSE register is used, the registers are taken in the
     // order from %xmm0 to %xmm7.
-  case SSE:
+  case SSE: {
+    const llvm::Type *IRType = CGT.ConvertTypeRecursive(Ty);
+    if (Hi != NoClass || !UseX86_MMXType(IRType))
+      ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
+    else
+      // This is an MMX type. Treat it as such.
+      ResType = llvm::Type::getX86_MMXTy(getVMContext());
+
     ++neededSSE;
-    ResType = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(Ty), 0, Ty, 0);
     break;
   }
+  }
 
   const llvm::Type *HighPart = 0;
   switch (Hi) {
@@ -1719,7 +1844,7 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned &neededInt,
   // first class struct aggregate with the high and low part: {low, high}
   if (HighPart)
     ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getTargetData());
-  
+
   return ABIArgInfo::getDirect(ResType);
 }
 
@@ -1965,76 +2090,48 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return ResAddr;
 }
 
-llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
-                                      CodeGenFunction &CGF) const {
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
+ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty) const {
 
-  CGBuilderTy &Builder = CGF.Builder;
-  llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
-                                                       "ap");
-  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
-  llvm::Type *PTy =
-    llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
-  llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
-
-  uint64_t Offset =
-    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 8);
-  llvm::Value *NextAddr =
-    Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
-                      "ap.next");
-  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
-
-  return AddrTyped;
-}
-
-//===----------------------------------------------------------------------===//
-// PIC16 ABI Implementation
-//===----------------------------------------------------------------------===//
+  if (Ty->isVoidType())
+    return ABIArgInfo::getIgnore();
 
-namespace {
+  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+    Ty = EnumTy->getDecl()->getIntegerType();
 
-class PIC16ABIInfo : public ABIInfo {
-public:
-  PIC16ABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
+  uint64_t Size = getContext().getTypeSize(Ty);
 
-  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    if (hasNonTrivialDestructorOrCopyConstructor(RT) ||
+        RT->getDecl()->hasFlexibleArrayMember())
+      return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
 
-  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+    // FIXME: mingw64-gcc emits 128-bit struct as i128
+    if (Size <= 128 &&
+        (Size & (Size - 1)) == 0)
+      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
+                                                          Size));
 
-  virtual void computeInfo(CGFunctionInfo &FI) const {
-    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
-    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
-         it != ie; ++it)
-      it->info = classifyArgumentType(it->type);
+    return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
   }
 
-  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
-                                 CodeGenFunction &CGF) const;
-};
-
-class PIC16TargetCodeGenInfo : public TargetCodeGenInfo {
-public:
-  PIC16TargetCodeGenInfo(CodeGenTypes &CGT)
-    : TargetCodeGenInfo(new PIC16ABIInfo(CGT)) {}
-};
+  if (Ty->isPromotableIntegerType())
+    return ABIArgInfo::getExtend();
 
+  return ABIArgInfo::getDirect();
 }
 
-ABIArgInfo PIC16ABIInfo::classifyReturnType(QualType RetTy) const {
-  if (RetTy->isVoidType()) {
-    return ABIArgInfo::getIgnore();
-  } else {
-    return ABIArgInfo::getDirect();
-  }
-}
+void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
-ABIArgInfo PIC16ABIInfo::classifyArgumentType(QualType Ty) const {
-  return ABIArgInfo::getDirect();
+  QualType RetTy = FI.getReturnType();
+  FI.getReturnInfo() = classify(RetTy);
+
+  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
+       it != ie; ++it)
+    it->info = classify(it->type);
 }
 
-llvm::Value *PIC16ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
-                                     CodeGenFunction &CGF) const {
+llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                      CodeGenFunction &CGF) const {
   const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
   const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
 
@@ -2046,18 +2143,16 @@ llvm::Value *PIC16ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
     llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
   llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
 
-  uint64_t Offset = CGF.getContext().getTypeSize(Ty) / 8;
-
+  uint64_t Offset =
+    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 8);
   llvm::Value *NextAddr =
-    Builder.CreateGEP(Addr, llvm::ConstantInt::get(
-                          llvm::Type::getInt32Ty(CGF.getLLVMContext()), Offset),
+    Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
                       "ap.next");
   Builder.CreateStore(NextAddr, VAListAddrAsBPP);
 
   return AddrTyped;
 }
 
-
 // PowerPC-32
 
 namespace {
@@ -2213,6 +2308,8 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
   if (isRecordWithNonTrivialDestructorOrCopyConstructor(Ty))
     return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
 
+  // Otherwise, pass by coercing to a structure of the appropriate size.
+  //
   // FIXME: This is kind of nasty... but there isn't much choice because the ARM
   // backend doesn't support byval.
   // FIXME: This doesn't handle alignment > 64 bits.
@@ -2321,6 +2418,10 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy) const {
   if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
 
+  // Large vector types should be returned via memory.
+  if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)
+    return ABIArgInfo::getIndirect(0);
+
   if (!isAggregateTypeForABI(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
@@ -2407,21 +2508,6 @@ llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return AddrTyped;
 }
 
-ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
-  if (RetTy->isVoidType())
-    return ABIArgInfo::getIgnore();
-
-  if (isAggregateTypeForABI(RetTy))
-    return ABIArgInfo::getIndirect(0);
-
-  // Treat an enum type as its underlying type.
-  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-    RetTy = EnumTy->getDecl()->getIntegerType();
-
-  return (RetTy->isPromotableIntegerType() ?
-          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
-}
-
 //===----------------------------------------------------------------------===//
 // SystemZ ABI Implementation
 //===----------------------------------------------------------------------===//
@@ -2502,6 +2588,116 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
 }
 
 //===----------------------------------------------------------------------===//
+// MBlaze ABI Implementation
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class MBlazeABIInfo : public ABIInfo {
+public:
+  MBlazeABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  bool isPromotableIntegerType(QualType Ty) const;
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
+         it != ie; ++it)
+      it->info = classifyArgumentType(it->type);
+  }
+
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                 CodeGenFunction &CGF) const;
+};
+
+class MBlazeTargetCodeGenInfo : public TargetCodeGenInfo {
+public:
+  MBlazeTargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new MBlazeABIInfo(CGT)) {}
+  void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
+                           CodeGen::CodeGenModule &M) const;
+};
+
+}
+
+bool MBlazeABIInfo::isPromotableIntegerType(QualType Ty) const {
+  // MBlaze ABI requires all 8 and 16 bit quantities to be extended.
+  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
+    switch (BT->getKind()) {
+    case BuiltinType::Bool:
+    case BuiltinType::Char_S:
+    case BuiltinType::Char_U:
+    case BuiltinType::SChar:
+    case BuiltinType::UChar:
+    case BuiltinType::Short:
+    case BuiltinType::UShort:
+      return true;
+    default:
+      return false;
+    }
+  return false;
+}
+
+llvm::Value *MBlazeABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                      CodeGenFunction &CGF) const {
+  // FIXME: Implement
+  return 0;
+}
+
+
+ABIArgInfo MBlazeABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
+    return ABIArgInfo::getIgnore();
+  if (isAggregateTypeForABI(RetTy))
+    return ABIArgInfo::getIndirect(0);
+
+  return (isPromotableIntegerType(RetTy) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
+ABIArgInfo MBlazeABIInfo::classifyArgumentType(QualType Ty) const {
+  if (isAggregateTypeForABI(Ty))
+    return ABIArgInfo::getIndirect(0);
+
+  return (isPromotableIntegerType(Ty) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
+void MBlazeTargetCodeGenInfo::SetTargetAttributes(const Decl *D,
+                                                  llvm::GlobalValue *GV,
+                                                  CodeGen::CodeGenModule &M)
+                                                  const {
+  const FunctionDecl *FD = dyn_cast<FunctionDecl>(D);
+  if (!FD) return;
+
+  llvm::CallingConv::ID CC = llvm::CallingConv::C;
+  if (FD->hasAttr<MBlazeInterruptHandlerAttr>())
+    CC = llvm::CallingConv::MBLAZE_INTR;
+  else if (FD->hasAttr<MBlazeSaveVolatilesAttr>())
+    CC = llvm::CallingConv::MBLAZE_SVOL;
+
+  if (CC != llvm::CallingConv::C) {
+      // Handle 'interrupt_handler' attribute:
+      llvm::Function *F = cast<llvm::Function>(GV);
+
+      // Step 1: Set ISR calling convention.
+      F->setCallingConv(CC);
+
+      // Step 2: Add attributes goodness.
+      F->addFnAttr(llvm::Attribute::NoInline);
+  }
+
+  // Step 3: Emit _interrupt_handler alias.
+  if (CC == llvm::CallingConv::MBLAZE_INTR)
+    new llvm::GlobalAlias(GV->getType(), llvm::Function::ExternalLinkage,
+                          "_interrupt_handler", GV, &M.getModule());
+}
+
+
+//===----------------------------------------------------------------------===//
 // MSP430 ABI Implementation
 //===----------------------------------------------------------------------===//
 
@@ -2534,8 +2730,7 @@ void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
       // Step 3: Emit ISR vector alias.
       unsigned Num = attr->getNumber() + 0xffe0;
       new llvm::GlobalAlias(GV->getType(), llvm::Function::ExternalLinkage,
-                            "vector_" +
-                            llvm::LowercaseString(llvm::utohexstr(Num)),
+                            "vector_" + llvm::Twine::utohexstr(Num),
                             GV, &M.getModule());
     }
   }
@@ -2621,15 +2816,15 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
     return *(TheTargetCodeGenInfo =
              new ARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS));
 
-  case llvm::Triple::pic16:
-    return *(TheTargetCodeGenInfo = new PIC16TargetCodeGenInfo(Types));
-
   case llvm::Triple::ppc:
     return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types));
 
   case llvm::Triple::systemz:
     return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types));
 
+  case llvm::Triple::mblaze:
+    return *(TheTargetCodeGenInfo = new MBlazeTargetCodeGenInfo(Types));
+
   case llvm::Triple::msp430:
     return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo(Types));
 
@@ -2644,6 +2839,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
     case llvm::Triple::DragonFly:
     case llvm::Triple::FreeBSD:
     case llvm::Triple::OpenBSD:
+    case llvm::Triple::NetBSD:
       return *(TheTargetCodeGenInfo =
                new X86_32TargetCodeGenInfo(Types, false, true));
 
@@ -2655,7 +2851,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   case llvm::Triple::x86_64:
     switch (Triple.getOS()) {
     case llvm::Triple::Win32:
-    case llvm::Triple::MinGW64:
+    case llvm::Triple::MinGW32:
     case llvm::Triple::Cygwin:
       return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types));
     default:
diff --git a/lib/CodeGen/TargetInfo.h b/lib/CodeGen/TargetInfo.h
index 9d4cf1610308..4f59eb619e9d 100644
--- a/lib/CodeGen/TargetInfo.h
+++ b/lib/CodeGen/TargetInfo.h
@@ -15,8 +15,11 @@
 #ifndef CLANG_CODEGEN_TARGETINFO_H
 #define CLANG_CODEGEN_TARGETINFO_H
 
+#include "llvm/ADT/StringRef.h"
+
 namespace llvm {
   class GlobalValue;
+  class Type;
   class Value;
 }
 
@@ -102,6 +105,12 @@ namespace clang {
                                              llvm::Value *Address) const {
       return Address;
     }
+
+    virtual const llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
+                                                  llvm::StringRef Constraint, 
+                                                  const llvm::Type* Ty) const {
+      return Ty;
+    }
   };
 }
 
diff --git a/lib/Driver/Action.cpp b/lib/Driver/Action.cpp
index f34971bfa0c7..549ce0a7f41b 100644
--- a/lib/Driver/Action.cpp
+++ b/lib/Driver/Action.cpp
@@ -1,4 +1,4 @@
-//===--- Action.cpp - Abstract compilation steps ------------------------*-===//
+//===--- Action.cpp - Abstract compilation steps --------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/Arg.cpp b/lib/Driver/Arg.cpp
index 83d0d26c256d..f1177cf97e28 100644
--- a/lib/Driver/Arg.cpp
+++ b/lib/Driver/Arg.cpp
@@ -1,4 +1,4 @@
-//===--- Arg.cpp - Argument Implementations -----------------------------*-===//
+//===--- Arg.cpp - Argument Implementations -------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/ArgList.cpp b/lib/Driver/ArgList.cpp
index 910152376714..596e2a7529cb 100644
--- a/lib/Driver/ArgList.cpp
+++ b/lib/Driver/ArgList.cpp
@@ -1,4 +1,4 @@
-//===--- ArgList.cpp - Argument List Management -------------------------*-===//
+//===--- ArgList.cpp - Argument List Management ---------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -55,62 +55,59 @@ Arg *ArgList::getLastArgNoClaim(OptSpecifier Id) const {
 }
 
 Arg *ArgList::getLastArg(OptSpecifier Id) const {
-  Arg *A = getLastArgNoClaim(Id);
-  if (A)
-    A->claim();
-  return A;
+  Arg *Res = 0;
+  for (const_iterator it = begin(), ie = end(); it != ie; ++it) {
+    if ((*it)->getOption().matches(Id)) {
+      Res = *it;
+      Res->claim();
+    }
+  }
+
+  return Res;
 }
 
 Arg *ArgList::getLastArg(OptSpecifier Id0, OptSpecifier Id1) const {
   Arg *Res = 0;
-  for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) {
+  for (const_iterator it = begin(), ie = end(); it != ie; ++it) {
     if ((*it)->getOption().matches(Id0) ||
         (*it)->getOption().matches(Id1)) {
       Res = *it;
-      break;
+      Res->claim();
+
     }
   }
 
-  if (Res)
-    Res->claim();
-
   return Res;
 }
 
 Arg *ArgList::getLastArg(OptSpecifier Id0, OptSpecifier Id1,
                          OptSpecifier Id2) const {
   Arg *Res = 0;
-  for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) {
+  for (const_iterator it = begin(), ie = end(); it != ie; ++it) {
     if ((*it)->getOption().matches(Id0) ||
         (*it)->getOption().matches(Id1) ||
         (*it)->getOption().matches(Id2)) {
       Res = *it;
-      break;
+      Res->claim();
     }
   }
 
-  if (Res)
-    Res->claim();
-
   return Res;
 }
 
 Arg *ArgList::getLastArg(OptSpecifier Id0, OptSpecifier Id1,
                          OptSpecifier Id2, OptSpecifier Id3) const {
   Arg *Res = 0;
-  for (const_reverse_iterator it = rbegin(), ie = rend(); it != ie; ++it) {
+  for (const_iterator it = begin(), ie = end(); it != ie; ++it) {
     if ((*it)->getOption().matches(Id0) ||
         (*it)->getOption().matches(Id1) ||
         (*it)->getOption().matches(Id2) ||
         (*it)->getOption().matches(Id3)) {
       Res = *it;
-      break;
+      Res->claim();
     }
   }
 
-  if (Res)
-    Res->claim();
-
   return Res;
 }
 
@@ -214,7 +211,8 @@ const char *ArgList::GetOrMakeJoinedArgString(unsigned Index,
 
 //
 
-InputArgList::InputArgList(const char **ArgBegin, const char **ArgEnd)
+InputArgList::InputArgList(const char* const *ArgBegin,
+                           const char* const *ArgEnd)
   : NumInputArgStrings(ArgEnd - ArgBegin) {
   ArgStrings.append(ArgBegin, ArgEnd);
 }
diff --git a/lib/Driver/CMakeLists.txt b/lib/Driver/CMakeLists.txt
index 00d076bb7eb8..ee7ded94d4db 100644
--- a/lib/Driver/CMakeLists.txt
+++ b/lib/Driver/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST clangParse)
 
 add_clang_library(clangDriver
   Action.cpp
diff --git a/lib/Driver/Compilation.cpp b/lib/Driver/Compilation.cpp
index c059afd45439..5619212d38bb 100644
--- a/lib/Driver/Compilation.cpp
+++ b/lib/Driver/Compilation.cpp
@@ -1,4 +1,4 @@
-//===--- Compilation.cpp - Compilation Task Implementation --------------*-===//
+//===--- Compilation.cpp - Compilation Task Implementation ----------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -17,7 +17,7 @@
 #include "clang/Driver/ToolChain.h"
 
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Program.h"
+#include "llvm/Support/Program.h"
 #include <sys/stat.h>
 #include <errno.h>
 using namespace clang::driver;
@@ -101,21 +101,15 @@ bool Compilation::CleanupFileList(const ArgStringList &Files,
     llvm::sys::Path P(*it);
     std::string Error;
 
-    if (!P.isRegularFile()) {
-      // If we have a special file in our list, i.e. /dev/null
-      //  then don't call eraseFromDisk() and just continue.
-      continue;
-    }
-
     if (P.eraseFromDisk(false, &Error)) {
-      // Failure is only failure if the file doesn't exist. There is a
-      // race condition here due to the limited interface of
-      // llvm::sys::Path, we want to know if the removal gave E_NOENT.
+      // Failure is only failure if the file exists and is "regular". There is
+      // a race condition here due to the limited interface of
+      // llvm::sys::Path, we want to know if the removal gave ENOENT.
 
       // FIXME: Grumble, P.exists() is broken. PR3837.
       struct stat buf;
-      if (::stat(P.c_str(), &buf) == 0
-          || errno != ENOENT) {
+      if (::stat(P.c_str(), &buf) == 0 ? (buf.st_mode & S_IFMT) == S_IFREG :
+                                         (errno != ENOENT)) {
         if (IssueErrors)
           getDriver().Diag(clang::diag::err_drv_unable_to_remove_file)
             << Error;
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
index 82f913484f48..5a5986b5bc2b 100644
--- a/lib/Driver/Driver.cpp
+++ b/lib/Driver/Driver.cpp
@@ -1,4 +1,4 @@
-//===--- Driver.cpp - Clang GCC Compatible Driver -----------------------*-===//
+//===--- Driver.cpp - Clang GCC Compatible Driver -------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,6 +7,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifdef HAVE_CLANG_CONFIG_H
+# include "clang/Config/config.h"
+#endif
+
 #include "clang/Driver/Driver.h"
 
 #include "clang/Driver/Action.h"
@@ -30,13 +34,21 @@
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
-#include "llvm/System/Program.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
 
 #include "InputInfo.h"
 
 #include <map>
 
+#ifdef __CYGWIN__
+#include <cygwin/version.h>
+#if defined(CYGWIN_VERSION_DLL_MAJOR) && CYGWIN_VERSION_DLL_MAJOR<1007
+#define IS_CYGWIN15 1
+#endif
+#endif
+
 using namespace clang::driver;
 using namespace clang;
 
@@ -50,8 +62,9 @@ Driver::Driver(llvm::StringRef _ClangExecutable,
     DefaultImageName(_DefaultImageName),
     DriverTitle("clang \"gcc-compatible\" driver"),
     Host(0),
-    CCCGenericGCCName("gcc"), CCPrintOptionsFilename(0), CCCIsCXX(false),
+    CCPrintOptionsFilename(0), CCPrintHeadersFilename(0), CCCIsCXX(false),
     CCCEcho(false), CCCPrintBindings(false), CCPrintOptions(false),
+    CCPrintHeaders(false), CCCGenericGCCName("gcc"),
     CheckInputsExist(true), CCCUseClang(true), CCCUseClangCXX(true),
     CCCUseClangCPP(true), CCCUsePCH(true), SuppressMissingInputWarning(false) {
   if (IsProduction) {
@@ -69,16 +82,16 @@ Driver::Driver(llvm::StringRef _ClangExecutable,
       CCCUseClangCXX = false;
   }
 
-  llvm::sys::Path Executable(ClangExecutable);
-  Name = Executable.getBasename();
-  Dir = Executable.getDirname();
+  Name = llvm::sys::path::stem(ClangExecutable);
+  Dir  = llvm::sys::path::parent_path(ClangExecutable);
 
   // Compute the path to the resource directory.
-  llvm::sys::Path P(Dir);
-  P.eraseComponent(); // Remove /bin from foo/bin
-  P.appendComponent("lib");
-  P.appendComponent("clang");
-  P.appendComponent(CLANG_VERSION_STRING);
+  llvm::StringRef ClangResourceDir(CLANG_RESOURCE_DIR);
+  llvm::SmallString<128> P(Dir);
+  if (ClangResourceDir != "")
+    llvm::sys::path::append(P, ClangResourceDir);
+  else
+    llvm::sys::path::append(P, "..", "lib", "clang", CLANG_VERSION_STRING);
   ResourceDir = P.str();
 }
 
@@ -115,6 +128,7 @@ InputArgList *Driver::ParseArgStrings(const char **ArgBegin,
 DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
   DerivedArgList *DAL = new DerivedArgList(Args);
 
+  bool HasNostdlib = Args.hasArg(options::OPT_nostdlib);
   for (ArgList::const_iterator it = Args.begin(),
          ie = Args.end(); it != ie; ++it) {
     const Arg *A = *it;
@@ -157,6 +171,25 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
       continue;
     }
 
+    // Rewrite reserved library names.
+    if (A->getOption().matches(options::OPT_l)) {
+      llvm::StringRef Value = A->getValue(Args);
+
+      // Rewrite unless -nostdlib is present.
+      if (!HasNostdlib && Value == "stdc++") {
+        DAL->AddFlagArg(A, Opts->getOption(
+                              options::OPT_Z_reserved_lib_stdcxx));
+        continue;
+      }
+
+      // Rewrite unconditionally.
+      if (Value == "cc_kext") {
+        DAL->AddFlagArg(A, Opts->getOption(
+                              options::OPT_Z_reserved_lib_cckext));
+        continue;
+      }
+    }
+
     DAL->append(*it);
   }
 
@@ -205,6 +238,13 @@ Compilation *Driver::BuildCompilation(int argc, const char **argv) {
   CCCPrintActions = Args->hasArg(options::OPT_ccc_print_phases);
   CCCPrintBindings = Args->hasArg(options::OPT_ccc_print_bindings);
   CCCIsCXX = Args->hasArg(options::OPT_ccc_cxx) || CCCIsCXX;
+  if (CCCIsCXX) {
+#ifdef IS_CYGWIN15
+    CCCGenericGCCName = "g++-4";
+#else
+    CCCGenericGCCName = "g++";
+#endif
+  }
   CCCEcho = Args->hasArg(options::OPT_ccc_echo);
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_gcc_name))
     CCCGenericGCCName = A->getValue(*Args);
@@ -241,8 +281,12 @@ Compilation *Driver::BuildCompilation(int argc, const char **argv) {
     DefaultHostTriple = A->getValue(*Args);
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_install_dir))
     Dir = InstalledDir = A->getValue(*Args);
-  if (const Arg *A = Args->getLastArg(options::OPT_B))
-    PrefixDir = A->getValue(*Args);
+  for (arg_iterator it = Args->filtered_begin(options::OPT_B),
+         ie = Args->filtered_end(); it != ie; ++it) {
+    const Arg *A = *it;
+    A->claim();
+    PrefixDirs.push_back(A->getValue(*Args, 0));
+  }
 
   Host = GetHostInfo(DefaultHostTriple.c_str());
 
@@ -287,7 +331,7 @@ int Driver::ExecuteCompilation(const Compilation &C) const {
   }
 
   // If there were errors building the compilation, quit now.
-  if (getDiags().getNumErrors())
+  if (getDiags().hasErrorOccurred())
     return 1;
 
   const Command *FailingCommand = 0;
@@ -365,7 +409,7 @@ void Driver::PrintVersion(const Compilation &C, llvm::raw_ostream &OS) const {
 /// option.
 static void PrintDiagnosticCategories(llvm::raw_ostream &OS) {
   for (unsigned i = 1; // Skip the empty category.
-       const char *CategoryName = Diagnostic::getCategoryNameFromID(i); ++i)
+       const char *CategoryName = DiagnosticIDs::getCategoryNameFromID(i); ++i)
     OS << i << ',' << CategoryName << '\n';
 }
 
@@ -373,8 +417,19 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
   // The order these options are handled in gcc is all over the place, but we
   // don't expect inconsistencies w.r.t. that to matter in practice.
 
+  if (C.getArgs().hasArg(options::OPT_dumpmachine)) {
+    llvm::outs() << C.getDefaultToolChain().getTripleString() << '\n';
+    return false;
+  }
+
   if (C.getArgs().hasArg(options::OPT_dumpversion)) {
-    llvm::outs() << CLANG_VERSION_STRING "\n";
+    // Since -dumpversion is only implemented for pedantic GCC compatibility, we
+    // return an answer which matches our definition of __VERSION__.
+    //
+    // If we want to return a more correct answer some day, then we should
+    // introduce a non-pedantically GCC compatible mode to Clang in which we
+    // provide sensible definitions for -dumpversion, __VERSION__, etc.
+    llvm::outs() << "4.2.1\n";
     return false;
   }
 
@@ -709,10 +764,20 @@ void Driver::BuildActions(const ToolChain &TC, const ArgList &Args,
       }
 
       // Check that the file exists, if enabled.
-      if (CheckInputsExist && memcmp(Value, "-", 2) != 0 &&
-          !llvm::sys::Path(Value).exists())
-        Diag(clang::diag::err_drv_no_such_file) << A->getValue(Args);
-      else
+      if (CheckInputsExist && memcmp(Value, "-", 2) != 0) {
+        llvm::SmallString<64> Path(Value);
+        if (Arg *WorkDir = Args.getLastArg(options::OPT_working_directory))
+          if (llvm::sys::path::is_absolute(Path.str())) {
+            Path = WorkDir->getValue(Args);
+            llvm::sys::path::append(Path, Value);
+          }
+
+        bool exists = false;
+        if (/*error_code ec =*/llvm::sys::fs::exists(Value, exists) || !exists)
+          Diag(clang::diag::err_drv_no_such_file) << Path.str();
+        else
+          Inputs.push_back(std::make_pair(Ty, A));
+      } else
         Inputs.push_back(std::make_pair(Ty, A));
 
     } else if (A->getOption().isLinkerInput()) {
@@ -726,7 +791,7 @@ void Driver::BuildActions(const ToolChain &TC, const ArgList &Args,
 
       // Follow gcc behavior and treat as linker input for invalid -x
       // options. Its not clear why we shouldn't just revert to unknown; but
-      // this isn't very important, we might as well be bug comatible.
+      // this isn't very important, we might as well be bug compatible.
       if (!InputType) {
         Diag(clang::diag::err_drv_unknown_language) << A->getValue(Args);
         InputType = types::TY_Object;
@@ -747,8 +812,7 @@ void Driver::BuildActions(const ToolChain &TC, const ArgList &Args,
 
   // -{E,M,MM} only run the preprocessor.
   if ((FinalPhaseArg = Args.getLastArg(options::OPT_E)) ||
-      (FinalPhaseArg = Args.getLastArg(options::OPT_M)) ||
-      (FinalPhaseArg = Args.getLastArg(options::OPT_MM))) {
+      (FinalPhaseArg = Args.getLastArg(options::OPT_M, options::OPT_MM))) {
     FinalPhase = phases::Preprocess;
 
     // -{fsyntax-only,-analyze,emit-ast,S} only run up to the compiler.
@@ -856,7 +920,7 @@ Action *Driver::ConstructPhaseAction(const ArgList &Args, phases::ID Phase,
   case phases::Preprocess: {
     types::ID OutputTy;
     // -{M, MM} alter the output type.
-    if (Args.hasArg(options::OPT_M) || Args.hasArg(options::OPT_MM)) {
+    if (Args.hasArg(options::OPT_M, options::OPT_MM)) {
       OutputTy = types::TY_Dependencies;
     } else {
       OutputTy = types::getPreprocessedType(Input->getType());
@@ -945,7 +1009,7 @@ void Driver::BuildJobs(Compilation &C) const {
 
   // If the user passed -Qunused-arguments or there were errors, don't warn
   // about any unused arguments.
-  if (Diags.getNumErrors() ||
+  if (Diags.hasErrorOccurred() ||
       C.getArgs().hasArg(options::OPT_Qunused_arguments))
     return;
 
@@ -1141,8 +1205,8 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
     return C.addTempFile(C.getArgs().MakeArgString(TmpName.c_str()));
   }
 
-  llvm::sys::Path BasePath(BaseInput);
-  std::string BaseName(BasePath.getLast());
+  llvm::SmallString<128> BasePath(BaseInput);
+  llvm::StringRef BaseName = llvm::sys::path::filename(BasePath);
 
   // Determine what the derived output name should be.
   const char *NamedOutput;
@@ -1163,11 +1227,11 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
 
   // As an annoying special case, PCH generation doesn't strip the pathname.
   if (JA.getType() == types::TY_PCH) {
-    BasePath.eraseComponent();
-    if (BasePath.isEmpty())
+    llvm::sys::path::remove_filename(BasePath);
+    if (BasePath.empty())
       BasePath = NamedOutput;
     else
-      BasePath.appendComponent(NamedOutput);
+      llvm::sys::path::append(BasePath, NamedOutput);
     return C.addResultFile(C.getArgs().MakeArgString(BasePath.c_str()));
   } else {
     return C.addResultFile(NamedOutput);
@@ -1177,10 +1241,12 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
 std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
   // Respect a limited subset of the '-Bprefix' functionality in GCC by
   // attempting to use this prefix when lokup up program paths.
-  if (!PrefixDir.empty()) {
-    llvm::sys::Path P(PrefixDir);
+  for (Driver::prefix_list::const_iterator it = PrefixDirs.begin(),
+       ie = PrefixDirs.end(); it != ie; ++it) {
+    llvm::sys::Path P(*it);
     P.appendComponent(Name);
-    if (P.exists())
+    bool Exists;
+    if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
       return P.str();
   }
 
@@ -1189,7 +1255,8 @@ std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
          it = List.begin(), ie = List.end(); it != ie; ++it) {
     llvm::sys::Path P(*it);
     P.appendComponent(Name);
-    if (P.exists())
+    bool Exists;
+    if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
       return P.str();
   }
 
@@ -1200,10 +1267,13 @@ std::string Driver::GetProgramPath(const char *Name, const ToolChain &TC,
                                    bool WantFile) const {
   // Respect a limited subset of the '-Bprefix' functionality in GCC by
   // attempting to use this prefix when lokup up program paths.
-  if (!PrefixDir.empty()) {
-    llvm::sys::Path P(PrefixDir);
+  for (Driver::prefix_list::const_iterator it = PrefixDirs.begin(),
+       ie = PrefixDirs.end(); it != ie; ++it) {
+    llvm::sys::Path P(*it);
     P.appendComponent(Name);
-    if (WantFile ? P.exists() : P.canExecute())
+    bool Exists;
+    if (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
+                 : P.canExecute())
       return P.str();
   }
 
@@ -1212,7 +1282,9 @@ std::string Driver::GetProgramPath(const char *Name, const ToolChain &TC,
          it = List.begin(), ie = List.end(); it != ie; ++it) {
     llvm::sys::Path P(*it);
     P.appendComponent(Name);
-    if (WantFile ? P.exists() : P.canExecute())
+    bool Exists;
+    if (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
+                 : P.canExecute())
       return P.str();
   }
 
@@ -1266,6 +1338,8 @@ const HostInfo *Driver::GetHostInfo(const char *TripleStr) const {
     return createDragonFlyHostInfo(*this, Triple);
   case llvm::Triple::OpenBSD:
     return createOpenBSDHostInfo(*this, Triple);
+  case llvm::Triple::NetBSD:
+    return createNetBSDHostInfo(*this, Triple);
   case llvm::Triple::FreeBSD:
     return createFreeBSDHostInfo(*this, Triple);
   case llvm::Triple::Minix:
@@ -1275,7 +1349,6 @@ const HostInfo *Driver::GetHostInfo(const char *TripleStr) const {
   case llvm::Triple::Win32:
     return createWindowsHostInfo(*this, Triple);
   case llvm::Triple::MinGW32:
-  case llvm::Triple::MinGW64:
     return createMinGWHostInfo(*this, Triple);
   default:
     return createUnknownHostInfo(*this, Triple);
diff --git a/lib/Driver/DriverOptions.cpp b/lib/Driver/DriverOptions.cpp
index 72aaf56ad3e1..f2d9af86d08d 100644
--- a/lib/Driver/DriverOptions.cpp
+++ b/lib/Driver/DriverOptions.cpp
@@ -1,4 +1,4 @@
-//===--- DriverOptions.cpp - Driver Options Table -----------------------*-===//
+//===--- DriverOptions.cpp - Driver Options Table -------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/HostInfo.cpp b/lib/Driver/HostInfo.cpp
index 7c5e430bb7d5..cd413180f77b 100644
--- a/lib/Driver/HostInfo.cpp
+++ b/lib/Driver/HostInfo.cpp
@@ -1,4 +1,4 @@
-//===--- HostInfo.cpp - Host specific information -----------------------*-===//
+//===--- HostInfo.cpp - Host specific information -------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -115,6 +115,7 @@ ToolChain *DarwinHostInfo::CreateToolChain(const ArgList &Args,
     // If we recognized the arch, match it to the toolchains we support.
     const char *UseNewToolChain = ::getenv("CCC_ENABLE_NEW_DARWIN_TOOLCHAIN");
     if (UseNewToolChain || 
+        Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64 ||
         Arch == llvm::Triple::arm || Arch == llvm::Triple::thumb) {
       TC = new toolchains::DarwinClang(*this, TCTriple);
     } else if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64) {
@@ -371,6 +372,65 @@ ToolChain *FreeBSDHostInfo::CreateToolChain(const ArgList &Args,
   return TC;
 }
 
+// NetBSD Host Info
+
+/// NetBSDHostInfo -  NetBSD host information implementation.
+class NetBSDHostInfo : public HostInfo {
+  /// Cache of tool chains we have created.
+  mutable llvm::StringMap<ToolChain*> ToolChains;
+
+public:
+  NetBSDHostInfo(const Driver &D, const llvm::Triple& Triple)
+    : HostInfo(D, Triple) {}
+  ~NetBSDHostInfo();
+
+  virtual bool useDriverDriver() const;
+
+  virtual ToolChain *CreateToolChain(const ArgList &Args,
+                                     const char *ArchName) const;
+};
+
+NetBSDHostInfo::~NetBSDHostInfo() {
+  for (llvm::StringMap<ToolChain*>::iterator
+         it = ToolChains.begin(), ie = ToolChains.end(); it != ie; ++it)
+    delete it->second;
+}
+
+bool NetBSDHostInfo::useDriverDriver() const {
+  return false;
+}
+
+ToolChain *NetBSDHostInfo::CreateToolChain(const ArgList &Args,
+                                            const char *ArchName) const {
+  assert(!ArchName &&
+         "Unexpected arch name on platform without driver driver support.");
+
+  // Automatically handle some instances of -m32/-m64 we know about.
+  std::string Arch = getArchName();
+  ArchName = Arch.c_str();
+  if (Arg *A = Args.getLastArg(options::OPT_m32, options::OPT_m64)) {
+    if (Triple.getArch() == llvm::Triple::x86 ||
+        Triple.getArch() == llvm::Triple::x86_64) {
+      ArchName =
+        (A->getOption().matches(options::OPT_m32)) ? "i386" : "x86_64";
+    } else if (Triple.getArch() == llvm::Triple::ppc ||
+               Triple.getArch() == llvm::Triple::ppc64) {
+      ArchName =
+        (A->getOption().matches(options::OPT_m32)) ? "powerpc" : "powerpc64";
+    }
+  }
+
+  ToolChain *&TC = ToolChains[ArchName];
+  if (!TC) {
+    llvm::Triple TCTriple(getTriple());
+    TCTriple.setArchName(ArchName);
+
+    TC = new toolchains::NetBSD(*this, TCTriple);
+  }
+
+  return TC;
+}
+
 // Minix Host Info
 
 /// MinixHostInfo -  Minix host information implementation.
@@ -623,6 +683,12 @@ clang::driver::createFreeBSDHostInfo(const Driver &D,
 }
 
 const HostInfo *
+clang::driver::createNetBSDHostInfo(const Driver &D,
+                                     const llvm::Triple& Triple) {
+  return new NetBSDHostInfo(D, Triple);
+}
+
+const HostInfo *
 clang::driver::createMinixHostInfo(const Driver &D,
                                      const llvm::Triple& Triple) {
   return new MinixHostInfo(D, Triple);
diff --git a/lib/Driver/Job.cpp b/lib/Driver/Job.cpp
index fa7d0607728e..51055e93f558 100644
--- a/lib/Driver/Job.cpp
+++ b/lib/Driver/Job.cpp
@@ -1,4 +1,4 @@
-//===--- Job.cpp - Command to Execute -----------------------------------*-===//
+//===--- Job.cpp - Command to Execute -------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/OptTable.cpp b/lib/Driver/OptTable.cpp
index 3c363142d7fd..c3d3048c12fe 100644
--- a/lib/Driver/OptTable.cpp
+++ b/lib/Driver/OptTable.cpp
@@ -1,4 +1,4 @@
-//===--- OptTable.cpp - Option Table Implementation ---------------------*-===//
+//===--- OptTable.cpp - Option Table Implementation -----------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -226,7 +226,8 @@ Arg *OptTable::ParseOneArg(const ArgList &Args, unsigned &Index) const {
   return new Arg(TheUnknownOption, Index++, Str);
 }
 
-InputArgList *OptTable::ParseArgs(const char **ArgBegin, const char **ArgEnd,
+InputArgList *OptTable::ParseArgs(const char* const *ArgBegin,
+                                  const char* const *ArgEnd,
                                   unsigned &MissingArgIndex,
                                   unsigned &MissingArgCount) const {
   InputArgList *Args = new InputArgList(ArgBegin, ArgEnd);
diff --git a/lib/Driver/Option.cpp b/lib/Driver/Option.cpp
index 5396250dfae1..a992cef3d239 100644
--- a/lib/Driver/Option.cpp
+++ b/lib/Driver/Option.cpp
@@ -1,4 +1,4 @@
-//===--- Option.cpp - Abstract Driver Options ---------------------------*-===//
+//===--- Option.cpp - Abstract Driver Options -----------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/Phases.cpp b/lib/Driver/Phases.cpp
index df4cdee77500..f36000214ab0 100644
--- a/lib/Driver/Phases.cpp
+++ b/lib/Driver/Phases.cpp
@@ -1,4 +1,4 @@
-//===--- Phases.cpp - Transformations on Driver Types -------------------*-===//
+//===--- Phases.cpp - Transformations on Driver Types ---------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/Tool.cpp b/lib/Driver/Tool.cpp
index fe01531d9a2a..b93864ff8bf2 100644
--- a/lib/Driver/Tool.cpp
+++ b/lib/Driver/Tool.cpp
@@ -1,4 +1,4 @@
-//===--- Tool.cpp - Compilation Tools -----------------------------------*-===//
+//===--- Tool.cpp - Compilation Tools -------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Driver/ToolChain.cpp b/lib/Driver/ToolChain.cpp
index 94c1c6beb99f..e4051a165b2d 100644
--- a/lib/Driver/ToolChain.cpp
+++ b/lib/Driver/ToolChain.cpp
@@ -1,4 +1,4 @@
-//===--- ToolChain.cpp - Collections of tools for one platform ----------*-===//
+//===--- ToolChain.cpp - Collections of tools for one platform ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -43,6 +43,10 @@ types::ID ToolChain::LookupTypeForExtension(const char *Ext) const {
   return types::lookupTypeForExtension(Ext);
 }
 
+bool ToolChain::HasNativeLLVMSupport() const {
+  return false;
+}
+
 /// getARMTargetCPU - Get the (LLVM) name of the ARM cpu we are targetting.
 //
 // FIXME: tblgen this.
@@ -174,3 +178,52 @@ std::string ToolChain::ComputeEffectiveClangTriple(const ArgList &Args) const {
   return ComputeLLVMTriple(Args);
 }
 
+ToolChain::CXXStdlibType ToolChain::GetCXXStdlibType(const ArgList &Args) const{
+  if (Arg *A = Args.getLastArg(options::OPT_stdlib_EQ)) {
+    llvm::StringRef Value = A->getValue(Args);
+    if (Value == "libc++")
+      return ToolChain::CST_Libcxx;
+    if (Value == "libstdc++")
+      return ToolChain::CST_Libstdcxx;
+    getDriver().Diag(clang::diag::err_drv_invalid_stdlib_name)
+      << A->getAsString(Args);
+  }
+
+  return ToolChain::CST_Libstdcxx;
+}
+
+void ToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args,
+                                             ArgStringList &CmdArgs) const {
+  CXXStdlibType Type = GetCXXStdlibType(Args);
+
+  switch (Type) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-cxx-system-include");
+    CmdArgs.push_back("/usr/include/c++/v1");
+    break;
+
+  case ToolChain::CST_Libstdcxx:
+    // Currently handled by the mass of goop in InitHeaderSearch.
+    break;
+  }
+}
+
+void ToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
+                                    ArgStringList &CmdArgs) const {
+  CXXStdlibType Type = GetCXXStdlibType(Args);
+
+  switch (Type) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-lc++");
+    break;
+
+  case ToolChain::CST_Libstdcxx:
+    CmdArgs.push_back("-lstdc++");
+    break;
+  }
+}
+
+void ToolChain::AddCCKextLibArgs(const ArgList &Args,
+                                 ArgStringList &CmdArgs) const {
+  CmdArgs.push_back("-lcc_kext");
+}
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
index 471c47dd2b50..13b8b4609ec0 100644
--- a/lib/Driver/ToolChains.cpp
+++ b/lib/Driver/ToolChains.cpp
@@ -1,4 +1,4 @@
-//===--- ToolChains.cpp - ToolChain Implementations ---------------------*-===//
+//===--- ToolChains.cpp - ToolChain Implementations -----------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -23,8 +23,11 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/system_error.h"
 
 #include <cstdlib> // ::getenv
 
@@ -59,6 +62,10 @@ types::ID Darwin::LookupTypeForExtension(const char *Ext) const {
   return Ty;
 }
 
+bool Darwin::HasNativeLLVMSupport() const {
+  return true;
+}
+
 // FIXME: Can we tablegen this?
 static const char *GetArmArchForMArch(llvm::StringRef Value) {
   if (Value == "armv6k")
@@ -142,7 +149,8 @@ DarwinGCC::DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple)
 
   // Try the next major version if that tool chain dir is invalid.
   std::string Tmp = "/usr/lib/gcc/" + ToolChainDir;
-  if (!llvm::sys::Path(Tmp).exists()) {
+  bool Exists;
+  if (llvm::sys::fs::exists(Tmp, Exists) || Exists) {
     std::string Next = "i686-apple-darwin";
     Next += llvm::utostr(DarwinVersion[0] + 1);
     Next += "/";
@@ -156,7 +164,7 @@ DarwinGCC::DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple)
     //
     // FIXME: Drop dependency on gcc's tool chain.
     Tmp = "/usr/lib/gcc/" + Next;
-    if (llvm::sys::Path(Tmp).exists())
+    if (!llvm::sys::fs::exists(Tmp, Exists) && Exists)
       ToolChainDir = Next;
   }
 
@@ -301,19 +309,20 @@ void DarwinGCC::AddLinkSearchPathArgs(const ArgList &Args,
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/" + ToolChainDir));
 
   Tmp = getDriver().Dir + "/../lib/gcc/" + ToolChainDir;
-  if (llvm::sys::Path(Tmp).exists())
+  bool Exists;
+  if (!llvm::sys::fs::exists(Tmp, Exists) && Exists)
     CmdArgs.push_back(Args.MakeArgString("-L" + Tmp));
   Tmp = getDriver().Dir + "/../lib/gcc";
-  if (llvm::sys::Path(Tmp).exists())
+  if (!llvm::sys::fs::exists(Tmp, Exists) && Exists)
     CmdArgs.push_back(Args.MakeArgString("-L" + Tmp));
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/gcc/" + ToolChainDir));
   // Intentionally duplicated for (temporary) gcc bug compatibility.
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/gcc/" + ToolChainDir));
   Tmp = getDriver().Dir + "/../lib/" + ToolChainDir;
-  if (llvm::sys::Path(Tmp).exists())
+  if (!llvm::sys::fs::exists(Tmp, Exists) && Exists)
     CmdArgs.push_back(Args.MakeArgString("-L" + Tmp));
   Tmp = getDriver().Dir + "/../lib";
-  if (llvm::sys::Path(Tmp).exists())
+  if (!llvm::sys::fs::exists(Tmp, Exists) && Exists)
     CmdArgs.push_back(Args.MakeArgString("-L" + Tmp));
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/gcc/" + ToolChainDir +
                                        "/../../../" + ToolChainDir));
@@ -369,10 +378,30 @@ void DarwinGCC::AddLinkRuntimeLibArgs(const ArgList &Args,
 DarwinClang::DarwinClang(const HostInfo &Host, const llvm::Triple& Triple)
   : Darwin(Host, Triple)
 {
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir)
+    getProgramPaths().push_back(getDriver().Dir);
+
   // We expect 'as', 'ld', etc. to be adjacent to our install dir.
   getProgramPaths().push_back(getDriver().getInstalledDir());
   if (getDriver().getInstalledDir() != getDriver().Dir)
     getProgramPaths().push_back(getDriver().Dir);
+
+  // For fallback, we need to know how to find the GCC cc1 executables, so we
+  // also add the GCC libexec paths. This is legiy code that can be removed once
+  // fallback is no longer useful.
+  std::string ToolChainDir = "i686-apple-darwin";
+  ToolChainDir += llvm::utostr(DarwinVersion[0]);
+  ToolChainDir += "/4.2.1";
+
+  std::string Path = getDriver().Dir;
+  Path += "/../libexec/gcc/";
+  Path += ToolChainDir;
+  getProgramPaths().push_back(Path);
+
+  Path = "/usr/libexec/gcc/";
+  Path += ToolChainDir;
+  getProgramPaths().push_back(Path);
 }
 
 void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
@@ -431,12 +460,14 @@ void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
 
   if (ArchSpecificDir) {
     P.appendComponent(ArchSpecificDir);
-    if (P.exists())
+    bool Exists;
+    if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
       CmdArgs.push_back(Args.MakeArgString("-L" + P.str()));
     P.eraseComponent();
   }
 
-  if (P.exists())
+  bool Exists;
+  if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
     CmdArgs.push_back(Args.MakeArgString("-L" + P.str()));
 }
 
@@ -477,11 +508,20 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
     else if (isMacosxVersionLT(10, 6))
       CmdArgs.push_back("-lgcc_s.10.5");
 
-    // For OS X, we only need a static runtime library when targetting 10.4, to
-    // provide versions of the static functions which were omitted from
-    // 10.4.dylib.
-    if (isMacosxVersionLT(10, 5))
+    // For OS X, we thought we would only need a static runtime library when
+    // targetting 10.4, to provide versions of the static functions which were
+    // omitted from 10.4.dylib.
+    //
+    // Unfortunately, that turned out to not be true, because Darwin system
+    // headers can still use eprintf on i386, and it is not exported from
+    // libSystem. Therefore, we still must provide a runtime library just for
+    // the tiny tiny handful of projects that *might* use that symbol.
+    if (isMacosxVersionLT(10, 5)) {
       DarwinStaticLib = "libclang_rt.10.4.a";
+    } else {
+      if (getTriple().getArch() == llvm::Triple::x86)
+        DarwinStaticLib = "libclang_rt.eprintf.a";
+    }
   }
 
   /// Add the target specific static library, if needed.
@@ -493,10 +533,8 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
 
     // For now, allow missing resource libraries to support developers who may
     // not have compiler-rt checked out or integrated into their build.
-    if (!P.exists())
-      getDriver().Diag(clang::diag::warn_drv_missing_resource_library)
-        << P.str();
-    else
+    bool Exists;
+    if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
       CmdArgs.push_back(Args.MakeArgString(P.str()));
   }
 }
@@ -574,6 +612,72 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   setTarget(iPhoneVersion, Major, Minor, Micro);
 }
 
+void DarwinClang::AddCXXStdlibLibArgs(const ArgList &Args,
+                                      ArgStringList &CmdArgs) const {
+  CXXStdlibType Type = GetCXXStdlibType(Args);
+
+  switch (Type) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-lc++");
+    break;
+
+  case ToolChain::CST_Libstdcxx: {
+    // Unfortunately, -lstdc++ doesn't always exist in the standard search path;
+    // it was previously found in the gcc lib dir. However, for all the Darwin
+    // platforms we care about it was -lstdc++.6, so we search for that
+    // explicitly if we can't see an obvious -lstdc++ candidate.
+
+    // Check in the sysroot first.
+    bool Exists;
+    if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
+      llvm::sys::Path P(A->getValue(Args));
+      P.appendComponent("usr");
+      P.appendComponent("lib");
+      P.appendComponent("libstdc++.dylib");
+
+      if (llvm::sys::fs::exists(P.str(), Exists) || !Exists) {
+        P.eraseComponent();
+        P.appendComponent("libstdc++.6.dylib");
+        if (!llvm::sys::fs::exists(P.str(), Exists) && Exists) {
+          CmdArgs.push_back(Args.MakeArgString(P.str()));
+          return;
+        }
+      }
+    }
+
+    // Otherwise, look in the root.
+    if ((llvm::sys::fs::exists("/usr/lib/libstdc++.dylib", Exists) || !Exists)&&
+      (!llvm::sys::fs::exists("/usr/lib/libstdc++.6.dylib", Exists) && Exists)){
+      CmdArgs.push_back("/usr/lib/libstdc++.6.dylib");
+      return;
+    }
+
+    // Otherwise, let the linker search.
+    CmdArgs.push_back("-lstdc++");
+    break;
+  }
+  }
+}
+
+void DarwinClang::AddCCKextLibArgs(const ArgList &Args,
+                                   ArgStringList &CmdArgs) const {
+
+  // For Darwin platforms, use the compiler-rt-based support library
+  // instead of the gcc-provided one (which is also incidentally
+  // only present in the gcc lib dir, which makes it hard to find).
+
+  llvm::sys::Path P(getDriver().ResourceDir);
+  P.appendComponent("lib");
+  P.appendComponent("darwin");
+  P.appendComponent("libclang_rt.cc_kext.a");
+  
+  // For now, allow missing resource libraries to support developers who may
+  // not have compiler-rt checked out or integrated into their build.
+  bool Exists;
+  if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
+    CmdArgs.push_back(Args.MakeArgString(P.str()));
+}
+
 DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
                                       const char *BoundArch) const {
   DerivedArgList *DAL = new DerivedArgList(Args.getBaseArgs());
@@ -595,6 +699,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       if (getArchName() != A->getValue(Args, 0))
         continue;
 
+      Arg *OriginalArg = A;
       unsigned Index = Args.getBaseArgs().MakeIndex(A->getValue(Args, 1));
       unsigned Prev = Index;
       Arg *XarchArg = Opts.ParseOneArg(Args, Index);
@@ -618,6 +723,20 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       A = XarchArg;
 
       DAL->AddSynthesizedArg(A);
+
+      // Linker input arguments require custom handling. The problem is that we
+      // have already constructed the phase actions, so we can not treat them as
+      // "input arguments".
+      if (A->getOption().isLinkerInput()) {
+        // Convert the argument into individual Zlinker_input_args.
+        for (unsigned i = 0, e = A->getNumValues(); i != e; ++i) {
+          DAL->AddSeparateArg(OriginalArg,
+                              Opts.getOption(options::OPT_Zlinker_input),
+                              A->getValue(Args, i));
+          
+        }
+        continue;
+      }
     }
 
     // Sob. These is strictly gcc compatible for the time being. Apple
@@ -937,7 +1056,7 @@ Tool &TCEToolChain::SelectTool(const Compilation &C,
 /// OpenBSD - OpenBSD tool chain which can call as(1) and ld(1) directly.
 
 OpenBSD::OpenBSD(const HostInfo &Host, const llvm::Triple& Triple)
-  : Generic_GCC(Host, Triple) {
+  : Generic_ELF(Host, Triple) {
   getFilePaths().push_back(getDriver().Dir + "/../lib");
   getFilePaths().push_back("/usr/lib");
 }
@@ -949,11 +1068,20 @@ Tool &OpenBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
   else
     Key = JA.getKind();
 
+  bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
+                                             options::OPT_no_integrated_as,
+                                             IsIntegratedAssemblerDefault());
+
   Tool *&T = Tools[Key];
   if (!T) {
     switch (Key) {
-    case Action::AssembleJobClass:
-      T = new tools::openbsd::Assemble(*this); break;
+    case Action::AssembleJobClass: {
+      if (UseIntegratedAs)
+        T = new tools::ClangAs(*this);
+      else
+        T = new tools::openbsd::Assemble(*this);
+      break;
+    }
     case Action::LinkJobClass:
       T = new tools::openbsd::Link(*this); break;
     default:
@@ -967,7 +1095,7 @@ Tool &OpenBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
 /// FreeBSD - FreeBSD tool chain which can call as(1) and ld(1) directly.
 
 FreeBSD::FreeBSD(const HostInfo &Host, const llvm::Triple& Triple)
-  : Generic_GCC(Host, Triple) {
+  : Generic_ELF(Host, Triple) {
 
   // Determine if we are compiling 32-bit code on an x86_64 platform.
   bool Lib32 = false;
@@ -994,11 +1122,19 @@ Tool &FreeBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
   else
     Key = JA.getKind();
 
+  bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
+                                             options::OPT_no_integrated_as,
+                                             IsIntegratedAssemblerDefault());
+
   Tool *&T = Tools[Key];
   if (!T) {
     switch (Key) {
     case Action::AssembleJobClass:
-      T = new tools::freebsd::Assemble(*this); break;
+      if (UseIntegratedAs)
+        T = new tools::ClangAs(*this);
+      else
+        T = new tools::freebsd::Assemble(*this);
+      break;
     case Action::LinkJobClass:
       T = new tools::freebsd::Link(*this); break;
     default:
@@ -1009,6 +1145,57 @@ Tool &FreeBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
   return *T;
 }
 
+/// NetBSD - NetBSD tool chain which can call as(1) and ld(1) directly.
+
+NetBSD::NetBSD(const HostInfo &Host, const llvm::Triple& Triple)
+  : Generic_ELF(Host, Triple) {
+
+  // Determine if we are compiling 32-bit code on an x86_64 platform.
+  bool Lib32 = false;
+  if (Triple.getArch() == llvm::Triple::x86 &&
+      llvm::Triple(getDriver().DefaultHostTriple).getArch() ==
+        llvm::Triple::x86_64)
+    Lib32 = true;
+
+  getProgramPaths().push_back(getDriver().Dir + "/../libexec");
+  getProgramPaths().push_back("/usr/libexec");
+  if (Lib32) {
+    getFilePaths().push_back("/usr/lib/i386");
+  } else {
+    getFilePaths().push_back("/usr/lib");
+  }
+}
+
+Tool &NetBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
+  Action::ActionClass Key;
+  if (getDriver().ShouldUseClangCompiler(C, JA, getTriple()))
+    Key = Action::AnalyzeJobClass;
+  else
+    Key = JA.getKind();
+
+  bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
+                                             options::OPT_no_integrated_as,
+                                             IsIntegratedAssemblerDefault());
+
+  Tool *&T = Tools[Key];
+  if (!T) {
+    switch (Key) {
+    case Action::AssembleJobClass:
+      if (UseIntegratedAs)
+        T = new tools::ClangAs(*this);
+      else
+        T = new tools::netbsd::Assemble(*this);
+      break;
+    case Action::LinkJobClass:
+      T = new tools::netbsd::Link(*this); break;
+    default:
+      T = &Generic_GCC::SelectTool(C, JA);
+    }
+  }
+
+  return *T;
+}
+
 /// Minix - Minix tool chain which can call as(1) and ld(1) directly.
 
 Minix::Minix(const HostInfo &Host, const llvm::Triple& Triple)
@@ -1083,28 +1270,253 @@ Tool &AuroraUX::SelectTool(const Compilation &C, const JobAction &JA) const {
 
 /// Linux toolchain (very bare-bones at the moment).
 
-Linux::Linux(const HostInfo &Host, const llvm::Triple& Triple)
-  : Generic_GCC(Host, Triple) {
-  getFilePaths().push_back(getDriver().Dir +
-                           "/../lib/clang/" CLANG_VERSION_STRING "/");
-  getFilePaths().push_back("/lib/");
-  getFilePaths().push_back("/usr/lib/");
-
-  // Depending on the Linux distribution, any combination of lib{,32,64} is
-  // possible. E.g. Debian uses lib and lib32 for mixed i386/x86-64 systems,
-  // openSUSE uses lib and lib64 for the same purpose.
-  getFilePaths().push_back("/lib32/");
-  getFilePaths().push_back("/usr/lib32/");
-  getFilePaths().push_back("/lib64/");
-  getFilePaths().push_back("/usr/lib64/");
-
-  // FIXME: Figure out some way to get gcc's libdir
-  // (e.g. /usr/lib/gcc/i486-linux-gnu/4.3/ for Ubuntu 32-bit); we need
-  // crtbegin.o/crtend.o/etc., and want static versions of various
-  // libraries. If we had our own crtbegin.o/crtend.o/etc, we could probably
-  // get away with using shared versions in /usr/lib, though.
-  // We could fall back to the approach we used for includes (a massive
-  // list), but that's messy at best.
+enum LinuxDistro {
+  DebianLenny,
+  DebianSqueeze,
+  Exherbo,
+  Fedora13,
+  Fedora14,
+  OpenSuse11_3,
+  UbuntuJaunty,
+  UbuntuKarmic,
+  UbuntuLucid,
+  UbuntuMaverick,
+  UnknownDistro
+};
+
+static bool IsFedora(enum LinuxDistro Distro) {
+  return Distro == Fedora13 || Distro == Fedora14;
+}
+
+static bool IsOpenSuse(enum LinuxDistro Distro) {
+  return Distro == OpenSuse11_3;
+}
+
+static bool IsDebian(enum LinuxDistro Distro) {
+  return Distro == DebianLenny || Distro == DebianSqueeze;
+}
+
+static bool IsUbuntu(enum LinuxDistro Distro) {
+  return Distro == UbuntuLucid || Distro == UbuntuMaverick || 
+         Distro == UbuntuJaunty || Distro == UbuntuKarmic;
+}
+
+static bool IsDebianBased(enum LinuxDistro Distro) {
+  return IsDebian(Distro) || IsUbuntu(Distro);
+}
+
+static bool HasMultilib(llvm::Triple::ArchType Arch, enum LinuxDistro Distro) {
+  if (Arch == llvm::Triple::x86_64) {
+    bool Exists;
+    if (Distro == Exherbo &&
+        (llvm::sys::fs::exists("/usr/lib32/libc.so", Exists) || !Exists))
+      return false;
+
+    return true;
+  }
+  if (Arch == llvm::Triple::x86 && IsDebianBased(Distro))
+    return true;
+  return false;
+}
+
+static LinuxDistro DetectLinuxDistro(llvm::Triple::ArchType Arch) {
+  llvm::OwningPtr<llvm::MemoryBuffer> File;
+  if (!llvm::MemoryBuffer::getFile("/etc/lsb-release", File)) {
+    llvm::StringRef Data = File.get()->getBuffer();
+    llvm::SmallVector<llvm::StringRef, 8> Lines;
+    Data.split(Lines, "\n");
+    for (unsigned int i = 0, s = Lines.size(); i < s; ++ i) {
+      if (Lines[i] == "DISTRIB_CODENAME=maverick")
+        return UbuntuMaverick;
+      else if (Lines[i] == "DISTRIB_CODENAME=lucid")
+        return UbuntuLucid;
+      else if (Lines[i] == "DISTRIB_CODENAME=jaunty")
+        return UbuntuJaunty;
+      else if (Lines[i] == "DISTRIB_CODENAME=karmic")
+        return UbuntuKarmic;
+    }
+    return UnknownDistro;
+  }
+
+  if (!llvm::MemoryBuffer::getFile("/etc/redhat-release", File)) {
+    llvm::StringRef Data = File.get()->getBuffer();
+    if (Data.startswith("Fedora release 14 (Laughlin)"))
+      return Fedora14;
+    else if (Data.startswith("Fedora release 13 (Goddard)"))
+      return Fedora13;
+    return UnknownDistro;
+  }
+
+  if (!llvm::MemoryBuffer::getFile("/etc/debian_version", File)) {
+    llvm::StringRef Data = File.get()->getBuffer();
+    if (Data[0] == '5')
+      return DebianLenny;
+    else if (Data.startswith("squeeze/sid"))
+      return DebianSqueeze;
+    return UnknownDistro;
+  }
+
+  if (!llvm::MemoryBuffer::getFile("/etc/SuSE-release", File)) {
+    llvm::StringRef Data = File.get()->getBuffer();
+    if (Data.startswith("openSUSE 11.3"))
+      return OpenSuse11_3;
+    return UnknownDistro;
+  }
+
+  bool Exists;
+  if (!llvm::sys::fs::exists("/etc/exherbo-release", Exists) && Exists)
+    return Exherbo;
+
+  return UnknownDistro;
+}
+
+Linux::Linux(const HostInfo &Host, const llvm::Triple &Triple)
+  : Generic_ELF(Host, Triple) {
+  llvm::Triple::ArchType Arch =
+    llvm::Triple(getDriver().DefaultHostTriple).getArch();
+
+  std::string Suffix32  = "";
+  if (Arch == llvm::Triple::x86_64)
+    Suffix32 = "/32";
+
+  std::string Suffix64  = "";
+  if (Arch == llvm::Triple::x86)
+    Suffix64 = "/64";
+
+  std::string Lib32 = "lib";
+
+  bool Exists;
+  if (!llvm::sys::fs::exists("/lib32", Exists) && Exists)
+    Lib32 = "lib32";
+
+  std::string Lib64 = "lib";
+  bool Symlink;
+  if (!llvm::sys::fs::exists("/lib64", Exists) && Exists &&
+      (llvm::sys::fs::is_symlink("/lib64", Symlink) || !Symlink))
+    Lib64 = "lib64";
+
+  std::string GccTriple = "";
+  if (Arch == llvm::Triple::arm) {
+    if (!llvm::sys::fs::exists("/usr/lib/gcc/arm-linux-gnueabi", Exists) &&
+        Exists)
+      GccTriple = "arm-linux-gnueabi";
+  } else if (Arch == llvm::Triple::x86_64) {
+    if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-linux-gnu", Exists) &&
+        Exists)
+      GccTriple = "x86_64-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-unknown-linux-gnu",
+             Exists) && Exists)
+      GccTriple = "x86_64-unknown-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-pc-linux-gnu",
+             Exists) && Exists)
+      GccTriple = "x86_64-pc-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-redhat-linux",
+             Exists) && Exists)
+      GccTriple = "x86_64-redhat-linux";
+    else if (!llvm::sys::fs::exists("/usr/lib64/gcc/x86_64-suse-linux",
+             Exists) && Exists)
+      GccTriple = "x86_64-suse-linux";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/x86_64-manbo-linux-gnu",
+             Exists) && Exists)
+      GccTriple = "x86_64-manbo-linux-gnu";
+  } else if (Arch == llvm::Triple::x86) {
+    if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-linux-gnu", Exists) && Exists)
+      GccTriple = "i686-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-pc-linux-gnu", Exists) &&
+             Exists)
+      GccTriple = "i686-pc-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i486-linux-gnu", Exists) &&
+             Exists)
+      GccTriple = "i486-linux-gnu";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i686-redhat-linux", Exists) &&
+             Exists)
+      GccTriple = "i686-redhat-linux";
+    else if (!llvm::sys::fs::exists("/usr/lib/gcc/i586-suse-linux", Exists) &&
+             Exists)
+      GccTriple = "i586-suse-linux";
+  }
+
+  const char* GccVersions[] = {"4.5.1", "4.5", "4.4.5", "4.4.4", "4.4.3", "4.4",
+                               "4.3.4", "4.3.3", "4.3.2"};
+  std::string Base = "";
+  for (unsigned i = 0; i < sizeof(GccVersions)/sizeof(char*); ++i) {
+    std::string Suffix = GccTriple + "/" + GccVersions[i];
+    std::string t1 = "/usr/lib/gcc/" + Suffix;
+    if (!llvm::sys::fs::exists(t1 + "/crtbegin.o", Exists) && Exists) {
+      Base = t1;
+      break;
+    }
+    std::string t2 = "/usr/lib64/gcc/" + Suffix;
+    if (!llvm::sys::fs::exists(t2 + "/crtbegin.o", Exists) && Exists) {
+      Base = t2;
+      break;
+    }
+  }
+
+  path_list &Paths = getFilePaths();
+  bool Is32Bits = getArch() == llvm::Triple::x86;
+
+  std::string Suffix;
+  std::string Lib;
+
+  if (Is32Bits) {
+    Suffix = Suffix32;
+    Lib = Lib32;
+  } else {
+    Suffix = Suffix64;
+    Lib = Lib64;
+  }
+
+  llvm::sys::Path LinkerPath(Base + "/../../../../" + GccTriple + "/bin/ld");
+  if (!llvm::sys::fs::exists(LinkerPath.str(), Exists) && Exists)
+    Linker = LinkerPath.str();
+  else
+    Linker = GetProgramPath("ld");
+
+  LinuxDistro Distro = DetectLinuxDistro(Arch);
+
+  if (IsUbuntu(Distro)) {
+    ExtraOpts.push_back("-z");
+    ExtraOpts.push_back("relro");
+  }
+
+  if (Arch == llvm::Triple::arm)
+    ExtraOpts.push_back("-X");
+
+  if (IsFedora(Distro) || Distro == UbuntuMaverick)
+    ExtraOpts.push_back("--hash-style=gnu");
+
+  if (IsDebian(Distro) || Distro == UbuntuLucid || Distro == UbuntuJaunty ||
+      Distro == UbuntuKarmic)
+    ExtraOpts.push_back("--hash-style=both");
+
+  if (IsFedora(Distro))
+    ExtraOpts.push_back("--no-add-needed");
+
+  if (Distro == DebianSqueeze || IsOpenSuse(Distro) ||
+      IsFedora(Distro) || Distro == UbuntuLucid || Distro == UbuntuMaverick ||
+      Distro == UbuntuKarmic)
+    ExtraOpts.push_back("--build-id");
+
+  Paths.push_back(Base + Suffix);
+  if (HasMultilib(Arch, Distro)) {
+    if (IsOpenSuse(Distro) && Is32Bits)
+      Paths.push_back(Base + "/../../../../" + GccTriple + "/lib/../lib");
+    Paths.push_back(Base + "/../../../../" + Lib);
+    Paths.push_back("/lib/../" + Lib);
+    Paths.push_back("/usr/lib/../" + Lib);
+  }
+  if (!Suffix.empty())
+    Paths.push_back(Base);
+  if (IsOpenSuse(Distro))
+    Paths.push_back(Base + "/../../../../" + GccTriple + "/lib");
+  Paths.push_back(Base + "/../../..");
+  if (Arch == getArch() && IsUbuntu(Distro))
+    Paths.push_back("/usr/lib/" + GccTriple);
+}
+
+bool Linux::HasNativeLLVMSupport() const {
+  return true;
 }
 
 Tool &Linux::SelectTool(const Compilation &C, const JobAction &JA) const {
@@ -1114,11 +1526,21 @@ Tool &Linux::SelectTool(const Compilation &C, const JobAction &JA) const {
   else
     Key = JA.getKind();
 
+  bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
+                                             options::OPT_no_integrated_as,
+                                             IsIntegratedAssemblerDefault());
+
   Tool *&T = Tools[Key];
   if (!T) {
     switch (Key) {
     case Action::AssembleJobClass:
-      T = new tools::linuxtools::Assemble(*this); break;
+      if (UseIntegratedAs)
+        T = new tools::ClangAs(*this);
+      else
+        T = new tools::linuxtools::Assemble(*this);
+      break;
+    case Action::LinkJobClass:
+      T = new tools::linuxtools::Link(*this); break;
     default:
       T = &Generic_GCC::SelectTool(C, JA);
     }
@@ -1130,7 +1552,7 @@ Tool &Linux::SelectTool(const Compilation &C, const JobAction &JA) const {
 /// DragonFly - DragonFly tool chain which can call as(1) and ld(1) directly.
 
 DragonFly::DragonFly(const HostInfo &Host, const llvm::Triple& Triple)
-  : Generic_GCC(Host, Triple) {
+  : Generic_ELF(Host, Triple) {
 
   // Path mangling to find libexec
   getProgramPaths().push_back(getDriver().getInstalledDir());
diff --git a/lib/Driver/ToolChains.h b/lib/Driver/ToolChains.h
index d1f15565e6c6..0e3645cb8f7e 100644
--- a/lib/Driver/ToolChains.h
+++ b/lib/Driver/ToolChains.h
@@ -155,6 +155,8 @@ public:
 
   virtual types::ID LookupTypeForExtension(const char *Ext) const;
 
+  virtual bool HasNativeLLVMSupport() const;
+
   virtual DerivedArgList *TranslateArgs(const DerivedArgList &Args,
                                         const char *BoundArch) const;
 
@@ -174,6 +176,18 @@ public:
             getTriple().getArch() == llvm::Triple::x86_64);
 #endif
   }
+  virtual bool IsStrictAliasingDefault() const {
+#ifdef DISABLE_DEFAULT_STRICT_ALIASING
+    return false;
+#else
+    return ToolChain::IsStrictAliasingDefault();
+#endif
+  }
+  
+  virtual bool IsObjCDefaultSynthPropertiesDefault() const {
+    return false;
+  }
+
   virtual bool IsObjCNonFragileABIDefault() const {
     // Non-fragile ABI is default for everything but i386.
     return getTriple().getArch() != llvm::Triple::x86;
@@ -221,6 +235,12 @@ public:
   virtual void AddLinkRuntimeLibArgs(const ArgList &Args,
                                      ArgStringList &CmdArgs) const;
 
+  virtual void AddCXXStdlibLibArgs(const ArgList &Args,
+                                   ArgStringList &CmdArgs) const;
+
+  virtual void AddCCKextLibArgs(const ArgList &Args,
+                                ArgStringList &CmdArgs) const;
+
   /// }
 };
 
@@ -258,6 +278,18 @@ public:
   virtual const char *GetDefaultRelocationModel() const { return "pic"; }
 };
 
+class LLVM_LIBRARY_VISIBILITY Generic_ELF : public Generic_GCC {
+ public:
+  Generic_ELF(const HostInfo &Host, const llvm::Triple& Triple)
+    : Generic_GCC(Host, Triple) {}
+
+  virtual bool IsIntegratedAssemblerDefault() const {
+    // Default integrated assembler to on for x86.
+    return (getTriple().getArch() == llvm::Triple::x86 ||
+            getTriple().getArch() == llvm::Triple::x86_64);
+  }
+};
+
 class LLVM_LIBRARY_VISIBILITY AuroraUX : public Generic_GCC {
 public:
   AuroraUX(const HostInfo &Host, const llvm::Triple& Triple);
@@ -265,20 +297,27 @@ public:
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
-class LLVM_LIBRARY_VISIBILITY OpenBSD : public Generic_GCC {
+class LLVM_LIBRARY_VISIBILITY OpenBSD : public Generic_ELF {
 public:
   OpenBSD(const HostInfo &Host, const llvm::Triple& Triple);
 
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
-class LLVM_LIBRARY_VISIBILITY FreeBSD : public Generic_GCC {
+class LLVM_LIBRARY_VISIBILITY FreeBSD : public Generic_ELF {
 public:
   FreeBSD(const HostInfo &Host, const llvm::Triple& Triple);
 
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
+class LLVM_LIBRARY_VISIBILITY NetBSD : public Generic_ELF {
+public:
+  NetBSD(const HostInfo &Host, const llvm::Triple& Triple);
+
+  virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
+};
+
 class LLVM_LIBRARY_VISIBILITY Minix : public Generic_GCC {
 public:
   Minix(const HostInfo &Host, const llvm::Triple& Triple);
@@ -286,18 +325,23 @@ public:
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
-class LLVM_LIBRARY_VISIBILITY DragonFly : public Generic_GCC {
+class LLVM_LIBRARY_VISIBILITY DragonFly : public Generic_ELF {
 public:
   DragonFly(const HostInfo &Host, const llvm::Triple& Triple);
 
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
-class LLVM_LIBRARY_VISIBILITY Linux : public Generic_GCC {
+class LLVM_LIBRARY_VISIBILITY Linux : public Generic_ELF {
 public:
   Linux(const HostInfo &Host, const llvm::Triple& Triple);
 
+  virtual bool HasNativeLLVMSupport() const;
+
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
+
+  std::string Linker;
+  std::vector<std::string> ExtraOpts;
 };
 
 
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index 8436561e6ef9..6717349b9765 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -1,4 +1,4 @@
-//===--- Tools.cpp - Tools Implementations ------------------------------*-===//
+//===--- Tools.cpp - Tools Implementations --------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -25,10 +25,11 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Host.h"
-#include "llvm/System/Process.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/Process.h"
 
 #include "InputInfo.h"
 #include "ToolChains.h"
@@ -85,6 +86,48 @@ static void QuoteTarget(llvm::StringRef Target,
   }
 }
 
+static void AddLinkerInputs(const ToolChain &TC,
+                            const InputInfoList &Inputs, const ArgList &Args,
+                            ArgStringList &CmdArgs) {
+  const Driver &D = TC.getDriver();
+
+  // Add extra linker input arguments which are not treated as inputs
+  // (constructed via -Xarch_).
+  Args.AddAllArgValues(CmdArgs, options::OPT_Zlinker_input);
+
+  for (InputInfoList::const_iterator
+         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
+    const InputInfo &II = *it;
+
+    if (!TC.HasNativeLLVMSupport()) {
+      // Don't try to pass LLVM inputs unless we have native support.
+      if (II.getType() == types::TY_LLVM_IR ||
+          II.getType() == types::TY_LTO_IR ||
+          II.getType() == types::TY_LLVM_BC ||
+          II.getType() == types::TY_LTO_BC)
+        D.Diag(clang::diag::err_drv_no_linker_llvm_support)
+          << TC.getTripleString();
+    }
+
+    // Add filenames immediately.
+    if (II.isFilename()) {
+      CmdArgs.push_back(II.getFilename());
+      continue;
+    }
+
+    // Otherwise, this is a linker input argument.
+    const Arg &A = II.getInputArg();
+
+    // Handle reserved library options.
+    if (A.getOption().matches(options::OPT_Z_reserved_lib_stdcxx)) {
+      TC.AddCXXStdlibLibArgs(Args, CmdArgs);
+    } else if (A.getOption().matches(options::OPT_Z_reserved_lib_cckext)) {
+      TC.AddCCKextLibArgs(Args, CmdArgs);
+    } else
+      A.renderAsInput(Args, CmdArgs);
+  }
+}
+
 void Clang::AddPreprocessingOptions(const Driver &D,
                                     const ArgList &Args,
                                     ArgStringList &CmdArgs,
@@ -98,8 +141,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   Args.AddLastArg(CmdArgs, options::OPT_CC);
 
   // Handle dependency file generation.
-  if ((A = Args.getLastArg(options::OPT_M)) ||
-      (A = Args.getLastArg(options::OPT_MM)) ||
+  if ((A = Args.getLastArg(options::OPT_M, options::OPT_MM)) ||
       (A = Args.getLastArg(options::OPT_MD)) ||
       (A = Args.getLastArg(options::OPT_MMD))) {
     // Determine the output location.
@@ -130,11 +172,9 @@ void Clang::AddPreprocessingOptions(const Driver &D,
         // Otherwise derive from the base input.
         //
         // FIXME: This should use the computed output file location.
-        llvm::sys::Path P(Inputs[0].getBaseInput());
-
-        P.eraseSuffix();
-        P.appendSuffix("o");
-        DepTarget = Args.MakeArgString(P.getLast());
+        llvm::SmallString<128> P(Inputs[0].getBaseInput());
+        llvm::sys::path::replace_extension(P, "o");
+        DepTarget = Args.MakeArgString(llvm::sys::path::filename(P));
       }
 
       CmdArgs.push_back("-MT");
@@ -174,20 +214,25 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   // wonky, but we include looking for .gch so we can support seamless
   // replacement into a build system already set up to be generating
   // .gch files.
+  bool RenderedImplicitInclude = false;
   for (arg_iterator it = Args.filtered_begin(options::OPT_clang_i_Group),
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = it;
 
     if (A->getOption().matches(options::OPT_include)) {
+      bool IsFirstImplicitInclude = !RenderedImplicitInclude;
+      RenderedImplicitInclude = true;
+
       // Use PCH if the user requested it.
       bool UsePCH = D.CCCUsePCH;
 
       bool FoundPTH = false;
       bool FoundPCH = false;
       llvm::sys::Path P(A->getValue(Args));
+      bool Exists;
       if (UsePCH) {
         P.appendSuffix("pch");
-        if (P.exists())
+        if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
           FoundPCH = true;
         else
           P.eraseSuffix();
@@ -195,7 +240,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
 
       if (!FoundPCH) {
         P.appendSuffix("pth");
-        if (P.exists())
+        if (!llvm::sys::fs::exists(P.str(), Exists) && Exists)
           FoundPTH = true;
         else
           P.eraseSuffix();
@@ -203,7 +248,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
 
       if (!FoundPCH && !FoundPTH) {
         P.appendSuffix("gch");
-        if (P.exists()) {
+        if (!llvm::sys::fs::exists(P.str(), Exists) && Exists) {
           FoundPCH = UsePCH;
           FoundPTH = !UsePCH;
         }
@@ -212,13 +257,19 @@ void Clang::AddPreprocessingOptions(const Driver &D,
       }
 
       if (FoundPCH || FoundPTH) {
-        A->claim();
-        if (UsePCH)
-          CmdArgs.push_back("-include-pch");
-        else
-          CmdArgs.push_back("-include-pth");
-        CmdArgs.push_back(Args.MakeArgString(P.str()));
-        continue;
+        if (IsFirstImplicitInclude) {
+          A->claim();
+          if (UsePCH)
+            CmdArgs.push_back("-include-pch");
+          else
+            CmdArgs.push_back("-include-pth");
+          CmdArgs.push_back(Args.MakeArgString(P.str()));
+          continue;
+        } else {
+          // Ignore the PCH if not first on command line and emit warning.
+          D.Diag(clang::diag::warn_drv_pch_not_first_include)
+              << P.str() << A->getAsString(Args);
+        }
       }
     }
 
@@ -230,6 +281,11 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   Args.AddAllArgs(CmdArgs, options::OPT_D, options::OPT_U);
   Args.AddAllArgs(CmdArgs, options::OPT_I_Group, options::OPT_F);
 
+  // Add C++ include arguments, if needed.
+  types::ID InputType = Inputs[0].getType();
+  if (types::isCXX(InputType))
+    getToolChain().AddClangCXXStdlibIncludeArgs(Args, CmdArgs);
+
   // Add -Wp, and -Xassembler if using the preprocessor.
 
   // FIXME: There is a very unfortunate problem here, some troubled
@@ -242,6 +298,15 @@ void Clang::AddPreprocessingOptions(const Driver &D,
   // -I- is a deprecated GCC feature, reject it.
   if (Arg *A = Args.getLastArg(options::OPT_I_))
     D.Diag(clang::diag::err_drv_I_dash_not_supported) << A->getAsString(Args);
+
+  // If we have a --sysroot, and don't have an explicit -isysroot flag, add an
+  // -isysroot to the CC1 invocation.
+  if (Arg *A = Args.getLastArg(options::OPT__sysroot_EQ)) {
+    if (!Args.hasArg(options::OPT_isysroot)) {
+      CmdArgs.push_back("-isysroot");
+      CmdArgs.push_back(A->getValue(Args));
+    }
+  }
 }
 
 /// getARMTargetCPU - Get the (LLVM) name of the ARM cpu we are targetting.
@@ -367,13 +432,16 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     ABIName = A->getValue(Args);
   } else {
     // Select the default based on the platform.
-    llvm::StringRef env = Triple.getEnvironmentName();
-    if (env == "gnueabi")
+    switch(Triple.getEnvironment()) {
+    case llvm::Triple::GNUEABI:
       ABIName = "aapcs-linux";
-    else if (env == "eabi")
+      break;
+    case llvm::Triple::EABI:
       ABIName = "aapcs";
-    else
+      break;
+    default:
       ABIName = "apcs-gnu";
+    }
   }
   CmdArgs.push_back("-target-abi");
   CmdArgs.push_back(ABIName);
@@ -420,8 +488,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     }
 
     case llvm::Triple::Linux: {
-      llvm::StringRef Env = getToolChain().getTriple().getEnvironmentName();
-      if (Env == "gnueabi") {
+      if (getToolChain().getTriple().getEnvironment() == llvm::Triple::GNUEABI) {
         FloatABI = "softfp";
         break;
       }
@@ -429,10 +496,20 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     // fall through
 
     default:
-      // Assume "soft", but warn the user we are guessing.
-      FloatABI = "soft";
-      D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
-      break;
+      switch(Triple.getEnvironment()) {
+      case llvm::Triple::GNUEABI:
+        FloatABI = "softfp";
+        break;
+      case llvm::Triple::EABI:
+        // EABI is always AAPCS, and if it was not marked 'hard', it's softfp
+        FloatABI = "softfp";
+        break;
+      default:
+        // Assume "soft", but warn the user we are guessing.
+        FloatABI = "soft";
+        D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
+        break;
+      }
     }
   }
 
@@ -557,6 +634,51 @@ void Clang::AddMIPSTargetArgs(const ArgList &Args,
   }
 }
 
+void Clang::AddSparcTargetArgs(const ArgList &Args,
+                             ArgStringList &CmdArgs) const {
+  const Driver &D = getToolChain().getDriver();
+
+  if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
+    llvm::StringRef MArch = A->getValue(Args);
+    CmdArgs.push_back("-target-cpu");
+    CmdArgs.push_back(MArch.str().c_str());
+  }
+
+  // Select the float ABI as determined by -msoft-float, -mhard-float, and
+  llvm::StringRef FloatABI;
+  if (Arg *A = Args.getLastArg(options::OPT_msoft_float,
+                               options::OPT_mhard_float)) {
+    if (A->getOption().matches(options::OPT_msoft_float))
+      FloatABI = "soft";
+    else if (A->getOption().matches(options::OPT_mhard_float))
+      FloatABI = "hard";
+  }
+
+  // If unspecified, choose the default based on the platform.
+  if (FloatABI.empty()) {
+    switch (getToolChain().getTriple().getOS()) {
+    default:
+      // Assume "soft", but warn the user we are guessing.
+      FloatABI = "soft";
+      D.Diag(clang::diag::warn_drv_assuming_mfloat_abi_is) << "soft";
+      break;
+    }
+  }
+
+  if (FloatABI == "soft") {
+    // Floating point operations and argument passing are soft.
+    //
+    // FIXME: This changes CPP defines, we need -target-soft-float.
+    CmdArgs.push_back("-msoft-float");
+    CmdArgs.push_back("soft");
+    CmdArgs.push_back("-target-feature");
+    CmdArgs.push_back("+soft-float");
+  } else {
+    assert(FloatABI == "hard" && "Invalid float abi!");
+    CmdArgs.push_back("-mhard-float");
+  }
+}
+
 void Clang::AddX86TargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
   if (!Args.hasFlag(options::OPT_mred_zone,
@@ -604,6 +726,11 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
         CPUName = "x86-64";
       else if (getToolChain().getArchName() == "i386")
         CPUName = "i486";
+    } else if (getToolChain().getOS().startswith("netbsd"))  {
+      if (getToolChain().getArchName() == "x86_64")
+        CPUName = "x86-64";
+      else if (getToolChain().getArchName() == "i386")
+        CPUName = "i486";
     } else {
       if (getToolChain().getArchName() == "x86_64")
         CPUName = "x86-64";
@@ -637,6 +764,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
 
 static bool needsExceptions(const ArgList &Args,  types::ID InputType,
                             const llvm::Triple &Triple) {
+  // Handle -fno-exceptions.
   if (Arg *A = Args.getLastArg(options::OPT_fexceptions,
                                options::OPT_fno_exceptions)) {
     if (A->getOption().matches(options::OPT_fexceptions))
@@ -644,25 +772,24 @@ static bool needsExceptions(const ArgList &Args,  types::ID InputType,
     else
       return false;
   }
-  switch (InputType) {
-  case types::TY_CXX: case types::TY_CXXHeader:
-  case types::TY_PP_CXX: case types::TY_PP_CXXHeader:
-  case types::TY_ObjCXX: case types::TY_ObjCXXHeader:
-  case types::TY_PP_ObjCXX: case types::TY_PP_ObjCXXHeader:
+
+  // Otherwise, C++ inputs use exceptions.
+  if (types::isCXX(InputType))
     return true;
 
-  case types::TY_ObjC: case types::TY_ObjCHeader:
-  case types::TY_PP_ObjC: case types::TY_PP_ObjCHeader:
+  // As do Objective-C non-fragile ABI inputs and all Objective-C inputs on
+  // x86_64 and ARM after SnowLeopard.
+  if (types::isObjC(InputType)) {
     if (Args.hasArg(options::OPT_fobjc_nonfragile_abi))
       return true;
     if (Triple.getOS() != llvm::Triple::Darwin)
       return false;
     return (Triple.getDarwinMajorNumber() >= 9 &&
-            Triple.getArch() == llvm::Triple::x86_64);
-
-  default:
-    return false;
+            (Triple.getArch() == llvm::Triple::x86_64 ||
+             Triple.getArch() == llvm::Triple::arm));
   }
+
+  return false;
 }
 
 void Clang::ConstructJob(Compilation &C, const JobAction &JA,
@@ -709,10 +836,33 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                       !IsOpt))
       CmdArgs.push_back("-mrelax-all");
 
-    // When using an integrated assembler, we send -Wa, and -Xassembler options
-    // to -cc1.
-    Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
-                         options::OPT_Xassembler);
+    // When using an integrated assembler, translate -Wa, and -Xassembler
+    // options.
+    for (arg_iterator it = Args.filtered_begin(options::OPT_Wa_COMMA,
+                                               options::OPT_Xassembler),
+           ie = Args.filtered_end(); it != ie; ++it) {
+      const Arg *A = *it;
+      A->claim();
+
+      for (unsigned i = 0, e = A->getNumValues(); i != e; ++i) {
+        llvm::StringRef Value = A->getValue(Args, i);
+
+        if (Value == "-force_cpusubtype_ALL") {
+          // Do nothing, this is the default and we don't support anything else.
+        } else if (Value == "-L") {
+          // We don't support -L yet, but it isn't important enough to error
+          // on. No one should really be using it for a semantic change.
+          D.Diag(clang::diag::warn_drv_unsupported_option_argument)
+            << A->getOption().getName() << Value;
+        } else {
+          D.Diag(clang::diag::err_drv_unsupported_option_argument)
+            << A->getOption().getName() << Value;
+        }
+      }
+    }
+
+    // Also ignore explicit -force_cpusubtype_ALL option.
+    (void) Args.hasArg(options::OPT_force__cpusubtype__ALL);
   } else if (isa<PrecompileJobAction>(JA)) {
     // Use PCH if the user requested it.
     bool UsePCH = D.CCCUsePCH;
@@ -772,17 +922,27 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     // Add default argument set.
     if (!Args.hasArg(options::OPT__analyzer_no_default_checks)) {
-      CmdArgs.push_back("-analyzer-check-dead-stores");
-      // Do not enable the security-syntatic check since it
-      // it needs to be refined (known issues).
-      // CmdArgs.push_back("-analyzer-check-security-syntactic");
+      types::ID InputType = Inputs[0].getType();
+
+      // Checks to perform for all language types.
+
+      CmdArgs.push_back("-analyzer-checker=core");
+      if (getToolChain().getTriple().getOS() != llvm::Triple::Win32)
+        CmdArgs.push_back("-analyzer-checker=unix");
+      if (getToolChain().getTriple().getVendor() == llvm::Triple::Apple)
+        CmdArgs.push_back("-analyzer-checker=macosx");
+
+      // Checks to perform for Objective-C/Objective-C++.
+      if (types::isObjC(InputType)) {
+        // Enable all checkers in 'cocoa' package.
+        CmdArgs.push_back("-analyzer-checker=cocoa");
+      }
+
+      // NOTE: Leaving -analyzer-check-objc-mem here is intentional.
+      // It also checks C code.
       CmdArgs.push_back("-analyzer-check-objc-mem");
+
       CmdArgs.push_back("-analyzer-eagerly-assume");
-      CmdArgs.push_back("-analyzer-check-objc-methodsigs");
-      // Do not enable the missing -dealloc check.
-      // '-analyzer-check-objc-missing-dealloc',
-      CmdArgs.push_back("-analyzer-check-objc-unused-ivars");
-      CmdArgs.push_back("-analyzer-check-idempotent-operations");
     }
 
     // Set the output format. The default is plist, for (lame) historical
@@ -850,6 +1010,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // LLVM Code Generator Options.
 
+  if (Arg *A = Args.getLastArg(options::OPT_mregparm_EQ)) {
+    CmdArgs.push_back("-mregparm");
+    CmdArgs.push_back(A->getValue(Args));
+  }
+
   // FIXME: Set --enable-unsafe-fp-math.
   if (Args.hasFlag(options::OPT_fno_omit_frame_pointer,
                    options::OPT_fomit_frame_pointer))
@@ -857,12 +1022,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasFlag(options::OPT_fzero_initialized_in_bss,
                     options::OPT_fno_zero_initialized_in_bss))
     CmdArgs.push_back("-mno-zero-initialized-in-bss");
+  if (!Args.hasFlag(options::OPT_fstrict_aliasing,
+                    options::OPT_fno_strict_aliasing,
+                    getToolChain().IsStrictAliasingDefault()))
+    CmdArgs.push_back("-relaxed-aliasing");
 
   // Decide whether to use verbose asm. Verbose assembly is the default on
   // toolchains which have the integrated assembler on by default.
   bool IsVerboseAsmDefault = getToolChain().IsIntegratedAssemblerDefault();
   if (Args.hasFlag(options::OPT_fverbose_asm, options::OPT_fno_verbose_asm,
-                   IsVerboseAsmDefault) || 
+                   IsVerboseAsmDefault) ||
       Args.hasArg(options::OPT_dA))
     CmdArgs.push_back("-masm-verbose");
 
@@ -880,6 +1049,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (getToolChain().getTriple().getOS() != llvm::Triple::Darwin)
     CmdArgs.push_back("-mconstructor-aliases");
 
+  if (Args.hasArg(options::OPT_mms_bitfields)) {
+    CmdArgs.push_back("-mms-bitfields");
+  }
+
   // This is a coarse approximation of what llvm-gcc actually does, both
   // -fasynchronous-unwind-tables and -fnon-call-exceptions interact in more
   // complicated ways.
@@ -920,6 +1093,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     AddMIPSTargetArgs(Args, CmdArgs);
     break;
 
+  case llvm::Triple::sparc:
+    AddSparcTargetArgs(Args, CmdArgs);
+    break;
+
   case llvm::Triple::x86:
   case llvm::Triple::x86_64:
     AddX86TargetArgs(Args, CmdArgs);
@@ -932,9 +1109,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(A->getValue(Args));
   }
 
-  // -mno-omit-leaf-frame-pointer is default.
+  // -mno-omit-leaf-frame-pointer is the default on Darwin.
   if (Args.hasFlag(options::OPT_momit_leaf_frame_pointer,
-                   options::OPT_mno_omit_leaf_frame_pointer, false))
+                   options::OPT_mno_omit_leaf_frame_pointer,
+                   getToolChain().getTriple().getOS() != llvm::Triple::Darwin))
     CmdArgs.push_back("-momit-leaf-frame-pointer");
 
   // -fno-math-errno is default.
@@ -946,23 +1124,29 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Explicitly error on some things we know we don't support and can't just
   // ignore.
   types::ID InputType = Inputs[0].getType();
-  Arg *Unsupported;
-  if ((Unsupported = Args.getLastArg(options::OPT_MG)) ||
-      (Unsupported = Args.getLastArg(options::OPT_iframework)) ||
-      (Unsupported = Args.getLastArg(options::OPT_fshort_enums)))
-    D.Diag(clang::diag::err_drv_clang_unsupported)
-      << Unsupported->getOption().getName();
-
-  if (types::isCXX(InputType) &&
-      getToolChain().getTriple().getOS() == llvm::Triple::Darwin &&
-      getToolChain().getTriple().getArch() == llvm::Triple::x86) {
-    if ((Unsupported = Args.getLastArg(options::OPT_fapple_kext)))
-      D.Diag(clang::diag::err_drv_clang_unsupported_opt_cxx_darwin_i386)
+  if (!Args.hasArg(options::OPT_fallow_unsupported)) {
+    Arg *Unsupported;
+    if ((Unsupported = Args.getLastArg(options::OPT_MG)) ||
+        (Unsupported = Args.getLastArg(options::OPT_iframework)))
+      D.Diag(clang::diag::err_drv_clang_unsupported)
         << Unsupported->getOption().getName();
+
+    if (types::isCXX(InputType) &&
+        getToolChain().getTriple().getOS() == llvm::Triple::Darwin &&
+        getToolChain().getTriple().getArch() == llvm::Triple::x86) {
+      if ((Unsupported = Args.getLastArg(options::OPT_fapple_kext)))
+        D.Diag(clang::diag::err_drv_clang_unsupported_opt_cxx_darwin_i386)
+          << Unsupported->getOption().getName();
+    }
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_v);
   Args.AddLastArg(CmdArgs, options::OPT_H);
+  if (D.CCPrintHeaders) {
+    CmdArgs.push_back("-header-include-file");
+    CmdArgs.push_back(D.CCPrintHeadersFilename ?
+                      D.CCPrintHeadersFilename : "-");
+  }
   Args.AddLastArg(CmdArgs, options::OPT_P);
   Args.AddLastArg(CmdArgs, options::OPT_print_ivar_layout);
 
@@ -986,6 +1170,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-resource-dir");
   CmdArgs.push_back(D.ResourceDir.c_str());
 
+  Args.AddLastArg(CmdArgs, options::OPT_working_directory);
+
   // Add preprocessing options like -I, -D, etc. if we are using the
   // preprocessor.
   //
@@ -1001,6 +1187,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     else if (A->getOption().matches(options::OPT_O) &&
              A->getValue(Args)[0] == '\0')
       CmdArgs.push_back("-O2");
+    else if (A->getOption().matches(options::OPT_O) &&
+             A->getValue(Args)[0] == 'z' &&
+             A->getValue(Args)[1] == '\0')
+      CmdArgs.push_back("-Os");
     else
       A->render(Args, CmdArgs);
   }
@@ -1054,6 +1244,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(A->getValue(Args));
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_Wlarge_by_value_copy_EQ,
+                               options::OPT_Wlarge_by_value_copy_def)) {
+    CmdArgs.push_back("-Wlarge-by-value-copy");
+    if (A->getNumValues())
+      CmdArgs.push_back(A->getValue(Args));
+    else
+      CmdArgs.push_back("64"); // default value for -Wlarge-by-value-copy.
+  }
+
   if (Args.hasArg(options::OPT__relocatable_pch))
     CmdArgs.push_back("-relocatable-pch");
 
@@ -1100,7 +1299,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   Args.AddLastArg(CmdArgs, options::OPT_fvisibility_inlines_hidden);
-                        
+
   // -fhosted is default.
   if (KernelOrKext || Args.hasFlag(options::OPT_ffreestanding,
                                    options::OPT_fhosted,
@@ -1111,6 +1310,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fcatch_undefined_behavior);
   Args.AddLastArg(CmdArgs, options::OPT_femit_all_decls);
   Args.AddLastArg(CmdArgs, options::OPT_fheinous_gnu_extensions);
+  Args.AddLastArg(CmdArgs, options::OPT_flimit_debug_info);
+  Args.AddLastArg(CmdArgs, options::OPT_pg);
 
   // -flax-vector-conversions is default.
   if (!Args.hasFlag(options::OPT_flax_vector_conversions,
@@ -1132,12 +1333,21 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
+  if (Args.getLastArg(options::OPT_fapple_kext))
+    CmdArgs.push_back("-fapple-kext");
+
   Args.AddLastArg(CmdArgs, options::OPT_fno_show_column);
   Args.AddLastArg(CmdArgs, options::OPT_fobjc_sender_dependent_dispatch);
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_print_source_range_info);
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_parseable_fixits);
   Args.AddLastArg(CmdArgs, options::OPT_ftime_report);
   Args.AddLastArg(CmdArgs, options::OPT_ftrapv);
+
+  if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
+    CmdArgs.push_back("-ftrapv-handler");
+    CmdArgs.push_back(A->getValue(Args));
+  }
+
   Args.AddLastArg(CmdArgs, options::OPT_fwrapv);
   Args.AddLastArg(CmdArgs, options::OPT_fwritable_strings);
   Args.AddLastArg(CmdArgs, options::OPT_funroll_loops);
@@ -1163,8 +1373,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Forward -f options with positive and negative forms; we translate
   // these by hand.
 
+  if (Args.hasArg(options::OPT_mkernel)) {
+    if (!Args.hasArg(options::OPT_fapple_kext) && types::isCXX(InputType))
+      CmdArgs.push_back("-fapple-kext");
+    if (!Args.hasArg(options::OPT_fbuiltin))
+      CmdArgs.push_back("-fno-builtin");
+  }
   // -fbuiltin is default.
-  if (!Args.hasFlag(options::OPT_fbuiltin, options::OPT_fno_builtin))
+  else if (!Args.hasFlag(options::OPT_fbuiltin, options::OPT_fno_builtin))
     CmdArgs.push_back("-fno-builtin");
 
   if (!Args.hasFlag(options::OPT_fassume_sane_operator_new,
@@ -1183,8 +1399,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    false))
     CmdArgs.push_back("-fno-access-control");
 
+  // -felide-constructors is the default.
+  if (Args.hasFlag(options::OPT_fno_elide_constructors,
+                   options::OPT_felide_constructors,
+                   false))
+    CmdArgs.push_back("-fno-elide-constructors");
+
   // -fexceptions=0 is default.
-  if (needsExceptions(Args, InputType, getToolChain().getTriple()))
+  if (!KernelOrKext &&
+      needsExceptions(Args, InputType, getToolChain().getTriple()))
     CmdArgs.push_back("-fexceptions");
 
   if (getToolChain().UseSjLjExceptions())
@@ -1195,19 +1418,27 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasFlag(options::OPT_frtti, options::OPT_fno_rtti))
     CmdArgs.push_back("-fno-rtti");
 
+  // -fshort-enums=0 is default.
+  // FIXME: Are there targers where -fshort-enums is on by default ?
+  if (Args.hasFlag(options::OPT_fshort_enums,
+                   options::OPT_fno_short_enums, false))
+    CmdArgs.push_back("-fshort-enums");
+
   // -fsigned-char is default.
   if (!Args.hasFlag(options::OPT_fsigned_char, options::OPT_funsigned_char,
                     isSignedCharDefault(getToolChain().getTriple())))
     CmdArgs.push_back("-fno-signed-char");
 
   // -fthreadsafe-static is default.
-  if (!Args.hasFlag(options::OPT_fthreadsafe_statics, 
+  if (!Args.hasFlag(options::OPT_fthreadsafe_statics,
                     options::OPT_fno_threadsafe_statics))
     CmdArgs.push_back("-fno-threadsafe-statics");
 
   // -fuse-cxa-atexit is default.
-  if (KernelOrKext || !Args.hasFlag(options::OPT_fuse_cxa_atexit,
-                                    options::OPT_fno_use_cxa_atexit))
+  if (KernelOrKext ||
+    !Args.hasFlag(options::OPT_fuse_cxa_atexit, options::OPT_fno_use_cxa_atexit,
+                  getToolChain().getTriple().getOS() != llvm::Triple::Cygwin &&
+                  getToolChain().getTriple().getOS() != llvm::Triple::MinGW32))
     CmdArgs.push_back("-fno-use-cxa-atexit");
 
   // -fms-extensions=0 is default.
@@ -1215,6 +1446,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    getToolChain().getTriple().getOS() == llvm::Triple::Win32))
     CmdArgs.push_back("-fms-extensions");
 
+  // -fmsc-version=1300 is default.
+  if (Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions,
+                   getToolChain().getTriple().getOS() == llvm::Triple::Win32) ||
+      Args.hasArg(options::OPT_fmsc_version)) {
+    llvm::StringRef msc_ver = Args.getLastArgValue(options::OPT_fmsc_version);
+    if (msc_ver.empty())
+      CmdArgs.push_back("-fmsc-version=1300");
+    else
+      CmdArgs.push_back(Args.MakeArgString("-fmsc-version=" + msc_ver));
+  }
+
+
   // -fborland-extensions=0 is default.
   if (Args.hasFlag(options::OPT_fborland_extensions,
                    options::OPT_fno_borland_extensions, false))
@@ -1228,7 +1471,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // -fnext-runtime defaults to on Darwin and when rewriting Objective-C, and is
   // -the -cc1 default.
-  bool NeXTRuntimeIsDefault = 
+  bool NeXTRuntimeIsDefault =
     IsRewriter || getToolChain().getTriple().getOS() == llvm::Triple::Darwin;
   if (!Args.hasFlag(options::OPT_fnext_runtime, options::OPT_fgnu_runtime,
                     NeXTRuntimeIsDefault))
@@ -1236,20 +1479,52 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // -fobjc-nonfragile-abi=0 is default.
   if (types::isObjC(InputType)) {
+    // Compute the Objective-C ABI "version" to use. Version numbers are
+    // slightly confusing for historical reasons:
+    //   1 - Traditional "fragile" ABI
+    //   2 - Non-fragile ABI, version 1
+    //   3 - Non-fragile ABI, version 2
     unsigned Version = 1;
-    if (Args.hasArg(options::OPT_fobjc_nonfragile_abi) ||
-        getToolChain().IsObjCNonFragileABIDefault())
-      Version = 2;
+    // If -fobjc-abi-version= is present, use that to set the version.
     if (Arg *A = Args.getLastArg(options::OPT_fobjc_abi_version_EQ)) {
       if (llvm::StringRef(A->getValue(Args)) == "1")
         Version = 1;
       else if (llvm::StringRef(A->getValue(Args)) == "2")
         Version = 2;
+      else if (llvm::StringRef(A->getValue(Args)) == "3")
+        Version = 3;
       else
         D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
+    } else {
+      // Otherwise, determine if we are using the non-fragile ABI.
+      if (Args.hasFlag(options::OPT_fobjc_nonfragile_abi,
+                       options::OPT_fno_objc_nonfragile_abi,
+                       getToolChain().IsObjCNonFragileABIDefault())) {
+        // Determine the non-fragile ABI version to use.
+#ifdef DISABLE_DEFAULT_NONFRAGILEABI_TWO
+        unsigned NonFragileABIVersion = 1;
+#else
+        unsigned NonFragileABIVersion = 2;
+#endif
+
+        if (Arg *A = Args.getLastArg(
+              options::OPT_fobjc_nonfragile_abi_version_EQ)) {
+          if (llvm::StringRef(A->getValue(Args)) == "1")
+            NonFragileABIVersion = 1;
+          else if (llvm::StringRef(A->getValue(Args)) == "2")
+            NonFragileABIVersion = 2;
+          else
+            D.Diag(clang::diag::err_drv_clang_unsupported)
+              << A->getAsString(Args);
+        }
+
+        Version = 1 + NonFragileABIVersion;
+      } else {
+        Version = 1;
+      }
     }
 
-    if (Version == 2) {
+    if (Version == 2 || Version == 3) {
       CmdArgs.push_back("-fobjc-nonfragile-abi");
 
       // -fobjc-dispatch-method is only relevant with the nonfragile-abi, and
@@ -1264,10 +1539,17 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       }
     }
 
-    // FIXME: -fobjc-nonfragile-abi2 is a transient option meant to expose
-    // features in testing.  It will eventually be removed.
-    if (Args.hasArg(options::OPT_fobjc_nonfragile_abi2))
-      CmdArgs.push_back("-fobjc-nonfragile-abi2");
+    // -fobjc-default-synthesize-properties=0 is default.
+    if (Args.hasFlag(options::OPT_fobjc_default_synthesize_properties,
+                     options::OPT_fno_objc_default_synthesize_properties,
+                     getToolChain().IsObjCDefaultSynthPropertiesDefault())) {
+      CmdArgs.push_back("-fobjc-default-synthesize-properties");
+    }
+
+    // -fobjc-exceptions is default.
+    if (!Args.hasFlag(options::OPT_fobjc_exceptions, 
+                      options::OPT_fno_objc_exceptions))
+      CmdArgs.push_back("-fno-objc-exceptions");
   }
 
   if (!Args.hasFlag(options::OPT_fassume_sane_operator_new,
@@ -1300,17 +1582,28 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    false))
     CmdArgs.push_back("-fpascal-strings");
 
+  if (Args.hasArg(options::OPT_mkernel) ||
+      Args.hasArg(options::OPT_fapple_kext)) {
+    if (!Args.hasArg(options::OPT_fcommon))
+      CmdArgs.push_back("-fno-common");
+  }
   // -fcommon is default, only pass non-default.
-  if (!Args.hasFlag(options::OPT_fcommon, options::OPT_fno_common))
+  else if (!Args.hasFlag(options::OPT_fcommon, options::OPT_fno_common))
     CmdArgs.push_back("-fno-common");
 
   // -fsigned-bitfields is default, and clang doesn't yet support
-  // --funsigned-bitfields.
+  // -funsigned-bitfields.
   if (!Args.hasFlag(options::OPT_fsigned_bitfields,
                     options::OPT_funsigned_bitfields))
     D.Diag(clang::diag::warn_drv_clang_unsupported)
       << Args.getLastArg(options::OPT_funsigned_bitfields)->getAsString(Args);
 
+  // -fsigned-bitfields is default, and clang doesn't support -fno-for-scope.
+  if (!Args.hasFlag(options::OPT_ffor_scope,
+                    options::OPT_fno_for_scope))
+    D.Diag(clang::diag::err_drv_clang_unsupported)
+      << Args.getLastArg(options::OPT_fno_for_scope)->getAsString(Args);
+
   // -fcaret-diagnostics is default.
   if (!Args.hasFlag(options::OPT_fcaret_diagnostics,
                     options::OPT_fno_caret_diagnostics, true))
@@ -1321,8 +1614,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                     options::OPT_fno_diagnostics_fixit_info))
     CmdArgs.push_back("-fno-diagnostics-fixit-info");
 
-  Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_binary);
-
   // Enable -fdiagnostics-show-option by default.
   if (Args.hasFlag(options::OPT_fdiagnostics_show_option,
                    options::OPT_fno_diagnostics_show_option))
@@ -1337,8 +1628,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Color diagnostics are the default, unless the terminal doesn't support
   // them.
   if (Args.hasFlag(options::OPT_fcolor_diagnostics,
-                   options::OPT_fno_color_diagnostics) &&
-      llvm::sys::Process::StandardErrHasColors())
+                   options::OPT_fno_color_diagnostics,
+                   llvm::sys::Process::StandardErrHasColors()))
     CmdArgs.push_back("-fcolor-diagnostics");
 
   if (!Args.hasFlag(options::OPT_fshow_source_location,
@@ -1349,6 +1640,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                     options::OPT_fno_spell_checking))
     CmdArgs.push_back("-fno-spell-checking");
 
+
+  // Silently ignore -fasm-blocks for now.
+  (void) Args.hasFlag(options::OPT_fasm_blocks, options::OPT_fno_asm_blocks,
+                      false);
+
   if (Arg *A = Args.getLastArg(options::OPT_fshow_overloads_EQ))
     A->render(Args, CmdArgs);
 
@@ -1450,13 +1746,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 
-  // Explicitly warn that these options are unsupported, even though
-  // we are allowing compilation to continue.
-  for (arg_iterator it = Args.filtered_begin(options::OPT_pg),
-         ie = Args.filtered_end(); it != ie; ++it) {
-    (*it)->claim();
-    D.Diag(clang::diag::warn_drv_clang_unsupported) << (*it)->getAsString(Args);
-  }
+  if (Arg *A = Args.getLastArg(options::OPT_pg))
+    if (Args.hasArg(options::OPT_fomit_frame_pointer))
+      D.Diag(clang::diag::err_drv_argument_not_allowed_with)
+        << "-fomit-frame-pointer" << A->getAsString(Args);
 
   // Claim some arguments which clang supports automatically.
 
@@ -1481,6 +1774,9 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
   assert(Inputs.size() == 1 && "Unexpected number of inputs.");
   const InputInfo &Input = Inputs[0];
 
+  // Don't warn about "clang -w -c foo.s"
+  Args.ClaimAllArgs(options::OPT_w);
+
   // Invoke ourselves in -cc1as mode.
   //
   // FIXME: Implement custom jobs for internal actions.
@@ -1614,12 +1910,21 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
 
     if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
-    else
+    else {
+      const Arg &A = II.getInputArg();
+
+      // Reverse translate some rewritten options.
+      if (A.getOption().matches(options::OPT_Z_reserved_lib_stdcxx)) {
+        CmdArgs.push_back("-lstdc++");
+        continue;
+      }
+
       // Don't render as input, we need gcc to do the translations.
-      II.getInputArg().render(Args, CmdArgs);
+      A.render(Args, CmdArgs);
+    }
   }
 
-  const char *GCCName = getToolChain().getDriver().CCCGenericGCCName.c_str();
+  const char *GCCName = getToolChain().getDriver().getCCCGenericGCCName().c_str();
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(GCCName));
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
@@ -1647,7 +1952,7 @@ void gcc::Compile::RenderExtraToolArgs(const JobAction &JA,
     if (JA.getType() != types::TY_PP_Asm)
       D.Diag(clang::diag::err_drv_invalid_gcc_output_type)
         << getTypeName(JA.getType());
-      
+
     CmdArgs.push_back("-S");
   }
 }
@@ -1684,15 +1989,15 @@ const char *darwin::CC1::getCC1Name(types::ID Type) const {
 
 const char *darwin::CC1::getBaseInputName(const ArgList &Args,
                                           const InputInfoList &Inputs) {
-  llvm::sys::Path P(Inputs[0].getBaseInput());
-  return Args.MakeArgString(P.getLast());
+  return Args.MakeArgString(
+    llvm::sys::path::filename(Inputs[0].getBaseInput()));
 }
 
 const char *darwin::CC1::getBaseInputStem(const ArgList &Args,
                                           const InputInfoList &Inputs) {
   const char *Str = getBaseInputName(Args, Inputs);
 
-  if (const char *End = strchr(Str, '.'))
+  if (const char *End = strrchr(Str, '.'))
     return Args.MakeArgString(std::string(Str, End));
 
   return Str;
@@ -2187,7 +2492,8 @@ void darwin::DarwinTool::AddDarwinArch(const ArgList &Args,
     CmdArgs.push_back("-force_cpusubtype_ALL");
 }
 
-void darwin::Link::AddLinkArgs(const ArgList &Args,
+void darwin::Link::AddLinkArgs(Compilation &C,
+                               const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   const Driver &D = getToolChain().getDriver();
 
@@ -2203,8 +2509,30 @@ void darwin::Link::AddLinkArgs(const ArgList &Args,
 
   // Newer linkers support -demangle, pass it if supported and not disabled by
   // the user.
-  if (Version[0] >= 100 && !Args.hasArg(options::OPT_Z_Xlinker__no_demangle)) {
-    CmdArgs.push_back("-demangle");
+  //
+  // FIXME: We temporarily avoid passing -demangle to any iOS linker, because
+  // unfortunately we can't be guaranteed that the linker version used there
+  // will match the linker version detected at configure time. We need the
+  // universal driver.
+  if (Version[0] >= 100 && !Args.hasArg(options::OPT_Z_Xlinker__no_demangle) &&
+      !getDarwinToolChain().isTargetIPhoneOS()) {
+    // Don't pass -demangle to ld_classic.
+    //
+    // FIXME: This is a temporary workaround, ld should be handling this.
+    bool UsesLdClassic = (getToolChain().getArch() == llvm::Triple::x86 &&
+                          Args.hasArg(options::OPT_static));
+    if (getToolChain().getArch() == llvm::Triple::x86) {
+      for (arg_iterator it = Args.filtered_begin(options::OPT_Xlinker,
+                                                 options::OPT_Wl_COMMA),
+             ie = Args.filtered_end(); it != ie; ++it) {
+        const Arg *A = *it;
+        for (unsigned i = 0, e = A->getNumValues(); i != e; ++i)
+          if (llvm::StringRef(A->getValue(Args, i)) == "-kext")
+            UsesLdClassic = true;
+      }
+    }
+    if (!UsesLdClassic)
+      CmdArgs.push_back("-demangle");
   }
 
   // Derived from the "link" spec.
@@ -2361,7 +2689,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
-  AddLinkArgs(Args, CmdArgs);
+  AddLinkArgs(C, Args, CmdArgs);
 
   Args.AddAllArgs(CmdArgs, options::OPT_d_Flag);
   Args.AddAllArgs(CmdArgs, options::OPT_s);
@@ -2373,6 +2701,12 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_m_Separate);
   Args.AddAllArgs(CmdArgs, options::OPT_r);
 
+  // Forward -ObjC when either -ObjC or -ObjC++ is used, to force loading
+  // members of static archive libraries which implement Objective-C classes or
+  // categories.
+  if (Args.hasArg(options::OPT_ObjC) || Args.hasArg(options::OPT_ObjCXX))
+    CmdArgs.push_back("-ObjC");
+
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
@@ -2458,14 +2792,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   getDarwinToolChain().AddLinkSearchPathArgs(Args, CmdArgs);
 
-  for (InputInfoList::const_iterator
-         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   if (LinkingOutput) {
     CmdArgs.push_back("-arch_multiple");
@@ -2484,10 +2811,8 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
-    // FIXME: g++ is more complicated here, it tries to put -lstdc++
-    // before -lm, for example.
     if (getToolChain().getDriver().CCCIsCXX)
-      CmdArgs.push_back("-lstdc++");
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
 
     // link_ssp spec is empty.
 
@@ -2582,7 +2907,6 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
                                   const InputInfoList &Inputs,
                                   const ArgList &Args,
                                   const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
   ArgStringList CmdArgs;
 
   if ((!Args.hasArg(options::OPT_nostdlib)) &&
@@ -2637,21 +2961,7 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
 
-  for (InputInfoList::const_iterator
-         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-
-    // Don't try to pass LLVM inputs to a generic gcc.
-    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
-        II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-        << getToolChain().getTripleString();
-
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
@@ -2719,6 +3029,8 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_static)) {
     CmdArgs.push_back("-Bstatic");
   } else {
+    if (Args.hasArg(options::OPT_rdynamic))
+      CmdArgs.push_back("-export-dynamic");
     CmdArgs.push_back("--eh-frame-hdr");
     CmdArgs.push_back("-Bdynamic");
     if (Args.hasArg(options::OPT_shared)) {
@@ -2759,26 +3071,12 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
 
-  for (InputInfoList::const_iterator
-         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-
-    // Don't try to pass LLVM inputs to a generic gcc.
-    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
-        II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-        << getToolChain().getTripleString();
-
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     if (D.CCCIsCXX) {
-      CmdArgs.push_back("-lstdc++");
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
 
@@ -2820,7 +3118,7 @@ void freebsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   if (getToolChain().getArchName() == "i386")
     CmdArgs.push_back("--32");
 
-  
+
   // Set byte order explicitly
   if (getToolChain().getArchName() == "mips")
     CmdArgs.push_back("-EB");
@@ -2855,6 +3153,8 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_static)) {
     CmdArgs.push_back("-Bstatic");
   } else {
+    if (Args.hasArg(options::OPT_rdynamic))
+      CmdArgs.push_back("-export-dynamic");
     CmdArgs.push_back("--eh-frame-hdr");
     if (Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back("-Bshareable");
@@ -2881,8 +3181,12 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles)) {
     if (!Args.hasArg(options::OPT_shared)) {
-      CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath("crt1.o")));
+      if (Args.hasArg(options::OPT_pg))
+        CmdArgs.push_back(Args.MakeArgString(
+                                getToolChain().GetFilePath("gcrt1.o")));
+      else
+        CmdArgs.push_back(Args.MakeArgString(
+                                getToolChain().GetFilePath("crt1.o")));
       CmdArgs.push_back(Args.MakeArgString(
                               getToolChain().GetFilePath("crti.o")));
       CmdArgs.push_back(Args.MakeArgString(
@@ -2896,6 +3200,7 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
+  CmdArgs.push_back("-L/usr/lib");
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
   Args.AddAllArgs(CmdArgs, options::OPT_s);
@@ -2903,26 +3208,182 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
   Args.AddAllArgs(CmdArgs, options::OPT_r);
 
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
+
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+      !Args.hasArg(options::OPT_nodefaultlibs)) {
+    if (D.CCCIsCXX) {
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
+      if (Args.hasArg(options::OPT_pg))
+        CmdArgs.push_back("-lm_p");
+      else
+        CmdArgs.push_back("-lm");
+    }
+    // FIXME: For some reason GCC passes -lgcc and -lgcc_s before adding
+    // the default system libraries. Just mimic this for now.
+    if (Args.hasArg(options::OPT_pg))
+      CmdArgs.push_back("-lgcc_p");
+    else
+      CmdArgs.push_back("-lgcc");
+    if (Args.hasArg(options::OPT_static)) {
+      CmdArgs.push_back("-lgcc_eh");
+    } else if (Args.hasArg(options::OPT_pg)) {
+      CmdArgs.push_back("-lgcc_eh_p");
+    } else {
+      CmdArgs.push_back("--as-needed");
+      CmdArgs.push_back("-lgcc_s");
+      CmdArgs.push_back("--no-as-needed");
+    }
+
+    if (Args.hasArg(options::OPT_pthread)) {
+      if (Args.hasArg(options::OPT_pg))
+        CmdArgs.push_back("-lpthread_p");
+      else
+        CmdArgs.push_back("-lpthread");
+    }
+
+    if (Args.hasArg(options::OPT_pg)) {
+      if (Args.hasArg(options::OPT_shared))
+        CmdArgs.push_back("-lc");
+      else
+        CmdArgs.push_back("-lc_p");
+      CmdArgs.push_back("-lgcc_p");
+    } else {
+      CmdArgs.push_back("-lc");
+      CmdArgs.push_back("-lgcc");
+    }
+
+    if (Args.hasArg(options::OPT_static)) {
+      CmdArgs.push_back("-lgcc_eh");
+    } else if (Args.hasArg(options::OPT_pg)) {
+      CmdArgs.push_back("-lgcc_eh_p");
+    } else {
+      CmdArgs.push_back("--as-needed");
+      CmdArgs.push_back("-lgcc_s");
+      CmdArgs.push_back("--no-as-needed");
+    }
+  }
+
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+      !Args.hasArg(options::OPT_nostartfiles)) {
+    if (!Args.hasArg(options::OPT_shared))
+      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
+                                                                  "crtend.o")));
+    else
+      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
+                                                                 "crtendS.o")));
+    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
+                                                                    "crtn.o")));
+  }
+
+  const char *Exec =
+    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+}
+
+void netbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
+                                     const InputInfo &Output,
+                                     const InputInfoList &Inputs,
+                                     const ArgList &Args,
+                                     const char *LinkingOutput) const {
+  ArgStringList CmdArgs;
+
+  // When building 32-bit code on NetBSD/amd64, we have to explicitly
+  // instruct as in the base system to assemble 32-bit code.
+  if (getToolChain().getArchName() == "i386")
+    CmdArgs.push_back("--32");
+
+
+  // Set byte order explicitly
+  if (getToolChain().getArchName() == "mips")
+    CmdArgs.push_back("-EB");
+  else if (getToolChain().getArchName() == "mipsel")
+    CmdArgs.push_back("-EL");
+
+  Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
+                       options::OPT_Xassembler);
+
+  CmdArgs.push_back("-o");
+  CmdArgs.push_back(Output.getFilename());
+
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
+    CmdArgs.push_back(II.getFilename());
+  }
 
-    // Don't try to pass LLVM inputs to a generic gcc.
-    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
-        II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-        << getToolChain().getTripleString();
+  const char *Exec =
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+}
 
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
+void netbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
+                                 const InputInfo &Output,
+                                 const InputInfoList &Inputs,
+                                 const ArgList &Args,
+                                 const char *LinkingOutput) const {
+  const Driver &D = getToolChain().getDriver();
+  ArgStringList CmdArgs;
+
+  if (Args.hasArg(options::OPT_static)) {
+    CmdArgs.push_back("-Bstatic");
+  } else {
+    if (Args.hasArg(options::OPT_rdynamic))
+      CmdArgs.push_back("-export-dynamic");
+    CmdArgs.push_back("--eh-frame-hdr");
+    if (Args.hasArg(options::OPT_shared)) {
+      CmdArgs.push_back("-Bshareable");
+    } else {
+      CmdArgs.push_back("-dynamic-linker");
+      CmdArgs.push_back("/libexec/ld.elf_so");
+    }
   }
 
+  // When building 32-bit code on NetBSD/amd64, we have to explicitly
+  // instruct ld in the base system to link 32-bit code.
+  if (getToolChain().getArchName() == "i386") {
+    CmdArgs.push_back("-m");
+    CmdArgs.push_back("elf_i386");
+  }
+
+  if (Output.isFilename()) {
+    CmdArgs.push_back("-o");
+    CmdArgs.push_back(Output.getFilename());
+  } else {
+    assert(Output.isNothing() && "Invalid output.");
+  }
+
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+      !Args.hasArg(options::OPT_nostartfiles)) {
+    if (!Args.hasArg(options::OPT_shared)) {
+      CmdArgs.push_back(Args.MakeArgString(
+                              getToolChain().GetFilePath("crt0.o")));
+      CmdArgs.push_back(Args.MakeArgString(
+                              getToolChain().GetFilePath("crti.o")));
+      CmdArgs.push_back(Args.MakeArgString(
+                              getToolChain().GetFilePath("crtbegin.o")));
+    } else {
+      CmdArgs.push_back(Args.MakeArgString(
+                              getToolChain().GetFilePath("crti.o")));
+      CmdArgs.push_back(Args.MakeArgString(
+                              getToolChain().GetFilePath("crtbeginS.o")));
+    }
+  }
+
+  Args.AddAllArgs(CmdArgs, options::OPT_L);
+  Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
+  Args.AddAllArgs(CmdArgs, options::OPT_e);
+  Args.AddAllArgs(CmdArgs, options::OPT_s);
+  Args.AddAllArgs(CmdArgs, options::OPT_t);
+  Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
+  Args.AddAllArgs(CmdArgs, options::OPT_r);
+
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
+
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     if (D.CCCIsCXX) {
-      CmdArgs.push_back("-lstdc++");
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
     // FIXME: For some reason GCC passes -lgcc and -lgcc_s before adding
@@ -3003,6 +3464,185 @@ void linuxtools::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
+void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
+                                    const InputInfo &Output,
+                                    const InputInfoList &Inputs,
+                                    const ArgList &Args,
+                                    const char *LinkingOutput) const {
+  const toolchains::Linux& ToolChain =
+    static_cast<const toolchains::Linux&>(getToolChain());
+  const Driver &D = ToolChain.getDriver();
+  ArgStringList CmdArgs;
+
+  // Silence warning for "clang -g foo.o -o foo"
+  Args.ClaimAllArgs(options::OPT_g_Group);
+  // and for "clang -g foo.o -o foo". Other warning options are already
+  // handled somewhere else.
+  Args.ClaimAllArgs(options::OPT_w);
+
+  if (Arg *A = Args.getLastArg(options::OPT__sysroot_EQ)) {
+    CmdArgs.push_back("--sysroot");
+    CmdArgs.push_back(A->getValue(Args));
+  }
+
+  if (Args.hasArg(options::OPT_pie))
+    CmdArgs.push_back("-pie");
+
+  if (Args.hasArg(options::OPT_rdynamic))
+    CmdArgs.push_back("-export-dynamic");
+
+  if (Args.hasArg(options::OPT_s))
+    CmdArgs.push_back("-s");
+
+  for (std::vector<std::string>::const_iterator i = ToolChain.ExtraOpts.begin(),
+         e = ToolChain.ExtraOpts.end();
+       i != e; ++i)
+    CmdArgs.push_back(i->c_str());
+
+  if (!Args.hasArg(options::OPT_static)) {
+    CmdArgs.push_back("--eh-frame-hdr");
+  }
+
+  CmdArgs.push_back("-m");
+  if (ToolChain.getArch() == llvm::Triple::x86)
+    CmdArgs.push_back("elf_i386");
+  else if (ToolChain.getArch() == llvm::Triple::arm)
+    CmdArgs.push_back("armelf_linux_eabi");
+  else
+    CmdArgs.push_back("elf_x86_64");
+
+  if (Args.hasArg(options::OPT_static)) {
+    if (ToolChain.getArch() == llvm::Triple::arm)
+      CmdArgs.push_back("-Bstatic");
+    else
+      CmdArgs.push_back("-static");
+  } else if (Args.hasArg(options::OPT_shared)) {
+    CmdArgs.push_back("-shared");
+  }
+
+  if (ToolChain.getArch() == llvm::Triple::arm ||
+      (!Args.hasArg(options::OPT_static) &&
+       !Args.hasArg(options::OPT_shared))) {
+    CmdArgs.push_back("-dynamic-linker");
+    if (ToolChain.getArch() == llvm::Triple::x86)
+      CmdArgs.push_back("/lib/ld-linux.so.2");
+    else if (ToolChain.getArch() == llvm::Triple::arm)
+      CmdArgs.push_back("/lib/ld-linux.so.3");
+    else
+      CmdArgs.push_back("/lib64/ld-linux-x86-64.so.2");
+  }
+
+  CmdArgs.push_back("-o");
+  CmdArgs.push_back(Output.getFilename());
+
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+      !Args.hasArg(options::OPT_nostartfiles)) {
+    const char *crt1 = NULL;
+    if (!Args.hasArg(options::OPT_shared)){
+      if (Args.hasArg(options::OPT_pie))
+        crt1 = "Scrt1.o";
+      else
+        crt1 = "crt1.o";
+    }
+    if (crt1)
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crt1)));
+
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
+
+    const char *crtbegin;
+    if (Args.hasArg(options::OPT_static))
+      crtbegin = "crtbeginT.o";
+    else if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+      crtbegin = "crtbeginS.o";
+    else
+      crtbegin = "crtbegin.o";
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
+  }
+
+  Args.AddAllArgs(CmdArgs, options::OPT_L);
+
+  const ToolChain::path_list Paths = ToolChain.getFilePaths();
+
+  for (ToolChain::path_list::const_iterator i = Paths.begin(),
+         e = Paths.end();
+       i != e; ++i) {
+    const std::string &s = *i;
+    CmdArgs.push_back(Args.MakeArgString(std::string("-L") + s));
+  }
+
+  AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs);
+
+  if (D.CCCIsCXX && !Args.hasArg(options::OPT_nostdlib)) {
+    ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
+    CmdArgs.push_back("-lm");
+  }
+
+  if (Args.hasArg(options::OPT_static))
+    CmdArgs.push_back("--start-group");
+
+  if (!Args.hasArg(options::OPT_nostdlib)) {
+    if (!D.CCCIsCXX)
+      CmdArgs.push_back("-lgcc");
+
+    if (Args.hasArg(options::OPT_static)) {
+      if (D.CCCIsCXX)
+        CmdArgs.push_back("-lgcc");
+    } else {
+      if (!D.CCCIsCXX)
+        CmdArgs.push_back("--as-needed");
+      CmdArgs.push_back("-lgcc_s");
+      if (!D.CCCIsCXX)
+        CmdArgs.push_back("--no-as-needed");
+    }
+
+    if (Args.hasArg(options::OPT_static))
+      CmdArgs.push_back("-lgcc_eh");
+    else if (!Args.hasArg(options::OPT_shared) && D.CCCIsCXX)
+      CmdArgs.push_back("-lgcc");
+
+    if (Args.hasArg(options::OPT_pthread) ||
+        Args.hasArg(options::OPT_pthreads))
+      CmdArgs.push_back("-lpthread");
+
+    CmdArgs.push_back("-lc");
+
+    if (Args.hasArg(options::OPT_static))
+      CmdArgs.push_back("--end-group");
+    else {
+      if (!D.CCCIsCXX)
+        CmdArgs.push_back("-lgcc");
+
+      if (!D.CCCIsCXX)
+        CmdArgs.push_back("--as-needed");
+      CmdArgs.push_back("-lgcc_s");
+      if (!D.CCCIsCXX)
+        CmdArgs.push_back("--no-as-needed");
+
+      if (!Args.hasArg(options::OPT_shared) && D.CCCIsCXX)
+        CmdArgs.push_back("-lgcc");
+    }
+
+
+    if (!Args.hasArg(options::OPT_nostartfiles)) {
+      const char *crtend;
+      if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+        crtend = "crtendS.o";
+      else
+        crtend = "crtend.o";
+
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtend)));
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
+    }
+  }
+
+  if (Args.hasArg(options::OPT_use_gold_plugin)) {
+    CmdArgs.push_back("-plugin");
+    std::string Plugin = ToolChain.getDriver().Dir + "/../lib/LLVMgold.so";
+    CmdArgs.push_back(Args.MakeArgString(Plugin));
+  }
+
+  C.addCommand(new Command(JA, *this, ToolChain.Linker.c_str(), CmdArgs));
+}
 
 void minix::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                                    const InputInfo &Output,
@@ -3052,26 +3692,12 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
 
-  for (InputInfoList::const_iterator
-         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-
-    // Don't try to pass LLVM inputs to a generic gcc.
-    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
-        II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-        << getToolChain().getTripleString();
-
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     if (D.CCCIsCXX) {
-      CmdArgs.push_back("-lstdc++");
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
 
@@ -3183,21 +3809,7 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
 
-  for (InputInfoList::const_iterator
-         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-
-    // Don't try to pass LLVM inputs to a generic gcc.
-    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||
-        II.getType() == types::TY_LLVM_BC || II.getType() == types::TY_LTO_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-        << getToolChain().getTripleString();
-
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
@@ -3220,7 +3832,7 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
     }
 
     if (D.CCCIsCXX) {
-      CmdArgs.push_back("-lstdc++");
+      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
 
@@ -3267,11 +3879,11 @@ void visualstudio::Link::ConstructJob(Compilation &C, const JobAction &JA,
                                       const InputInfoList &Inputs,
                                       const ArgList &Args,
                                       const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
   ArgStringList CmdArgs;
 
   if (Output.isFilename()) {
-    CmdArgs.push_back(Args.MakeArgString(std::string("-out:") + Output.getFilename()));
+    CmdArgs.push_back(Args.MakeArgString(std::string("-out:") +
+                                         Output.getFilename()));
   } else {
     assert(Output.isNothing() && "Invalid output.");
   }
@@ -3283,22 +3895,9 @@ void visualstudio::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   CmdArgs.push_back("-nologo");
 
-  for (InputInfoList::const_iterator
-    it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
-    const InputInfo &II = *it;
-
-    // Don't try to pass LLVM inputs to visual studio linker.
-    if (II.getType() == types::TY_LLVM_BC)
-      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
-      << getToolChain().getTripleString();
-
-    if (II.isFilename())
-      CmdArgs.push_back(II.getFilename());
-    else
-      II.getInputArg().renderAsInput(Args, CmdArgs);
-  }
+  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
 
   const char *Exec =
-  Args.MakeArgString(getToolChain().GetProgramPath("link.exe"));
+    Args.MakeArgString(getToolChain().GetProgramPath("link.exe"));
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
diff --git a/lib/Driver/Tools.h b/lib/Driver/Tools.h
index b5defa4569cf..10c883900933 100644
--- a/lib/Driver/Tools.h
+++ b/lib/Driver/Tools.h
@@ -36,6 +36,7 @@ namespace tools {
 
     void AddARMTargetArgs(const ArgList &Args, ArgStringList &CmdArgs) const;
     void AddMIPSTargetArgs(const ArgList &Args, ArgStringList &CmdArgs) const;
+    void AddSparcTargetArgs(const ArgList &Args, ArgStringList &CmdArgs) const;
     void AddX86TargetArgs(const ArgList &Args, ArgStringList &CmdArgs) const;
 
   public:
@@ -232,7 +233,8 @@ namespace darwin {
   };
 
   class LLVM_LIBRARY_VISIBILITY Link : public DarwinTool  {
-    void AddLinkArgs(const ArgList &Args, ArgStringList &CmdArgs) const;
+    void AddLinkArgs(Compilation &C, const ArgList &Args,
+                     ArgStringList &CmdArgs) const;
 
   public:
     Link(const ToolChain &TC) : DarwinTool("darwin::Link", "linker", TC) {}
@@ -332,6 +334,35 @@ namespace freebsd {
   };
 } // end namespace freebsd
 
+  /// netbsd -- Directly call GNU Binutils assembler and linker
+namespace netbsd {
+  class LLVM_LIBRARY_VISIBILITY Assemble : public Tool  {
+  public:
+    Assemble(const ToolChain &TC) : Tool("netbsd::Assemble", "assembler",
+                                         TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+                              const InputInfo &Output,
+                              const InputInfoList &Inputs,
+                              const ArgList &TCArgs,
+                              const char *LinkingOutput) const;
+  };
+  class LLVM_LIBRARY_VISIBILITY Link : public Tool  {
+  public:
+    Link(const ToolChain &TC) : Tool("netbsd::Link", "linker", TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+                              const InputInfo &Output,
+                              const InputInfoList &Inputs,
+                              const ArgList &TCArgs,
+                              const char *LinkingOutput) const;
+  };
+} // end namespace netbsd
+
   /// linux -- Directly call GNU Binutils assembler and linker
 namespace linuxtools {
   class LLVM_LIBRARY_VISIBILITY Assemble : public Tool  {
@@ -347,6 +378,18 @@ namespace linuxtools {
                               const ArgList &TCArgs,
                               const char *LinkingOutput) const;
   };
+  class LLVM_LIBRARY_VISIBILITY Link : public Tool  {
+  public:
+    Link(const ToolChain &TC) : Tool("linux::Link", "linker", TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+                              const InputInfo &Output,
+                              const InputInfoList &Inputs,
+                              const ArgList &TCArgs,
+                              const char *LinkingOutput) const;
+  };
 }
   /// minix -- Directly call GNU Binutils assembler and linker
 namespace minix {
diff --git a/lib/Driver/Types.cpp b/lib/Driver/Types.cpp
index 3c07cf2898af..4a4312b77fc9 100644
--- a/lib/Driver/Types.cpp
+++ b/lib/Driver/Types.cpp
@@ -1,4 +1,4 @@
-//===--- Types.cpp - Driver input & temporary type information ----------*-===//
+//===--- Types.cpp - Driver input & temporary type information ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -78,6 +78,7 @@ bool types::isAcceptedByClang(ID Id) {
   case TY_Asm:
   case TY_C: case TY_PP_C:
   case TY_CL:
+  case TY_CUDA:
   case TY_ObjC: case TY_PP_ObjC:
   case TY_CXX: case TY_PP_CXX:
   case TY_ObjCXX: case TY_PP_ObjCXX:
@@ -151,6 +152,7 @@ types::ID types::lookupTypeForExtension(const char *Ext) {
            .Case("CC", TY_CXX)
            .Case("cl", TY_CL)
            .Case("cp", TY_CXX)
+           .Case("cu", TY_CUDA)
            .Case("hh", TY_CXXHeader)
            .Case("ll", TY_LLVM_IR)
            .Case("hpp", TY_CXXHeader)
diff --git a/lib/Frontend/ASTConsumers.cpp b/lib/Frontend/ASTConsumers.cpp
index eb7f270ae8fb..92fb1e8cbeb0 100644
--- a/lib/Frontend/ASTConsumers.cpp
+++ b/lib/Frontend/ASTConsumers.cpp
@@ -24,7 +24,7 @@
 #include "llvm/Module.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -354,8 +354,18 @@ void DeclContextPrinter::PrintDeclContext(const DeclContext* DC,
       PrintDeclContext(DC, Indentation+2);
       break;
     }
+    case Decl::IndirectField: {
+      IndirectFieldDecl* IFD = cast<IndirectFieldDecl>(*I);
+      Out << "<IndirectField> " << IFD << '\n';
+      break;
+    }
+    case Decl::Label: {
+      LabelDecl *LD = cast<LabelDecl>(*I);
+      Out << "<Label> " << LD << '\n';
+      break;
+    }
     case Decl::Field: {
-      FieldDecl* FD = cast<FieldDecl>(*I);
+      FieldDecl *FD = cast<FieldDecl>(*I);
       Out << "<field> " << FD << '\n';
       break;
     }
@@ -423,29 +433,21 @@ ASTConsumer *clang::CreateDeclContextPrinter() {
 }
 
 //===----------------------------------------------------------------------===//
-/// InheritanceViewer - C++ Inheritance Visualization
+/// ASTDumperXML - In-depth XML dumping.
 
 namespace {
-class InheritanceViewer : public ASTConsumer {
-  const std::string clsname;
+class ASTDumpXML : public ASTConsumer {
+  llvm::raw_ostream &OS;
+
 public:
-  InheritanceViewer(const std::string& cname) : clsname(cname) {}
+  ASTDumpXML(llvm::raw_ostream &OS) : OS(OS) {}
 
   void HandleTranslationUnit(ASTContext &C) {
-    for (ASTContext::type_iterator I=C.types_begin(),E=C.types_end(); I!=E; ++I)
-      if (RecordType *T = dyn_cast<RecordType>(*I)) {
-        if (CXXRecordDecl *D = dyn_cast<CXXRecordDecl>(T->getDecl())) {
-          // FIXME: This lookup needs to be generalized to handle namespaces and
-          // (when we support them) templates.
-          if (D->getNameAsString() == clsname) {
-            D->viewInheritance(C);
-          }
-        }
-      }
-  }
+    C.getTranslationUnitDecl()->dumpXML(OS);
+  }  
 };
 }
 
-ASTConsumer *clang::CreateInheritanceViewer(const std::string& clsname) {
-  return new InheritanceViewer(clsname);
+ASTConsumer *clang::CreateASTDumperXML(llvm::raw_ostream &OS) {
+  return new ASTDumpXML(OS);
 }
diff --git a/lib/Frontend/ASTMerge.cpp b/lib/Frontend/ASTMerge.cpp
index b46212fedaf4..3905b99b02a4 100644
--- a/lib/Frontend/ASTMerge.cpp
+++ b/lib/Frontend/ASTMerge.cpp
@@ -38,23 +38,22 @@ void ASTMergeAction::ExecuteAction() {
                                          CI.getASTContext().getLangOptions());
   CI.getDiagnostics().SetArgToStringFn(&FormatASTNodeDiagnosticArgument,
                                        &CI.getASTContext());
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(&CI.getDiagnostics());
+  llvm::IntrusiveRefCntPtr<DiagnosticIDs>
+      DiagIDs(CI.getDiagnostics().getDiagnosticIDs());
   for (unsigned I = 0, N = ASTFiles.size(); I != N; ++I) {
-    ASTUnit *Unit = ASTUnit::LoadFromASTFile(ASTFiles[I], Diags, false);
+    llvm::IntrusiveRefCntPtr<Diagnostic>
+        Diags(new Diagnostic(DiagIDs, CI.getDiagnostics().getClient(),
+                             /*ShouldOwnClient=*/false));
+    ASTUnit *Unit = ASTUnit::LoadFromASTFile(ASTFiles[I], Diags,
+                                             CI.getFileSystemOpts(), false);
     if (!Unit)
       continue;
 
-    // Reset the argument -> string function so that it has the AST
-    // context we want, since the Sema object created by
-    // LoadFromASTFile will override it.
-    CI.getDiagnostics().SetArgToStringFn(&FormatASTNodeDiagnosticArgument,
-                                         &CI.getASTContext());
-
-    ASTImporter Importer(CI.getDiagnostics(),
-                         CI.getASTContext(), 
+    ASTImporter Importer(CI.getASTContext(), 
                          CI.getFileManager(),
                          Unit->getASTContext(), 
-                         Unit->getFileManager());
+                         Unit->getFileManager(),
+                         /*MinimalImport=*/false);
 
     TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
     for (DeclContext::decl_iterator D = TU->decls_begin(), 
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index c76488b2c62c..4a5a51d9f1dc 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -25,17 +25,21 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendOptions.h"
+#include "clang/Frontend/Utils.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTSerializationListener.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/TargetOptions.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Atomic.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/System/Host.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Timer.h"
 #include <cstdlib>
@@ -43,20 +47,63 @@
 #include <sys/stat.h>
 using namespace clang;
 
+using llvm::TimeRecord;
+
+namespace {
+  class SimpleTimer {
+    bool WantTiming;
+    TimeRecord Start;
+    std::string Output;
+
+  public:
+    explicit SimpleTimer(bool WantTiming) : WantTiming(WantTiming) {
+      if (WantTiming)
+        Start = TimeRecord::getCurrentTime();
+    }
+
+    void setOutput(const llvm::Twine &Output) {
+      if (WantTiming)
+        this->Output = Output.str();
+    }
+
+    ~SimpleTimer() {
+      if (WantTiming) {
+        TimeRecord Elapsed = TimeRecord::getCurrentTime();
+        Elapsed -= Start;
+        llvm::errs() << Output << ':';
+        Elapsed.print(Elapsed, llvm::errs());
+        llvm::errs() << '\n';
+      }
+    }
+  };
+}
+
 /// \brief After failing to build a precompiled preamble (due to
 /// errors in the source that occurs in the preamble), the number of
 /// reparses during which we'll skip even trying to precompile the
 /// preamble.
 const unsigned DefaultPreambleRebuildInterval = 5;
 
+/// \brief Tracks the number of ASTUnit objects that are currently active.
+///
+/// Used for debugging purposes only.
+static llvm::sys::cas_flag ActiveASTUnitObjects;
+
 ASTUnit::ASTUnit(bool _MainFileIsAST)
   : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), 
-    CompleteTranslationUnit(true), ConcurrencyCheckValue(CheckUnlocked), 
+    CompleteTranslationUnit(true), WantTiming(getenv("LIBCLANG_TIMING")),
+    NumStoredDiagnosticsFromDriver(0),
+    ConcurrencyCheckValue(CheckUnlocked), 
     PreambleRebuildCounter(0), SavedMainFileBuffer(0), PreambleBuffer(0),
     ShouldCacheCodeCompletionResults(false),
-    NumTopLevelDeclsAtLastCompletionCache(0),
-    CacheCodeCompletionCoolDown(0),
+    CompletionCacheTopLevelHashValue(0),
+    PreambleTopLevelHashValue(0),
+    CurrentTopLevelHashValue(0),
     UnsafeToFree(false) { 
+  if (getenv("LIBCLANG_OBJTRACKING")) {
+    llvm::sys::AtomicIncrement(&ActiveASTUnitObjects);
+    fprintf(stderr, "+++ %d translation units\n", ActiveASTUnitObjects);
+  }    
 }
 
 ASTUnit::~ASTUnit() {
@@ -82,10 +129,12 @@ ASTUnit::~ASTUnit() {
   delete SavedMainFileBuffer;
   delete PreambleBuffer;
 
-  ClearCachedCompletionResults();
+  ClearCachedCompletionResults();  
   
-  for (unsigned I = 0, N = Timers.size(); I != N; ++I)
-    delete Timers[I];
+  if (getenv("LIBCLANG_OBJTRACKING")) {
+    llvm::sys::AtomicDecrement(&ActiveASTUnitObjects);
+    fprintf(stderr, "--- %d translation units\n", ActiveASTUnitObjects);
+  }    
 }
 
 void ASTUnit::CleanTemporaryFiles() {
@@ -115,7 +164,8 @@ static unsigned getDeclShowContexts(NamedDecl *ND,
                 | (1 << (CodeCompletionContext::CCC_ObjCIvarList - 1))
                 | (1 << (CodeCompletionContext::CCC_ClassStructUnion - 1))
                 | (1 << (CodeCompletionContext::CCC_Statement - 1))
-                | (1 << (CodeCompletionContext::CCC_Type - 1));
+                | (1 << (CodeCompletionContext::CCC_Type - 1))
+              | (1 << (CodeCompletionContext::CCC_ParenthesizedExpression - 1));
 
     // In C++, types can appear in expressions contexts (for functional casts).
     if (LangOpts.CPlusPlus)
@@ -141,12 +191,13 @@ static unsigned getDeclShowContexts(NamedDecl *ND,
       
       if (LangOpts.CPlusPlus)
         IsNestedNameSpecifier = true;
-    } else if (isa<ClassTemplateDecl>(ND) || isa<TemplateTemplateParmDecl>(ND))
+    } else if (isa<ClassTemplateDecl>(ND))
       IsNestedNameSpecifier = true;
   } else if (isa<ValueDecl>(ND) || isa<FunctionTemplateDecl>(ND)) {
     // Values can appear in these contexts.
     Contexts = (1 << (CodeCompletionContext::CCC_Statement - 1))
              | (1 << (CodeCompletionContext::CCC_Expression - 1))
+             | (1 << (CodeCompletionContext::CCC_ParenthesizedExpression - 1))
              | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1));
   } else if (isa<ObjCProtocolDecl>(ND)) {
     Contexts = (1 << (CodeCompletionContext::CCC_ObjCProtocolName - 1));
@@ -164,13 +215,8 @@ void ASTUnit::CacheCodeCompletionResults() {
   if (!TheSema)
     return;
   
-  llvm::Timer *CachingTimer = 0;
-  if (TimerGroup.get()) {
-    CachingTimer = new llvm::Timer("Cache global code completions", 
-                                   *TimerGroup);
-    CachingTimer->startTimer();
-    Timers.push_back(CachingTimer);
-  }
+  SimpleTimer Timer(WantTiming);
+  Timer.setOutput("Cache global code completions for " + getMainFileName());
 
   // Clear out the previous results.
   ClearCachedCompletionResults();
@@ -178,7 +224,8 @@ void ASTUnit::CacheCodeCompletionResults() {
   // Gather the set of global code completions.
   typedef CodeCompletionResult Result;
   llvm::SmallVector<Result, 8> Results;
-  TheSema->GatherGlobalCodeCompletions(Results);
+  CachedCompletionAllocator = new GlobalCodeCompletionAllocator;
+  TheSema->GatherGlobalCodeCompletions(*CachedCompletionAllocator, Results);
   
   // Translate global code completions into cached completions.
   llvm::DenseMap<CanQualType, unsigned> CompletionTypes;
@@ -188,7 +235,8 @@ void ASTUnit::CacheCodeCompletionResults() {
     case Result::RK_Declaration: {
       bool IsNestedNameSpecifier = false;
       CachedCodeCompletionResult CachedResult;
-      CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+      CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema,
+                                                    *CachedCompletionAllocator);
       CachedResult.ShowInContexts = getDeclShowContexts(Results[I].Declaration,
                                                         Ctx->getLangOptions(),
                                                         IsNestedNameSpecifier);
@@ -237,7 +285,8 @@ void ASTUnit::CacheCodeCompletionResults() {
           | (1 << (CodeCompletionContext::CCC_UnionTag - 1))
           | (1 << (CodeCompletionContext::CCC_ClassOrStructTag - 1))
           | (1 << (CodeCompletionContext::CCC_Type - 1))
-          | (1 << (CodeCompletionContext::CCC_PotentiallyQualifiedName - 1));
+          | (1 << (CodeCompletionContext::CCC_PotentiallyQualifiedName - 1))
+          | (1 << (CodeCompletionContext::CCC_ParenthesizedExpression - 1));
 
         if (isa<NamespaceDecl>(Results[I].Declaration) ||
             isa<NamespaceAliasDecl>(Results[I].Declaration))
@@ -249,7 +298,9 @@ void ASTUnit::CacheCodeCompletionResults() {
           // nested-name-specifier but isn't already an option, create a 
           // nested-name-specifier completion.
           Results[I].StartsNestedNameSpecifier = true;
-          CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+          CachedResult.Completion 
+            = Results[I].CreateCodeCompletionString(*TheSema,
+                                                    *CachedCompletionAllocator);
           CachedResult.ShowInContexts = RemainingContexts;
           CachedResult.Priority = CCP_NestedNameSpecifier;
           CachedResult.TypeClass = STC_Void;
@@ -268,7 +319,9 @@ void ASTUnit::CacheCodeCompletionResults() {
       
     case Result::RK_Macro: {
       CachedCodeCompletionResult CachedResult;
-      CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+      CachedResult.Completion 
+        = Results[I].CreateCodeCompletionString(*TheSema,
+                                                *CachedCompletionAllocator);
       CachedResult.ShowInContexts
         = (1 << (CodeCompletionContext::CCC_TopLevel - 1))
         | (1 << (CodeCompletionContext::CCC_ObjCInterface - 1))
@@ -279,7 +332,9 @@ void ASTUnit::CacheCodeCompletionResults() {
         | (1 << (CodeCompletionContext::CCC_Expression - 1))
         | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1))
         | (1 << (CodeCompletionContext::CCC_MacroNameUse - 1))
-        | (1 << (CodeCompletionContext::CCC_PreprocessorExpression - 1));
+        | (1 << (CodeCompletionContext::CCC_PreprocessorExpression - 1))
+        | (1 << (CodeCompletionContext::CCC_ParenthesizedExpression - 1))
+        | (1 << (CodeCompletionContext::CCC_OtherWithMacros - 1));
       
       CachedResult.Priority = Results[I].Priority;
       CachedResult.Kind = Results[I].CursorKind;
@@ -290,22 +345,16 @@ void ASTUnit::CacheCodeCompletionResults() {
       break;
     }
     }
-    Results[I].Destroy();
   }
-
-  if (CachingTimer)
-    CachingTimer->stopTimer();
   
-  // Make a note of the state when we performed this caching.
-  NumTopLevelDeclsAtLastCompletionCache = top_level_size();
-  CacheCodeCompletionCoolDown = 15;
+  // Save the current top-level hash value.
+  CompletionCacheTopLevelHashValue = CurrentTopLevelHashValue;
 }
 
 void ASTUnit::ClearCachedCompletionResults() {
-  for (unsigned I = 0, N = CachedCompletionResults.size(); I != N; ++I)
-    delete CachedCompletionResults[I].Completion;
   CachedCompletionResults.clear();
   CachedCompletionTypes.clear();
+  CachedCompletionAllocator = 0;
 }
 
 namespace {
@@ -378,7 +427,7 @@ class CaptureDroppedDiagnostics {
 
 public:
   CaptureDroppedDiagnostics(bool RequestCapture, Diagnostic &Diags, 
-                           llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags)
+                          llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags)
     : Diags(Diags), Client(StoredDiags), PreviousClient(0)
   {
     if (RequestCapture || Diags.getClient() == 0) {
@@ -399,6 +448,9 @@ public:
 
 void StoredDiagnosticClient::HandleDiagnostic(Diagnostic::Level Level,
                                               const DiagnosticInfo &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticClient::HandleDiagnostic(Level, Info);
+
   StoredDiags.push_back(StoredDiagnostic(Level, Info));
 }
 
@@ -411,32 +463,48 @@ const std::string &ASTUnit::getASTFileName() {
   return static_cast<ASTReader *>(Ctx->getExternalSource())->getFileName();
 }
 
+llvm::MemoryBuffer *ASTUnit::getBufferForFile(llvm::StringRef Filename,
+                                              std::string *ErrorStr) {
+  assert(FileMgr);
+  return FileMgr->getBufferForFile(Filename, ErrorStr);
+}
+
+/// \brief Configure the diagnostics object for use with ASTUnit.
+void ASTUnit::ConfigureDiags(llvm::IntrusiveRefCntPtr<Diagnostic> &Diags,
+                             const char **ArgBegin, const char **ArgEnd,
+                             ASTUnit &AST, bool CaptureDiagnostics) {
+  if (!Diags.getPtr()) {
+    // No diagnostics engine was provided, so create our own diagnostics object
+    // with the default options.
+    DiagnosticOptions DiagOpts;
+    DiagnosticClient *Client = 0;
+    if (CaptureDiagnostics)
+      Client = new StoredDiagnosticClient(AST.StoredDiagnostics);
+    Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgEnd- ArgBegin, 
+                                                ArgBegin, Client);
+  } else if (CaptureDiagnostics) {
+    Diags->setClient(new StoredDiagnosticClient(AST.StoredDiagnostics));
+  }
+}
+
 ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                                  const FileSystemOptions &FileSystemOpts,
                                   bool OnlyLocalDecls,
                                   RemappedFile *RemappedFiles,
                                   unsigned NumRemappedFiles,
                                   bool CaptureDiagnostics) {
   llvm::OwningPtr<ASTUnit> AST(new ASTUnit(true));
-  
-  if (!Diags.getPtr()) {
-    // No diagnostics engine was provided, so create our own diagnostics object
-    // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
-  }
+  ConfigureDiags(Diags, 0, 0, *AST, CaptureDiagnostics);
 
-  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->OnlyLocalDecls = OnlyLocalDecls;
+  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->Diagnostics = Diags;
-  AST->FileMgr.reset(new FileManager);
-  AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics()));
+  AST->FileMgr.reset(new FileManager(FileSystemOpts));
+  AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics(),
+                                         AST->getFileManager()));
   AST->HeaderInfo.reset(new HeaderSearch(AST->getFileManager()));
   
-  // If requested, capture diagnostics in the ASTUnit.
-  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, AST->getDiagnostics(),
-                                    AST->StoredDiagnostics);
-
   for (unsigned I = 0; I != NumRemappedFiles; ++I) {
     // Create the file entry for the file that we're mapping from.
     const FileEntry *FromFile
@@ -471,7 +539,7 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   Reader->setListener(new ASTInfoCollector(LangInfo, HeaderInfo, TargetTriple,
                                            Predefines, Counter));
 
-  switch (Reader->ReadAST(Filename)) {
+  switch (Reader->ReadAST(Filename, ASTReader::MainFile)) {
   case ASTReader::Success:
     break;
 
@@ -538,12 +606,69 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
 
 namespace {
 
+/// \brief Preprocessor callback class that updates a hash value with the names 
+/// of all macros that have been defined by the translation unit.
+class MacroDefinitionTrackerPPCallbacks : public PPCallbacks {
+  unsigned &Hash;
+  
+public:
+  explicit MacroDefinitionTrackerPPCallbacks(unsigned &Hash) : Hash(Hash) { }
+  
+  virtual void MacroDefined(const Token &MacroNameTok, const MacroInfo *MI) {
+    Hash = llvm::HashString(MacroNameTok.getIdentifierInfo()->getName(), Hash);
+  }
+};
+
+/// \brief Add the given declaration to the hash of all top-level entities.
+void AddTopLevelDeclarationToHash(Decl *D, unsigned &Hash) {
+  if (!D)
+    return;
+  
+  DeclContext *DC = D->getDeclContext();
+  if (!DC)
+    return;
+  
+  if (!(DC->isTranslationUnit() || DC->getLookupParent()->isTranslationUnit()))
+    return;
+
+  if (NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
+    if (ND->getIdentifier())
+      Hash = llvm::HashString(ND->getIdentifier()->getName(), Hash);
+    else if (DeclarationName Name = ND->getDeclName()) {
+      std::string NameStr = Name.getAsString();
+      Hash = llvm::HashString(NameStr, Hash);
+    }
+    return;
+  }
+  
+  if (ObjCForwardProtocolDecl *Forward 
+      = dyn_cast<ObjCForwardProtocolDecl>(D)) {
+    for (ObjCForwardProtocolDecl::protocol_iterator 
+         P = Forward->protocol_begin(),
+         PEnd = Forward->protocol_end();
+         P != PEnd; ++P)
+      AddTopLevelDeclarationToHash(*P, Hash);
+    return;
+  }
+  
+  if (ObjCClassDecl *Class = llvm::dyn_cast<ObjCClassDecl>(D)) {
+    for (ObjCClassDecl::iterator I = Class->begin(), IEnd = Class->end();
+         I != IEnd; ++I)
+      AddTopLevelDeclarationToHash(I->getInterface(), Hash);
+    return;
+  }
+}
+
 class TopLevelDeclTrackerConsumer : public ASTConsumer {
   ASTUnit &Unit;
-
+  unsigned &Hash;
+  
 public:
-  TopLevelDeclTrackerConsumer(ASTUnit &_Unit) : Unit(_Unit) {}
-
+  TopLevelDeclTrackerConsumer(ASTUnit &_Unit, unsigned &Hash)
+    : Unit(_Unit), Hash(Hash) {
+    Hash = 0;
+  }
+  
   void HandleTopLevelDecl(DeclGroupRef D) {
     for (DeclGroupRef::iterator it = D.begin(), ie = D.end(); it != ie; ++it) {
       Decl *D = *it;
@@ -553,6 +678,8 @@ public:
       // fundamental problem in the parser right now.
       if (isa<ObjCMethodDecl>(D))
         continue;
+
+      AddTopLevelDeclarationToHash(D, Hash);
       Unit.addTopLevelDecl(D);
     }
   }
@@ -567,7 +694,10 @@ public:
 
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
                                          llvm::StringRef InFile) {
-    return new TopLevelDeclTrackerConsumer(Unit);
+    CI.getPreprocessor().addPPCallbacks(
+     new MacroDefinitionTrackerPPCallbacks(Unit.getCurrentTopLevelHashValue()));
+    return new TopLevelDeclTrackerConsumer(Unit, 
+                                           Unit.getCurrentTopLevelHashValue());
   }
 
 public:
@@ -579,15 +709,20 @@ public:
   }
 };
 
-class PrecompilePreambleConsumer : public PCHGenerator {
+class PrecompilePreambleConsumer : public PCHGenerator, 
+                                   public ASTSerializationListener {
   ASTUnit &Unit;
+  unsigned &Hash;                                   
   std::vector<Decl *> TopLevelDecls;
-
+                                     
 public:
   PrecompilePreambleConsumer(ASTUnit &Unit,
                              const Preprocessor &PP, bool Chaining,
                              const char *isysroot, llvm::raw_ostream *Out)
-    : PCHGenerator(PP, Chaining, isysroot, Out), Unit(Unit) { }
+    : PCHGenerator(PP, "", Chaining, isysroot, Out), Unit(Unit),
+      Hash(Unit.getCurrentTopLevelHashValue()) {
+    Hash = 0;
+  }
 
   virtual void HandleTopLevelDecl(DeclGroupRef D) {
     for (DeclGroupRef::iterator it = D.begin(), ie = D.end(); it != ie; ++it) {
@@ -598,6 +733,7 @@ public:
       // fundamental problem in the parser right now.
       if (isa<ObjCMethodDecl>(D))
         continue;
+      AddTopLevelDeclarationToHash(D, Hash);
       TopLevelDecls.push_back(D);
     }
   }
@@ -614,6 +750,15 @@ public:
                                       getWriter().getDeclID(TopLevelDecls[I]));
     }
   }
+                                     
+  virtual void SerializedPreprocessedEntity(PreprocessedEntity *Entity,
+                                            uint64_t Offset) {
+    Unit.addPreprocessedEntityFromPreamble(Offset);
+  }
+                                     
+  virtual ASTSerializationListener *GetASTSerializationListener() {
+    return this;
+  }
 };
 
 class PrecompilePreambleAction : public ASTFrontendAction {
@@ -625,14 +770,18 @@ public:
   virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
                                          llvm::StringRef InFile) {
     std::string Sysroot;
+    std::string OutputFile;
     llvm::raw_ostream *OS = 0;
     bool Chaining;
-    if (GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot, 
+    if (GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
+                                                       OutputFile,
                                                        OS, Chaining))
       return 0;
     
     const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
                              Sysroot.c_str() : 0;  
+    CI.getPreprocessor().addPPCallbacks(
+     new MacroDefinitionTrackerPPCallbacks(Unit.getCurrentTopLevelHashValue()));
     return new PrecompilePreambleConsumer(Unit, CI.getPreprocessor(), Chaining,
                                           isysroot, OS);
   }
@@ -666,11 +815,9 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   // Set up diagnostics, capturing any diagnostics that would
   // otherwise be dropped.
   Clang.setDiagnostics(&getDiagnostics());
-  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
-                                    getDiagnostics(),
-                                    StoredDiagnostics);
   
   // Create the target instance.
+  Clang.getTargetOpts().Features = TargetFeatures;
   Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
                                                Clang.getTargetOpts()));
   if (!Clang.hasTarget()) {
@@ -693,20 +840,25 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
 
   // Configure the various subsystems.
   // FIXME: Should we retain the previous file manager?
-  FileMgr.reset(new FileManager);
-  SourceMgr.reset(new SourceManager(getDiagnostics()));
+  FileSystemOpts = Clang.getFileSystemOpts();
+  FileMgr.reset(new FileManager(Clang.getFileSystemOpts()));
+  SourceMgr.reset(new SourceManager(getDiagnostics(), *FileMgr));
   TheSema.reset();
   Ctx.reset();
   PP.reset();
   
   // Clear out old caches and data.
   TopLevelDecls.clear();
+  PreprocessedEntities.clear();
   CleanTemporaryFiles();
   PreprocessedEntitiesByFile.clear();
 
   if (!OverrideMainBuffer) {
-    StoredDiagnostics.clear();
+    StoredDiagnostics.erase(
+                    StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
+                            StoredDiagnostics.end());
     TopLevelDeclsInPreamble.clear();
+    PreprocessedEntitiesInPreamble.clear();
   }
 
   // Create a file manager object to provide access to and cache the filesystem.
@@ -728,19 +880,21 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
     PreprocessorOpts.ImplicitPCHInclude = PreambleFile;
     PreprocessorOpts.DisablePCHValidation = true;
     
-    // Keep track of the override buffer;
-    SavedMainFileBuffer = OverrideMainBuffer;
-
     // The stored diagnostic has the old source manager in it; update
     // the locations to refer into the new source manager. Since we've
     // been careful to make sure that the source manager's state
     // before and after are identical, so that we can reuse the source
     // location itself.
-    for (unsigned I = 0, N = StoredDiagnostics.size(); I != N; ++I) {
+    for (unsigned I = NumStoredDiagnosticsFromDriver, 
+                  N = StoredDiagnostics.size(); 
+         I < N; ++I) {
       FullSourceLoc Loc(StoredDiagnostics[I].getLocation(),
                         getSourceManager());
       StoredDiagnostics[I].setLocation(Loc);
     }
+
+    // Keep track of the override buffer;
+    SavedMainFileBuffer = OverrideMainBuffer;
   } else {
     PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
     PreprocessorOpts.PrecompiledPreambleBytes.second = false;
@@ -774,12 +928,6 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   }
 
   Invocation.reset(Clang.takeInvocation());
-  
-  // If we were asked to cache code-completion results and don't have any
-  // results yet, do so now.
-  if (ShouldCacheCodeCompletionResults && CachedCompletionResults.empty())
-    CacheCodeCompletionResults();
-  
   return false;
   
 error:
@@ -787,11 +935,12 @@ error:
   if (OverrideMainBuffer) {
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
-    PreprocessorOpts.DisablePCHValidation = true;
     PreprocessorOpts.ImplicitPCHInclude = PriorImplicitPCHInclude;
     delete OverrideMainBuffer;
+    SavedMainFileBuffer = 0;
   }
   
+  StoredDiagnostics.clear();
   Clang.takeSourceManager();
   Clang.takeFileManager();
   Invocation.reset(Clang.takeInvocation());
@@ -803,15 +952,27 @@ static std::string GetPreamblePCHPath() {
   // FIXME: This is lame; sys::Path should provide this function (in particular,
   // it should know how to find the temporary files dir).
   // FIXME: This is really lame. I copied this code from the Driver!
+  // FIXME: This is a hack so that we can override the preamble file during
+  // crash-recovery testing, which is the only case where the preamble files
+  // are not necessarily cleaned up. 
+  const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE");
+  if (TmpFile)
+    return TmpFile;
+  
   std::string Error;
   const char *TmpDir = ::getenv("TMPDIR");
   if (!TmpDir)
     TmpDir = ::getenv("TEMP");
   if (!TmpDir)
     TmpDir = ::getenv("TMP");
+#ifdef LLVM_ON_WIN32
+  if (!TmpDir)
+    TmpDir = ::getenv("USERPROFILE");
+#endif
   if (!TmpDir)
     TmpDir = "/tmp";
   llvm::sys::Path P(TmpDir);
+  P.createDirectoryOnDisk(true);
   P.appendComponent("preamble");
   P.appendSuffix("pch");
   if (P.createTemporaryFileOnDisk())
@@ -827,8 +988,7 @@ std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> >
 ASTUnit::ComputePreamble(CompilerInvocation &Invocation, 
                          unsigned MaxLines, bool &CreatedBuffer) {
   FrontendOptions &FrontendOpts = Invocation.getFrontendOpts();
-  PreprocessorOptions &PreprocessorOpts
-    = Invocation.getPreprocessorOpts();
+  PreprocessorOptions &PreprocessorOpts = Invocation.getPreprocessorOpts();
   CreatedBuffer = false;
   
   // Try to determine if the main file has been remapped, either from the 
@@ -852,17 +1012,11 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
             CreatedBuffer = false;
           }
           
-          Buffer = llvm::MemoryBuffer::getFile(M->second);
+          Buffer = getBufferForFile(M->second);
           if (!Buffer)
             return std::make_pair((llvm::MemoryBuffer*)0, 
                                   std::make_pair(0, true));
           CreatedBuffer = true;
-          
-          // Remove this remapping. We've captured the buffer already.
-          M = PreprocessorOpts.eraseRemappedFile(M);
-          E = PreprocessorOpts.remapped_file_end();
-          if (M == E)
-            break;
         }
       }
     }
@@ -884,12 +1038,6 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
           }
           
           Buffer = const_cast<llvm::MemoryBuffer *>(M->second);
-
-          // Remove this remapping. We've captured the buffer already.
-          M = PreprocessorOpts.eraseRemappedFile(M);
-          E = PreprocessorOpts.remapped_file_buffer_end();
-          if (M == E)
-            break;
         }
       }
     }
@@ -897,7 +1045,7 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
   
   // If the main source file was not remapped, load it now.
   if (!Buffer) {
-    Buffer = llvm::MemoryBuffer::getFile(FrontendOpts.Inputs[0].second);
+    Buffer = getBufferForFile(FrontendOpts.Inputs[0].second);
     if (!Buffer)
       return std::make_pair((llvm::MemoryBuffer*)0, std::make_pair(0, true));    
     
@@ -908,7 +1056,6 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
 }
 
 static llvm::MemoryBuffer *CreatePaddedMainFileBuffer(llvm::MemoryBuffer *Old,
-                                                      bool DeleteOld,
                                                       unsigned NewSize,
                                                       llvm::StringRef NewName) {
   llvm::MemoryBuffer *Result
@@ -919,9 +1066,6 @@ static llvm::MemoryBuffer *CreatePaddedMainFileBuffer(llvm::MemoryBuffer *Old,
          ' ', NewSize - Old->getBufferSize() - 1);
   const_cast<char*>(Result->getBufferEnd())[-1] = '\n';  
   
-  if (DeleteOld)
-    delete Old;
-  
   return Result;
 }
 
@@ -957,6 +1101,11 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> > NewPreamble 
     = ComputePreamble(PreambleInvocation, MaxLines, CreatedPreambleBuffer);
 
+  // If ComputePreamble() Take ownership of the
+  llvm::OwningPtr<llvm::MemoryBuffer> OwnedPreambleBuffer;
+  if (CreatedPreambleBuffer)
+    OwnedPreambleBuffer.reset(NewPreamble.first);
+
   if (!NewPreamble.second.first) {
     // We couldn't find a preamble in the main source. Clear out the current
     // preamble, if we have one. It's obviously no good any more.
@@ -965,8 +1114,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
       llvm::sys::Path(PreambleFile).eraseFromDisk();
       PreambleFile.clear();
     }
-    if (CreatedPreambleBuffer)
-      delete NewPreamble.first;
 
     // The next time we actually see a preamble, precompile it.
     PreambleRebuildCounter = 1;
@@ -1049,7 +1196,11 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
 
         // Set the state of the diagnostic object to mimic its state
         // after parsing the preamble.
+        // FIXME: This won't catch any #pragma push warning changes that
+        // have occurred in the preamble.
         getDiagnostics().Reset();
+        ProcessWarningOptions(getDiagnostics(), 
+                              PreambleInvocation.getDiagnosticOpts());
         getDiagnostics().setNumWarnings(NumWarningsInPreamble);
         if (StoredDiagnostics.size() > NumStoredDiagnosticsInPreamble)
           StoredDiagnostics.erase(
@@ -1059,7 +1210,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
         // Create a version of the main file buffer that is padded to
         // buffer size we reserved when creating the preamble.
         return CreatePaddedMainFileBuffer(NewPreamble.first, 
-                                          CreatedPreambleBuffer,
                                           PreambleReservedSize,
                                           FrontendOpts.Inputs[0].second);
       }
@@ -1069,7 +1219,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     // return now.
     if (!AllowRebuild)
       return 0;
-    
+
     // We can't reuse the previously-computed preamble. Build a new one.
     Preamble.clear();
     llvm::sys::Path(PreambleFile).eraseFromDisk();
@@ -1088,14 +1238,19 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     return 0;
   }
 
-  // We did not previously compute a preamble, or it can't be reused anyway.
-  llvm::Timer *PreambleTimer = 0;
-  if (TimerGroup.get()) {
-    PreambleTimer = new llvm::Timer("Precompiling preamble", *TimerGroup);
-    PreambleTimer->startTimer();
-    Timers.push_back(PreambleTimer);
+  // Create a temporary file for the precompiled preamble. In rare 
+  // circumstances, this can fail.
+  std::string PreamblePCHPath = GetPreamblePCHPath();
+  if (PreamblePCHPath.empty()) {
+    // Try again next time.
+    PreambleRebuildCounter = 1;
+    return 0;
   }
   
+  // We did not previously compute a preamble, or it can't be reused anyway.
+  SimpleTimer PreambleTimer(WantTiming);
+  PreambleTimer.setOutput("Precompiling preamble");
+  
   // Create a new buffer that stores the preamble. The buffer also contains
   // extra space for the original contents of the file (which will be present
   // when we actually parse the file) along with more room in case the file
@@ -1129,11 +1284,11 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   
   // Tell the compiler invocation to generate a temporary precompiled header.
   FrontendOpts.ProgramAction = frontend::GeneratePCH;
-  // FIXME: Set ChainedPCH unconditionally, once it is ready.
-  if (::getenv("LIBCLANG_CHAINING"))
-    FrontendOpts.ChainedPCH = true;
+  FrontendOpts.ChainedPCH = true;
   // FIXME: Generate the precompiled header into memory?
-  FrontendOpts.OutputFile = GetPreamblePCHPath();
+  FrontendOpts.OutputFile = PreamblePCHPath;
+  PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
+  PreprocessorOpts.PrecompiledPreambleBytes.second = false;
   
   // Create the compiler instance to use for building the precompiled preamble.
   CompilerInstance Clang;
@@ -1142,20 +1297,14 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   
   // Set up diagnostics, capturing all of the diagnostics produced.
   Clang.setDiagnostics(&getDiagnostics());
-  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
-                                    getDiagnostics(),
-                                    StoredDiagnostics);
   
   // Create the target instance.
+  Clang.getTargetOpts().Features = TargetFeatures;
   Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
                                                Clang.getTargetOpts()));
   if (!Clang.hasTarget()) {
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
-    if (CreatedPreambleBuffer)
-      delete NewPreamble.first;
-    if (PreambleTimer)
-      PreambleTimer->stopTimer();
     PreambleRebuildCounter = DefaultPreambleRebuildInterval;
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
@@ -1176,15 +1325,22 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
          "IR inputs not support here!");
   
   // Clear out old caches and data.
-  StoredDiagnostics.clear();
+  getDiagnostics().Reset();
+  ProcessWarningOptions(getDiagnostics(), Clang.getDiagnosticOpts());
+  StoredDiagnostics.erase(
+                    StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver,
+                          StoredDiagnostics.end());
   TopLevelDecls.clear();
   TopLevelDeclsInPreamble.clear();
+  PreprocessedEntities.clear();
+  PreprocessedEntitiesInPreamble.clear();
   
   // Create a file manager object to provide access to and cache the filesystem.
-  Clang.setFileManager(new FileManager);
+  Clang.setFileManager(new FileManager(Clang.getFileSystemOpts()));
   
   // Create the source manager.
-  Clang.setSourceManager(new SourceManager(getDiagnostics()));
+  Clang.setSourceManager(new SourceManager(getDiagnostics(),
+                                           Clang.getFileManager()));
   
   llvm::OwningPtr<PrecompilePreambleAction> Act;
   Act.reset(new PrecompilePreambleAction(*this));
@@ -1193,10 +1349,6 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     Clang.takeInvocation();
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
-    if (CreatedPreambleBuffer)
-      delete NewPreamble.first;
-    if (PreambleTimer)
-      PreambleTimer->stopTimer();
     PreambleRebuildCounter = DefaultPreambleRebuildInterval;
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
@@ -1213,11 +1365,9 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     // FIXME: Should we leave a note for ourselves to try again?
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
-    if (CreatedPreambleBuffer)
-      delete NewPreamble.first;
-    if (PreambleTimer)
-      PreambleTimer->stopTimer();
     TopLevelDeclsInPreamble.clear();
+    PreprocessedEntities.clear();
+    PreprocessedEntitiesInPreamble.clear();
     PreambleRebuildCounter = DefaultPreambleRebuildInterval;
     PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
@@ -1247,14 +1397,19 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
       = std::make_pair(F->second->getSize(), File->getModificationTime());
   }
   
-  if (PreambleTimer)
-    PreambleTimer->stopTimer();
-  
   PreambleRebuildCounter = 1;
   PreprocessorOpts.eraseRemappedFile(
                                PreprocessorOpts.remapped_file_buffer_end() - 1);
+  
+  // If the hash of top-level entities differs from the hash of the top-level
+  // entities the last time we rebuilt the preamble, clear out the completion
+  // cache.
+  if (CurrentTopLevelHashValue != PreambleTopLevelHashValue) {
+    CompletionCacheTopLevelHashValue = 0;
+    PreambleTopLevelHashValue = CurrentTopLevelHashValue;
+  }
+  
   return CreatePaddedMainFileBuffer(NewPreamble.first, 
-                                    CreatedPreambleBuffer,
                                     PreambleReservedSize,
                                     FrontendOpts.Inputs[0].second);
 }
@@ -1274,14 +1429,90 @@ void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
   TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
 }
 
+void ASTUnit::RealizePreprocessedEntitiesFromPreamble() {
+  if (!PP)
+    return;
+  
+  PreprocessingRecord *PPRec = PP->getPreprocessingRecord();
+  if (!PPRec)
+    return;
+  
+  ExternalPreprocessingRecordSource *External = PPRec->getExternalSource();
+  if (!External)
+    return;
+
+  for (unsigned I = 0, N = PreprocessedEntitiesInPreamble.size(); I != N; ++I) {
+    if (PreprocessedEntity *PE
+          = External->ReadPreprocessedEntityAtOffset(
+                                            PreprocessedEntitiesInPreamble[I]))
+      PreprocessedEntities.push_back(PE);
+  }
+  
+  if (PreprocessedEntities.empty())
+    return;
+  
+  PreprocessedEntities.insert(PreprocessedEntities.end(), 
+                              PPRec->begin(true), PPRec->end(true));
+}
+
+ASTUnit::pp_entity_iterator ASTUnit::pp_entity_begin() {
+  if (!PreprocessedEntitiesInPreamble.empty() &&
+      PreprocessedEntities.empty())
+    RealizePreprocessedEntitiesFromPreamble();
+  
+  if (PreprocessedEntities.empty())
+    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
+      return PPRec->begin(true);
+  
+  return PreprocessedEntities.begin();
+}
+
+ASTUnit::pp_entity_iterator ASTUnit::pp_entity_end() {
+  if (!PreprocessedEntitiesInPreamble.empty() &&
+      PreprocessedEntities.empty())
+    RealizePreprocessedEntitiesFromPreamble();
+  
+  if (PreprocessedEntities.empty())
+    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
+      return PPRec->end(true);
+  
+  return PreprocessedEntities.end();
+}
+
 unsigned ASTUnit::getMaxPCHLevel() const {
   if (!getOnlyLocalDecls())
     return Decl::MaxPCHLevel;
 
-  unsigned Result = 0;
-  if (isMainFileAST() || SavedMainFileBuffer)
-    ++Result;
-  return Result;
+  return 0;
+}
+
+llvm::StringRef ASTUnit::getMainFileName() const {
+  return Invocation->getFrontendOpts().Inputs[0].second;
+}
+
+bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) {
+  if (!Invocation)
+    return true;
+  
+  // We'll manage file buffers ourselves.
+  Invocation->getPreprocessorOpts().RetainRemappedFileBuffers = true;
+  Invocation->getFrontendOpts().DisableFree = false;
+  ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
+
+  // Save the target features.
+  TargetFeatures = Invocation->getTargetOpts().Features;
+  
+  llvm::MemoryBuffer *OverrideMainBuffer = 0;
+  if (PrecompilePreamble) {
+    PreambleRebuildCounter = 2;
+    OverrideMainBuffer
+      = getMainBufferWithPrecompiledPreamble(*Invocation);
+  }
+  
+  SimpleTimer ParsingTimer(WantTiming);
+  ParsingTimer.setOutput("Parsing " + getMainFileName());
+  
+  return Parse(OverrideMainBuffer);
 }
 
 ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
@@ -1290,50 +1521,19 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
                                              bool CaptureDiagnostics,
                                              bool PrecompilePreamble,
                                              bool CompleteTranslationUnit,
-                                             bool CacheCodeCompletionResults) {
-  if (!Diags.getPtr()) {
-    // No diagnostics engine was provided, so create our own diagnostics object
-    // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
-  }
-  
+                                             bool CacheCodeCompletionResults) {  
   // Create the AST unit.
   llvm::OwningPtr<ASTUnit> AST;
   AST.reset(new ASTUnit(false));
+  ConfigureDiags(Diags, 0, 0, *AST, CaptureDiagnostics);
   AST->Diagnostics = Diags;
-  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->OnlyLocalDecls = OnlyLocalDecls;
+  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->CompleteTranslationUnit = CompleteTranslationUnit;
   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
   AST->Invocation.reset(CI);
-  CI->getPreprocessorOpts().RetainRemappedFileBuffers = true;
-  
-  if (getenv("LIBCLANG_TIMING"))
-    AST->TimerGroup.reset(
-                  new llvm::TimerGroup(CI->getFrontendOpts().Inputs[0].second));
   
-  
-  llvm::MemoryBuffer *OverrideMainBuffer = 0;
-  // FIXME: When C++ PCH is ready, allow use of it for a precompiled preamble.
-  if (PrecompilePreamble && !CI->getLangOpts().CPlusPlus) {
-    AST->PreambleRebuildCounter = 1;
-    OverrideMainBuffer
-      = AST->getMainBufferWithPrecompiledPreamble(*AST->Invocation);
-  }
-  
-  llvm::Timer *ParsingTimer = 0;
-  if (AST->TimerGroup.get()) {
-    ParsingTimer = new llvm::Timer("Initial parse", *AST->TimerGroup);
-    ParsingTimer->startTimer();
-    AST->Timers.push_back(ParsingTimer);
-  }
-  
-  bool Failed = AST->Parse(OverrideMainBuffer);
-  if (ParsingTimer)
-    ParsingTimer->stopTimer();
-  
-  return Failed? 0 : AST.take();
+  return AST->LoadFromCompilerInvocation(PrecompilePreamble)? 0 : AST.take();
 }
 
 ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
@@ -1341,20 +1541,20 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                     llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
                                       llvm::StringRef ResourceFilesPath,
                                       bool OnlyLocalDecls,
+                                      bool CaptureDiagnostics,
                                       RemappedFile *RemappedFiles,
                                       unsigned NumRemappedFiles,
-                                      bool CaptureDiagnostics,
                                       bool PrecompilePreamble,
                                       bool CompleteTranslationUnit,
-                                      bool CacheCodeCompletionResults) {
-  bool CreatedDiagnosticsObject = false;
-  
+                                      bool CacheCodeCompletionResults,
+                                      bool CXXPrecompilePreamble,
+                                      bool CXXChainedPCH) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
     DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
-    CreatedDiagnosticsObject = true;
+    Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgEnd - ArgBegin, 
+                                                ArgBegin);
   }
   
   llvm::SmallVector<const char *, 16> Args;
@@ -1365,40 +1565,49 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   // also want to force it to use clang.
   Args.push_back("-fsyntax-only");
 
-  // FIXME: We shouldn't have to pass in the path info.
-  driver::Driver TheDriver("clang", llvm::sys::getHostTriple(),
-                           "a.out", false, false, *Diags);
-
-  // Don't check that inputs exist, they have been remapped.
-  TheDriver.setCheckInputsExist(false);
-
-  llvm::OwningPtr<driver::Compilation> C(
-    TheDriver.BuildCompilation(Args.size(), Args.data()));
+  llvm::SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
+  
+  llvm::OwningPtr<CompilerInvocation> CI;
 
-  // We expect to get back exactly one command job, if we didn't something
-  // failed.
-  const driver::JobList &Jobs = C->getJobs();
-  if (Jobs.size() != 1 || !isa<driver::Command>(Jobs.begin())) {
-    llvm::SmallString<256> Msg;
-    llvm::raw_svector_ostream OS(Msg);
-    C->PrintJob(OS, C->getJobs(), "; ", true);
-    Diags->Report(diag::err_fe_expected_compiler_job) << OS.str();
-    return 0;
-  }
+  {
+    CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags, 
+                                      StoredDiagnostics);
+
+    // FIXME: We shouldn't have to pass in the path info.
+    driver::Driver TheDriver("clang", llvm::sys::getHostTriple(),
+                             "a.out", false, false, *Diags);
+
+    // Don't check that inputs exist, they have been remapped.
+    TheDriver.setCheckInputsExist(false);
+
+    llvm::OwningPtr<driver::Compilation> C(
+      TheDriver.BuildCompilation(Args.size(), Args.data()));
+
+    // We expect to get back exactly one command job, if we didn't something
+    // failed.
+    const driver::JobList &Jobs = C->getJobs();
+    if (Jobs.size() != 1 || !isa<driver::Command>(Jobs.begin())) {
+      llvm::SmallString<256> Msg;
+      llvm::raw_svector_ostream OS(Msg);
+      C->PrintJob(OS, C->getJobs(), "; ", true);
+      Diags->Report(diag::err_fe_expected_compiler_job) << OS.str();
+      return 0;
+    }
 
-  const driver::Command *Cmd = cast<driver::Command>(*Jobs.begin());
-  if (llvm::StringRef(Cmd->getCreator().getName()) != "clang") {
-    Diags->Report(diag::err_fe_expected_clang_command);
-    return 0;
-  }
+    const driver::Command *Cmd = cast<driver::Command>(*Jobs.begin());
+    if (llvm::StringRef(Cmd->getCreator().getName()) != "clang") {
+      Diags->Report(diag::err_fe_expected_clang_command);
+      return 0;
+    }
 
-  const driver::ArgStringList &CCArgs = Cmd->getArguments();
-  llvm::OwningPtr<CompilerInvocation> CI(new CompilerInvocation);
-  CompilerInvocation::CreateFromArgs(*CI,
+    const driver::ArgStringList &CCArgs = Cmd->getArguments();
+    CI.reset(new CompilerInvocation);
+    CompilerInvocation::CreateFromArgs(*CI,
                                      const_cast<const char **>(CCArgs.data()),
                                      const_cast<const char **>(CCArgs.data()) +
-                                     CCArgs.size(),
-                                     *Diags);
+                                       CCArgs.size(),
+                                       *Diags);
+  }
 
   // Override any files that need remapping
   for (unsigned I = 0; I != NumRemappedFiles; ++I)
@@ -1408,26 +1617,44 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   // Override the resources path.
   CI->getHeaderSearchOpts().ResourceDir = ResourceFilesPath;
 
-  CI->getFrontendOpts().DisableFree = false;
-  return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls,
-                                    CaptureDiagnostics, PrecompilePreamble,
-                                    CompleteTranslationUnit,
-                                    CacheCodeCompletionResults);
+  // Check whether we should precompile the preamble and/or use chained PCH.
+  // FIXME: This is a temporary hack while we debug C++ chained PCH.
+  if (CI->getLangOpts().CPlusPlus) {
+    PrecompilePreamble = PrecompilePreamble && CXXPrecompilePreamble;
+    
+    if (PrecompilePreamble && !CXXChainedPCH &&
+        !CI->getPreprocessorOpts().ImplicitPCHInclude.empty())
+      PrecompilePreamble = false;
+  }
+  
+  // Create the AST unit.
+  llvm::OwningPtr<ASTUnit> AST;
+  AST.reset(new ASTUnit(false));
+  ConfigureDiags(Diags, ArgBegin, ArgEnd, *AST, CaptureDiagnostics);
+  AST->Diagnostics = Diags;
+  
+  AST->FileMgr.reset(new FileManager(FileSystemOptions()));
+  AST->OnlyLocalDecls = OnlyLocalDecls;
+  AST->CaptureDiagnostics = CaptureDiagnostics;
+  AST->CompleteTranslationUnit = CompleteTranslationUnit;
+  AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
+  AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
+  AST->NumStoredDiagnosticsInPreamble = StoredDiagnostics.size();
+  AST->StoredDiagnostics.swap(StoredDiagnostics);
+  AST->Invocation.reset(CI.take());
+  return AST->LoadFromCompilerInvocation(PrecompilePreamble) ? 0 : AST.take();
 }
 
 bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
   if (!Invocation.get())
     return true;
   
-  llvm::Timer *ReparsingTimer = 0;
-  if (TimerGroup.get()) {
-    ReparsingTimer = new llvm::Timer("Reparse", *TimerGroup);
-    ReparsingTimer->startTimer();
-    Timers.push_back(ReparsingTimer);
-  }
-  
+  SimpleTimer ParsingTimer(WantTiming);
+  ParsingTimer.setOutput("Reparsing " + getMainFileName());
+
   // Remap files.
   PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
+  PPOpts.DisableStatCache = true;
   for (PreprocessorOptions::remapped_file_buffer_iterator 
          R = PPOpts.remapped_file_buffer_begin(),
          REnd = PPOpts.remapped_file_buffer_end();
@@ -1447,21 +1674,20 @@ bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
     OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(*Invocation);
     
   // Clear out the diagnostics state.
-  if (!OverrideMainBuffer)
+  if (!OverrideMainBuffer) {
     getDiagnostics().Reset();
+    ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
+  }
   
   // Parse the sources
-  bool Result = Parse(OverrideMainBuffer);  
-  if (ReparsingTimer)
-    ReparsingTimer->stopTimer();
-  
-  if (ShouldCacheCodeCompletionResults) {
-    if (CacheCodeCompletionCoolDown > 0)
-      --CacheCodeCompletionCoolDown;
-    else if (top_level_size() != NumTopLevelDeclsAtLastCompletionCache)
-      CacheCodeCompletionResults();
-  }
+  bool Result = Parse(OverrideMainBuffer);
   
+  // If we're caching global code-completion results, and the top-level 
+  // declarations have changed, clear out the code-completion cache.
+  if (!Result && ShouldCacheCodeCompletionResults &&
+      CurrentTopLevelHashValue != CompletionCacheTopLevelHashValue)
+    CacheCodeCompletionResults();
+
   return Result;
 }
 
@@ -1496,8 +1722,10 @@ namespace {
         | (1 << (CodeCompletionContext::CCC_Expression - 1))
         | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1))
         | (1 << (CodeCompletionContext::CCC_MemberAccess - 1))
-        | (1 << (CodeCompletionContext::CCC_ObjCProtocolName - 1));
-      
+        | (1 << (CodeCompletionContext::CCC_ObjCProtocolName - 1))
+        | (1 << (CodeCompletionContext::CCC_ParenthesizedExpression - 1))
+        | (1 << (CodeCompletionContext::CCC_Recovery - 1));
+
       if (AST.getASTContext().getLangOptions().CPlusPlus)
         NormalContexts |= (1 << (CodeCompletionContext::CCC_EnumTag - 1))
                     | (1 << (CodeCompletionContext::CCC_UnionTag - 1))
@@ -1514,19 +1742,23 @@ namespace {
                                            unsigned NumCandidates) { 
       Next.ProcessOverloadCandidates(S, CurrentArg, Candidates, NumCandidates);
     }
+    
+    virtual CodeCompletionAllocator &getAllocator() {
+      return Next.getAllocator();
+    }
   };
 }
 
 /// \brief Helper function that computes which global names are hidden by the
 /// local code-completion results.
-void CalculateHiddenNames(const CodeCompletionContext &Context,
-                          CodeCompletionResult *Results,
-                          unsigned NumResults,
-                          ASTContext &Ctx,
-                          llvm::StringSet<> &HiddenNames) {
+static void CalculateHiddenNames(const CodeCompletionContext &Context,
+                                 CodeCompletionResult *Results,
+                                 unsigned NumResults,
+                                 ASTContext &Ctx,
+                          llvm::StringSet<llvm::BumpPtrAllocator> &HiddenNames){
   bool OnlyTagNames = false;
   switch (Context.getKind()) {
-  case CodeCompletionContext::CCC_Other:
+  case CodeCompletionContext::CCC_Recovery:
   case CodeCompletionContext::CCC_TopLevel:
   case CodeCompletionContext::CCC_ObjCInterface:
   case CodeCompletionContext::CCC_ObjCImplementation:
@@ -1540,6 +1772,7 @@ void CalculateHiddenNames(const CodeCompletionContext &Context,
   case CodeCompletionContext::CCC_Type:
   case CodeCompletionContext::CCC_Name:
   case CodeCompletionContext::CCC_PotentiallyQualifiedName:
+  case CodeCompletionContext::CCC_ParenthesizedExpression:
     break;
     
   case CodeCompletionContext::CCC_EnumTag:
@@ -1556,6 +1789,8 @@ void CalculateHiddenNames(const CodeCompletionContext &Context,
   case CodeCompletionContext::CCC_NaturalLanguage:
   case CodeCompletionContext::CCC_SelectorName:
   case CodeCompletionContext::CCC_TypeQualifiers:
+  case CodeCompletionContext::CCC_Other:
+  case CodeCompletionContext::CCC_OtherWithMacros:
     // We're looking for nothing, or we're looking for names that cannot
     // be hidden.
     return;
@@ -1600,12 +1835,11 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
   // Merge the results we were given with the results we cached.
   bool AddedResult = false;
   unsigned InContexts  
-    = (Context.getKind() == CodeCompletionContext::CCC_Other? NormalContexts
+    = (Context.getKind() == CodeCompletionContext::CCC_Recovery? NormalContexts
                                             : (1 << (Context.getKind() - 1)));
 
   // Contains the set of names that are hidden by "local" completion results.
-  llvm::StringSet<> HiddenNames;
-  llvm::SmallVector<CodeCompletionString *, 4> StringsToDestroy;
+  llvm::StringSet<llvm::BumpPtrAllocator> HiddenNames;
   typedef CodeCompletionResult Result;
   llvm::SmallVector<Result, 8> AllResults;
   for (ASTUnit::cached_completion_iterator 
@@ -1638,6 +1872,7 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
     if (!Context.getPreferredType().isNull()) {
       if (C->Kind == CXCursor_MacroDefinition) {
         Priority = getMacroUsagePriority(C->Completion->getTypedText(),
+                                         S.getLangOptions(),
                                Context.getPreferredType()->isAnyPointerType());        
       } else if (C->Type) {
         CanQualType Expected
@@ -1663,11 +1898,12 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
         Context.getKind() == CodeCompletionContext::CCC_MacroNameUse) {
       // Create a new code-completion string that just contains the
       // macro name, without its arguments.
-      Completion = new CodeCompletionString;
-      Completion->AddTypedTextChunk(C->Completion->getTypedText());
-      StringsToDestroy.push_back(Completion);
+      CodeCompletionBuilder Builder(getAllocator(), CCP_CodePattern,
+                                    C->Availability);
+      Builder.AddTypedTextChunk(C->Completion->getTypedText());
       CursorKind = CXCursor_NotImplemented;
       Priority = CCP_CodePattern;
+      Completion = Builder.TakeString();
     }
     
     AllResults.push_back(Result(Completion, Priority, CursorKind, 
@@ -1683,9 +1919,6 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
   
   Next.ProcessCodeCompleteResults(S, Context, AllResults.data(),
                                   AllResults.size());
-  
-  for (unsigned I = 0, N = StringsToDestroy.size(); I != N; ++I)
-    delete StringsToDestroy[I];
 }
 
 
@@ -1703,16 +1936,9 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
   if (!Invocation.get())
     return;
 
-  llvm::Timer *CompletionTimer = 0;
-  if (TimerGroup.get()) {
-    llvm::SmallString<128> TimerName;
-    llvm::raw_svector_ostream TimerNameOut(TimerName);
-    TimerNameOut << "Code completion @ " << File << ":" << Line << ":" 
-                 << Column;
-    CompletionTimer = new llvm::Timer(TimerNameOut.str(), *TimerGroup);
-    CompletionTimer->startTimer();
-    Timers.push_back(CompletionTimer);
-  }
+  SimpleTimer CompletionTimer(WantTiming);
+  CompletionTimer.setOutput("Code completion @ " + File + ":" +
+                            llvm::Twine(Line) + ":" + llvm::Twine(Column));
 
   CompilerInvocation CCInvocation(*Invocation);
   FrontendOptions &FrontendOpts = CCInvocation.getFrontendOpts();
@@ -1727,11 +1953,6 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
   FrontendOpts.CodeCompletionAt.Line = Line;
   FrontendOpts.CodeCompletionAt.Column = Column;
 
-  // Turn on spell-checking when performing code completion. It leads
-  // to better results.
-  unsigned SpellChecking = CCInvocation.getLangOpts().SpellChecking;
-  CCInvocation.getLangOpts().SpellChecking = 1;
-
   // Set the language options appropriately.
   LangOpts = CCInvocation.getLangOpts();
 
@@ -1741,16 +1962,17 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
     
   // Set up diagnostics, capturing any diagnostics produced.
   Clang.setDiagnostics(&Diag);
+  ProcessWarningOptions(Diag, CCInvocation.getDiagnosticOpts());
   CaptureDroppedDiagnostics Capture(true, 
-                                    Clang.getDiagnostics(),
+                                    Clang.getDiagnostics(), 
                                     StoredDiagnostics);
   
   // Create the target instance.
+  Clang.getTargetOpts().Features = TargetFeatures;
   Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
                                                Clang.getTargetOpts()));
   if (!Clang.hasTarget()) {
     Clang.takeInvocation();
-    CCInvocation.getLangOpts().SpellChecking = SpellChecking;
     return;
   }
   
@@ -1783,11 +2005,12 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
   
   // Use the code completion consumer we were given, but adding any cached
   // code-completion results.
-  AugmentedCodeCompleteConsumer 
-  AugmentedConsumer(*this, Consumer, FrontendOpts.ShowMacrosInCodeCompletion,
-                    FrontendOpts.ShowCodePatternsInCodeCompletion,
-                    FrontendOpts.ShowGlobalSymbolsInCodeCompletion);
-  Clang.setCodeCompletionConsumer(&AugmentedConsumer);
+  AugmentedCodeCompleteConsumer *AugmentedConsumer
+    = new AugmentedCodeCompleteConsumer(*this, Consumer, 
+                                        FrontendOpts.ShowMacrosInCodeCompletion,
+                                FrontendOpts.ShowCodePatternsInCodeCompletion,
+                                FrontendOpts.ShowGlobalSymbolsInCodeCompletion);
+  Clang.setCodeCompletionConsumer(AugmentedConsumer);
 
   // If we have a precompiled preamble, try to use it. We only allow
   // the use of the precompiled preamble if we're if the completion
@@ -1808,6 +2031,10 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
 
   // If the main file has been overridden due to the use of a preamble,
   // make that override happen and introduce the preamble.
+  PreprocessorOpts.DisableStatCache = true;
+  StoredDiagnostics.insert(StoredDiagnostics.end(),
+                           this->StoredDiagnostics.begin(),
+             this->StoredDiagnostics.begin() + NumStoredDiagnosticsFromDriver);
   if (OverrideMainBuffer) {
     PreprocessorOpts.addRemappedFile(OriginalSourceFile, OverrideMainBuffer);
     PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size();
@@ -1819,12 +2046,14 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
     // The stored diagnostics have the old source manager. Copy them
     // to our output set of stored diagnostics, updating the source
     // manager to the one we were given.
-    for (unsigned I = 0, N = this->StoredDiagnostics.size(); I != N; ++I) {
+    for (unsigned I = NumStoredDiagnosticsFromDriver, 
+                  N = this->StoredDiagnostics.size(); 
+         I < N; ++I) {
       StoredDiagnostics.push_back(this->StoredDiagnostics[I]);
       FullSourceLoc Loc(StoredDiagnostics[I].getLocation(), SourceMgr);
       StoredDiagnostics[I].setLocation(Loc);
     }
-    
+
     OwnedBuffers.push_back(OverrideMainBuffer);
   } else {
     PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
@@ -1839,15 +2068,10 @@ void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
     Act->EndSourceFile();
   }
 
-  if (CompletionTimer)
-    CompletionTimer->stopTimer();
-  
   // Steal back our resources. 
   Clang.takeFileManager();
   Clang.takeSourceManager();
   Clang.takeInvocation();
-  Clang.takeCodeCompletionConsumer();
-  CCInvocation.getLangOpts().SpellChecking = SpellChecking;
 }
 
 bool ASTUnit::Save(llvm::StringRef File) {
@@ -1865,7 +2089,7 @@ bool ASTUnit::Save(llvm::StringRef File) {
   std::vector<unsigned char> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
   ASTWriter Writer(Stream);
-  Writer.WriteAST(getSema(), 0, 0);
+  Writer.WriteAST(getSema(), 0, std::string(), 0);
   
   // Write the generated bitstream to "Out".
   if (!Buffer.empty())
diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt
index 5a31495397ae..9f197b4f8762 100644
--- a/lib/Frontend/CMakeLists.txt
+++ b/lib/Frontend/CMakeLists.txt
@@ -1,4 +1,12 @@
-set(LLVM_NO_RTTI 1)
+set( LLVM_USED_LIBS
+  clangAST
+  clangBasic
+  clangDriver
+  clangLex
+  clangParse
+  clangSema
+  clangSerialization
+  )
 
 add_clang_library(clangFrontend
   ASTConsumers.cpp
@@ -15,9 +23,11 @@ add_clang_library(clangFrontend
   FrontendAction.cpp
   FrontendActions.cpp
   FrontendOptions.cpp
+  HeaderIncludeGen.cpp
   InitHeaderSearch.cpp
   InitPreprocessor.cpp
   LangStandards.cpp
+  MultiplexConsumer.cpp
   PrintPreprocessedOutput.cpp
   StmtXML.cpp
   TextDiagnosticBuffer.cpp
@@ -38,6 +48,7 @@ ENDIF(MSVC)
 add_dependencies(clangFrontend 
   ClangAttrClasses
   ClangAttrList
+  ClangCC1Options
   ClangDiagnosticFrontend 
   ClangDiagnosticLex
   ClangDiagnosticSema
diff --git a/lib/Frontend/CacheTokens.cpp b/lib/Frontend/CacheTokens.cpp
index 53f7362ac276..ee3fdd834334 100644
--- a/lib/Frontend/CacheTokens.cpp
+++ b/lib/Frontend/CacheTokens.cpp
@@ -13,18 +13,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/Utils.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "clang/Basic/IdentifierTable.h"
-#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/OnDiskHashTable.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 
 // FIXME: put this somewhere else?
 #ifndef S_ISDIR
@@ -189,8 +191,6 @@ class PTHWriter {
 
   void Emit16(uint32_t V) { ::Emit16(Out, V); }
 
-  void Emit24(uint32_t V) { ::Emit24(Out, V); }
-
   void Emit32(uint32_t V) { ::Emit32(Out, V); }
 
   void EmitBuf(const char *Ptr, unsigned NumBytes) {
@@ -300,7 +300,7 @@ PTHEntry PTHWriter::LexTokens(Lexer& L) {
       ParsingPreprocessorDirective = false;
     }
 
-    if (Tok.is(tok::identifier)) {
+    if (Tok.is(tok::raw_identifier)) {
       PP.LookUpIdentifierInfo(Tok);
       EmitToken(Tok);
       continue;
@@ -320,13 +320,13 @@ PTHEntry PTHWriter::LexTokens(Lexer& L) {
       // this case, discard both tokens.
       if (NextTok.isAtStartOfLine())
         goto NextToken;
-      
+
       // The token is the start of a directive.  Emit it.
       EmitToken(Tok);
       Tok = NextTok;
 
       // Did we see 'include'/'import'/'include_next'?
-      if (Tok.isNot(tok::identifier)) {
+      if (Tok.isNot(tok::raw_identifier)) {
         EmitToken(Tok);
         continue;
       }
@@ -353,7 +353,7 @@ PTHEntry PTHWriter::LexTokens(Lexer& L) {
         L.LexIncludeFilename(Tok);
         L.setParsingPreprocessorDirective(false);
         assert(!Tok.isAtStartOfLine());
-        if (Tok.is(tok::identifier))
+        if (Tok.is(tok::raw_identifier))
           PP.LookUpIdentifierInfo(Tok);
 
         break;
@@ -476,8 +476,7 @@ void PTHWriter::GeneratePTH(const std::string &MainFile) {
     const FileEntry *FE = C.Entry;
 
     // FIXME: Handle files with non-absolute paths.
-    llvm::sys::Path P(FE->getName());
-    if (!P.isAbsolute())
+    if (llvm::sys::path::is_relative(FE->getName()))
       continue;
 
     const llvm::MemoryBuffer *B = C.getBuffer(PP.getDiagnostics(), SM);
@@ -512,26 +511,27 @@ namespace {
 /// as input to PTH generation.  StatListener populates the PTHWriter's
 /// file map with stat information for directories as well as negative stats.
 /// Stat information for files are populated elsewhere.
-class StatListener : public StatSysCallCache {
+class StatListener : public FileSystemStatCache {
   PTHMap &PM;
 public:
   StatListener(PTHMap &pm) : PM(pm) {}
   ~StatListener() {}
 
-  int stat(const char *path, struct stat *buf) {
-    int result = StatSysCallCache::stat(path, buf);
+  LookupResult getStat(const char *Path, struct stat &StatBuf,
+                       int *FileDescriptor) {
+    LookupResult Result = statChained(Path, StatBuf, FileDescriptor);
 
-    if (result != 0) // Failed 'stat'.
-      PM.insert(PTHEntryKeyVariant(path), PTHEntry());
-    else if (S_ISDIR(buf->st_mode)) {
+    if (Result == CacheMissing) // Failed 'stat'.
+      PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
+    else if (S_ISDIR(StatBuf.st_mode)) {
       // Only cache directories with absolute paths.
-      if (!llvm::sys::Path(path).isAbsolute())
-        return result;
+      if (llvm::sys::path::is_relative(Path))
+        return Result;
 
-      PM.insert(PTHEntryKeyVariant(buf, path), PTHEntry());
+      PM.insert(PTHEntryKeyVariant(&StatBuf, Path), PTHEntry());
     }
 
-    return result;
+    return Result;
   }
 };
 } // end anonymous namespace
@@ -541,9 +541,9 @@ void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) {
   // Get the name of the main file.
   const SourceManager &SrcMgr = PP.getSourceManager();
   const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
-  llvm::sys::Path MainFilePath(MainFile->getName());
+  llvm::SmallString<128> MainFilePath(MainFile->getName());
 
-  MainFilePath.makeAbsolute();
+  llvm::sys::fs::make_absolute(MainFilePath);
 
   // Create the PTHWriter.
   PTHWriter PW(*OS, PP);
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index ce0b07243b05..fd593de560c0 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -27,14 +27,16 @@
 #include "clang/Frontend/Utils.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
-#include "llvm/LLVMContext.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Timer.h"
-#include "llvm/System/Host.h"
-#include "llvm/System/Path.h"
-#include "llvm/System/Program.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/system_error.h"
 using namespace clang;
 
 CompilerInstance::CompilerInstance()
@@ -44,10 +46,6 @@ CompilerInstance::CompilerInstance()
 CompilerInstance::~CompilerInstance() {
 }
 
-void CompilerInstance::setLLVMContext(llvm::LLVMContext *Value) {
-  LLVMContext.reset(Value);
-}
-
 void CompilerInstance::setInvocation(CompilerInvocation *Value) {
   Invocation.reset(Value);
 }
@@ -89,26 +87,8 @@ void CompilerInstance::setCodeCompletionConsumer(CodeCompleteConsumer *Value) {
 }
 
 // Diagnostics
-namespace {
-  class BinaryDiagnosticSerializer : public DiagnosticClient {
-    llvm::raw_ostream &OS;
-    SourceManager *SourceMgr;
-  public:
-    explicit BinaryDiagnosticSerializer(llvm::raw_ostream &OS)
-      : OS(OS), SourceMgr(0) { }
-
-    virtual void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                  const DiagnosticInfo &Info);
-  };
-}
-
-void BinaryDiagnosticSerializer::HandleDiagnostic(Diagnostic::Level DiagLevel,
-                                                  const DiagnosticInfo &Info) {
-  StoredDiagnostic(DiagLevel, Info).Serialize(OS);
-}
-
 static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
-                              unsigned argc, char **argv,
+                              unsigned argc, const char* const *argv,
                               Diagnostic &Diags) {
   std::string ErrorInfo;
   llvm::OwningPtr<llvm::raw_ostream> OS(
@@ -130,33 +110,24 @@ static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
   Diags.setClient(new ChainedDiagnosticClient(Diags.takeClient(), Logger));
 }
 
-void CompilerInstance::createDiagnostics(int Argc, char **Argv) {
-  Diagnostics = createDiagnostics(getDiagnosticOpts(), Argc, Argv);
+void CompilerInstance::createDiagnostics(int Argc, const char* const *Argv,
+                                         DiagnosticClient *Client) {
+  Diagnostics = createDiagnostics(getDiagnosticOpts(), Argc, Argv, Client);
 }
 
 llvm::IntrusiveRefCntPtr<Diagnostic> 
 CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
-                                    int Argc, char **Argv) {
-  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(new Diagnostic());
+                                    int Argc, const char* const *Argv,
+                                    DiagnosticClient *Client) {
+  llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+  llvm::IntrusiveRefCntPtr<Diagnostic> Diags(new Diagnostic(DiagID));
 
   // Create the diagnostic client for reporting errors or for
   // implementing -verify.
-  llvm::OwningPtr<DiagnosticClient> DiagClient;
-  if (Opts.BinaryOutput) {
-    if (llvm::sys::Program::ChangeStderrToBinary()) {
-      // We weren't able to set standard error to binary, which is a
-      // bit of a problem. So, just create a text diagnostic printer
-      // to complain about this problem, and pretend that the user
-      // didn't try to use binary output.
-      Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
-      Diags->Report(diag::err_fe_stderr_binary);
-      return Diags;
-    } else {
-      Diags->setClient(new BinaryDiagnosticSerializer(llvm::errs()));
-    }
-  } else {
+  if (Client)
+    Diags->setClient(Client);
+  else
     Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
-  }
 
   // Chain in -verify checker, if requested.
   if (Opts.VerifyDiagnostics)
@@ -174,13 +145,13 @@ CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
 // File Manager
 
 void CompilerInstance::createFileManager() {
-  FileMgr.reset(new FileManager());
+  FileMgr.reset(new FileManager(getFileSystemOpts()));
 }
 
 // Source Manager
 
-void CompilerInstance::createSourceManager() {
-  SourceMgr.reset(new SourceManager(getDiagnostics()));
+void CompilerInstance::createSourceManager(FileManager &FileMgr) {
+  SourceMgr.reset(new SourceManager(getDiagnostics(), FileMgr));
 }
 
 // Preprocessor
@@ -231,6 +202,16 @@ CompilerInstance::createPreprocessor(Diagnostic &Diags,
   if (!DepOpts.OutputFile.empty())
     AttachDependencyFileGen(*PP, DepOpts);
 
+  // Handle generating header include information, if requested.
+  if (DepOpts.ShowHeaderIncludes)
+    AttachHeaderIncludeGen(*PP);
+  if (!DepOpts.HeaderIncludeOutputFile.empty()) {
+    llvm::StringRef OutputPath = DepOpts.HeaderIncludeOutputFile;
+    if (OutputPath == "-")
+      OutputPath = "";
+    AttachHeaderIncludeGen(*PP, /*ShowAllHeaders=*/true, OutputPath);
+  }
+
   return PP;
 }
 
@@ -248,12 +229,16 @@ void CompilerInstance::createASTContext() {
 
 void CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
                                                   bool DisablePCHValidation,
+                                                  bool DisableStatCache,
                                                  void *DeserializationListener){
   llvm::OwningPtr<ExternalASTSource> Source;
+  bool Preamble = getPreprocessorOpts().PrecompiledPreambleBytes.first != 0;
   Source.reset(createPCHExternalASTSource(Path, getHeaderSearchOpts().Sysroot,
-                                          DisablePCHValidation,
+                                          DisablePCHValidation, 
+                                          DisableStatCache,
                                           getPreprocessor(), getASTContext(),
-                                          DeserializationListener));
+                                          DeserializationListener,
+                                          Preamble));
   getASTContext().setExternalSource(Source);
 }
 
@@ -261,17 +246,20 @@ ExternalASTSource *
 CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
                                              const std::string &Sysroot,
                                              bool DisablePCHValidation,
+                                             bool DisableStatCache,
                                              Preprocessor &PP,
                                              ASTContext &Context,
-                                             void *DeserializationListener) {
+                                             void *DeserializationListener,
+                                             bool Preamble) {
   llvm::OwningPtr<ASTReader> Reader;
   Reader.reset(new ASTReader(PP, &Context,
                              Sysroot.empty() ? 0 : Sysroot.c_str(),
-                             DisablePCHValidation));
+                             DisablePCHValidation, DisableStatCache));
 
   Reader->setDeserializationListener(
             static_cast<ASTDeserializationListener *>(DeserializationListener));
-  switch (Reader->ReadAST(Path)) {
+  switch (Reader->ReadAST(Path,
+                          Preamble ? ASTReader::Preamble : ASTReader::PCH)) {
   case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader. Typically, the
     // predefines buffer will be empty.
@@ -316,7 +304,6 @@ void CompilerInstance::createCodeCompletionConsumer() {
     CompletionConsumer.reset(
       createCodeCompletionConsumer(getPreprocessor(),
                                    Loc.FileName, Loc.Line, Loc.Column,
-                                   getFrontendOpts().DebugCodeCompletionPrinter,
                                    getFrontendOpts().ShowMacrosInCodeCompletion,
                              getFrontendOpts().ShowCodePatternsInCodeCompletion,
                            getFrontendOpts().ShowGlobalSymbolsInCodeCompletion,
@@ -345,7 +332,6 @@ CompilerInstance::createCodeCompletionConsumer(Preprocessor &PP,
                                                const std::string &Filename,
                                                unsigned Line,
                                                unsigned Column,
-                                               bool UseDebugPrinter,
                                                bool ShowMacros,
                                                bool ShowCodePatterns,
                                                bool ShowGlobals,
@@ -354,11 +340,7 @@ CompilerInstance::createCodeCompletionConsumer(Preprocessor &PP,
     return 0;
 
   // Set up the creation routine for code-completion.
-  if (UseDebugPrinter)
-    return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
-                                            ShowGlobals, OS);
-  else
-    return new CIndexCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
+  return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
                                           ShowGlobals, OS);
 }
 
@@ -370,18 +352,34 @@ void CompilerInstance::createSema(bool CompleteTranslationUnit,
 
 // Output Files
 
-void CompilerInstance::addOutputFile(llvm::StringRef Path,
-                                     llvm::raw_ostream *OS) {
-  assert(OS && "Attempt to add empty stream to output list!");
-  OutputFiles.push_back(std::make_pair(Path, OS));
+void CompilerInstance::addOutputFile(const OutputFile &OutFile) {
+  assert(OutFile.OS && "Attempt to add empty stream to output list!");
+  OutputFiles.push_back(OutFile);
 }
 
 void CompilerInstance::clearOutputFiles(bool EraseFiles) {
-  for (std::list< std::pair<std::string, llvm::raw_ostream*> >::iterator
+  for (std::list<OutputFile>::iterator
          it = OutputFiles.begin(), ie = OutputFiles.end(); it != ie; ++it) {
-    delete it->second;
-    if (EraseFiles && !it->first.empty())
-      llvm::sys::Path(it->first).eraseFromDisk();
+    delete it->OS;
+    if (!it->TempFilename.empty()) {
+      llvm::sys::Path TempPath(it->TempFilename);
+      if (EraseFiles)
+        TempPath.eraseFromDisk();
+      else {
+        std::string Error;
+        llvm::sys::Path NewOutFile(it->Filename);
+        // If '-working-directory' was passed, the output filename should be
+        // relative to that.
+        FileManager::FixupRelativePath(NewOutFile, getFileSystemOpts());
+        if (TempPath.renamePathOnDisk(NewOutFile, &Error)) {
+          getDiagnostics().Report(diag::err_fe_unable_to_rename_temp)
+            << it->TempFilename << it->Filename << Error;
+          TempPath.eraseFromDisk();
+        }
+      }
+    } else if (!it->Filename.empty() && EraseFiles)
+      llvm::sys::Path(it->Filename).eraseFromDisk();
+      
   }
   OutputFiles.clear();
 }
@@ -391,18 +389,20 @@ CompilerInstance::createDefaultOutputFile(bool Binary,
                                           llvm::StringRef InFile,
                                           llvm::StringRef Extension) {
   return createOutputFile(getFrontendOpts().OutputFile, Binary,
-                          InFile, Extension);
+                          /*RemoveFileOnSignal=*/true, InFile, Extension);
 }
 
 llvm::raw_fd_ostream *
 CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
-                                   bool Binary,
+                                   bool Binary, bool RemoveFileOnSignal,
                                    llvm::StringRef InFile,
                                    llvm::StringRef Extension) {
-  std::string Error, OutputPathName;
+  std::string Error, OutputPathName, TempPathName;
   llvm::raw_fd_ostream *OS = createOutputFile(OutputPath, Error, Binary,
+                                              RemoveFileOnSignal,
                                               InFile, Extension,
-                                              &OutputPathName);
+                                              &OutputPathName,
+                                              &TempPathName);
   if (!OS) {
     getDiagnostics().Report(diag::err_fe_unable_to_open_output)
       << OutputPath << Error;
@@ -411,7 +411,8 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
 
   // Add the output file -- but don't try to remove "-", since this means we are
   // using stdin.
-  addOutputFile((OutputPathName != "-") ? OutputPathName : "", OS);
+  addOutputFile(OutputFile((OutputPathName != "-") ? OutputPathName : "",
+                TempPathName, OS));
 
   return OS;
 }
@@ -420,10 +421,12 @@ llvm::raw_fd_ostream *
 CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
                                    std::string &Error,
                                    bool Binary,
+                                   bool RemoveFileOnSignal,
                                    llvm::StringRef InFile,
                                    llvm::StringRef Extension,
-                                   std::string *ResultPathName) {
-  std::string OutFile;
+                                   std::string *ResultPathName,
+                                   std::string *TempPathName) {
+  std::string OutFile, TempFile;
   if (!OutputPath.empty()) {
     OutFile = OutputPath;
   } else if (InFile == "-") {
@@ -436,15 +439,39 @@ CompilerInstance::createOutputFile(llvm::StringRef OutputPath,
   } else {
     OutFile = "-";
   }
+  
+  if (OutFile != "-") {
+    llvm::sys::Path OutPath(OutFile);
+    // Only create the temporary if we can actually write to OutPath, otherwise
+    // we want to fail early.
+    bool Exists;
+    if ((llvm::sys::fs::exists(OutPath.str(), Exists) || !Exists) ||
+        (OutPath.isRegularFile() && OutPath.canWrite())) {
+      // Create a temporary file.
+      llvm::sys::Path TempPath(OutFile);
+      if (!TempPath.createTemporaryFileOnDisk())
+        TempFile = TempPath.str();
+    }
+  }
+
+  std::string OSFile = OutFile;
+  if (!TempFile.empty())
+    OSFile = TempFile;
 
   llvm::OwningPtr<llvm::raw_fd_ostream> OS(
-    new llvm::raw_fd_ostream(OutFile.c_str(), Error,
+    new llvm::raw_fd_ostream(OSFile.c_str(), Error,
                              (Binary ? llvm::raw_fd_ostream::F_Binary : 0)));
   if (!Error.empty())
     return 0;
 
+  // Make sure the out stream file gets removed if we crash.
+  if (RemoveFileOnSignal)
+    llvm::sys::RemoveFileOnSignal(llvm::sys::Path(OSFile));
+
   if (ResultPathName)
     *ResultPathName = OutFile;
+  if (TempPathName)
+    *TempPathName = TempFile;
 
   return OS.take();
 }
@@ -461,23 +488,32 @@ bool CompilerInstance::InitializeSourceManager(llvm::StringRef InputFile,
                                                FileManager &FileMgr,
                                                SourceManager &SourceMgr,
                                                const FrontendOptions &Opts) {
-  // Figure out where to get and map in the main file.
-  if (InputFile != "-") {
+  // Figure out where to get and map in the main file, unless it's already
+  // been created (e.g., by a precompiled preamble).
+  if (!SourceMgr.getMainFileID().isInvalid()) {
+    // Do nothing: the main file has already been set.
+  } else if (InputFile != "-") {
     const FileEntry *File = FileMgr.getFile(InputFile);
-    if (File) SourceMgr.createMainFileID(File);
-    if (SourceMgr.getMainFileID().isInvalid()) {
+    if (!File) {
       Diags.Report(diag::err_fe_error_reading) << InputFile;
       return false;
     }
+    SourceMgr.createMainFileID(File);
   } else {
-    llvm::MemoryBuffer *SB = llvm::MemoryBuffer::getSTDIN();
-    if (SB) SourceMgr.createMainFileIDForMemBuffer(SB);
-    if (SourceMgr.getMainFileID().isInvalid()) {
+    llvm::OwningPtr<llvm::MemoryBuffer> SB;
+    if (llvm::MemoryBuffer::getSTDIN(SB)) {
+      // FIXME: Give ec.message() in this diag.
       Diags.Report(diag::err_fe_error_reading_stdin);
       return false;
     }
+    const FileEntry *File = FileMgr.getVirtualFile(SB->getBufferIdentifier(),
+                                                   SB->getBufferSize(), 0);
+    SourceMgr.createMainFileID(File);
+    SourceMgr.overrideFileContents(File, SB.take());
   }
 
+  assert(!SourceMgr.getMainFileID().isInvalid() &&
+         "Couldn't establish MainFileID!");
   return true;
 }
 
@@ -529,9 +565,10 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
   }
 
   if (getDiagnosticOpts().ShowCarets) {
-    unsigned NumWarnings = getDiagnostics().getNumWarnings();
-    unsigned NumErrors = getDiagnostics().getNumErrors() - 
-                               getDiagnostics().getNumErrorsSuppressed();
+    // We can have multiple diagnostics sharing one diagnostic client.
+    // Get the total number of warnings/errors from the client.
+    unsigned NumWarnings = getDiagnostics().getClient()->getNumWarnings();
+    unsigned NumErrors = getDiagnostics().getClient()->getNumErrors();
     
     if (NumWarnings)
       OS << NumWarnings << " warning" << (NumWarnings == 1 ? "" : "s");
@@ -548,15 +585,7 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
     OS << "\n";
   }
 
-  // Return the appropriate status when verifying diagnostics.
-  //
-  // FIXME: If we could make getNumErrors() do the right thing, we wouldn't need
-  // this.
-  if (getDiagnosticOpts().VerifyDiagnostics)
-    return !static_cast<VerifyDiagnosticsClient&>(
-      getDiagnosticClient()).HadErrors();
-
-  return !getDiagnostics().getNumErrors();
+  return !getDiagnostics().getClient()->getNumErrors();
 }
 
 
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 8c644833b20e..103d251a88f5 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -10,6 +10,7 @@
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/Version.h"
+#include "clang/Basic/FileManager.h"
 #include "clang/Driver/Arg.h"
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/CC1Options.h"
@@ -25,8 +26,8 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/System/Host.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/Path.h"
 using namespace clang;
 
 static const char *getAnalysisName(Analyses Kind) {
@@ -99,6 +100,8 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
     Res.push_back("-analyzer-display-progress");
   if (Opts.AnalyzeNestedBlocks)
     Res.push_back("-analyzer-opt-analyze-nested-blocks");
+  if (Opts.AnalyzerStats)
+    Res.push_back("-analyzer-stats");
   if (Opts.EagerlyAssume)
     Res.push_back("-analyzer-eagerly-assume");
   if (!Opts.PurgeDead)
@@ -113,8 +116,17 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
     Res.push_back("-analyzer-experimental-checks");
   if (Opts.EnableExperimentalInternalChecks)
     Res.push_back("-analyzer-experimental-internal-checks");
-  if (Opts.IdempotentOps)
-      Res.push_back("-analyzer-check-idempotent-operations");
+  if (Opts.BufferOverflows)
+    Res.push_back("-analyzer-check-buffer-overflows");
+
+  for (unsigned i = 0, e = Opts.CheckersControlList.size(); i != e; ++i) {
+    const std::pair<std::string, bool> &opt = Opts.CheckersControlList[i];
+    if (opt.second)
+      Res.push_back("-analyzer-disable-checker");
+    else
+      Res.push_back("-analyzer-checker");
+    Res.push_back(opt.first);
+  }
 }
 
 static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
@@ -150,7 +162,7 @@ static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
   // TimePasses is only derived.
   // UnitAtATime is unused.
   // Inlining is only derived.
-  
+
   // UnrollLoops is derived, but also accepts an option, no
   // harm in pushing it back here.
   if (Opts.UnrollLoops)
@@ -195,6 +207,10 @@ static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
     Res.push_back("-fobjc-dispatch-method=non-legacy");
     break;
   }
+  if (Opts.NumRegisterParameters) {
+    Res.push_back("-mregparm");
+    Res.push_back(llvm::utostr(Opts.NumRegisterParameters));
+  }
   if (Opts.RelaxAll)
     Res.push_back("-mrelax-all");
   if (Opts.SoftFloat)
@@ -213,6 +229,12 @@ static void DependencyOutputOptsToArgs(const DependencyOutputOptions &Opts,
                                        std::vector<std::string> &Res) {
   if (Opts.IncludeSystemHeaders)
     Res.push_back("-sys-header-deps");
+  if (Opts.ShowHeaderIncludes)
+    Res.push_back("-H");
+  if (!Opts.HeaderIncludeOutputFile.empty()) {
+    Res.push_back("-header-include-file");
+    Res.push_back(Opts.HeaderIncludeOutputFile);
+  }
   if (Opts.UsePhonyTargets)
     Res.push_back("-MP");
   if (!Opts.OutputFile.empty()) {
@@ -251,8 +273,6 @@ static void DiagnosticOptsToArgs(const DiagnosticOptions &Opts,
     Res.push_back("-fcolor-diagnostics");
   if (Opts.VerifyDiagnostics)
     Res.push_back("-verify");
-  if (Opts.BinaryOutput)
-    Res.push_back("-fdiagnostics-binary");
   if (Opts.ShowOptionNames)
     Res.push_back("-fdiagnostics-show-option");
   if (Opts.ShowCategories == 1)
@@ -301,6 +321,7 @@ static const char *getInputKindName(InputKind Kind) {
   case IK_ObjC:              return "objective-c";
   case IK_ObjCXX:            return "objective-c++";
   case IK_OpenCL:            return "cl";
+  case IK_CUDA:              return "cuda";
   case IK_PreprocessedC:     return "cpp-output";
   case IK_PreprocessedCXX:   return "c++-cpp-output";
   case IK_PreprocessedObjC:  return "objective-c-cpp-output";
@@ -314,10 +335,10 @@ static const char *getInputKindName(InputKind Kind) {
 static const char *getActionName(frontend::ActionKind Kind) {
   switch (Kind) {
   case frontend::PluginAction:
-  case frontend::InheritanceView:
     llvm_unreachable("Invalid kind!");
 
   case frontend::ASTDump:                return "-ast-dump";
+  case frontend::ASTDumpXML:             return "-ast-dump-xml";
   case frontend::ASTPrint:               return "-ast-print";
   case frontend::ASTPrintXML:            return "-ast-print-xml";
   case frontend::ASTView:                return "-ast-view";
@@ -351,10 +372,16 @@ static const char *getActionName(frontend::ActionKind Kind) {
   return 0;
 }
 
+static void FileSystemOptsToArgs(const FileSystemOptions &Opts,
+                                 std::vector<std::string> &Res) {
+  if (!Opts.WorkingDir.empty()) {
+    Res.push_back("-working-directory");
+    Res.push_back(Opts.WorkingDir);
+  }
+}
+
 static void FrontendOptsToArgs(const FrontendOptions &Opts,
                                std::vector<std::string> &Res) {
-  if (!Opts.DebugCodeCompletionPrinter)
-    Res.push_back("-no-code-completion-debug-printer");
   if (Opts.DisableFree)
     Res.push_back("-disable-free");
   if (Opts.RelocatablePCH)
@@ -397,18 +424,13 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-o");
     Res.push_back(Opts.OutputFile);
   }
-  if (!Opts.ViewClassInheritance.empty()) {
-    Res.push_back("-cxx-inheritance-view");
-    Res.push_back(Opts.ViewClassInheritance);
-  }
   if (!Opts.CodeCompletionAt.FileName.empty()) {
     Res.push_back("-code-completion-at");
     Res.push_back(Opts.CodeCompletionAt.FileName + ":" +
                   llvm::utostr(Opts.CodeCompletionAt.Line) + ":" +
                   llvm::utostr(Opts.CodeCompletionAt.Column));
   }
-  if (Opts.ProgramAction != frontend::InheritanceView &&
-      Opts.ProgramAction != frontend::PluginAction)
+  if (Opts.ProgramAction != frontend::PluginAction)
     Res.push_back(getActionName(Opts.ProgramAction));
   if (!Opts.ActionName.empty()) {
     Res.push_back("-plugin");
@@ -422,6 +444,14 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-load");
     Res.push_back(Opts.Plugins[i]);
   }
+  for (unsigned i = 0, e = Opts.AddPluginActions.size(); i != e; ++i) {
+    Res.push_back("-add-plugin");
+    Res.push_back(Opts.AddPluginActions[i]);
+    for(unsigned ai = 0, ae = Opts.AddPluginArgs.size(); ai != ae; ++ai) {
+      Res.push_back("-plugin-arg-" + Opts.AddPluginActions[i]);
+      Res.push_back(Opts.AddPluginArgs[i][ai]);
+    }
+  }
   for (unsigned i = 0, e = Opts.ASTMergeFiles.size(); i != e; ++i) {
     Res.push_back("-ast-merge");
     Res.push_back(Opts.ASTMergeFiles[i]);
@@ -443,6 +473,11 @@ static void HeaderSearchOptsToArgs(const HeaderSearchOptions &Opts,
     Res.push_back(Opts.Sysroot);
   }
 
+  for (unsigned i = 0, e = Opts.CXXSystemIncludes.size(); i != e; ++i) {
+    Res.push_back("-cxx-system-include");
+    Res.push_back(Opts.CXXSystemIncludes[i]);
+  }
+
   /// User specified include entries.
   for (unsigned i = 0, e = Opts.UserEntries.size(); i != e; ++i) {
     const HeaderSearchOptions::Entry &E = Opts.UserEntries[i];
@@ -526,12 +561,16 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fgnu-keywords");
   if (Opts.Microsoft)
     Res.push_back("-fms-extensions");
+  if (Opts.MSCVersion != 0)
+    Res.push_back("-fmsc-version=" + llvm::utostr(Opts.MSCVersion));
   if (Opts.Borland)
     Res.push_back("-fborland-extensions");
   if (Opts.ObjCNonFragileABI)
     Res.push_back("-fobjc-nonfragile-abi");
   if (Opts.ObjCNonFragileABI2)
-    Res.push_back("-fobjc-nonfragile-abi2");
+    Res.push_back("-fobjc-nonfragile-abi");
+  if (Opts.ObjCDefaultSynthProperties)
+    Res.push_back("-fobjc-default-synthesize-properties");
   // NoInline is implicit.
   if (!Opts.CXXOperatorNames)
     Res.push_back("-fno-operator-names");
@@ -551,8 +590,12 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fexceptions");
   if (Opts.SjLjExceptions)
     Res.push_back("-fsjlj-exceptions");
+  if (!Opts.ObjCExceptions)
+    Res.push_back("-fno-objc-exceptions");
   if (!Opts.RTTI)
     Res.push_back("-fno-rtti");
+  if (Opts.MSBitfields)
+    Res.push_back("-mms-bitfields");
   if (!Opts.NeXTRuntime)
     Res.push_back("-fgnu-runtime");
   if (Opts.Freestanding)
@@ -574,7 +617,12 @@ static void LangOptsToArgs(const LangOptions &Opts,
   switch (Opts.getSignedOverflowBehavior()) {
   case LangOptions::SOB_Undefined: break;
   case LangOptions::SOB_Defined:   Res.push_back("-fwrapv"); break;
-  case LangOptions::SOB_Trapping:  Res.push_back("-ftrapv"); break;
+  case LangOptions::SOB_Trapping:
+    Res.push_back("-ftrapv"); break;
+    if (!Opts.OverflowHandler.empty()) {
+      Res.push_back("-ftrapv-handler");
+      Res.push_back(Opts.OverflowHandler);
+    }
   }
   if (Opts.HeinousExtensions)
     Res.push_back("-fheinous-gnu-extensions");
@@ -588,8 +636,6 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fdump-vtable-layouts");
   if (Opts.NoBitFieldTypeAlign)
     Res.push_back("-fno-bitfield-type-alignment");
-  if (Opts.SjLjExceptions)
-    Res.push_back("-fsjlj-exceptions");
   if (Opts.PICLevel) {
     Res.push_back("-pic-level");
     Res.push_back(llvm::utostr(Opts.PICLevel));
@@ -614,19 +660,22 @@ static void LangOptsToArgs(const LangOptions &Opts,
       Res.push_back("-fobjc-gc-only");
     }
   }
-  if (Opts.getVisibilityMode() != LangOptions::Default) {
+  if (Opts.AppleKext)
+    Res.push_back("-fapple-kext");
+  
+  if (Opts.getVisibilityMode() != DefaultVisibility) {
     Res.push_back("-fvisibility");
-    if (Opts.getVisibilityMode() == LangOptions::Hidden) {
+    if (Opts.getVisibilityMode() == HiddenVisibility) {
       Res.push_back("hidden");
     } else {
-      assert(Opts.getVisibilityMode() == LangOptions::Protected &&
+      assert(Opts.getVisibilityMode() == ProtectedVisibility &&
              "Invalid visibility!");
       Res.push_back("protected");
     }
   }
   if (Opts.InlineVisibilityHidden)
     Res.push_back("-fvisibility-inlines-hidden");
-  
+
   if (Opts.getStackProtectorMode() != 0) {
     Res.push_back("-stack-protector");
     Res.push_back(llvm::utostr(Opts.getStackProtectorMode()));
@@ -692,8 +741,6 @@ static void PreprocessorOutputOptsToArgs(const PreprocessorOutputOptions &Opts,
   else if (!Opts.ShowCPP && Opts.ShowMacros)
     Res.push_back("-dM");
 
-  if (Opts.ShowHeaderIncludes)
-    Res.push_back("-H");
   if (!Opts.ShowLineMarkers)
     Res.push_back("-P");
   if (Opts.ShowComments)
@@ -733,6 +780,7 @@ void CompilerInvocation::toArgs(std::vector<std::string> &Res) {
   CodeGenOptsToArgs(getCodeGenOpts(), Res);
   DependencyOutputOptsToArgs(getDependencyOutputOpts(), Res);
   DiagnosticOptsToArgs(getDiagnosticOpts(), Res);
+  FileSystemOptsToArgs(getFileSystemOpts(), Res);
   FrontendOptsToArgs(getFrontendOpts(), Res);
   HeaderSearchOptsToArgs(getHeaderSearchOpts(), Res);
   LangOptsToArgs(getLangOpts(), Res);
@@ -750,6 +798,16 @@ using namespace clang::driver::cc1options;
 
 //
 
+static unsigned getOptimizationLevel(ArgList &Args, InputKind IK,
+                                     Diagnostic &Diags) {
+  unsigned DefaultOpt = 0;
+  if (IK == IK_OpenCL && !Args.hasArg(OPT_cl_opt_disable))
+    DefaultOpt = 2;
+  // -Os implies -O2
+  return Args.hasArg(OPT_Os) ? 2 :
+    Args.getLastArgIntValue(OPT_O, DefaultOpt, Diags);
+}
+
 static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
                               Diagnostic &Diags) {
   using namespace cc1options;
@@ -810,33 +868,44 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   Opts.AnalyzerDisplayProgress = Args.hasArg(OPT_analyzer_display_progress);
   Opts.AnalyzeNestedBlocks =
     Args.hasArg(OPT_analyzer_opt_analyze_nested_blocks);
+  Opts.AnalyzerStats = Args.hasArg(OPT_analysis_AnalyzerStats);
   Opts.PurgeDead = !Args.hasArg(OPT_analyzer_no_purge_dead);
   Opts.EagerlyAssume = Args.hasArg(OPT_analyzer_eagerly_assume);
   Opts.AnalyzeSpecificFunction = Args.getLastArgValue(OPT_analyze_function);
   Opts.UnoptimizedCFG = Args.hasArg(OPT_analysis_UnoptimizedCFG);
+  Opts.CFGAddImplicitDtors = Args.hasArg(OPT_analysis_CFGAddImplicitDtors);
+  Opts.CFGAddInitializers = Args.hasArg(OPT_analysis_CFGAddInitializers);
   Opts.EnableExperimentalChecks = Args.hasArg(OPT_analyzer_experimental_checks);
   Opts.EnableExperimentalInternalChecks =
     Args.hasArg(OPT_analyzer_experimental_internal_checks);
   Opts.TrimGraph = Args.hasArg(OPT_trim_egraph);
   Opts.MaxNodes = Args.getLastArgIntValue(OPT_analyzer_max_nodes, 150000,Diags);
-  Opts.MaxLoop = Args.getLastArgIntValue(OPT_analyzer_max_loop, 3, Diags);
+  Opts.MaxLoop = Args.getLastArgIntValue(OPT_analyzer_max_loop, 4, Diags);
+  Opts.EagerlyTrimEGraph = !Args.hasArg(OPT_analyzer_no_eagerly_trim_egraph);
   Opts.InlineCall = Args.hasArg(OPT_analyzer_inline_call);
-  Opts.IdempotentOps = Args.hasArg(OPT_analysis_WarnIdempotentOps);
+  Opts.BufferOverflows = Args.hasArg(OPT_analysis_WarnBufferOverflows);
+
+  Opts.CheckersControlList.clear();
+  for (arg_iterator it = Args.filtered_begin(OPT_analyzer_checker,
+                                             OPT_analyzer_disable_checker),
+         ie = Args.filtered_end(); it != ie; ++it) {
+    const Arg *A = *it;
+    A->claim();
+    bool enable = (A->getOption().getID() == OPT_analyzer_checker);
+    Opts.CheckersControlList.push_back(std::make_pair(A->getValue(Args),
+                                                      enable));
+  }
 }
 
-static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
+static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
                              Diagnostic &Diags) {
   using namespace cc1options;
-  // -Os implies -O2
-  if (Args.hasArg(OPT_Os))
-    Opts.OptimizationLevel = 2;
-  else {
-    Opts.OptimizationLevel = Args.getLastArgIntValue(OPT_O, 0, Diags);
-    if (Opts.OptimizationLevel > 3) {
-      Diags.Report(diag::err_drv_invalid_value)
-        << Args.getLastArg(OPT_O)->getAsString(Args) << Opts.OptimizationLevel;
-      Opts.OptimizationLevel = 3;
-    }
+
+  Opts.OptimizationLevel = getOptimizationLevel(Args, IK, Diags);
+  if (Opts.OptimizationLevel > 3) {
+    Diags.Report(diag::err_drv_invalid_value)
+      << Args.getLastArg(OPT_O)->getAsString(Args) << Opts.OptimizationLevel;
+    Opts.OptimizationLevel = 3;
   }
 
   // We must always run at least the always inlining pass.
@@ -844,8 +913,10 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
     : CodeGenOptions::OnlyAlwaysInlining;
 
   Opts.DebugInfo = Args.hasArg(OPT_g);
+  Opts.LimitDebugInfo = Args.hasArg(OPT_flimit_debug_info);
   Opts.DisableLLVMOpts = Args.hasArg(OPT_disable_llvm_optzns);
   Opts.DisableRedZone = Args.hasArg(OPT_disable_red_zone);
+  Opts.RelaxedAliasing = Args.hasArg(OPT_relaxed_aliasing);
   Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
   Opts.MergeAllConstants = !Args.hasArg(OPT_fno_merge_all_constants);
   Opts.NoCommon = Args.hasArg(OPT_fno_common);
@@ -853,7 +924,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.OptimizeSize = Args.hasArg(OPT_Os);
   Opts.SimplifyLibCalls = !(Args.hasArg(OPT_fno_builtin) ||
                             Args.hasArg(OPT_ffreestanding));
-  Opts.UnrollLoops = Args.hasArg(OPT_funroll_loops) || 
+  Opts.UnrollLoops = Args.hasArg(OPT_funroll_loops) ||
                      (Opts.OptimizationLevel > 1 && !Opts.OptimizeSize);
 
   Opts.AsmVerbose = Args.hasArg(OPT_masm_verbose);
@@ -864,11 +935,17 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.DisableFPElim = Args.hasArg(OPT_mdisable_fp_elim);
   Opts.FloatABI = Args.getLastArgValue(OPT_mfloat_abi);
   Opts.HiddenWeakVTables = Args.hasArg(OPT_fhidden_weak_vtables);
+  Opts.LessPreciseFPMAD = Args.hasArg(OPT_cl_mad_enable);
   Opts.LimitFloatPrecision = Args.getLastArgValue(OPT_mlimit_float_precision);
+  Opts.NoInfsFPMath = Opts.NoNaNsFPMath = Args.hasArg(OPT_cl_finite_math_only)||
+                                          Args.hasArg(OPT_cl_fast_relaxed_math);
   Opts.NoZeroInitializedInBSS = Args.hasArg(OPT_mno_zero_initialized_in_bss);
+  Opts.NumRegisterParameters = Args.getLastArgIntValue(OPT_mregparm, 0, Diags);
   Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
   Opts.OmitLeafFramePointer = Args.hasArg(OPT_momit_leaf_frame_pointer);
   Opts.SoftFloat = Args.hasArg(OPT_msoft_float);
+  Opts.UnsafeFPMath = Args.hasArg(OPT_cl_unsafe_math_optimizations) ||
+                      Args.hasArg(OPT_cl_fast_relaxed_math);
   Opts.UnwindTables = Args.hasArg(OPT_munwind_tables);
   Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
 
@@ -879,6 +956,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.VerifyModule = !Args.hasArg(OPT_disable_llvm_verifier);
 
   Opts.InstrumentFunctions = Args.hasArg(OPT_finstrument_functions);
+  Opts.InstrumentForProfiling = Args.hasArg(OPT_pg);
 
   if (Arg *A = Args.getLastArg(OPT_fobjc_dispatch_method_EQ)) {
     llvm::StringRef Name = A->getValue(Args);
@@ -901,6 +979,8 @@ static void ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
   Opts.Targets = Args.getAllArgValues(OPT_MT);
   Opts.IncludeSystemHeaders = Args.hasArg(OPT_sys_header_deps);
   Opts.UsePhonyTargets = Args.hasArg(OPT_MP);
+  Opts.ShowHeaderIncludes = Args.hasArg(OPT_H);
+  Opts.HeaderIncludeOutputFile = Args.getLastArgValue(OPT_header_include_file);
 }
 
 static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
@@ -940,18 +1020,17 @@ static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
     Diags.Report(diag::err_drv_invalid_value)
       << Args.getLastArg(OPT_fdiagnostics_show_category)->getAsString(Args)
       << ShowCategory;
-  
+
   Opts.ShowSourceRanges = Args.hasArg(OPT_fdiagnostics_print_source_range_info);
   Opts.ShowParseableFixits = Args.hasArg(OPT_fdiagnostics_parseable_fixits);
   Opts.VerifyDiagnostics = Args.hasArg(OPT_verify);
-  Opts.BinaryOutput = Args.hasArg(OPT_fdiagnostics_binary);
   Opts.ErrorLimit = Args.getLastArgIntValue(OPT_ferror_limit, 0, Diags);
   Opts.MacroBacktraceLimit
-    = Args.getLastArgIntValue(OPT_fmacro_backtrace_limit, 
+    = Args.getLastArgIntValue(OPT_fmacro_backtrace_limit,
                          DiagnosticOptions::DefaultMacroBacktraceLimit, Diags);
   Opts.TemplateBacktraceLimit
-    = Args.getLastArgIntValue(OPT_ftemplate_backtrace_limit, 
-                         DiagnosticOptions::DefaultTemplateBacktraceLimit, 
+    = Args.getLastArgIntValue(OPT_ftemplate_backtrace_limit,
+                         DiagnosticOptions::DefaultTemplateBacktraceLimit,
                          Diags);
   Opts.TabStop = Args.getLastArgIntValue(OPT_ftabstop,
                                     DiagnosticOptions::DefaultTabStop, Diags);
@@ -965,6 +1044,10 @@ static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   Opts.Warnings = Args.getAllArgValues(OPT_W);
 }
 
+static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) {
+  Opts.WorkingDir = Args.getLastArgValue(OPT_working_directory);
+}
+
 static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
                                    Diagnostic &Diags) {
   using namespace cc1options;
@@ -975,6 +1058,8 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       assert(0 && "Invalid option in group!");
     case OPT_ast_dump:
       Opts.ProgramAction = frontend::ASTDump; break;
+    case OPT_ast_dump_xml:
+      Opts.ProgramAction = frontend::ASTDumpXML; break;
     case OPT_ast_print:
       Opts.ProgramAction = frontend::ASTPrint; break;
     case OPT_ast_print_xml:
@@ -1047,6 +1132,16 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     }
   }
 
+  Opts.AddPluginActions = Args.getAllArgValues(OPT_add_plugin);
+  Opts.AddPluginArgs.resize(Opts.AddPluginActions.size());
+  for (int i = 0, e = Opts.AddPluginActions.size(); i != e; ++i) {
+    for (arg_iterator it = Args.filtered_begin(OPT_plugin_arg),
+           end = Args.filtered_end(); it != end; ++it) {
+      if ((*it)->getValue(Args, 0) == Opts.AddPluginActions[i])
+        Opts.AddPluginArgs[i].push_back((*it)->getValue(Args, 1));
+    }
+  }
+
   if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) {
     Opts.CodeCompletionAt =
       ParsedSourceLocation::FromString(A->getValue(Args));
@@ -1054,8 +1149,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value)
         << A->getAsString(Args) << A->getValue(Args);
   }
-  Opts.DebugCodeCompletionPrinter =
-    !Args.hasArg(OPT_no_code_completion_debug_printer);
   Opts.DisableFree = Args.hasArg(OPT_disable_free);
 
   Opts.OutputFile = Args.getLastArgValue(OPT_o);
@@ -1071,7 +1164,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.ShowStats = Args.hasArg(OPT_print_stats);
   Opts.ShowTimers = Args.hasArg(OPT_ftime_report);
   Opts.ShowVersion = Args.hasArg(OPT_version);
-  Opts.ViewClassInheritance = Args.getLastArgValue(OPT_cxx_inheritance_view);
   Opts.ASTMergeFiles = Args.getAllArgValues(OPT_ast_merge);
   Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
   Opts.FixWhatYouCan = Args.hasArg(OPT_fix_what_you_can);
@@ -1084,6 +1176,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       .Case("cl", IK_OpenCL)
       .Case("c", IK_C)
       .Case("cl", IK_OpenCL)
+      .Case("cuda", IK_CUDA)
       .Case("c++", IK_CXX)
       .Case("objective-c", IK_ObjC)
       .Case("objective-c++", IK_ObjCXX)
@@ -1143,6 +1236,7 @@ std::string CompilerInvocation::GetResourcesPath(const char *Argv0,
 
 static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   using namespace cc1options;
+  Opts.CXXSystemIncludes = Args.getAllArgValues(OPT_cxx_system_include);
   Opts.Sysroot = Args.getLastArgValue(OPT_isysroot, "/");
   Opts.Verbose = Args.hasArg(OPT_v);
   Opts.UseBuiltinIncludes = !Args.hasArg(OPT_nobuiltininc);
@@ -1186,10 +1280,8 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   // FIXME: Need options for the various environment variables!
 }
 
-static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
-                          Diagnostic &Diags) {
-  // FIXME: Cleanup per-file based stuff.
-
+void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
+                                         LangStandard::Kind LangStd) {
   // Set some properties which depend soley on the input kind; it would be nice
   // to move these to the language standard, and have the driver resolve the
   // input kind + language standard.
@@ -1202,18 +1294,6 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     Opts.ObjC1 = Opts.ObjC2 = 1;
   }
 
-  LangStandard::Kind LangStd = LangStandard::lang_unspecified;
-  if (const Arg *A = Args.getLastArg(OPT_std_EQ)) {
-    LangStd = llvm::StringSwitch<LangStandard::Kind>(A->getValue(Args))
-#define LANGSTANDARD(id, name, desc, features) \
-      .Case(name, LangStandard::lang_##id)
-#include "clang/Frontend/LangStandards.def"
-      .Default(LangStandard::lang_unspecified);
-    if (LangStd == LangStandard::lang_unspecified)
-      Diags.Report(diag::err_drv_invalid_value)
-        << A->getAsString(Args) << A->getValue(Args);
-  }
-
   if (LangStd == LangStandard::lang_unspecified) {
     // Based on the base language, pick one.
     switch (IK) {
@@ -1224,6 +1304,9 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     case IK_OpenCL:
       LangStd = LangStandard::lang_opencl;
       break;
+    case IK_CUDA:
+      LangStd = LangStandard::lang_cuda;
+      break;
     case IK_Asm:
     case IK_C:
     case IK_PreprocessedC:
@@ -1257,26 +1340,71 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     Opts.AltiVec = 1;
     Opts.CXXOperatorNames = 1;
     Opts.LaxVectorConversions = 1;
+    Opts.DefaultFPContract = 1;
   }
 
+  if (LangStd == LangStandard::lang_cuda)
+    Opts.CUDA = 1;
+
   // OpenCL and C++ both have bool, true, false keywords.
   Opts.Bool = Opts.OpenCL || Opts.CPlusPlus;
 
+  Opts.GNUKeywords = Opts.GNUMode;
+  Opts.CXXOperatorNames = Opts.CPlusPlus;
+
+  // Mimicing gcc's behavior, trigraphs are only enabled if -trigraphs
+  // is specified, or -std is set to a conforming mode.
+  Opts.Trigraphs = !Opts.GNUMode;
+
+  Opts.DollarIdents = !Opts.AsmPreprocessor;
+}
+
+static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
+                          Diagnostic &Diags) {
+  // FIXME: Cleanup per-file based stuff.
+  LangStandard::Kind LangStd = LangStandard::lang_unspecified;
+  if (const Arg *A = Args.getLastArg(OPT_std_EQ)) {
+    LangStd = llvm::StringSwitch<LangStandard::Kind>(A->getValue(Args))
+#define LANGSTANDARD(id, name, desc, features) \
+      .Case(name, LangStandard::lang_##id)
+#include "clang/Frontend/LangStandards.def"
+      .Default(LangStandard::lang_unspecified);
+    if (LangStd == LangStandard::lang_unspecified)
+      Diags.Report(diag::err_drv_invalid_value)
+        << A->getAsString(Args) << A->getValue(Args);
+  }
+
+  if (const Arg *A = Args.getLastArg(OPT_cl_std_EQ)) {
+    if (strcmp(A->getValue(Args), "CL1.1") != 0) {
+      Diags.Report(diag::err_drv_invalid_value)
+        << A->getAsString(Args) << A->getValue(Args);
+    }
+  }
+
+  CompilerInvocation::setLangDefaults(Opts, IK, LangStd);
+
   // We abuse '-f[no-]gnu-keywords' to force overriding all GNU-extension
   // keywords. This behavior is provided by GCC's poorly named '-fasm' flag,
   // while a subset (the non-C++ GNU keywords) is provided by GCC's
   // '-fgnu-keywords'. Clang conflates the two for simplicity under the single
   // name, as it doesn't seem a useful distinction.
   Opts.GNUKeywords = Args.hasFlag(OPT_fgnu_keywords, OPT_fno_gnu_keywords,
-                                  Opts.GNUMode);
+                                  Opts.GNUKeywords);
 
-  if (Opts.CPlusPlus)
-    Opts.CXXOperatorNames = !Args.hasArg(OPT_fno_operator_names);
+  if (Args.hasArg(OPT_fno_operator_names))
+    Opts.CXXOperatorNames = 0;
 
   if (Args.hasArg(OPT_fobjc_gc_only))
     Opts.setGCMode(LangOptions::GCOnly);
   else if (Args.hasArg(OPT_fobjc_gc))
     Opts.setGCMode(LangOptions::HybridGC);
+  
+  if (Args.hasArg(OPT_fapple_kext)) {
+    if (!Opts.CPlusPlus)
+      Diags.Report(diag::warn_c_kext);
+    else
+      Opts.AppleKext = 1;
+  }
 
   if (Args.hasArg(OPT_print_ivar_layout))
     Opts.ObjCGCBitmapPrint = 1;
@@ -1291,34 +1419,36 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
 
   llvm::StringRef Vis = Args.getLastArgValue(OPT_fvisibility, "default");
   if (Vis == "default")
-    Opts.setVisibilityMode(LangOptions::Default);
+    Opts.setVisibilityMode(DefaultVisibility);
   else if (Vis == "hidden")
-    Opts.setVisibilityMode(LangOptions::Hidden);
+    Opts.setVisibilityMode(HiddenVisibility);
   else if (Vis == "protected")
-    Opts.setVisibilityMode(LangOptions::Protected);
+    Opts.setVisibilityMode(ProtectedVisibility);
   else
     Diags.Report(diag::err_drv_invalid_value)
       << Args.getLastArg(OPT_fvisibility)->getAsString(Args) << Vis;
 
   if (Args.hasArg(OPT_fvisibility_inlines_hidden))
     Opts.InlineVisibilityHidden = 1;
-  
-  if (Args.hasArg(OPT_ftrapv))
-    Opts.setSignedOverflowBehavior(LangOptions::SOB_Trapping); 
+
+  if (Args.hasArg(OPT_ftrapv)) {
+    Opts.setSignedOverflowBehavior(LangOptions::SOB_Trapping);
+    // Set the handler, if one is specified.
+    Opts.OverflowHandler =
+        Args.getLastArgValue(OPT_ftrapv_handler);
+  }
   else if (Args.hasArg(OPT_fwrapv))
-    Opts.setSignedOverflowBehavior(LangOptions::SOB_Defined); 
+    Opts.setSignedOverflowBehavior(LangOptions::SOB_Defined);
 
-  // Mimicing gcc's behavior, trigraphs are only enabled if -trigraphs
-  // is specified, or -std is set to a conforming mode.
-  Opts.Trigraphs = !Opts.GNUMode;
   if (Args.hasArg(OPT_trigraphs))
     Opts.Trigraphs = 1;
 
   Opts.DollarIdents = Args.hasFlag(OPT_fdollars_in_identifiers,
                                    OPT_fno_dollars_in_identifiers,
-                                   !Opts.AsmPreprocessor);
+                                   Opts.DollarIdents);
   Opts.PascalStrings = Args.hasArg(OPT_fpascal_strings);
   Opts.Microsoft = Args.hasArg(OPT_fms_extensions);
+  Opts.MSCVersion = Args.getLastArgIntValue(OPT_fmsc_version, 0, Diags);
   Opts.Borland = Args.hasArg(OPT_fborland_extensions);
   Opts.WritableStrings = Args.hasArg(OPT_fwritable_strings);
   Opts.ConstStrings = Args.hasArg(OPT_Wwrite_strings);
@@ -1327,10 +1457,12 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Args.hasArg(OPT_fno_threadsafe_statics))
     Opts.ThreadsafeStatics = 0;
   Opts.Exceptions = Args.hasArg(OPT_fexceptions);
+  Opts.ObjCExceptions = !Args.hasArg(OPT_fno_objc_exceptions);
   Opts.RTTI = !Args.hasArg(OPT_fno_rtti);
   Opts.Blocks = Args.hasArg(OPT_fblocks);
   Opts.CharIsSigned = !Args.hasArg(OPT_fno_signed_char);
   Opts.ShortWChar = Args.hasArg(OPT_fshort_wchar);
+  Opts.ShortEnums = Args.hasArg(OPT_fshort_enums);
   Opts.Freestanding = Args.hasArg(OPT_ffreestanding);
   Opts.NoBuiltin = Args.hasArg(OPT_fno_builtin) || Opts.Freestanding;
   Opts.AssumeSaneOperatorNew = !Args.hasArg(OPT_fno_assume_sane_operator_new);
@@ -1340,27 +1472,33 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.MathErrno = Args.hasArg(OPT_fmath_errno);
   Opts.InstantiationDepth = Args.getLastArgIntValue(OPT_ftemplate_depth, 1024,
                                                Diags);
+  Opts.NumLargeByValueCopy = Args.getLastArgIntValue(OPT_Wlarge_by_value_copy,
+                                                    0, Diags);
+  Opts.MSBitfields = Args.hasArg(OPT_mms_bitfields);
   Opts.NeXTRuntime = !Args.hasArg(OPT_fgnu_runtime);
   Opts.ObjCConstantStringClass =
     Args.getLastArgValue(OPT_fconstant_string_class);
   Opts.ObjCNonFragileABI = Args.hasArg(OPT_fobjc_nonfragile_abi);
-  Opts.ObjCNonFragileABI2 = Args.hasArg(OPT_fobjc_nonfragile_abi2);
-  if (Opts.ObjCNonFragileABI2)
-    Opts.ObjCNonFragileABI = true;
+  if (Opts.ObjCNonFragileABI)
+    Opts.ObjCNonFragileABI2 = true;
+  Opts.ObjCDefaultSynthProperties =
+    Args.hasArg(OPT_fobjc_default_synthesize_properties);
   Opts.CatchUndefined = Args.hasArg(OPT_fcatch_undefined_behavior);
   Opts.EmitAllDecls = Args.hasArg(OPT_femit_all_decls);
   Opts.PICLevel = Args.getLastArgIntValue(OPT_pic_level, 0, Diags);
   Opts.SjLjExceptions = Args.hasArg(OPT_fsjlj_exceptions);
+  Opts.ObjCExceptions = !Args.hasArg(OPT_fno_objc_exceptions);
   Opts.Static = Args.hasArg(OPT_static_define);
   Opts.DumpRecordLayouts = Args.hasArg(OPT_fdump_record_layouts);
   Opts.DumpVTableLayouts = Args.hasArg(OPT_fdump_vtable_layouts);
   Opts.SpellChecking = !Args.hasArg(OPT_fno_spell_checking);
   Opts.NoBitFieldTypeAlign = Args.hasArg(OPT_fno_bitfield_type_align);
+  Opts.SinglePrecisionConstants = Args.hasArg(OPT_cl_single_precision_constant);
+  Opts.FastRelaxedMath = Args.hasArg(OPT_cl_fast_relaxed_math);
   Opts.OptimizeSize = 0;
 
   // FIXME: Eliminate this dependency.
-  unsigned Opt =
-    Args.hasArg(OPT_Os) ? 2 : Args.getLastArgIntValue(OPT_O, 0, Diags);
+  unsigned Opt = getOptimizationLevel(Args, IK, Diags);
   Opts.Optimize = Opt != 0;
 
   // This is the __NO_INLINE__ define, which just depends on things like the
@@ -1383,6 +1521,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
 }
 
 static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
+                                  FileManager &FileMgr,
                                   Diagnostic &Diags) {
   using namespace cc1options;
   Opts.ImplicitPCHInclude = Args.getLastArgValue(OPT_include_pch);
@@ -1395,12 +1534,19 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   Opts.DetailedRecord = Args.hasArg(OPT_detailed_preprocessing_record);
   Opts.DisablePCHValidation = Args.hasArg(OPT_fno_validate_pch);
 
+  Opts.DumpDeserializedPCHDecls = Args.hasArg(OPT_dump_deserialized_pch_decls);
+  for (arg_iterator it = Args.filtered_begin(OPT_error_on_deserialized_pch_decl),
+         ie = Args.filtered_end(); it != ie; ++it) {
+    const Arg *A = *it;
+    Opts.DeserializedPCHDeclsToErrorOn.insert(A->getValue(Args));
+  }
+
   if (const Arg *A = Args.getLastArg(OPT_preamble_bytes_EQ)) {
     llvm::StringRef Value(A->getValue(Args));
     size_t Comma = Value.find(',');
     unsigned Bytes = 0;
     unsigned EndOfLine = 0;
-    
+
     if (Comma == llvm::StringRef::npos ||
         Value.substr(0, Comma).getAsInteger(10, Bytes) ||
         Value.substr(Comma + 1).getAsInteger(10, EndOfLine))
@@ -1410,7 +1556,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
       Opts.PrecompiledPreambleBytes.second = (EndOfLine != 0);
     }
   }
-    
+
   // Add macros from the command line.
   for (arg_iterator it = Args.filtered_begin(OPT_D, OPT_U),
          ie = Args.filtered_end(); it != ie; ++it) {
@@ -1430,7 +1576,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
     // PCH is handled specially, we need to extra the original include path.
     if (A->getOption().matches(OPT_include_pch)) {
       std::string OriginalFile =
-        ASTReader::getOriginalSourceFile(A->getValue(Args), Diags);
+        ASTReader::getOriginalSourceFile(A->getValue(Args), FileMgr, Diags);
       if (OriginalFile.empty())
         continue;
 
@@ -1463,7 +1609,6 @@ static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
   using namespace cc1options;
   Opts.ShowCPP = !Args.hasArg(OPT_dM);
   Opts.ShowComments = Args.hasArg(OPT_C);
-  Opts.ShowHeaderIncludes = Args.hasArg(OPT_H);
   Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
   Opts.ShowMacroComments = Args.hasArg(OPT_CC);
   Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
@@ -1486,8 +1631,8 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
 //
 
 void CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
-                                        const char **ArgBegin,
-                                        const char **ArgEnd,
+                                        const char *const *ArgBegin,
+                                        const char *const *ArgEnd,
                                         Diagnostic &Diags) {
   // Parse the arguments.
   llvm::OwningPtr<OptTable> Opts(createCC1OptTable());
@@ -1506,14 +1651,21 @@ void CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
     Diags.Report(diag::err_drv_unknown_argument) << (*it)->getAsString(*Args);
 
   ParseAnalyzerArgs(Res.getAnalyzerOpts(), *Args, Diags);
-  ParseCodeGenArgs(Res.getCodeGenOpts(), *Args, Diags);
   ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), *Args);
   ParseDiagnosticArgs(Res.getDiagnosticOpts(), *Args, Diags);
+  ParseFileSystemArgs(Res.getFileSystemOpts(), *Args);
+  // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = ParseFrontendArgs(Res.getFrontendOpts(), *Args, Diags);
+  ParseCodeGenArgs(Res.getCodeGenOpts(), *Args, DashX, Diags);
   ParseHeaderSearchArgs(Res.getHeaderSearchOpts(), *Args);
   if (DashX != IK_AST && DashX != IK_LLVM_IR)
     ParseLangArgs(Res.getLangOpts(), *Args, DashX, Diags);
-  ParsePreprocessorArgs(Res.getPreprocessorOpts(), *Args, Diags);
+  // FIXME: ParsePreprocessorArgs uses the FileManager to read the contents of
+  // PCH file and find the original header name. Remove the need to do that in
+  // ParsePreprocessorArgs and remove the FileManager 
+  // parameters from the function and the "FileManager.h" #include.
+  FileManager FileMgr(Res.getFileSystemOpts());
+  ParsePreprocessorArgs(Res.getPreprocessorOpts(), *Args, FileMgr, Diags);
   ParsePreprocessorOutputArgs(Res.getPreprocessorOutputOpts(), *Args);
   ParseTargetArgs(Res.getTargetOpts(), *Args);
 }
diff --git a/lib/Frontend/DeclXML.cpp b/lib/Frontend/DeclXML.cpp
index 97a7f5558392..8d3d225a4b38 100644
--- a/lib/Frontend/DeclXML.cpp
+++ b/lib/Frontend/DeclXML.cpp
@@ -38,10 +38,12 @@ class DocumentXML::DeclPrinter : public DeclVisitor<DocumentXML::DeclPrinter> {
   }
 
   void addSubNodes(RecordDecl* RD) {
-    for (RecordDecl::field_iterator i = RD->field_begin(),
-                                    e = RD->field_end(); i != e; ++i) {
-      Visit(*i);
-      Doc.toParent();
+    for (RecordDecl::decl_iterator i = RD->decls_begin(),
+                                   e = RD->decls_end(); i != e; ++i) {
+      if (!(*i)->isImplicit()) {
+        Visit(*i);
+        Doc.toParent();
+      }
     }
   }
 
@@ -71,15 +73,7 @@ class DocumentXML::DeclPrinter : public DeclVisitor<DocumentXML::DeclPrinter> {
         Doc.addAttribute("is_virtual", base->isVirtual());
         Doc.toParent();
       }
-
-      for (CXXRecordDecl::method_iterator i = RD->method_begin(),
-             e = RD->method_end(); i != e; ++i) {
-        Visit(*i);
-        Doc.toParent();
-      }
-
     }
-
   }
 
   void addSubNodes(EnumDecl* ED) {
@@ -110,7 +104,7 @@ class DocumentXML::DeclPrinter : public DeclVisitor<DocumentXML::DeclPrinter> {
       Doc.PrintStmt(argDecl->getDefaultArg());
   }
 
-  void addSubNodes(NamespaceDecl* ns) {
+  void addSubNodes(DeclContext* ns) {
 
     for (DeclContext::decl_iterator 
            d    = ns->decls_begin(), 
diff --git a/lib/Frontend/DependencyFile.cpp b/lib/Frontend/DependencyFile.cpp
index cdff8077ee47..bc5a55df0860 100644
--- a/lib/Frontend/DependencyFile.cpp
+++ b/lib/Frontend/DependencyFile.cpp
@@ -13,7 +13,6 @@
 
 #include "clang/Frontend/Utils.h"
 #include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/DependencyOutputOptions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
@@ -22,7 +21,6 @@
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/raw_ostream.h"
-#include <string>
 
 using namespace clang;
 
@@ -117,6 +115,16 @@ void DependencyFileCallback::FileChanged(SourceLocation Loc,
     Files.push_back(Filename);
 }
 
+/// PrintFilename - GCC escapes spaces, but apparently not ' or " or other
+/// scary characters.
+static void PrintFilename(llvm::raw_ostream &OS, llvm::StringRef Filename) {
+  for (unsigned i = 0, e = Filename.size(); i != e; ++i) {
+    if (Filename[i] == ' ')
+      OS << '\\';
+    OS << Filename[i];
+  }
+}
+
 void DependencyFileCallback::OutputDependencyFile() {
   // Write out the dependency targets, trying to avoid overly long
   // lines when possible. We try our best to emit exactly the same
@@ -130,14 +138,15 @@ void DependencyFileCallback::OutputDependencyFile() {
     unsigned N = I->length();
     if (Columns == 0) {
       Columns += N;
-      *OS << *I;
     } else if (Columns + N + 2 > MaxColumns) {
       Columns = N + 2;
-      *OS << " \\\n  " << *I;
+      *OS << " \\\n  ";
     } else {
       Columns += N + 1;
-      *OS << ' ' << *I;
+      *OS << ' ';
     }
+    // Targets already quoted as needed.
+    *OS << *I;
   }
 
   *OS << ':';
@@ -155,7 +164,8 @@ void DependencyFileCallback::OutputDependencyFile() {
       *OS << " \\\n ";
       Columns = 2;
     }
-    *OS << ' ' << *I;
+    *OS << ' ';
+    PrintFilename(*OS, *I);
     Columns += N + 1;
   }
   *OS << '\n';
@@ -166,7 +176,8 @@ void DependencyFileCallback::OutputDependencyFile() {
     for (std::vector<std::string>::iterator I = Files.begin() + 1,
            E = Files.end(); I != E; ++I) {
       *OS << '\n';
-      *OS << *I << ":\n";
+      PrintFilename(*OS, *I);
+      *OS << ":\n";
     }
   }
 }
diff --git a/lib/Frontend/DocumentXML.cpp b/lib/Frontend/DocumentXML.cpp
index 894f230216f6..b24ece5119d6 100644
--- a/lib/Frontend/DocumentXML.cpp
+++ b/lib/Frontend/DocumentXML.cpp
@@ -17,6 +17,8 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Config/config.h"
+#include <cstdio>
 
 namespace clang {
 
@@ -104,7 +106,11 @@ std::string DocumentXML::escapeString(const char* pStr,
       if (isprint(C))
         value += C;
       else {
+#ifdef LLVM_ON_WIN32
         sprintf(buffer, "\\%03o", C);
+#else
+        snprintf(buffer, sizeof(buffer), "\\%03o", C);
+#endif
         value += buffer;
       }
       break;
@@ -321,9 +327,11 @@ PresumedLoc DocumentXML::addLocation(const SourceLocation& Loc)
   if (!SpellingLoc.isInvalid())
   {
     PLoc = SM.getPresumedLoc(SpellingLoc);
-    addSourceFileAttribute(PLoc.getFilename());
-    addAttribute("line", PLoc.getLine());
-    addAttribute("col", PLoc.getColumn());
+    if (PLoc.isValid()) {
+      addSourceFileAttribute(PLoc.getFilename());
+      addAttribute("line", PLoc.getLine());
+      addAttribute("col", PLoc.getColumn());
+    }
   }
   // else there is no error in some cases (eg. CXXThisExpr)
   return PLoc;
@@ -340,8 +348,9 @@ void DocumentXML::addLocationRange(const SourceRange& R)
     if (!SpellingLoc.isInvalid())
     {
       PresumedLoc PLoc = SM.getPresumedLoc(SpellingLoc);
-      if (PStartLoc.isInvalid() ||
-          strcmp(PLoc.getFilename(), PStartLoc.getFilename()) != 0) {
+      if (PLoc.isInvalid()) {
+      } else if (PStartLoc.isInvalid() ||
+                 strcmp(PLoc.getFilename(), PStartLoc.getFilename()) != 0) {
         addToMap(SourceFiles, PLoc.getFilename(), ID_FILE);
         addAttribute("endfile", PLoc.getFilename());
         addAttribute("endline", PLoc.getLine());
diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp
index b244c5ce0225..e3d8b8594190 100644
--- a/lib/Frontend/FrontendAction.cpp
+++ b/lib/Frontend/FrontendAction.cpp
@@ -10,18 +10,73 @@
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclGroup.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/FrontendPluginRegistry.h"
+#include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Parse/ParseAST.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
+namespace {
+
+/// \brief Dumps deserialized declarations.
+class DeserializedDeclsDumper : public ASTDeserializationListener {
+  ASTDeserializationListener *Previous;
+
+public:
+  DeserializedDeclsDumper(ASTDeserializationListener *Previous)
+    : Previous(Previous) { }
+
+  virtual void DeclRead(serialization::DeclID ID, const Decl *D) {
+    llvm::outs() << "PCH DECL: " << D->getDeclKindName();
+    if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
+      llvm::outs() << " - " << ND->getNameAsString();
+    llvm::outs() << "\n";
+
+    if (Previous)
+      Previous->DeclRead(ID, D);
+  }
+};
+
+  /// \brief Checks deserialized declarations and emits error if a name
+  /// matches one given in command-line using -error-on-deserialized-decl.
+  class DeserializedDeclsChecker : public ASTDeserializationListener {
+    ASTContext &Ctx;
+    std::set<std::string> NamesToCheck;
+    ASTDeserializationListener *Previous;
+
+  public:
+    DeserializedDeclsChecker(ASTContext &Ctx,
+                             const std::set<std::string> &NamesToCheck, 
+                             ASTDeserializationListener *Previous)
+      : Ctx(Ctx), NamesToCheck(NamesToCheck), Previous(Previous) { }
+
+    virtual void DeclRead(serialization::DeclID ID, const Decl *D) {
+      if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
+        if (NamesToCheck.find(ND->getNameAsString()) != NamesToCheck.end()) {
+          unsigned DiagID
+            = Ctx.getDiagnostics().getCustomDiagID(Diagnostic::Error,
+                                                   "%0 was deserialized");
+          Ctx.getDiagnostics().Report(Ctx.getFullLoc(D->getLocation()), DiagID)
+              << ND->getNameAsString();
+        }
+
+      if (Previous)
+        Previous->DeclRead(ID, D);
+    }
+};
+
+} // end anonymous namespace
+
 FrontendAction::FrontendAction() : Instance(0) {}
 
 FrontendAction::~FrontendAction() {}
@@ -33,6 +88,39 @@ void FrontendAction::setCurrentFile(llvm::StringRef Value, InputKind Kind,
   CurrentASTUnit.reset(AST);
 }
 
+ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
+                                                      llvm::StringRef InFile) {
+  ASTConsumer* Consumer = CreateASTConsumer(CI, InFile);
+  if (!Consumer)
+    return 0;
+
+  if (CI.getFrontendOpts().AddPluginActions.size() == 0)
+    return Consumer;
+
+  // Make sure the non-plugin consumer is first, so that plugins can't
+  // modifiy the AST.
+  std::vector<ASTConsumer*> Consumers(1, Consumer);
+
+  for (size_t i = 0, e = CI.getFrontendOpts().AddPluginActions.size();
+       i != e; ++i) { 
+    // This is O(|plugins| * |add_plugins|), but since both numbers are
+    // way below 50 in practice, that's ok.
+    for (FrontendPluginRegistry::iterator
+        it = FrontendPluginRegistry::begin(),
+        ie = FrontendPluginRegistry::end();
+        it != ie; ++it) {
+      if (it->getName() == CI.getFrontendOpts().AddPluginActions[i]) {
+        llvm::OwningPtr<PluginASTAction> P(it->instantiate());
+        FrontendAction* c = P.get();
+        if (P->ParseArgs(CI, CI.getFrontendOpts().AddPluginArgs[i]))
+          Consumers.push_back(c->CreateASTConsumer(CI, InFile));
+      }
+    }
+  }
+
+  return new MultiplexConsumer(Consumers);
+}
+
 bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
                                      llvm::StringRef Filename,
                                      InputKind InputKind) {
@@ -51,7 +139,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
 
     llvm::IntrusiveRefCntPtr<Diagnostic> Diags(&CI.getDiagnostics());
     std::string Error;
-    ASTUnit *AST = ASTUnit::LoadFromASTFile(Filename, Diags);
+    ASTUnit *AST = ASTUnit::LoadFromASTFile(Filename, Diags,
+                                            CI.getFileSystemOpts());
     if (!AST)
       goto failure;
 
@@ -69,7 +158,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
       goto failure;
 
     /// Create the AST consumer.
-    CI.setASTConsumer(CreateASTConsumer(CI, Filename));
+    CI.setASTConsumer(CreateWrappedASTConsumer(CI, Filename));
     if (!CI.hasASTConsumer())
       goto failure;
 
@@ -80,7 +169,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   if (!CI.hasFileManager())
     CI.createFileManager();
   if (!CI.hasSourceManager())
-    CI.createSourceManager();
+    CI.createSourceManager(CI.getFileManager());
 
   // IR files bypass the rest of initialization.
   if (InputKind == IK_LLVM_IR) {
@@ -113,16 +202,30 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   if (!usesPreprocessorOnly()) {
     CI.createASTContext();
 
-    llvm::OwningPtr<ASTConsumer> Consumer(CreateASTConsumer(CI, Filename));
+    llvm::OwningPtr<ASTConsumer> Consumer(
+        CreateWrappedASTConsumer(CI, Filename));
+    if (!Consumer)
+      goto failure;
+
+    CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener());
 
     /// Use PCH?
     if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
       assert(hasPCHSupport() && "This action does not have PCH support!");
+      ASTDeserializationListener *DeserialListener
+          = CI.getInvocation().getFrontendOpts().ChainedPCH ?
+                  Consumer->GetASTDeserializationListener() : 0;
+      if (CI.getPreprocessorOpts().DumpDeserializedPCHDecls)
+        DeserialListener = new DeserializedDeclsDumper(DeserialListener);
+      if (!CI.getPreprocessorOpts().DeserializedPCHDeclsToErrorOn.empty())
+        DeserialListener = new DeserializedDeclsChecker(CI.getASTContext(),
+                         CI.getPreprocessorOpts().DeserializedPCHDeclsToErrorOn,
+                                                        DeserialListener);
       CI.createPCHExternalASTSource(
                                 CI.getPreprocessorOpts().ImplicitPCHInclude,
                                 CI.getPreprocessorOpts().DisablePCHValidation,
-                                CI.getInvocation().getFrontendOpts().ChainedPCH?
-                                 Consumer->GetASTDeserializationListener() : 0);
+                                CI.getPreprocessorOpts().DisableStatCache,
+                                DeserialListener);
       if (!CI.getASTContext().getExternalSource())
         goto failure;
     }
@@ -137,7 +240,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   if (!CI.hasASTContext() || !CI.getASTContext().getExternalSource()) {
     Preprocessor &PP = CI.getPreprocessor();
     PP.getBuiltinInfo().InitializeBuiltins(PP.getIdentifierTable(),
-                                           PP.getLangOptions().NoBuiltin);
+                                           PP.getLangOptions());
   }
 
   return true;
@@ -187,6 +290,9 @@ void FrontendAction::Execute() {
 void FrontendAction::EndSourceFile() {
   CompilerInstance &CI = getCompilerInstance();
 
+  // Inform the diagnostic client we are done with this source file.
+  CI.getDiagnosticClient().EndSourceFile();
+
   // Finalize the action.
   EndSourceFileAction();
 
@@ -223,10 +329,7 @@ void FrontendAction::EndSourceFile() {
 
   // Cleanup the output streams, and erase the output files if we encountered
   // an error.
-  CI.clearOutputFiles(/*EraseFiles=*/CI.getDiagnostics().getNumErrors());
-
-  // Inform the diagnostic client we are done with this source file.
-  CI.getDiagnosticClient().EndSourceFile();
+  CI.clearOutputFiles(/*EraseFiles=*/CI.getDiagnostics().hasErrorOccurred());
 
   if (isCurrentFileAST()) {
     CI.takeSema();
diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp
index 5bc6506e1fd9..d8e7d2904500 100644
--- a/lib/Frontend/FrontendActions.cpp
+++ b/lib/Frontend/FrontendActions.cpp
@@ -59,6 +59,17 @@ ASTConsumer *ASTDumpAction::CreateASTConsumer(CompilerInstance &CI,
   return CreateASTDumper();
 }
 
+ASTConsumer *ASTDumpXMLAction::CreateASTConsumer(CompilerInstance &CI,
+                                                 llvm::StringRef InFile) {
+  llvm::raw_ostream *OS;
+  if (CI.getFrontendOpts().OutputFile.empty())
+    OS = &llvm::outs();
+  else
+    OS = CI.createDefaultOutputFile(false, InFile);
+  if (!OS) return 0;
+  return CreateASTDumperXML(*OS);
+}
+
 ASTConsumer *ASTViewAction::CreateASTConsumer(CompilerInstance &CI,
                                               llvm::StringRef InFile) {
   return CreateASTViewer();
@@ -72,19 +83,21 @@ ASTConsumer *DeclContextPrintAction::CreateASTConsumer(CompilerInstance &CI,
 ASTConsumer *GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI,
                                                   llvm::StringRef InFile) {
   std::string Sysroot;
+  std::string OutputFile;
   llvm::raw_ostream *OS = 0;
   bool Chaining;
-  if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OS, Chaining))
+  if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile, OS, Chaining))
     return 0;
 
   const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
                              Sysroot.c_str() : 0;  
-  return new PCHGenerator(CI.getPreprocessor(), Chaining, isysroot, OS);
+  return new PCHGenerator(CI.getPreprocessor(), OutputFile, Chaining, isysroot, OS);
 }
 
 bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
                                                     llvm::StringRef InFile,
                                                     std::string &Sysroot,
+                                                    std::string &OutputFile,
                                                     llvm::raw_ostream *&OS,
                                                     bool &Chaining) {
   Sysroot = CI.getHeaderSearchOpts().Sysroot;
@@ -93,20 +106,19 @@ bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
     return true;
   }
 
-  OS = CI.createDefaultOutputFile(true, InFile);
+  // We use createOutputFile here because this is exposed via libclang, and we
+  // must disable the RemoveFileOnSignal behavior.
+  OS = CI.createOutputFile(CI.getFrontendOpts().OutputFile, /*Binary=*/true,
+                           /*RemoveFileOnSignal=*/false, InFile);
   if (!OS)
     return true;
 
+  OutputFile = CI.getFrontendOpts().OutputFile;
   Chaining = CI.getInvocation().getFrontendOpts().ChainedPCH &&
              !CI.getPreprocessorOpts().ImplicitPCHInclude.empty();
   return false;
 }
 
-ASTConsumer *InheritanceViewAction::CreateASTConsumer(CompilerInstance &CI,
-                                                      llvm::StringRef InFile) {
-  return CreateInheritanceViewer(CI.getFrontendOpts().ViewClassInheritance);
-}
-
 ASTConsumer *SyntaxOnlyAction::CreateASTConsumer(CompilerInstance &CI,
                                                  llvm::StringRef InFile) {
   return new ASTConsumer();
@@ -193,6 +205,7 @@ void PrintPreambleAction::ExecuteAction() {
   case IK_ObjC:
   case IK_ObjCXX:
   case IK_OpenCL:
+  case IK_CUDA:
     break;
       
   case IK_None:
@@ -207,7 +220,9 @@ void PrintPreambleAction::ExecuteAction() {
     return;
   }
   
-  llvm::MemoryBuffer *Buffer = llvm::MemoryBuffer::getFile(getCurrentFile());
+  CompilerInstance &CI = getCompilerInstance();
+  llvm::MemoryBuffer *Buffer
+      = CI.getFileManager().getBufferForFile(getCurrentFile());
   if (Buffer) {
     unsigned Preamble = Lexer::ComputePreamble(Buffer).first;
     llvm::outs().write(Buffer->getBufferStart(), Preamble);
diff --git a/lib/Frontend/FrontendOptions.cpp b/lib/Frontend/FrontendOptions.cpp
index 9dfee244ea36..0a2005192a19 100644
--- a/lib/Frontend/FrontendOptions.cpp
+++ b/lib/Frontend/FrontendOptions.cpp
@@ -26,6 +26,7 @@ InputKind FrontendOptions::getInputKindForExtension(llvm::StringRef Extension) {
     .Cases("C", "cc", "cp", IK_CXX)
     .Cases("cpp", "CPP", "c++", "cxx", "hpp", IK_CXX)
     .Case("cl", IK_OpenCL)
+    .Case("cu", IK_CUDA)
     .Cases("ll", "bc", IK_LLVM_IR)
     .Default(IK_C);
 }
diff --git a/lib/Frontend/HeaderIncludeGen.cpp b/lib/Frontend/HeaderIncludeGen.cpp
new file mode 100644
index 000000000000..45ff1d2e412f
--- /dev/null
+++ b/lib/Frontend/HeaderIncludeGen.cpp
@@ -0,0 +1,113 @@
+//===--- HeaderIncludes.cpp - Generate Header Includes --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/Utils.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace clang;
+
+namespace {
+class HeaderIncludesCallback : public PPCallbacks {
+  SourceManager &SM;
+  llvm::raw_ostream *OutputFile;
+  unsigned CurrentIncludeDepth;
+  bool HasProcessedPredefines;
+  bool OwnsOutputFile;
+  bool ShowAllHeaders;
+
+public:
+  HeaderIncludesCallback(const Preprocessor *PP, bool ShowAllHeaders_,
+                         llvm::raw_ostream *OutputFile_, bool OwnsOutputFile_)
+    : SM(PP->getSourceManager()), OutputFile(OutputFile_),
+      CurrentIncludeDepth(0), HasProcessedPredefines(false),
+      OwnsOutputFile(OwnsOutputFile_), ShowAllHeaders(ShowAllHeaders_) {}
+
+  ~HeaderIncludesCallback() {
+    if (OwnsOutputFile)
+      delete OutputFile;
+  }
+
+  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+                           SrcMgr::CharacteristicKind FileType);
+};
+}
+
+void clang::AttachHeaderIncludeGen(Preprocessor &PP, bool ShowAllHeaders,
+                                   llvm::StringRef OutputPath) {
+  llvm::raw_ostream *OutputFile = &llvm::errs();
+  bool OwnsOutputFile = false;
+
+  // Open the output file, if used.
+  if (!OutputPath.empty()) {
+    std::string Error;
+    llvm::raw_fd_ostream *OS = new llvm::raw_fd_ostream(
+      OutputPath.str().c_str(), Error, llvm::raw_fd_ostream::F_Append);
+    if (!Error.empty()) {
+      PP.getDiagnostics().Report(
+        clang::diag::warn_fe_cc_print_header_failure) << Error;
+      delete OS;
+    } else {
+      OS->SetUnbuffered();
+      OS->SetUseAtomicWrites(true);
+      OutputFile = OS;
+      OwnsOutputFile = true;
+    }
+  }
+
+  PP.addPPCallbacks(new HeaderIncludesCallback(&PP, ShowAllHeaders,
+                                               OutputFile, OwnsOutputFile));
+}
+
+void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
+                                         FileChangeReason Reason,
+                                       SrcMgr::CharacteristicKind NewFileType) {
+  // Unless we are exiting a #include, make sure to skip ahead to the line the
+  // #include directive was at.
+  PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
+  if (UserLoc.isInvalid())
+    return;
+
+  // Adjust the current include depth.
+  if (Reason == PPCallbacks::EnterFile) {
+    ++CurrentIncludeDepth;
+  } else {
+    if (CurrentIncludeDepth)
+      --CurrentIncludeDepth;
+
+    // We track when we are done with the predefines by watching for the first
+    // place where we drop back to a nesting depth of 0.
+    if (CurrentIncludeDepth == 0 && !HasProcessedPredefines)
+      HasProcessedPredefines = true;
+  }
+
+  // Show the header if we are (a) past the predefines, or (b) showing all
+  // headers and in the predefines at a depth past the initial file and command
+  // line buffers.
+  bool ShowHeader = (HasProcessedPredefines ||
+                     (ShowAllHeaders && CurrentIncludeDepth > 2));
+
+  // Dump the header include information we are past the predefines buffer or
+  // are showing all headers.
+  if (ShowHeader && Reason == PPCallbacks::EnterFile) {
+    // Write to a temporary string to avoid unnecessary flushing on errs().
+    llvm::SmallString<512> Filename(UserLoc.getFilename());
+    Lexer::Stringify(Filename);
+
+    llvm::SmallString<256> Msg;
+    for (unsigned i = 0; i != CurrentIncludeDepth; ++i)
+      Msg += '.';
+    Msg += ' ';
+    Msg += Filename;
+    Msg += '\n';
+
+    OutputFile->write(Msg.data(), Msg.size());
+  }
+}
diff --git a/lib/Frontend/InitHeaderSearch.cpp b/lib/Frontend/InitHeaderSearch.cpp
index df917136203f..4855b62b747d 100644
--- a/lib/Frontend/InitHeaderSearch.cpp
+++ b/lib/Frontend/InitHeaderSearch.cpp
@@ -11,6 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifdef HAVE_CLANG_CONFIG_H
+# include "clang/Config/config.h"
+#endif
+
 #include "clang/Frontend/Utils.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangOptions.h"
@@ -23,7 +27,7 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Config/config.h"
 #ifdef _MSC_VER
   #define WIN32_LEAN_AND_MEAN 1
@@ -41,13 +45,15 @@ class InitHeaderSearch {
   std::vector<DirectoryLookup> IncludeGroup[4];
   HeaderSearch& Headers;
   bool Verbose;
-  std::string isysroot;
+  std::string IncludeSysroot;
+  bool IsNotEmptyOrRoot;
 
 public:
 
-  InitHeaderSearch(HeaderSearch &HS,
-      bool verbose = false, const std::string &iSysroot = "")
-    : Headers(HS), Verbose(verbose), isysroot(iSysroot) {}
+  InitHeaderSearch(HeaderSearch &HS, bool verbose, llvm::StringRef sysroot)
+    : Headers(HS), Verbose(verbose), IncludeSysroot(sysroot),
+      IsNotEmptyOrRoot(!(sysroot.empty() || sysroot == "/")) {
+  }
 
   /// AddPath - Add the specified path to the specified group list.
   void AddPath(const llvm::Twine &Path, IncludeDirGroup Group,
@@ -62,7 +68,7 @@ public:
                                    llvm::StringRef Dir64,
                                    const llvm::Triple &triple);
 
-  /// AddMinGWCPlusPlusIncludePaths - Add the necessary paths to suport a MinGW
+  /// AddMinGWCPlusPlusIncludePaths - Add the necessary paths to support a MinGW
   ///  libstdc++.
   void AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
                                      llvm::StringRef Arch,
@@ -101,19 +107,18 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
   FileManager &FM = Headers.getFileMgr();
 
   // Compute the actual path, taking into consideration -isysroot.
-  llvm::SmallString<256> MappedPathStr;
-  llvm::raw_svector_ostream MappedPath(MappedPathStr);
+  llvm::SmallString<256> MappedPathStorage;
+  llvm::StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
 
   // Handle isysroot.
-  if (Group == System && !IgnoreSysRoot) {
-    // FIXME: Portability.  This should be a sys::Path interface, this doesn't
-    // handle things like C:\ right, nor win32 \\network\device\blah.
-    if (isysroot.size() != 1 || isysroot[0] != '/') // Add isysroot if present.
-      MappedPath << isysroot;
+  if (Group == System && !IgnoreSysRoot &&
+      llvm::sys::path::is_absolute(MappedPathStr) &&
+      IsNotEmptyOrRoot) {
+    MappedPathStorage.clear();
+    MappedPathStr =
+      (IncludeSysroot + Path).toStringRef(MappedPathStorage);
   }
 
-  Path.print(MappedPath);
-
   // Compute the DirectoryLookup type.
   SrcMgr::CharacteristicKind Type;
   if (Group == Quoted || Group == Angled)
@@ -125,7 +130,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
 
 
   // If the directory exists, add it.
-  if (const DirectoryEntry *DE = FM.getDirectory(MappedPath.str())) {
+  if (const DirectoryEntry *DE = FM.getDirectory(MappedPathStr)) {
     IncludeGroup[Group].push_back(DirectoryLookup(DE, Type, isUserSupplied,
                                                   isFramework));
     return;
@@ -134,7 +139,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
   // Check to see if this is an apple-style headermap (which are not allowed to
   // be frameworks).
   if (!isFramework) {
-    if (const FileEntry *FE = FM.getFile(MappedPath.str())) {
+    if (const FileEntry *FE = FM.getFile(MappedPathStr)) {
       if (const HeaderMap *HM = Headers.CreateHeaderMap(FE)) {
         // It is a headermap, add it to the search path.
         IncludeGroup[Group].push_back(DirectoryLookup(HM, Type,isUserSupplied));
@@ -145,7 +150,7 @@ void InitHeaderSearch::AddPath(const llvm::Twine &Path,
 
   if (Verbose)
     llvm::errs() << "ignoring nonexistent directory \""
-                 << MappedPath.str() << "\"\n";
+                 << MappedPathStr << "\"\n";
 }
 
 
@@ -191,8 +196,6 @@ void InitHeaderSearch::AddGnuCPlusPlusIncludePaths(llvm::StringRef Base,
 void InitHeaderSearch::AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
                                                      llvm::StringRef Arch,
                                                      llvm::StringRef Version) {
-  AddPath(Base + "/" + Arch + "/" + Version + "/include",
-          System, true, false, false);
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++",
           System, true, false, false);
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++/" + Arch,
@@ -344,7 +347,7 @@ static bool getVisualStudioDir(std::string &path) {
   bool hasVCExpressDir = getSystemRegistryString(
     "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VCExpress\\$VERSION",
     "InstallDir", vsExpressIDEInstallDir, sizeof(vsExpressIDEInstallDir) - 1);
-    // If we have both vc80 and vc90, pick version we were compiled with. 
+    // If we have both vc80 and vc90, pick version we were compiled with.
   if (hasVCDir && vsIDEInstallDir[0]) {
     char *p = (char*)strstr(vsIDEInstallDir, "\\Common7\\IDE");
     if (p)
@@ -366,7 +369,7 @@ static bool getVisualStudioDir(std::string &path) {
     const char* vs80comntools = getenv("VS80COMNTOOLS");
     const char* vscomntools = NULL;
 
-    // Try to find the version that we were compiled with 
+    // Try to find the version that we were compiled with
     if(false) {}
     #if (_MSC_VER >= 1600)  // VC100
     else if(vs100comntools) {
@@ -409,7 +412,7 @@ static bool getWindowsSDKDir(std::string &path) {
   bool hasSDKDir = getSystemRegistryString(
    "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Microsoft SDKs\\Windows\\$VERSION",
     "InstallationFolder", windowsSDKInstallDir, sizeof(windowsSDKInstallDir) - 1);
-    // If we have both vc80 and vc90, pick version we were compiled with. 
+    // If we have both vc80 and vc90, pick version we were compiled with.
   if (hasSDKDir && windowsSDKInstallDir[0]) {
     path = windowsSDKInstallDir;
     return(true);
@@ -419,8 +422,16 @@ static bool getWindowsSDKDir(std::string &path) {
 
 void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
                                             const HeaderSearchOptions &HSOpts) {
-  // FIXME: temporary hack: hard-coded paths.
-  AddPath("/usr/local/include", System, true, false, false);
+  llvm::Triple::OSType os = triple.getOS();
+
+  switch (os) {
+  case llvm::Triple::NetBSD:
+    break;
+  default:
+    // FIXME: temporary hack: hard-coded paths.
+    AddPath("/usr/local/include", System, true, false, false);
+    break;
+  }
 
   // Builtin includes use #include_next directives and should be positioned
   // just prior C include dirs.
@@ -439,47 +450,39 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     CIncludeDirs.split(dirs, ":");
     for (llvm::SmallVectorImpl<llvm::StringRef>::iterator i = dirs.begin();
          i != dirs.end();
-         ++i) 
+         ++i)
       AddPath(*i, System, false, false, false);
     return;
   }
-  llvm::Triple::OSType os = triple.getOS();
+
   switch (os) {
-  case llvm::Triple::Win32:
-    {
-      std::string VSDir;
-      std::string WindowsSDKDir;
-      if (getVisualStudioDir(VSDir)) {
-        AddPath(VSDir + "\\VC\\include", System, false, false, false);
-        if (getWindowsSDKDir(WindowsSDKDir))
-          AddPath(WindowsSDKDir + "\\include", System, false, false, false);
-        else
-          AddPath(VSDir + "\\VC\\PlatformSDK\\Include",
-            System, false, false, false);
-      }
-      else {
-          // Default install paths.
-        AddPath("C:/Program Files/Microsoft Visual Studio 10.0/VC/include",
-          System, false, false, false);
-        AddPath("C:/Program Files/Microsoft Visual Studio 9.0/VC/include",
-          System, false, false, false);
-        AddPath(
+  case llvm::Triple::Win32: {
+    std::string VSDir;
+    std::string WindowsSDKDir;
+    if (getVisualStudioDir(VSDir)) {
+      AddPath(VSDir + "\\VC\\include", System, false, false, false);
+      if (getWindowsSDKDir(WindowsSDKDir))
+        AddPath(WindowsSDKDir + "\\include", System, false, false, false);
+      else
+        AddPath(VSDir + "\\VC\\PlatformSDK\\Include",
+                System, false, false, false);
+    } else {
+      // Default install paths.
+      AddPath("C:/Program Files/Microsoft Visual Studio 10.0/VC/include",
+              System, false, false, false);
+      AddPath("C:/Program Files/Microsoft Visual Studio 9.0/VC/include",
+              System, false, false, false);
+      AddPath(
         "C:/Program Files/Microsoft Visual Studio 9.0/VC/PlatformSDK/Include",
-          System, false, false, false);
-        AddPath("C:/Program Files/Microsoft Visual Studio 8/VC/include",
-          System, false, false, false);
-        AddPath(
+        System, false, false, false);
+      AddPath("C:/Program Files/Microsoft Visual Studio 8/VC/include",
+              System, false, false, false);
+      AddPath(
         "C:/Program Files/Microsoft Visual Studio 8/VC/PlatformSDK/Include",
-          System, false, false, false);
-          // For some clang developers.
-        AddPath("G:/Program Files/Microsoft Visual Studio 9.0/VC/include",
-          System, false, false, false);
-        AddPath(
-        "G:/Program Files/Microsoft Visual Studio 9.0/VC/PlatformSDK/Include",
-          System, false, false, false);
-      }
+        System, false, false, false);
     }
     break;
+  }
   case llvm::Triple::Haiku:
     AddPath("/boot/common/include", System, true, false, false);
     AddPath("/boot/develop/headers/os", System, true, false, false);
@@ -487,7 +490,7 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath("/boot/develop/headers/os/arch", System, true, false, false);
     AddPath("/boot/develop/headers/os/device", System, true, false, false);
     AddPath("/boot/develop/headers/os/drivers", System, true, false, false);
-    AddPath("/boot/develop/headers/os/game", System, true, false, false); 
+    AddPath("/boot/develop/headers/os/game", System, true, false, false);
     AddPath("/boot/develop/headers/os/interface", System, true, false, false);
     AddPath("/boot/develop/headers/os/kernel", System, true, false, false);
     AddPath("/boot/develop/headers/os/locale", System, true, false, false);
@@ -500,13 +503,13 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath("/boot/develop/headers/os/support", System, true, false, false);
     AddPath("/boot/develop/headers/os/translation",
       System, true, false, false);
-    AddPath("/boot/develop/headers/os/add-ons/graphics", 
+    AddPath("/boot/develop/headers/os/add-ons/graphics",
       System, true, false, false);
-    AddPath("/boot/develop/headers/os/add-ons/input_server", 
+    AddPath("/boot/develop/headers/os/add-ons/input_server",
       System, true, false, false);
-    AddPath("/boot/develop/headers/os/add-ons/screen_saver", 
+    AddPath("/boot/develop/headers/os/add-ons/screen_saver",
       System, true, false, false);
-    AddPath("/boot/develop/headers/os/add-ons/tracker", 
+    AddPath("/boot/develop/headers/os/add-ons/tracker",
       System, true, false, false);
     AddPath("/boot/develop/headers/os/be_apps/Deskbar",
       System, true, false, false);
@@ -515,7 +518,7 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath("/boot/develop/headers/os/be_apps/Tracker",
       System, true, false, false);
     AddPath("/boot/develop/headers/cpp", System, true, false, false);
-    AddPath("/boot/develop/headers/cpp/i586-pc-haiku", 
+    AddPath("/boot/develop/headers/cpp/i586-pc-haiku",
       System, true, false, false);
     AddPath("/boot/develop/headers/3rdparty", System, true, false, false);
     AddPath("/boot/develop/headers/bsd", System, true, false, false);
@@ -523,7 +526,9 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath("/boot/develop/headers/posix", System, true, false, false);
     AddPath("/boot/develop/headers",  System, true, false, false);
     break;
-  case llvm::Triple::MinGW64:
+  case llvm::Triple::Cygwin:
+    AddPath("/usr/include/w32api", System, true, false, false);
+    break;
   case llvm::Triple::MinGW32:
     AddPath("c:/mingw/include", System, true, false, false);
     break;
@@ -553,24 +558,24 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
   // FIXME: temporary hack: hard-coded paths.
   switch (os) {
   case llvm::Triple::Cygwin:
-    AddPath("/lib/gcc/i686-pc-cygwin/3.4.4/include",
-        System, true, false, false);
-    AddPath("/lib/gcc/i686-pc-cygwin/3.4.4/include/c++",
-        System, true, false, false);
-    AddPath("/lib/gcc/i686-pc-cygwin/3.4.4/include/c++/i686-pc-cygwin",
-        System, true, false, false);
+    // Cygwin-1.7
+    AddMinGWCPlusPlusIncludePaths("/usr/lib/gcc", "i686-pc-cygwin", "4.3.4");
+    // g++-4 / Cygwin-1.5
+    AddMinGWCPlusPlusIncludePaths("/usr/lib/gcc", "i686-pc-cygwin", "4.3.2");
+    // FIXME: Do we support g++-3.4.4?
+    AddMinGWCPlusPlusIncludePaths("/usr/lib/gcc", "i686-pc-cygwin", "3.4.4");
     break;
-  case llvm::Triple::MinGW64:
-    // Try gcc 4.5.0
-    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.5.0");
-    // Try gcc 4.4.0
-    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.4.0");
-    // Try gcc 4.3.0
-    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.3.0");
-    // Fall through.
   case llvm::Triple::MinGW32:
-   // Try gcc 4.5.0
-    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.5.0"); 
+    // mingw-w64-20110207
+    AddPath("c:/MinGW/include/c++/4.5.3", System, true, false, false);
+    AddPath("c:/MinGW/include/c++/4.5.3/x86_64-w64-mingw32", System, true, false, false);
+    AddPath("c:/MinGW/include/c++/4.5.3/backward", System, true, false, false);
+    // mingw-w64-20101129
+    AddPath("c:/MinGW/include/c++/4.5.2", System, true, false, false);
+    AddPath("c:/MinGW/include/c++/4.5.2/x86_64-w64-mingw32", System, true, false, false);
+    AddPath("c:/MinGW/include/c++/4.5.2/backward", System, true, false, false);
+    // Try gcc 4.5.0
+    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.5.0");
     // Try gcc 4.4.0
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.4.0");
     // Try gcc 4.3.0
@@ -580,13 +585,13 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     switch (triple.getArch()) {
     default: break;
 
-    case llvm::Triple::ppc: 
+    case llvm::Triple::ppc:
     case llvm::Triple::ppc64:
       AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.2.1",
-                                  "powerpc-apple-darwin10", "", "ppc64", 
+                                  "powerpc-apple-darwin10", "", "ppc64",
                                   triple);
       AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.0.0",
-                                  "powerpc-apple-darwin10", "", "ppc64", 
+                                  "powerpc-apple-darwin10", "", "ppc64",
                                   triple);
       break;
 
@@ -615,6 +620,11 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     // Debian based distros.
     // Note: these distros symlink /usr/include/c++/X.Y.Z -> X.Y
     //===------------------------------------------------------------------===//
+    // Ubuntu 10.10 "Maverick Meerkat" -- gcc-4.4.5
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.4",
+                                "i686-linux-gnu", "", "64", triple);
+    // The rest of 10.10 is the same as previous versions.
+
     // Ubuntu 10.04 LTS "Lucid Lynx" -- gcc-4.4.3
     // Ubuntu 9.10 "Karmic Koala"    -- gcc-4.4.1
     // Debian 6.0 "squeeze"          -- gcc-4.4.2
@@ -622,6 +632,8 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
                                 "x86_64-linux-gnu", "32", "", triple);
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.4",
                                 "i486-linux-gnu", "", "64", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.4",
+                                "arm-linux-gnueabi", "", "", triple);
     // Ubuntu 9.04 "Jaunty Jackalope" -- gcc-4.3.3
     // Ubuntu 8.10 "Intrepid Ibex"    -- gcc-4.3.2
     // Debian 5.0 "lenny"             -- gcc-4.3.2
@@ -646,6 +658,11 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     //===------------------------------------------------------------------===//
     // Redhat based distros.
     //===------------------------------------------------------------------===//
+    // Fedora 14
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.1",
+                                "x86_64-redhat-linux", "32", "", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5.1",
+                                "i686-redhat-linux", "", "", triple);
     // Fedora 13
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.4.4",
                                 "x86_64-redhat-linux", "32", "", triple);
@@ -701,11 +718,22 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
                                 "i586-suse-linux", "", "", triple);
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.4",
                                 "x86_64-suse-linux", "", "", triple);
+
+    // openSUSE 11.4
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5",
+                                "i586-suse-linux", "", "", triple);
+    AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.5",
+                                "x86_64-suse-linux", "", "", triple);
+
     // Arch Linux 2008-06-24
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.3.1",
                                 "i686-pc-linux-gnu", "", "", triple);
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.3.1",
                                 "x86_64-unknown-linux-gnu", "", "", triple);
+    // Gentoo x86 2010.0 stable
+    AddGnuCPlusPlusIncludePaths(
+      "/usr/lib/gcc/i686-pc-linux-gnu/4.4.3/include/g++-v4",
+      "i686-pc-linux-gnu", "", "", triple);
     // Gentoo x86 2009.1 stable
     AddGnuCPlusPlusIncludePaths(
       "/usr/lib/gcc/i686-pc-linux-gnu/4.3.4/include/g++-v4",
@@ -718,26 +746,33 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     AddGnuCPlusPlusIncludePaths(
       "/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include/g++-v4",
       "i686-pc-linux-gnu", "", "", triple);
-    // Gentoo amd64 stable
+
+    // Gentoo amd64 gcc 4.4.5
     AddGnuCPlusPlusIncludePaths(
-        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.1.2/include/g++-v4",
-        "i686-pc-linux-gnu", "", "", triple);
-        
-    // Gentoo amd64 gcc 4.3.2
+        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.5/include/g++-v4",
+        "x86_64-pc-linux-gnu", "32", "", triple);
+    // Gentoo amd64 gcc 4.4.4
     AddGnuCPlusPlusIncludePaths(
-        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.3.2/include/g++-v4",
-        "x86_64-pc-linux-gnu", "", "", triple);
-        
+        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.4/include/g++-v4",
+        "x86_64-pc-linux-gnu", "32", "", triple);
     // Gentoo amd64 gcc 4.4.3
     AddGnuCPlusPlusIncludePaths(
         "/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.3/include/g++-v4",
         "x86_64-pc-linux-gnu", "32", "", triple);
+    // Gentoo amd64 gcc 4.3.2
+    AddGnuCPlusPlusIncludePaths(
+        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.3.2/include/g++-v4",
+        "x86_64-pc-linux-gnu", "", "", triple);
+    // Gentoo amd64 stable
+    AddGnuCPlusPlusIncludePaths(
+        "/usr/lib/gcc/x86_64-pc-linux-gnu/4.1.2/include/g++-v4",
+        "i686-pc-linux-gnu", "", "", triple);
 
     // Gentoo amd64 llvm-gcc trunk
     AddGnuCPlusPlusIncludePaths(
         "/usr/lib/llvm-gcc-4.2-9999/include/c++/4.2.1",
         "x86_64-pc-linux-gnu", "", "", triple);
-    
+
     break;
   case llvm::Triple::FreeBSD:
     // FreeBSD 8.0
@@ -774,8 +809,13 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
 void InitHeaderSearch::AddDefaultSystemIncludePaths(const LangOptions &Lang,
                                                     const llvm::Triple &triple,
                                             const HeaderSearchOptions &HSOpts) {
-  if (Lang.CPlusPlus && HSOpts.UseStandardCXXIncludes)
-    AddDefaultCPlusPlusIncludePaths(triple);
+  if (Lang.CPlusPlus && HSOpts.UseStandardCXXIncludes) {
+    if (!HSOpts.CXXSystemIncludes.empty()) {
+      for (unsigned i = 0, e = HSOpts.CXXSystemIncludes.size(); i != e; ++i)
+        AddPath(HSOpts.CXXSystemIncludes[i], System, true, false, false);
+    } else
+      AddDefaultCPlusPlusIncludePaths(triple);
+  }
 
   AddDefaultCIncludePaths(triple, HSOpts);
 
diff --git a/lib/Frontend/InitPreprocessor.cpp b/lib/Frontend/InitPreprocessor.cpp
index 0d07192b50e5..d0111a5d26a9 100644
--- a/lib/Frontend/InitPreprocessor.cpp
+++ b/lib/Frontend/InitPreprocessor.cpp
@@ -22,8 +22,9 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/APFloat.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 using namespace clang;
 
 // Append a #define line to Buf for Macro.  Macro should be of the form XXX,
@@ -55,9 +56,10 @@ std::string clang::NormalizeDashIncludePath(llvm::StringRef File) {
   // it has not file entry. For now, workaround this by using an
   // absolute path if we find the file here, and otherwise letting
   // header search handle it.
-  llvm::sys::Path Path(File);
-  Path.makeAbsolute();
-  if (!Path.exists())
+  llvm::SmallString<128> Path(File);
+  llvm::sys::fs::make_absolute(Path);
+  bool exists;
+  if (llvm::sys::fs::exists(Path.str(), exists) || !exists)
     Path = File;
 
   return Lexer::Stringify(Path.str());
@@ -342,6 +344,10 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
       Builder.defineMacro("_NATIVE_WCHAR_T_DEFINED");
       Builder.append("class type_info;");
     }
+
+    if (LangOpts.CPlusPlus0x) {
+      Builder.defineMacro("_HAS_CHAR16_T_LANGUAGE_SUPPORT", "1");
+    }
   }
 
   if (LangOpts.Optimize)
@@ -465,6 +471,9 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   if (FEOpts.ProgramAction == frontend::RunAnalysis)
     Builder.defineMacro("__clang_analyzer__");
 
+  if (LangOpts.FastRelaxedMath)
+    Builder.defineMacro("__FAST_RELAXED_MATH__");
+
   // Get other target #defines.
   TI.getTargetDefines(LangOpts, Builder);
 }
@@ -515,8 +524,7 @@ static void InitializeFileRemapping(Diagnostic &Diags,
     
     // Create the file entry for the file that we're mapping from.
     const FileEntry *FromFile = FileMgr.getVirtualFile(Remap->first,
-                                                       ToFile->getSize(),
-                                                       0);
+                                                       ToFile->getSize(), 0);
     if (!FromFile) {
       Diags.Report(diag::err_fe_remap_missing_from_file)
       << Remap->first;
@@ -526,7 +534,7 @@ static void InitializeFileRemapping(Diagnostic &Diags,
     // Load the contents of the file we're mapping to.
     std::string ErrorStr;
     const llvm::MemoryBuffer *Buffer
-    = llvm::MemoryBuffer::getFile(ToFile->getName(), &ErrorStr);
+      = FileMgr.getBufferForFile(ToFile->getName(), &ErrorStr);
     if (!Buffer) {
       Diags.Report(diag::err_fe_error_opening)
         << Remap->second << ErrorStr;
diff --git a/lib/Frontend/MultiplexConsumer.cpp b/lib/Frontend/MultiplexConsumer.cpp
new file mode 100644
index 000000000000..3649c3c99728
--- /dev/null
+++ b/lib/Frontend/MultiplexConsumer.cpp
@@ -0,0 +1,221 @@
+//===- MultiplexConsumer.cpp - AST Consumer for PCH Generation --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the MultiplexConsumer class. It also declares and defines
+//  MultiplexASTDeserializationListener and  MultiplexASTMutationListener, which
+//  are implementation details of MultiplexConsumer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/MultiplexConsumer.h"
+
+#include "clang/AST/ASTMutationListener.h"
+#include "clang/AST/DeclGroup.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
+
+using namespace clang;
+
+namespace clang {
+
+// This ASTDeserializationListener forwards its notifications to a set of
+// child listeners.
+class MultiplexASTDeserializationListener
+    : public ASTDeserializationListener {
+public:
+  // Does NOT take ownership of the elements in L.
+  MultiplexASTDeserializationListener(
+      const std::vector<ASTDeserializationListener*>& L);
+  virtual void ReaderInitialized(ASTReader *Reader);
+  virtual void IdentifierRead(serialization::IdentID ID,
+                              IdentifierInfo *II);
+  virtual void TypeRead(serialization::TypeIdx Idx, QualType T);
+  virtual void DeclRead(serialization::DeclID ID, const Decl *D);
+  virtual void SelectorRead(serialization::SelectorID iD, Selector Sel);
+  virtual void MacroDefinitionRead(serialization::MacroID, 
+                                   MacroDefinition *MD);
+private:
+  std::vector<ASTDeserializationListener*> Listeners;
+};
+
+MultiplexASTDeserializationListener::MultiplexASTDeserializationListener(
+      const std::vector<ASTDeserializationListener*>& L)
+    : Listeners(L) {
+}
+
+void MultiplexASTDeserializationListener::ReaderInitialized(
+    ASTReader *Reader) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->ReaderInitialized(Reader);
+}
+
+void MultiplexASTDeserializationListener::IdentifierRead(
+    serialization::IdentID ID, IdentifierInfo *II) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->IdentifierRead(ID, II);
+}
+
+void MultiplexASTDeserializationListener::TypeRead(
+    serialization::TypeIdx Idx, QualType T) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->TypeRead(Idx, T);
+}
+
+void MultiplexASTDeserializationListener::DeclRead(
+    serialization::DeclID ID, const Decl *D) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->DeclRead(ID, D);
+}
+
+void MultiplexASTDeserializationListener::SelectorRead(
+    serialization::SelectorID ID, Selector Sel) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->SelectorRead(ID, Sel);
+}
+
+void MultiplexASTDeserializationListener::MacroDefinitionRead(
+    serialization::MacroID ID, MacroDefinition *MD) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->MacroDefinitionRead(ID, MD);
+}
+
+// This ASTMutationListener forwards its notifications to a set of
+// child listeners.
+class MultiplexASTMutationListener : public ASTMutationListener {
+public:
+  // Does NOT take ownership of the elements in L.
+  MultiplexASTMutationListener(const std::vector<ASTMutationListener*>& L);
+  virtual void CompletedTagDefinition(const TagDecl *D);
+  virtual void AddedVisibleDecl(const DeclContext *DC, const Decl *D);
+  virtual void AddedCXXImplicitMember(const CXXRecordDecl *RD, const Decl *D);
+  virtual void AddedCXXTemplateSpecialization(const ClassTemplateDecl *TD,
+                                    const ClassTemplateSpecializationDecl *D);
+private:
+  std::vector<ASTMutationListener*> Listeners;
+};
+
+MultiplexASTMutationListener::MultiplexASTMutationListener(
+    const std::vector<ASTMutationListener*>& L)
+    : Listeners(L) {
+}
+
+void MultiplexASTMutationListener::CompletedTagDefinition(const TagDecl *D) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->CompletedTagDefinition(D);
+}
+
+void MultiplexASTMutationListener::AddedVisibleDecl(
+    const DeclContext *DC, const Decl *D) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->AddedVisibleDecl(DC, D);
+}
+
+void MultiplexASTMutationListener::AddedCXXImplicitMember(
+    const CXXRecordDecl *RD, const Decl *D) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->AddedCXXImplicitMember(RD, D);
+}
+void MultiplexASTMutationListener::AddedCXXTemplateSpecialization(
+    const ClassTemplateDecl *TD, const ClassTemplateSpecializationDecl *D) {
+  for (size_t i = 0, e = Listeners.size(); i != e; ++i)
+    Listeners[i]->AddedCXXTemplateSpecialization(TD, D);
+}
+
+}  // end namespace clang
+
+
+MultiplexConsumer::MultiplexConsumer(const std::vector<ASTConsumer*>& C)
+    : Consumers(C), MutationListener(0), DeserializationListener(0) {
+  // Collect the mutation listeners and deserialization listeners of all
+  // children, and create a multiplex listener each if so.
+  std::vector<ASTMutationListener*> mutationListeners;
+  std::vector<ASTDeserializationListener*> serializationListeners;
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i) {
+    ASTMutationListener* mutationListener =
+        Consumers[i]->GetASTMutationListener();
+    if (mutationListener)
+      mutationListeners.push_back(mutationListener);
+    ASTDeserializationListener* serializationListener =
+        Consumers[i]->GetASTDeserializationListener();
+    if (serializationListener)
+      serializationListeners.push_back(serializationListener);
+  }
+  if (mutationListeners.size()) {
+    MutationListener.reset(new MultiplexASTMutationListener(mutationListeners));
+  }
+  if (serializationListeners.size()) {
+    DeserializationListener.reset(
+        new MultiplexASTDeserializationListener(serializationListeners));
+  }
+}
+
+MultiplexConsumer::~MultiplexConsumer() {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    delete Consumers[i];
+}
+
+void MultiplexConsumer::Initialize(ASTContext &Context) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->Initialize(Context);
+}
+
+void MultiplexConsumer::HandleTopLevelDecl(DeclGroupRef D) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->HandleTopLevelDecl(D);
+}
+
+void MultiplexConsumer::HandleInterestingDecl(DeclGroupRef D) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->HandleInterestingDecl(D);
+}
+
+void MultiplexConsumer::HandleTranslationUnit(ASTContext &Ctx) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->HandleTranslationUnit(Ctx);
+}
+
+void MultiplexConsumer::HandleTagDeclDefinition(TagDecl *D) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->HandleTagDeclDefinition(D);
+}
+
+void MultiplexConsumer::CompleteTentativeDefinition(VarDecl *D) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->CompleteTentativeDefinition(D);
+}
+
+void MultiplexConsumer::HandleVTable(
+    CXXRecordDecl *RD, bool DefinitionRequired) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->HandleVTable(RD, DefinitionRequired);
+}
+
+ASTMutationListener *MultiplexConsumer::GetASTMutationListener() {
+  return MutationListener.get();
+}
+
+ASTDeserializationListener *MultiplexConsumer::GetASTDeserializationListener() {
+  return DeserializationListener.get();
+}
+
+void MultiplexConsumer::PrintStats() {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    Consumers[i]->PrintStats();
+}
+
+void MultiplexConsumer::InitializeSema(Sema &S) {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    if (SemaConsumer *SC = dyn_cast<SemaConsumer>(Consumers[i]))
+      SC->InitializeSema(S);
+}
+
+void MultiplexConsumer::ForgetSema() {
+  for (size_t i = 0, e = Consumers.size(); i != e; ++i)
+    if (SemaConsumer *SC = dyn_cast<SemaConsumer>(Consumers[i]))
+      SC->ForgetSema();
+}
diff --git a/lib/Frontend/PrintPreprocessedOutput.cpp b/lib/Frontend/PrintPreprocessedOutput.cpp
index cfaf8a23b118..922d743adf4e 100644
--- a/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -86,9 +86,6 @@ public:
 private:
   unsigned CurLine;
 
-  /// The current include nesting level, used by header include dumping (-H).
-  unsigned CurrentIncludeDepth;
-
   bool EmittedTokensOnThisLine;
   bool EmittedMacroOnThisLine;
   SrcMgr::CharacteristicKind FileType;
@@ -96,22 +93,19 @@ private:
   bool Initialized;
   bool DisableLineMarkers;
   bool DumpDefines;
-  bool DumpHeaderIncludes;
   bool UseLineDirective;
-  bool HasProcessedPredefines;
 public:
   PrintPPOutputPPCallbacks(Preprocessor &pp, llvm::raw_ostream &os,
-                           bool lineMarkers, bool defines, bool headers)
+                           bool lineMarkers, bool defines)
      : PP(pp), SM(PP.getSourceManager()),
        ConcatInfo(PP), OS(os), DisableLineMarkers(lineMarkers),
-       DumpDefines(defines), DumpHeaderIncludes(headers) {
-    CurLine = CurrentIncludeDepth = 0;
+       DumpDefines(defines) {
+    CurLine = 0;
     CurFilename += "<uninit>";
     EmittedTokensOnThisLine = false;
     EmittedMacroOnThisLine = false;
     FileType = SrcMgr::C_User;
     Initialized = false;
-    HasProcessedPredefines = false;
 
     // If we're in microsoft mode, use normal #line instead of line markers.
     UseLineDirective = PP.getLangOptions().Microsoft;
@@ -120,6 +114,8 @@ public:
   void SetEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
   bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
 
+  bool StartNewLineIfNeeded();
+  
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                            SrcMgr::CharacteristicKind FileType);
   virtual void Ident(SourceLocation Loc, const std::string &str);
@@ -129,7 +125,10 @@ public:
 
   bool HandleFirstTokOnLine(Token &Tok);
   bool MoveToLine(SourceLocation Loc) {
-    return MoveToLine(SM.getPresumedLoc(Loc).getLine());
+    PresumedLoc PLoc = SM.getPresumedLoc(Loc);
+    if (PLoc.isInvalid())
+      return false;
+    return MoveToLine(PLoc.getLine());
   }
   bool MoveToLine(unsigned LineNo);
 
@@ -138,15 +137,14 @@ public:
     return ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok);
   }
   void WriteLineInfo(unsigned LineNo, const char *Extra=0, unsigned ExtraLen=0);
-
+  bool LineMarkersAreDisabled() const { return DisableLineMarkers; }
   void HandleNewlinesInToken(const char *TokStr, unsigned Len);
 
   /// MacroDefined - This hook is called whenever a macro definition is seen.
-  void MacroDefined(const IdentifierInfo *II, const MacroInfo *MI);
+  void MacroDefined(const Token &MacroNameTok, const MacroInfo *MI);
 
   /// MacroUndefined - This hook is called whenever a macro #undef is seen.
-  void MacroUndefined(SourceLocation Loc, const IdentifierInfo *II,
-                      const MacroInfo *MI);
+  void MacroUndefined(const Token &MacroNameTok, const MacroInfo *MI);
 };
 }  // end anonymous namespace
 
@@ -162,11 +160,11 @@ void PrintPPOutputPPCallbacks::WriteLineInfo(unsigned LineNo,
   // Emit #line directives or GNU line markers depending on what mode we're in.
   if (UseLineDirective) {
     OS << "#line" << ' ' << LineNo << ' ' << '"';
-    OS.write(&CurFilename[0], CurFilename.size());
+    OS.write(CurFilename.data(), CurFilename.size());
     OS << '"';
   } else {
     OS << '#' << ' ' << LineNo << ' ' << '"';
-    OS.write(&CurFilename[0], CurFilename.size());
+    OS.write(CurFilename.data(), CurFilename.size());
     OS << '"';
 
     if (ExtraLen)
@@ -213,6 +211,17 @@ bool PrintPPOutputPPCallbacks::MoveToLine(unsigned LineNo) {
   return true;
 }
 
+bool PrintPPOutputPPCallbacks::StartNewLineIfNeeded() {
+  if (EmittedTokensOnThisLine || EmittedMacroOnThisLine) {
+    OS << '\n';
+    EmittedTokensOnThisLine = false;
+    EmittedMacroOnThisLine = false;
+    ++CurLine;
+    return true;
+  }
+  
+  return false;
+}
 
 /// FileChanged - Whenever the preprocessor enters or exits a #include file
 /// it invokes this handler.  Update our conception of the current source
@@ -225,10 +234,13 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
   SourceManager &SourceMgr = SM;
   
   PresumedLoc UserLoc = SourceMgr.getPresumedLoc(Loc);
+  if (UserLoc.isInvalid())
+    return;
+  
   unsigned NewLine = UserLoc.getLine();
 
   if (Reason == PPCallbacks::EnterFile) {
-    SourceLocation IncludeLoc = SourceMgr.getPresumedLoc(Loc).getIncludeLoc();
+    SourceLocation IncludeLoc = UserLoc.getIncludeLoc();
     if (IncludeLoc.isValid())
       MoveToLine(IncludeLoc);
   } else if (Reason == PPCallbacks::SystemHeaderPragma) {
@@ -238,19 +250,6 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
     // directive and emits a bunch of spaces that aren't needed.  Emulate this
     // strange behavior.
   }
-
-  // Adjust the current include depth.
-  if (Reason == PPCallbacks::EnterFile) {
-    ++CurrentIncludeDepth;
-  } else {
-    if (CurrentIncludeDepth)
-      --CurrentIncludeDepth;
-
-    // We track when we are done with the predefines by watching for the first
-    // place where we drop back to a nesting depth of 0.
-    if (CurrentIncludeDepth == 0 && !HasProcessedPredefines)
-      HasProcessedPredefines = true;
-  }
   
   CurLine = NewLine;
 
@@ -259,18 +258,6 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
   Lexer::Stringify(CurFilename);
   FileType = NewFileType;
 
-  // Dump the header include information, if enabled and we are past the
-  // predefines buffer.
-  if (DumpHeaderIncludes && HasProcessedPredefines &&
-      Reason == PPCallbacks::EnterFile) {
-    llvm::SmallString<256> Msg;
-    llvm::raw_svector_ostream OS(Msg);
-    for (unsigned i = 0; i != CurrentIncludeDepth; ++i)
-      OS << '.';
-    OS << ' ' << CurFilename << '\n';
-    llvm::errs() << OS.str();
-  }
-
   if (DisableLineMarkers) return;
   
   if (!Initialized) {
@@ -303,7 +290,7 @@ void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
 }
 
 /// MacroDefined - This hook is called whenever a macro definition is seen.
-void PrintPPOutputPPCallbacks::MacroDefined(const IdentifierInfo *II,
+void PrintPPOutputPPCallbacks::MacroDefined(const Token &MacroNameTok,
                                             const MacroInfo *MI) {
   // Only print out macro definitions in -dD mode.
   if (!DumpDefines ||
@@ -311,18 +298,17 @@ void PrintPPOutputPPCallbacks::MacroDefined(const IdentifierInfo *II,
       MI->isBuiltinMacro()) return;
 
   MoveToLine(MI->getDefinitionLoc());
-  PrintMacroDefinition(*II, *MI, PP, OS);
+  PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS);
   EmittedMacroOnThisLine = true;
 }
 
-void PrintPPOutputPPCallbacks::MacroUndefined(SourceLocation Loc,
-                                              const IdentifierInfo *II,
+void PrintPPOutputPPCallbacks::MacroUndefined(const Token &MacroNameTok,
                                               const MacroInfo *MI) {
   // Only print out macro definitions in -dD mode.
   if (!DumpDefines) return;
 
-  MoveToLine(Loc);
-  OS << "#undef " << II->getName();
+  MoveToLine(MacroNameTok.getLocation());
+  OS << "#undef " << MacroNameTok.getIdentifierInfo()->getName();
   EmittedMacroOnThisLine = true;
 }
 
@@ -437,12 +423,14 @@ struct UnknownPragmaHandler : public PragmaHandler {
 
   UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks)
     : Prefix(prefix), Callbacks(callbacks) {}
-  virtual void HandlePragma(Preprocessor &PP, Token &PragmaTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PragmaTok) {
     // Figure out what line we went to and insert the appropriate number of
     // newline characters.
+    Callbacks->StartNewLineIfNeeded();
     Callbacks->MoveToLine(PragmaTok.getLocation());
     Callbacks->OS.write(Prefix, strlen(Prefix));
-
+    Callbacks->SetEmittedTokensOnThisLine();
     // Read and print all of the pragma tokens.
     while (PragmaTok.isNot(tok::eom)) {
       if (PragmaTok.hasLeadingSpace())
@@ -451,7 +439,7 @@ struct UnknownPragmaHandler : public PragmaHandler {
       Callbacks->OS.write(&TokSpell[0], TokSpell.size());
       PP.LexUnexpandedToken(PragmaTok);
     }
-    Callbacks->OS << '\n';
+    Callbacks->StartNewLineIfNeeded();
   }
 };
 } // end anonymous namespace
@@ -561,10 +549,11 @@ void clang::DoPrintPreprocessedInput(Preprocessor &PP, llvm::raw_ostream *OS,
 
   PrintPPOutputPPCallbacks *Callbacks =
       new PrintPPOutputPPCallbacks(PP, *OS, !Opts.ShowLineMarkers,
-                                   Opts.ShowMacros, Opts.ShowHeaderIncludes);
+                                   Opts.ShowMacros);
   PP.AddPragmaHandler(new UnknownPragmaHandler("#pragma", Callbacks));
-  PP.AddPragmaHandler("GCC", new UnknownPragmaHandler("#pragma GCC",
-                                                      Callbacks));
+  PP.AddPragmaHandler("GCC", new UnknownPragmaHandler("#pragma GCC",Callbacks));
+  PP.AddPragmaHandler("clang",
+                      new UnknownPragmaHandler("#pragma clang", Callbacks));
 
   PP.addPPCallbacks(Callbacks);
 
@@ -576,13 +565,20 @@ void clang::DoPrintPreprocessedInput(Preprocessor &PP, llvm::raw_ostream *OS,
   // start.
   const SourceManager &SourceMgr = PP.getSourceManager();
   Token Tok;
-  do PP.Lex(Tok);
-  while (Tok.isNot(tok::eof) && Tok.getLocation().isFileID() &&
-         !strcmp(SourceMgr.getPresumedLoc(Tok.getLocation()).getFilename(),
-                 "<built-in>"));
+  do {
+    PP.Lex(Tok);
+    if (Tok.is(tok::eof) || !Tok.getLocation().isFileID())
+      break;
+
+    PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());
+    if (PLoc.isInvalid())
+      break;
+
+    if (strcmp(PLoc.getFilename(), "<built-in>"))
+      break;
+  } while (true);
 
   // Read all the preprocessed tokens, printing them out to the stream.
   PrintPreprocessedTokens(PP, Tok, Callbacks, *OS);
   *OS << '\n';
 }
-
diff --git a/lib/Frontend/StmtXML.cpp b/lib/Frontend/StmtXML.cpp
index b6607349d76d..c113cc18dc1c 100644
--- a/lib/Frontend/StmtXML.cpp
+++ b/lib/Frontend/StmtXML.cpp
@@ -61,8 +61,7 @@ namespace  {
             Doc.PrintDecl(*DI);
           }
         } else {
-          for (Stmt::child_iterator i = S->child_begin(), e = S->child_end();
-               i != e; ++i)
+          for (Stmt::child_range i = S->children(); i; ++i)
             DumpSubTree(*i);
         }
         Doc.toParent();
@@ -133,7 +132,6 @@ namespace  {
     void VisitBinaryOperator(BinaryOperator *Node);
     void VisitCompoundAssignOperator(CompoundAssignOperator *Node);
     void VisitAddrLabelExpr(AddrLabelExpr *Node);
-    void VisitTypesCompatibleExpr(TypesCompatibleExpr *Node);
 
     // C++
     void VisitCXXNamedCastExpr(CXXNamedCastExpr *Node);
@@ -150,7 +148,6 @@ namespace  {
     void VisitObjCImplicitSetterGetterRefExpr(
                         ObjCImplicitSetterGetterRefExpr *Node);
     void VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node);
-    void VisitObjCSuperExpr(ObjCSuperExpr *Node);
 #endif
   };
 }
@@ -357,12 +354,6 @@ void StmtXML::VisitAddrLabelExpr(AddrLabelExpr *Node) {
   Doc.addAttribute("name", Node->getLabel()->getName());
 }
 
-void StmtXML::VisitTypesCompatibleExpr(TypesCompatibleExpr *Node) {
-  DumpExpr(Node);
-  DumpTypeExpr(Node->getArgType1());
-  DumpTypeExpr(Node->getArgType2());
-}
-
 //===----------------------------------------------------------------------===//
 // C++ Expressions
 //===----------------------------------------------------------------------===//
@@ -428,11 +419,6 @@ void StmtXML::VisitObjCImplicitSetterGetterRefExpr(
   Doc.addAttribute("Setter", Setter ? Setter->getSelector().getAsString().c_str() : "(null)");
 }
 
-void StmtXML::VisitObjCSuperExpr(ObjCSuperExpr *Node) {
-  DumpExpr(Node);
-  Doc.addAttribute("super", "1");
-}
-
 void StmtXML::VisitObjCIvarRefExpr(ObjCIvarRefExpr *Node) {
   DumpExpr(Node);
   Doc.addAttribute("kind", Node->getDecl()->getDeclKindName());
diff --git a/lib/Frontend/TextDiagnosticBuffer.cpp b/lib/Frontend/TextDiagnosticBuffer.cpp
index fdf2ec8ccf5d..069c86de137f 100644
--- a/lib/Frontend/TextDiagnosticBuffer.cpp
+++ b/lib/Frontend/TextDiagnosticBuffer.cpp
@@ -20,6 +20,9 @@ using namespace clang;
 ///
 void TextDiagnosticBuffer::HandleDiagnostic(Diagnostic::Level Level,
                                             const DiagnosticInfo &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticClient::HandleDiagnostic(Level, Info);
+
   llvm::SmallString<100> Buf;
   Info.FormatDiagnostic(Buf);
   switch (Level) {
diff --git a/lib/Frontend/TextDiagnosticPrinter.cpp b/lib/Frontend/TextDiagnosticPrinter.cpp
index 1e453a08fdb9..04c6a68023d9 100644
--- a/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/DiagnosticOptions.h"
 #include "clang/Lex/Lexer.h"
@@ -57,7 +58,9 @@ PrintIncludeStack(SourceLocation Loc, const SourceManager &SM) {
   if (Loc.isInvalid()) return;
 
   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-
+  if (PLoc.isInvalid())
+    return;
+  
   // Print out the other include frames first.
   PrintIncludeStack(PLoc.getIncludeLoc(), SM);
 
@@ -137,8 +140,9 @@ void TextDiagnosticPrinter::HighlightRange(const CharSourceRange &R,
            (SourceLine[EndColNo-1] == ' ' || SourceLine[EndColNo-1] == '\t'))
       --EndColNo;
 
-    // If the start/end passed each other, then we are trying to highlight a range
-    // that just exists in whitespace, which must be some sort of other bug.
+    // If the start/end passed each other, then we are trying to highlight a
+    // range that just exists in whitespace, which must be some sort of other
+    // bug.
     assert(StartColNo <= EndColNo && "Trying to highlight whitespace??");
   }
 
@@ -328,7 +332,9 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
     if (!Suppressed) {
       // Get the pretty name, according to #line directives etc.
       PresumedLoc PLoc = SM.getPresumedLoc(Loc);
-
+      if (PLoc.isInvalid())
+        return;
+      
       // If this diagnostic is not in the main file, print out the
       // "included from" lines.
       if (LastWarningLoc != PLoc.getIncludeLoc()) {
@@ -567,6 +573,10 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
 
         // We specifically do not do word-wrapping or tab-expansion here,
         // because this is supposed to be easy to parse.
+        PresumedLoc PLoc = SM.getPresumedLoc(B);
+        if (PLoc.isInvalid())
+          break;
+        
         OS << "fix-it:\"";
         OS.write_escaped(SM.getPresumedLoc(B).getFilename());
         OS << "\":{" << SM.getLineNumber(BInfo.first, BInfo.second)
@@ -756,6 +766,9 @@ static bool PrintWordWrapped(llvm::raw_ostream &OS,
 
 void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
                                              const DiagnosticInfo &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticClient::HandleDiagnostic(Level, Info);
+
   // Keeps track of the the starting position of the location
   // information (e.g., "foo.c:10:4:") that precedes the error
   // message. We use this information to determine how long the
@@ -768,77 +781,96 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   // If the location is specified, print out a file/line/col and include trace
   // if enabled.
   if (Info.getLocation().isValid()) {
-    const SourceManager &SM = Info.getLocation().getManager();
+    const SourceManager &SM = Info.getSourceManager();
     PresumedLoc PLoc = SM.getPresumedLoc(Info.getLocation());
-    unsigned LineNo = PLoc.getLine();
-
-    // First, if this diagnostic is not in the main file, print out the
-    // "included from" lines.
-    if (LastWarningLoc != PLoc.getIncludeLoc()) {
-      LastWarningLoc = PLoc.getIncludeLoc();
-      PrintIncludeStack(LastWarningLoc, SM);
-      StartOfLocationInfo = OS.tell();
-    }
+    if (PLoc.isInvalid()) {
+      // At least print the file name if available:
+      FileID FID = SM.getFileID(Info.getLocation());
+      if (!FID.isInvalid()) {
+        const FileEntry* FE = SM.getFileEntryForID(FID);
+        if (FE && FE->getName()) {
+          OS << FE->getName();
+          if (FE->getDevice() == 0 && FE->getInode() == 0
+              && FE->getFileMode() == 0) {
+            // in PCH is a guess, but a good one:
+            OS << " (in PCH)";
+          }
+          OS << ": ";
+        }
+      }
+    } else {
+      unsigned LineNo = PLoc.getLine();
 
-    // Compute the column number.
-    if (DiagOpts->ShowLocation) {
-      if (DiagOpts->ShowColors)
-        OS.changeColor(savedColor, true);
-
-      // Emit a Visual Studio compatible line number syntax.
-      if (LangOpts && LangOpts->Microsoft) {
-        OS << PLoc.getFilename() << '(' << LineNo << ')';
-        OS << " : ";
-      } else {
-        OS << PLoc.getFilename() << ':' << LineNo << ':';
-        if (DiagOpts->ShowColumn)
-          if (unsigned ColNo = PLoc.getColumn())
-            OS << ColNo << ':';
+      // First, if this diagnostic is not in the main file, print out the
+      // "included from" lines.
+      if (LastWarningLoc != PLoc.getIncludeLoc()) {
+        LastWarningLoc = PLoc.getIncludeLoc();
+        PrintIncludeStack(LastWarningLoc, SM);
+        StartOfLocationInfo = OS.tell();
       }
-      if (DiagOpts->ShowSourceRanges && Info.getNumRanges()) {
-        FileID CaretFileID =
-          SM.getFileID(SM.getInstantiationLoc(Info.getLocation()));
-        bool PrintedRange = false;
-
-        for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i) {
-          // Ignore invalid ranges.
-          if (!Info.getRange(i).isValid()) continue;
-
-          SourceLocation B = Info.getRange(i).getBegin();
-          SourceLocation E = Info.getRange(i).getEnd();
-          B = SM.getInstantiationLoc(B);
-          E = SM.getInstantiationLoc(E);
-
-          // If the End location and the start location are the same and are a
-          // macro location, then the range was something that came from a macro
-          // expansion or _Pragma.  If this is an object-like macro, the best we
-          // can do is to highlight the range.  If this is a function-like
-          // macro, we'd also like to highlight the arguments.
-          if (B == E && Info.getRange(i).getEnd().isMacroID())
-            E = SM.getInstantiationRange(Info.getRange(i).getEnd()).second;
-
-          std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
-          std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
-
-          // If the start or end of the range is in another file, just discard
-          // it.
-          if (BInfo.first != CaretFileID || EInfo.first != CaretFileID)
-            continue;
-
-          // Add in the length of the token, so that we cover multi-char tokens.
-          unsigned TokSize = 0;
-          if (Info.getRange(i).isTokenRange())
-            TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
-
-          OS << '{' << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
-             << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
-             << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
-             << (SM.getColumnNumber(EInfo.first, EInfo.second)+TokSize) << '}';
-          PrintedRange = true;
-        }
 
-        if (PrintedRange)
-          OS << ':';
+      // Compute the column number.
+      if (DiagOpts->ShowLocation && PLoc.isValid()) {
+        if (DiagOpts->ShowColors)
+          OS.changeColor(savedColor, true);
+
+        // Emit a Visual Studio compatible line number syntax.
+        if (LangOpts && LangOpts->Microsoft) {
+          OS << PLoc.getFilename() << '(' << LineNo << ')';
+          OS << " : ";
+        } else {
+          OS << PLoc.getFilename() << ':' << LineNo << ':';
+          if (DiagOpts->ShowColumn)
+            if (unsigned ColNo = PLoc.getColumn())
+              OS << ColNo << ':';
+        }
+        if (DiagOpts->ShowSourceRanges && Info.getNumRanges()) {
+          FileID CaretFileID =
+            SM.getFileID(SM.getInstantiationLoc(Info.getLocation()));
+          bool PrintedRange = false;
+
+          for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i) {
+            // Ignore invalid ranges.
+            if (!Info.getRange(i).isValid()) continue;
+
+            SourceLocation B = Info.getRange(i).getBegin();
+            SourceLocation E = Info.getRange(i).getEnd();
+            B = SM.getInstantiationLoc(B);
+            E = SM.getInstantiationLoc(E);
+
+            // If the End location and the start location are the same and are a
+            // macro location, then the range was something that came from a
+            // macro expansion or _Pragma.  If this is an object-like macro, the
+            // best we can do is to highlight the range.  If this is a
+            // function-like macro, we'd also like to highlight the arguments.
+            if (B == E && Info.getRange(i).getEnd().isMacroID())
+              E = SM.getInstantiationRange(Info.getRange(i).getEnd()).second;
+
+            std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
+            std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
+
+            // If the start or end of the range is in another file, just discard
+            // it.
+            if (BInfo.first != CaretFileID || EInfo.first != CaretFileID)
+              continue;
+
+            // Add in the length of the token, so that we cover multi-char
+            // tokens.
+            unsigned TokSize = 0;
+            if (Info.getRange(i).isTokenRange())
+              TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
+
+            OS << '{' << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
+               << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
+               << SM.getLineNumber(EInfo.first, EInfo.second) << ':'
+               << (SM.getColumnNumber(EInfo.first, EInfo.second)+TokSize)
+               << '}';
+            PrintedRange = true;
+          }
+
+          if (PrintedRange)
+            OS << ':';
+        }
       }
       OS << ' ';
       if (DiagOpts->ShowColors)
@@ -873,7 +905,8 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
 
   std::string OptionName;
   if (DiagOpts->ShowOptionNames) {
-    if (const char *Opt = Diagnostic::getWarningOptionForDiag(Info.getID())) {
+    if (const char *
+          Opt = DiagnosticIDs::getWarningOptionForDiag(Info.getID())) {
       OptionName = "-W";
       OptionName += Opt;
     } else if (Info.getID() == diag::fatal_too_many_errors) {
@@ -882,7 +915,8 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
       // If the diagnostic is an extension diagnostic and not enabled by default
       // then it must have been turned on with -pedantic.
       bool EnabledByDefault;
-      if (Diagnostic::isBuiltinExtensionDiag(Info.getID(), EnabledByDefault) &&
+      if (DiagnosticIDs::isBuiltinExtensionDiag(Info.getID(),
+                                                EnabledByDefault) &&
           !EnabledByDefault)
         OptionName = "-pedantic";
     }
@@ -891,7 +925,7 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   // If the user wants to see category information, include it too.
   unsigned DiagCategory = 0;
   if (DiagOpts->ShowCategories)
-    DiagCategory = Diagnostic::getCategoryNumberForDiag(Info.getID());
+    DiagCategory = DiagnosticIDs::getCategoryNumberForDiag(Info.getID());
 
   // If there is any categorization information, include it.
   if (!OptionName.empty() || DiagCategory != 0) {
@@ -909,7 +943,7 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
         OutStr += llvm::utostr(DiagCategory);
       else {
         assert(DiagOpts->ShowCategories == 2 && "Invalid ShowCategories value");
-        OutStr += Diagnostic::getCategoryNameFromID(DiagCategory);
+        OutStr += DiagnosticIDs::getCategoryNameFromID(DiagCategory);
       }
     }
     
@@ -951,7 +985,7 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
        (LastCaretDiagnosticWasNote && Level != Diagnostic::Note) ||
        Info.getNumFixItHints())) {
     // Cache the LastLoc, it allows us to omit duplicate source/caret spewage.
-    LastLoc = Info.getLocation();
+    LastLoc = FullSourceLoc(Info.getLocation(), Info.getSourceManager());
     LastCaretDiagnosticWasNote = (Level == Diagnostic::Note);
 
     // Get the ranges into a local array we can hack on.
diff --git a/lib/Frontend/TypeXML.cpp b/lib/Frontend/TypeXML.cpp
index be9db420d2f8..a8c8f75d4b7f 100644
--- a/lib/Frontend/TypeXML.cpp
+++ b/lib/Frontend/TypeXML.cpp
@@ -29,7 +29,7 @@ public:
   TypeWriter(DocumentXML& doc) : Doc(doc) {}
 
 #define NODE_XML( CLASS, NAME )          \
-  void Visit##CLASS(CLASS* T) {          \
+  void Visit##CLASS(const CLASS* T) {          \
     Doc.addSubNode(NAME);
 
 #define ID_ATTRIBUTE_XML                // done by the Document class itself
@@ -82,7 +82,7 @@ public:
   TypeAdder(DocumentXML& doc) : Doc(doc) {}
 
 #define NODE_XML( CLASS, NAME )          \
-  void Visit##CLASS(CLASS* T)            \
+  void Visit##CLASS(const CLASS* T) \
   {
 
 #define ID_ATTRIBUTE_XML
@@ -101,7 +101,7 @@ public:
 
 //---------------------------------------------------------
 void DocumentXML::addParentTypes(const Type* pType) {
-  TypeAdder(*this).Visit(const_cast<Type*>(pType));
+  TypeAdder(*this).Visit(pType);
 }
 
 //---------------------------------------------------------
diff --git a/lib/Frontend/VerifyDiagnosticsClient.cpp b/lib/Frontend/VerifyDiagnosticsClient.cpp
index 31eb28f912ca..51b351f4a923 100644
--- a/lib/Frontend/VerifyDiagnosticsClient.cpp
+++ b/lib/Frontend/VerifyDiagnosticsClient.cpp
@@ -23,7 +23,7 @@ using namespace clang;
 VerifyDiagnosticsClient::VerifyDiagnosticsClient(Diagnostic &_Diags,
                                                  DiagnosticClient *_Primary)
   : Diags(_Diags), PrimaryClient(_Primary),
-    Buffer(new TextDiagnosticBuffer()), CurrentPreprocessor(0), NumErrors(0) {
+    Buffer(new TextDiagnosticBuffer()), CurrentPreprocessor(0) {
 }
 
 VerifyDiagnosticsClient::~VerifyDiagnosticsClient() {
@@ -57,14 +57,6 @@ void VerifyDiagnosticsClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
   Buffer->HandleDiagnostic(DiagLevel, Info);
 }
 
-// FIXME: It would be nice to just get this from the primary diagnostic client
-// or something.
-bool VerifyDiagnosticsClient::HadErrors() {
-  CheckDiagnostics();
-
-  return NumErrors != 0;
-}
-
 //===----------------------------------------------------------------------===//
 // Checking diagnostics implementation.
 //===----------------------------------------------------------------------===//
diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt
index 26c9fc7b5106..720ce2adf123 100644
--- a/lib/FrontendTool/CMakeLists.txt
+++ b/lib/FrontendTool/CMakeLists.txt
@@ -1,4 +1,5 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangDriver clangFrontend clangRewrite clangCodeGen 
+    clangStaticAnalyzerFrontend clangStaticAnalyzerCheckers clangStaticAnalyzerCore)
 
 add_clang_library(clangFrontendTool
   ExecuteCompilerInvocation.cpp
diff --git a/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 63c628780762..4bb85e7e6340 100644
--- a/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/FrontendTool/Utils.h"
-#include "clang/Checker/FrontendActions.h"
+#include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "clang/CodeGen/CodeGenAction.h"
 #include "clang/Driver/CC1Options.h"
 #include "clang/Driver/OptTable.h"
@@ -24,7 +24,7 @@
 #include "clang/Frontend/FrontendPluginRegistry.h"
 #include "clang/Rewrite/FrontendActions.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/System/DynamicLibrary.h"
+#include "llvm/Support/DynamicLibrary.h"
 using namespace clang;
 
 static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
@@ -35,6 +35,7 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
     llvm_unreachable("Invalid program action!");
 
   case ASTDump:                return new ASTDumpAction();
+  case ASTDumpXML:             return new ASTDumpXMLAction();
   case ASTPrint:               return new ASTPrintAction();
   case ASTPrintXML:            return new ASTPrintXMLAction();
   case ASTView:                return new ASTViewAction();
@@ -52,7 +53,6 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
   case FixIt:                  return new FixItAction();
   case GeneratePCH:            return new GeneratePCHAction();
   case GeneratePTH:            return new GeneratePTHAction();
-  case InheritanceView:        return new InheritanceViewAction();
   case InitOnly:               return new InitOnlyAction();
   case ParseSyntaxOnly:        return new SyntaxOnlyAction();
 
@@ -79,7 +79,7 @@ static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
   case RewriteMacros:          return new RewriteMacrosAction();
   case RewriteObjC:            return new RewriteObjCAction();
   case RewriteTest:            return new RewriteTestAction();
-  case RunAnalysis:            return new AnalysisAction();
+  case RunAnalysis:            return new ento::AnalysisAction();
   case RunPreprocessorOnly:    return new PreprocessOnlyAction();
   }
 }
@@ -141,7 +141,7 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
 
   // If there were errors in processing arguments, don't do anything else.
   bool Success = false;
-  if (!Clang->getDiagnostics().getNumErrors()) {
+  if (!Clang->getDiagnostics().hasErrorOccurred()) {
     // Create and execute the frontend action.
     llvm::OwningPtr<FrontendAction> Act(CreateFrontendAction(*Clang));
     if (Act) {
diff --git a/lib/Headers/CMakeLists.txt b/lib/Headers/CMakeLists.txt
index a1b5f50caa08..eef541e5b8ac 100644
--- a/lib/Headers/CMakeLists.txt
+++ b/lib/Headers/CMakeLists.txt
@@ -18,21 +18,20 @@ set(files
   tmmintrin.h
   xmmintrin.h)
 
-if (MSVC_IDE OR XCODE)
-  set(output_dir ${LLVM_BINARY_DIR}/bin/lib/clang/${CLANG_VERSION}/include)
-else ()
-  set(output_dir ${LLVM_BINARY_DIR}/lib/clang/${CLANG_VERSION}/include)
-endif ()
+set(output_dir ${LLVM_BINARY_DIR}/lib/clang/${CLANG_VERSION}/include)
+
+# If we are in an IDE that has a configuration directory, we need to
+# create a second copy of the headers so that 'clang' can find them if
+# it's run from the build directory.
+if(MSVC_IDE OR XCODE)
+   set(other_output_dir ${LLVM_BINARY_DIR}/bin/lib/clang/${CLANG_VERSION}/include)
+endif()
 
 # Generate arm_neon.h
 set(LLVM_TARGET_DEFINITIONS ${CLANG_SOURCE_DIR}/include/clang/Basic/arm_neon.td)
 tablegen(arm_neon.h.inc -gen-arm-neon)
 
-add_custom_command(OUTPUT ${output_dir}/arm_neon.h 
-  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc ${output_dir}/arm_neon.h
-  COMMENT "Copying clang's arm_neon.h...")
-
+set(out_files)
 foreach( f ${files} )
   set( src ${CMAKE_CURRENT_SOURCE_DIR}/${f} )
   set( dst ${output_dir}/${f} )
@@ -40,10 +39,34 @@ foreach( f ${files} )
     DEPENDS ${src}
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${dst}
     COMMENT "Copying clang's ${f}...")
+  list(APPEND out_files ${dst})
+
+  if(other_output_dir)
+   set(other_dst ${other_output_dir}/${f})
+    add_custom_command(OUTPUT ${other_dst}
+      DEPENDS ${src}
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${other_dst}
+      COMMENT "Copying clang's ${f}...")    
+    list(APPEND out_files ${other_dst})
+  endif()
 endforeach( f )
 
-add_custom_target(clang-headers ALL
-  DEPENDS ${files} ${output_dir}/arm_neon.h)
+add_custom_command(OUTPUT ${output_dir}/arm_neon.h 
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc ${output_dir}/arm_neon.h
+  COMMENT "Copying clang's arm_neon.h...")
+list(APPEND out_files ${output_dir}/arm_neon.h)
+
+if (other_output_dir)
+    set(other_dst ${other_output_dir}/arm_neon.h)
+    add_custom_command(OUTPUT ${other_dst}
+      DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/arm_neon.h.inc ${other_dst}
+      COMMENT "Copying clang's arm_neon.h...")
+    list(APPEND out_files ${other_dst})
+endif ()
+
+add_custom_target(clang-headers ALL DEPENDS ${out_files})
 
 install(FILES ${files} ${output_dir}/arm_neon.h
   PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
diff --git a/lib/Headers/altivec.h b/lib/Headers/altivec.h
index 89bd259b6733..a225378c3062 100644
--- a/lib/Headers/altivec.h
+++ b/lib/Headers/altivec.h
@@ -20,9 +20,6 @@
  *
 \*===----------------------------------------------------------------------===*/
 
-// TODO: add functions for 'vector bool ..' and 'vector pixel' argument types according to 
-// the 'AltiVec Technology Programming Interface Manual'
-
 #ifndef __ALTIVEC_H
 #define __ALTIVEC_H
 
@@ -43,7 +40,9 @@ static vector signed char __ATTRS_o_ai
 vec_perm(vector signed char a, vector signed char b, vector unsigned char c);
 
 static vector unsigned char __ATTRS_o_ai
-vec_perm(vector unsigned char a, vector unsigned char b, vector unsigned char c);
+vec_perm(vector unsigned char a,
+         vector unsigned char b, 
+         vector unsigned char c);
 
 static vector bool char __ATTRS_o_ai
 vec_perm(vector bool char a, vector bool char b, vector unsigned char c);
@@ -52,7 +51,9 @@ static vector short __ATTRS_o_ai
 vec_perm(vector short a, vector short b, vector unsigned char c);
 
 static vector unsigned short __ATTRS_o_ai
-vec_perm(vector unsigned short a, vector unsigned short b, vector unsigned char c);
+vec_perm(vector unsigned short a,
+         vector unsigned short b, 
+         vector unsigned char c);
 
 static vector bool short __ATTRS_o_ai
 vec_perm(vector bool short a, vector bool short b, vector unsigned char c);
@@ -99,7 +100,8 @@ vec_abs(vector signed int a)
 static vector float __ATTRS_o_ai
 vec_abs(vector float a)
 {
-  vector unsigned int res = (vector unsigned int)a & (vector unsigned int)(0x7FFFFFFF);
+  vector unsigned int res = (vector unsigned int)a 
+                            & (vector unsigned int)(0x7FFFFFFF);
   return (vector float)res;
 }
 
@@ -112,19 +114,22 @@ vec_abs(vector float a)
 static vector signed char __ATTRS_o_ai
 vec_abss(vector signed char a)
 {
-  return __builtin_altivec_vmaxsb(a, __builtin_altivec_vsubsbs((vector signed char)(0), a));
+  return __builtin_altivec_vmaxsb
+           (a, __builtin_altivec_vsubsbs((vector signed char)(0), a));
 }
 
 static vector signed short __ATTRS_o_ai
 vec_abss(vector signed short a)
 {
-  return __builtin_altivec_vmaxsh(a, __builtin_altivec_vsubshs((vector signed short)(0), a));
+  return __builtin_altivec_vmaxsh
+           (a, __builtin_altivec_vsubshs((vector signed short)(0), a));
 }
 
 static vector signed int __ATTRS_o_ai
 vec_abss(vector signed int a)
 {
-  return __builtin_altivec_vmaxsw(a, __builtin_altivec_vsubsws((vector signed int)(0), a));
+  return __builtin_altivec_vmaxsw
+           (a, __builtin_altivec_vsubsws((vector signed int)(0), a));
 }
 
 /* vec_add */
@@ -1634,7 +1639,7 @@ vec_dssall(void)
 /* vec_dst */
 
 static void __attribute__((__always_inline__))
-vec_dst(void *a, int b, int c)
+vec_dst(const void *a, int b, int c)
 {
   __builtin_altivec_dst(a, b, c);
 }
@@ -1642,7 +1647,7 @@ vec_dst(void *a, int b, int c)
 /* vec_dstst */
 
 static void __attribute__((__always_inline__))
-vec_dstst(void *a, int b, int c)
+vec_dstst(const void *a, int b, int c)
 {
   __builtin_altivec_dstst(a, b, c);
 }
@@ -1650,7 +1655,7 @@ vec_dstst(void *a, int b, int c)
 /* vec_dststt */
 
 static void __attribute__((__always_inline__))
-vec_dststt(void *a, int b, int c)
+vec_dststt(const void *a, int b, int c)
 {
   __builtin_altivec_dststt(a, b, c);
 }
@@ -1658,7 +1663,7 @@ vec_dststt(void *a, int b, int c)
 /* vec_dstt */
 
 static void __attribute__((__always_inline__))
-vec_dstt(void *a, int b, int c)
+vec_dstt(const void *a, int b, int c)
 {
   __builtin_altivec_dstt(a, b, c);
 }
@@ -1698,109 +1703,109 @@ vec_vrfim(vector float a)
 /* vec_ld */
 
 static vector signed char __ATTRS_o_ai
-vec_ld(int a, vector signed char *b)
+vec_ld(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvx(a, b);
 }
 
 static vector signed char __ATTRS_o_ai
-vec_ld(int a, signed char *b)
+vec_ld(int a, const signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_ld(int a, vector unsigned char *b)
+vec_ld(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_ld(int a, unsigned char *b)
+vec_ld(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool char __ATTRS_o_ai
-vec_ld(int a, vector bool char *b)
+vec_ld(int a, const vector bool char *b)
 {
   return (vector bool char)__builtin_altivec_lvx(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_ld(int a, vector short *b)
+vec_ld(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvx(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_ld(int a, short *b)
+vec_ld(int a, const short *b)
 {
   return (vector short)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_ld(int a, vector unsigned short *b)
+vec_ld(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_ld(int a, unsigned short *b)
+vec_ld(int a, const unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool short __ATTRS_o_ai
-vec_ld(int a, vector bool short *b)
+vec_ld(int a, const vector bool short *b)
 {
   return (vector bool short)__builtin_altivec_lvx(a, b);
 }
 
 static vector pixel __ATTRS_o_ai
-vec_ld(int a, vector pixel *b)
+vec_ld(int a, const vector pixel *b)
 {
   return (vector pixel)__builtin_altivec_lvx(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_ld(int a, vector int *b)
+vec_ld(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvx(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_ld(int a, int *b)
+vec_ld(int a, const int *b)
 {
   return (vector int)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_ld(int a, vector unsigned int *b)
+vec_ld(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_ld(int a, unsigned int *b)
+vec_ld(int a, const unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool int __ATTRS_o_ai
-vec_ld(int a, vector bool int *b)
+vec_ld(int a, const vector bool int *b)
 {
   return (vector bool int)__builtin_altivec_lvx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_ld(int a, vector float *b)
+vec_ld(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_ld(int a, float *b)
+vec_ld(int a, const float *b)
 {
   return (vector float)__builtin_altivec_lvx(a, b);
 }
@@ -1808,109 +1813,109 @@ vec_ld(int a, float *b)
 /* vec_lvx */
 
 static vector signed char __ATTRS_o_ai
-vec_lvx(int a, vector signed char *b)
+vec_lvx(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvx(a, b);
 }
 
 static vector signed char __ATTRS_o_ai
-vec_lvx(int a, signed char *b)
+vec_lvx(int a, const signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvx(int a, vector unsigned char *b)
+vec_lvx(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvx(int a, unsigned char *b)
+vec_lvx(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool char __ATTRS_o_ai
-vec_lvx(int a, vector bool char *b)
+vec_lvx(int a, const vector bool char *b)
 {
   return (vector bool char)__builtin_altivec_lvx(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_lvx(int a, vector short *b)
+vec_lvx(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvx(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_lvx(int a, short *b)
+vec_lvx(int a, const short *b)
 {
   return (vector short)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lvx(int a, vector unsigned short *b)
+vec_lvx(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lvx(int a, unsigned short *b)
+vec_lvx(int a, const unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool short __ATTRS_o_ai
-vec_lvx(int a, vector bool short *b)
+vec_lvx(int a, const vector bool short *b)
 {
   return (vector bool short)__builtin_altivec_lvx(a, b);
 }
 
 static vector pixel __ATTRS_o_ai
-vec_lvx(int a, vector pixel *b)
+vec_lvx(int a, const vector pixel *b)
 {
   return (vector pixel)__builtin_altivec_lvx(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_lvx(int a, vector int *b)
+vec_lvx(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvx(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_lvx(int a, int *b)
+vec_lvx(int a, const int *b)
 {
   return (vector int)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lvx(int a, vector unsigned int *b)
+vec_lvx(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lvx(int a, unsigned int *b)
+vec_lvx(int a, const unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
 static vector bool int __ATTRS_o_ai
-vec_lvx(int a, vector bool int *b)
+vec_lvx(int a, const vector bool int *b)
 {
   return (vector bool int)__builtin_altivec_lvx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lvx(int a, vector float *b)
+vec_lvx(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lvx(int a, float *b)
+vec_lvx(int a, const float *b)
 {
   return (vector float)__builtin_altivec_lvx(a, b);
 }
@@ -1918,43 +1923,43 @@ vec_lvx(int a, float *b)
 /* vec_lde */
 
 static vector signed char __ATTRS_o_ai
-vec_lde(int a, vector signed char *b)
+vec_lde(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvebx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lde(int a, vector unsigned char *b)
+vec_lde(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvebx(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_lde(int a, vector short *b)
+vec_lde(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvehx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lde(int a, vector unsigned short *b)
+vec_lde(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvehx(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_lde(int a, vector int *b)
+vec_lde(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvewx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lde(int a, vector unsigned int *b)
+vec_lde(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvewx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lde(int a, vector float *b)
+vec_lde(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvewx(a, b);
 }
@@ -1962,13 +1967,13 @@ vec_lde(int a, vector float *b)
 /* vec_lvebx */
 
 static vector signed char __ATTRS_o_ai
-vec_lvebx(int a, vector signed char *b)
+vec_lvebx(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvebx(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvebx(int a, vector unsigned char *b)
+vec_lvebx(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvebx(a, b);
 }
@@ -1976,13 +1981,13 @@ vec_lvebx(int a, vector unsigned char *b)
 /* vec_lvehx */
 
 static vector short __ATTRS_o_ai
-vec_lvehx(int a, vector short *b)
+vec_lvehx(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvehx(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lvehx(int a, vector unsigned short *b)
+vec_lvehx(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvehx(a, b);
 }
@@ -1990,19 +1995,19 @@ vec_lvehx(int a, vector unsigned short *b)
 /* vec_lvewx */
 
 static vector int __ATTRS_o_ai
-vec_lvewx(int a, vector int *b)
+vec_lvewx(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvewx(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lvewx(int a, vector unsigned int *b)
+vec_lvewx(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvewx(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lvewx(int a, vector float *b)
+vec_lvewx(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvewx(a, b);
 }
@@ -2010,109 +2015,109 @@ vec_lvewx(int a, vector float *b)
 /* vec_ldl */
 
 static vector signed char __ATTRS_o_ai
-vec_ldl(int a, vector signed char *b)
+vec_ldl(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector signed char __ATTRS_o_ai
-vec_ldl(int a, signed char *b)
+vec_ldl(int a, const signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_ldl(int a, vector unsigned char *b)
+vec_ldl(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_ldl(int a, unsigned char *b)
+vec_ldl(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool char __ATTRS_o_ai
-vec_ldl(int a, vector bool char *b)
+vec_ldl(int a, const vector bool char *b)
 {
   return (vector bool char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_ldl(int a, vector short *b)
+vec_ldl(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_ldl(int a, short *b)
+vec_ldl(int a, const short *b)
 {
   return (vector short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_ldl(int a, vector unsigned short *b)
+vec_ldl(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_ldl(int a, unsigned short *b)
+vec_ldl(int a, const unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool short __ATTRS_o_ai
-vec_ldl(int a, vector bool short *b)
+vec_ldl(int a, const vector bool short *b)
 {
   return (vector bool short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector pixel __ATTRS_o_ai
-vec_ldl(int a, vector pixel *b)
+vec_ldl(int a, const vector pixel *b)
 {
   return (vector pixel short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_ldl(int a, vector int *b)
+vec_ldl(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_ldl(int a, int *b)
+vec_ldl(int a, const int *b)
 {
   return (vector int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_ldl(int a, vector unsigned int *b)
+vec_ldl(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_ldl(int a, unsigned int *b)
+vec_ldl(int a, const unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool int __ATTRS_o_ai
-vec_ldl(int a, vector bool int *b)
+vec_ldl(int a, const vector bool int *b)
 {
   return (vector bool int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_ldl(int a, vector float *b)
+vec_ldl(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvxl(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_ldl(int a, float *b)
+vec_ldl(int a, const float *b)
 {
   return (vector float)__builtin_altivec_lvxl(a, b);
 }
@@ -2120,109 +2125,109 @@ vec_ldl(int a, float *b)
 /* vec_lvxl */
 
 static vector signed char __ATTRS_o_ai
-vec_lvxl(int a, vector signed char *b)
+vec_lvxl(int a, const vector signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector signed char __ATTRS_o_ai
-vec_lvxl(int a, signed char *b)
+vec_lvxl(int a, const signed char *b)
 {
   return (vector signed char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvxl(int a, vector unsigned char *b)
+vec_lvxl(int a, const vector unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvxl(int a, unsigned char *b)
+vec_lvxl(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool char __ATTRS_o_ai
-vec_lvxl(int a, vector bool char *b)
+vec_lvxl(int a, const vector bool char *b)
 {
   return (vector bool char)__builtin_altivec_lvxl(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_lvxl(int a, vector short *b)
+vec_lvxl(int a, const vector short *b)
 {
   return (vector short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector short __ATTRS_o_ai
-vec_lvxl(int a, short *b)
+vec_lvxl(int a, const short *b)
 {
   return (vector short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lvxl(int a, vector unsigned short *b)
+vec_lvxl(int a, const vector unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_lvxl(int a, unsigned short *b)
+vec_lvxl(int a, const unsigned short *b)
 {
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool short __ATTRS_o_ai
-vec_lvxl(int a, vector bool short *b)
+vec_lvxl(int a, const vector bool short *b)
 {
   return (vector bool short)__builtin_altivec_lvxl(a, b);
 }
 
 static vector pixel __ATTRS_o_ai
-vec_lvxl(int a, vector pixel *b)
+vec_lvxl(int a, const vector pixel *b)
 {
   return (vector pixel)__builtin_altivec_lvxl(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_lvxl(int a, vector int *b)
+vec_lvxl(int a, const vector int *b)
 {
   return (vector int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector int __ATTRS_o_ai
-vec_lvxl(int a, int *b)
+vec_lvxl(int a, const int *b)
 {
   return (vector int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lvxl(int a, vector unsigned int *b)
+vec_lvxl(int a, const vector unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_lvxl(int a, unsigned int *b)
+vec_lvxl(int a, const unsigned int *b)
 {
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector bool int __ATTRS_o_ai
-vec_lvxl(int a, vector bool int *b)
+vec_lvxl(int a, const vector bool int *b)
 {
   return (vector bool int)__builtin_altivec_lvxl(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lvxl(int a, vector float *b)
+vec_lvxl(int a, const vector float *b)
 {
   return (vector float)__builtin_altivec_lvxl(a, b);
 }
 
 static vector float __ATTRS_o_ai
-vec_lvxl(int a, float *b)
+vec_lvxl(int a, const float *b)
 {
   return (vector float)__builtin_altivec_lvxl(a, b);
 }
@@ -2246,43 +2251,43 @@ vec_vlogefp(vector float a)
 /* vec_lvsl */
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, signed char *b)
+vec_lvsl(int a, const signed char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, unsigned char *b)
+vec_lvsl(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, short *b)
+vec_lvsl(int a, const short *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, unsigned short *b)
+vec_lvsl(int a, const unsigned short *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, int *b)
+vec_lvsl(int a, const int *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, unsigned int *b)
+vec_lvsl(int a, const unsigned int *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsl(int a, float *b)
+vec_lvsl(int a, const float *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsl(a, b);
 }
@@ -2290,43 +2295,43 @@ vec_lvsl(int a, float *b)
 /* vec_lvsr */
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, signed char *b)
+vec_lvsr(int a, const signed char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, unsigned char *b)
+vec_lvsr(int a, const unsigned char *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, short *b)
+vec_lvsr(int a, const short *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, unsigned short *b)
+vec_lvsr(int a, const unsigned short *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, int *b)
+vec_lvsr(int a, const int *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, unsigned int *b)
+vec_lvsr(int a, const unsigned int *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
 
 static vector unsigned char __ATTRS_o_ai
-vec_lvsr(int a, float *b)
+vec_lvsr(int a, const float *b)
 {
   return (vector unsigned char)__builtin_altivec_lvsr(a, b);
 }
@@ -2357,7 +2362,9 @@ vec_madds(vector signed short a, vector signed short b, vector signed short c)
 
 /* vec_vmhaddshs */
 static vector signed short __attribute__((__always_inline__))
-vec_vmhaddshs(vector signed short a, vector signed short b, vector signed short c)
+vec_vmhaddshs(vector signed short a,
+              vector signed short b, 
+              vector signed short c)
 {
   return __builtin_altivec_vmhaddshs(a, b, c);
 }
@@ -3261,7 +3268,9 @@ vec_mladd(vector unsigned short a, vector short b, vector short c)
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_mladd(vector unsigned short a, vector unsigned short b, vector unsigned short c)
+vec_mladd(vector unsigned short a,
+          vector unsigned short b, 
+          vector unsigned short c)
 {
   return a * b + c;
 }
@@ -3287,7 +3296,9 @@ vec_vmladduhm(vector unsigned short a, vector short b, vector short c)
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_vmladduhm(vector unsigned short a, vector unsigned short b, vector unsigned short c)
+vec_vmladduhm(vector unsigned short a,
+              vector unsigned short b,
+              vector unsigned short c)
 {
   return a * b + c;
 }
@@ -3329,7 +3340,9 @@ vec_msum(vector short a, vector short b, vector int c)
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_msum(vector unsigned short a, vector unsigned short b, vector unsigned int c)
+vec_msum(vector unsigned short a,
+         vector unsigned short b,
+         vector unsigned int c)
 {
   return __builtin_altivec_vmsumuhm(a, b, c);
 }
@@ -3345,7 +3358,9 @@ vec_vmsummbm(vector signed char a, vector unsigned char b, vector int c)
 /* vec_vmsumubm */
 
 static vector unsigned int __attribute__((__always_inline__))
-vec_vmsumubm(vector unsigned char a, vector unsigned char b, vector unsigned int c)
+vec_vmsumubm(vector unsigned char a,
+             vector unsigned char b,
+             vector unsigned int c)
 {
   return __builtin_altivec_vmsumubm(a, b, c);
 }
@@ -3361,7 +3376,9 @@ vec_vmsumshm(vector short a, vector short b, vector int c)
 /* vec_vmsumuhm */
 
 static vector unsigned int __attribute__((__always_inline__))
-vec_vmsumuhm(vector unsigned short a, vector unsigned short b, vector unsigned int c)
+vec_vmsumuhm(vector unsigned short a,
+             vector unsigned short b,
+             vector unsigned int c)
 {
   return __builtin_altivec_vmsumuhm(a, b, c);
 }
@@ -3375,7 +3392,9 @@ vec_msums(vector short a, vector short b, vector int c)
 }
 
 static vector unsigned int __ATTRS_o_ai
-vec_msums(vector unsigned short a, vector unsigned short b, vector unsigned int c)
+vec_msums(vector unsigned short a,
+          vector unsigned short b,
+          vector unsigned int c)
 {
   return __builtin_altivec_vmsumuhs(a, b, c);
 }
@@ -3391,7 +3410,9 @@ vec_vmsumshs(vector short a, vector short b, vector int c)
 /* vec_vmsumuhs */
 
 static vector unsigned int __attribute__((__always_inline__))
-vec_vmsumuhs(vector unsigned short a, vector unsigned short b, vector unsigned int c)
+vec_vmsumuhs(vector unsigned short a,
+             vector unsigned short b,
+             vector unsigned int c)
 {
   return __builtin_altivec_vmsumuhs(a, b, c);
 }
@@ -4263,43 +4284,54 @@ vec_vpkswus(vector unsigned int a, vector unsigned int b)
 vector signed char __ATTRS_o_ai
 vec_perm(vector signed char a, vector signed char b, vector unsigned char c)
 {
-  return (vector signed char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector signed char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector unsigned char __ATTRS_o_ai
-vec_perm(vector unsigned char a, vector unsigned char b, vector unsigned char c)
+vec_perm(vector unsigned char a,
+         vector unsigned char b,
+         vector unsigned char c)
 {
-  return (vector unsigned char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool char __ATTRS_o_ai
 vec_perm(vector bool char a, vector bool char b, vector unsigned char c)
 {
-  return (vector bool char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector short __ATTRS_o_ai
 vec_perm(vector short a, vector short b, vector unsigned char c)
 {
-  return (vector short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector unsigned short __ATTRS_o_ai
-vec_perm(vector unsigned short a, vector unsigned short b, vector unsigned char c)
+vec_perm(vector unsigned short a,
+         vector unsigned short b,
+         vector unsigned char c)
 {
-  return (vector unsigned short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool short __ATTRS_o_ai
 vec_perm(vector bool short a, vector bool short b, vector unsigned char c)
 {
-  return (vector bool short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector pixel __ATTRS_o_ai
 vec_perm(vector pixel a, vector pixel b, vector unsigned char c)
 {
-  return (vector pixel)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector pixel)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector int __ATTRS_o_ai
@@ -4311,19 +4343,22 @@ vec_perm(vector int a, vector int b, vector unsigned char c)
 vector unsigned int __ATTRS_o_ai
 vec_perm(vector unsigned int a, vector unsigned int b, vector unsigned char c)
 {
-  return (vector unsigned int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned int)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool int __ATTRS_o_ai
 vec_perm(vector bool int a, vector bool int b, vector unsigned char c)
 {
-  return (vector bool int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool int)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector float __ATTRS_o_ai
 vec_perm(vector float a, vector float b, vector unsigned char c)
 {
-  return (vector float)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector float)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 /* vec_vperm */
@@ -4331,43 +4366,54 @@ vec_perm(vector float a, vector float b, vector unsigned char c)
 vector signed char __ATTRS_o_ai
 vec_vperm(vector signed char a, vector signed char b, vector unsigned char c)
 {
-  return (vector signed char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector signed char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector unsigned char __ATTRS_o_ai
-vec_vperm(vector unsigned char a, vector unsigned char b, vector unsigned char c)
+vec_vperm(vector unsigned char a,
+          vector unsigned char b,
+          vector unsigned char c)
 {
-  return (vector unsigned char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool char __ATTRS_o_ai
 vec_vperm(vector bool char a, vector bool char b, vector unsigned char c)
 {
-  return (vector bool char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool char)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector short __ATTRS_o_ai
 vec_vperm(vector short a, vector short b, vector unsigned char c)
 {
-  return (vector short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector unsigned short __ATTRS_o_ai
-vec_vperm(vector unsigned short a, vector unsigned short b, vector unsigned char c)
+vec_vperm(vector unsigned short a,
+          vector unsigned short b,
+          vector unsigned char c)
 {
-  return (vector unsigned short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool short __ATTRS_o_ai
 vec_vperm(vector bool short a, vector bool short b, vector unsigned char c)
 {
-  return (vector bool short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool short)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector pixel __ATTRS_o_ai
 vec_vperm(vector pixel a, vector pixel b, vector unsigned char c)
 {
-  return (vector pixel)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector pixel)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector int __ATTRS_o_ai
@@ -4379,19 +4425,22 @@ vec_vperm(vector int a, vector int b, vector unsigned char c)
 vector unsigned int __ATTRS_o_ai
 vec_vperm(vector unsigned int a, vector unsigned int b, vector unsigned char c)
 {
-  return (vector unsigned int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector unsigned int)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector bool int __ATTRS_o_ai
 vec_vperm(vector bool int a, vector bool int b, vector unsigned char c)
 {
-  return (vector bool int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector bool int)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 vector float __ATTRS_o_ai
 vec_vperm(vector float a, vector float b, vector unsigned char c)
 {
-  return (vector float)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+  return (vector float)
+           __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
 /* vec_re */
@@ -4575,7 +4624,9 @@ vec_sel(vector short a, vector short b, vector bool short c)
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_sel(vector unsigned short a, vector unsigned short b, vector unsigned short c)
+vec_sel(vector unsigned short a,
+        vector unsigned short b,
+        vector unsigned short c)
 {
   return (a & ~c) | (b & c);
 }
@@ -4637,14 +4688,16 @@ vec_sel(vector bool int a, vector bool int b, vector bool int c)
 static vector float __ATTRS_o_ai
 vec_sel(vector float a, vector float b, vector unsigned int c)
 {
-  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  vector int res = ((vector int)a & ~(vector int)c) 
+                   | ((vector int)b & (vector int)c);
   return (vector float)res;
 }
 
 static vector float __ATTRS_o_ai
 vec_sel(vector float a, vector float b, vector bool int c)
 {
-  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  vector int res = ((vector int)a & ~(vector int)c)
+                   | ((vector int)b & (vector int)c);
   return (vector float)res;
 }
 
@@ -4699,7 +4752,9 @@ vec_vsel(vector short a, vector short b, vector bool short c)
 }
 
 static vector unsigned short __ATTRS_o_ai
-vec_vsel(vector unsigned short a, vector unsigned short b, vector unsigned short c)
+vec_vsel(vector unsigned short a,
+         vector unsigned short b,
+         vector unsigned short c)
 {
   return (a & ~c) | (b & c);
 }
@@ -4761,14 +4816,16 @@ vec_vsel(vector bool int a, vector bool int b, vector bool int c)
 static vector float __ATTRS_o_ai
 vec_vsel(vector float a, vector float b, vector unsigned int c)
 {
-  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  vector int res = ((vector int)a & ~(vector int)c)
+                   | ((vector int)b & (vector int)c);
   return (vector float)res;
 }
 
 static vector float __ATTRS_o_ai
 vec_vsel(vector float a, vector float b, vector bool int c)
 {
-  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  vector int res = ((vector int)a & ~(vector int)c)
+                   | ((vector int)b & (vector int)c);
   return (vector float)res;
 }
 
@@ -4997,37 +5054,43 @@ vec_vsldoi(vector float a, vector float b, unsigned char c)
 static vector signed char __ATTRS_o_ai
 vec_sll(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_sll(vector signed char a, vector unsigned short b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_sll(vector signed char a, vector unsigned int b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_sll(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_sll(vector unsigned char a, vector unsigned short b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_sll(vector unsigned char a, vector unsigned int b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool char __ATTRS_o_ai
@@ -5069,19 +5132,22 @@ vec_sll(vector short a, vector unsigned int b)
 static vector unsigned short __ATTRS_o_ai
 vec_sll(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_sll(vector unsigned short a, vector unsigned short b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_sll(vector unsigned short a, vector unsigned int b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool short __ATTRS_o_ai
@@ -5141,19 +5207,22 @@ vec_sll(vector int a, vector unsigned int b)
 static vector unsigned int __ATTRS_o_ai
 vec_sll(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_sll(vector unsigned int a, vector unsigned short b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_sll(vector unsigned int a, vector unsigned int b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool int __ATTRS_o_ai
@@ -5179,37 +5248,43 @@ vec_sll(vector bool int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_vsl(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vsl(vector signed char a, vector unsigned short b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vsl(vector signed char a, vector unsigned int b)
 {
-  return (vector signed char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsl(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsl(vector unsigned char a, vector unsigned short b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsl(vector unsigned char a, vector unsigned int b)
 {
-  return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool char __ATTRS_o_ai
@@ -5251,19 +5326,22 @@ vec_vsl(vector short a, vector unsigned int b)
 static vector unsigned short __ATTRS_o_ai
 vec_vsl(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsl(vector unsigned short a, vector unsigned short b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsl(vector unsigned short a, vector unsigned int b)
 {
-  return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool short __ATTRS_o_ai
@@ -5323,19 +5401,22 @@ vec_vsl(vector int a, vector unsigned int b)
 static vector unsigned int __ATTRS_o_ai
 vec_vsl(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vsl(vector unsigned int a, vector unsigned short b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vsl(vector unsigned int a, vector unsigned int b)
 {
-  return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
 static vector bool int __ATTRS_o_ai
@@ -5361,25 +5442,29 @@ vec_vsl(vector bool int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_slo(vector signed char a, vector signed char b)
 {
-  return (vector signed char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_slo(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_slo(vector unsigned char a, vector signed char b)
 {
-  return (vector unsigned char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_slo(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector short __ATTRS_o_ai
@@ -5397,13 +5482,15 @@ vec_slo(vector short a, vector unsigned char b)
 static vector unsigned short __ATTRS_o_ai
 vec_slo(vector unsigned short a, vector signed char b)
 {
-  return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_slo(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector pixel __ATTRS_o_ai
@@ -5433,13 +5520,15 @@ vec_slo(vector int a, vector unsigned char b)
 static vector unsigned int __ATTRS_o_ai
 vec_slo(vector unsigned int a, vector signed char b)
 {
-  return (vector unsigned int)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_slo(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector float __ATTRS_o_ai
@@ -5459,25 +5548,29 @@ vec_slo(vector float a, vector unsigned char b)
 static vector signed char __ATTRS_o_ai
 vec_vslo(vector signed char a, vector signed char b)
 {
-  return (vector signed char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vslo(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vslo(vector unsigned char a, vector signed char b)
 {
-  return (vector unsigned char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vslo(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector short __ATTRS_o_ai
@@ -5495,13 +5588,15 @@ vec_vslo(vector short a, vector unsigned char b)
 static vector unsigned short __ATTRS_o_ai
 vec_vslo(vector unsigned short a, vector signed char b)
 {
-  return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vslo(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector pixel __ATTRS_o_ai
@@ -5531,13 +5626,15 @@ vec_vslo(vector int a, vector unsigned char b)
 static vector unsigned int __ATTRS_o_ai
 vec_vslo(vector unsigned int a, vector signed char b)
 {
-  return (vector unsigned int)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vslo(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vslo((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
 static vector float __ATTRS_o_ai
@@ -5576,64 +5673,72 @@ static vector short __ATTRS_o_ai
 vec_splat(vector short a, unsigned char b)
 { 
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_splat(vector unsigned short a, unsigned char b)
 { 
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector bool short __ATTRS_o_ai
 vec_splat(vector bool short a, unsigned char b)
 { 
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector pixel __ATTRS_o_ai
 vec_splat(vector pixel a, unsigned char b)
 { 
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector int __ATTRS_o_ai
 vec_splat(vector int a, unsigned char b)
 { 
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_splat(vector unsigned int a, unsigned char b)
 { 
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector bool int __ATTRS_o_ai
 vec_splat(vector bool int a, unsigned char b)
 { 
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector float __ATTRS_o_ai
 vec_splat(vector float a, unsigned char b)
 { 
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 /* vec_vspltb */
@@ -5666,32 +5771,36 @@ static vector short __ATTRS_o_ai
 vec_vsplth(vector short a, unsigned char b)
 {
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsplth(vector unsigned short a, unsigned char b)
 {
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector bool short __ATTRS_o_ai
 vec_vsplth(vector bool short a, unsigned char b)
 {
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 static vector pixel __ATTRS_o_ai
 vec_vsplth(vector pixel a, unsigned char b)
 {
   b *= 2;
+  unsigned char b1=b+1;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+    (b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1, b, b1));
 }
 
 /* vec_vspltw */
@@ -5702,32 +5811,36 @@ static vector int __ATTRS_o_ai
 vec_vspltw(vector int a, unsigned char b)
 {
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vspltw(vector unsigned int a, unsigned char b)
 {
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector bool int __ATTRS_o_ai
 vec_vspltw(vector bool int a, unsigned char b)
 {
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 static vector float __ATTRS_o_ai
 vec_vspltw(vector float a, unsigned char b)
 {
   b *= 4;
+  unsigned char b1=b+1, b2=b+2, b3=b+3;
   return vec_perm(a, a, (vector unsigned char)
-    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+    (b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3, b, b1, b2, b3));
 }
 
 /* vec_splat_s8 */
@@ -5988,37 +6101,43 @@ vec_vsraw(vector unsigned int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_srl(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_srl(vector signed char a, vector unsigned short b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_srl(vector signed char a, vector unsigned int b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_srl(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_srl(vector unsigned char a, vector unsigned short b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_srl(vector unsigned char a, vector unsigned int b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool char __ATTRS_o_ai
@@ -6060,19 +6179,22 @@ vec_srl(vector short a, vector unsigned int b)
 static vector unsigned short __ATTRS_o_ai
 vec_srl(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_srl(vector unsigned short a, vector unsigned short b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_srl(vector unsigned short a, vector unsigned int b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool short __ATTRS_o_ai
@@ -6132,19 +6254,22 @@ vec_srl(vector int a, vector unsigned int b)
 static vector unsigned int __ATTRS_o_ai
 vec_srl(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_srl(vector unsigned int a, vector unsigned short b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_srl(vector unsigned int a, vector unsigned int b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool int __ATTRS_o_ai
@@ -6170,37 +6295,43 @@ vec_srl(vector bool int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_vsr(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vsr(vector signed char a, vector unsigned short b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vsr(vector signed char a, vector unsigned int b)
 {
-  return (vector signed char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsr(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsr(vector unsigned char a, vector unsigned short b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsr(vector unsigned char a, vector unsigned int b)
 {
-  return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool char __ATTRS_o_ai
@@ -6242,19 +6373,22 @@ vec_vsr(vector short a, vector unsigned int b)
 static vector unsigned short __ATTRS_o_ai
 vec_vsr(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsr(vector unsigned short a, vector unsigned short b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsr(vector unsigned short a, vector unsigned int b)
 {
-  return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool short __ATTRS_o_ai
@@ -6314,19 +6448,22 @@ vec_vsr(vector int a, vector unsigned int b)
 static vector unsigned int __ATTRS_o_ai
 vec_vsr(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vsr(vector unsigned int a, vector unsigned short b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vsr(vector unsigned int a, vector unsigned int b)
 {
-  return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
 static vector bool int __ATTRS_o_ai
@@ -6352,25 +6489,29 @@ vec_vsr(vector bool int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_sro(vector signed char a, vector signed char b)
 {
-  return (vector signed char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_sro(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_sro(vector unsigned char a, vector signed char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_sro(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector short __ATTRS_o_ai
@@ -6388,13 +6529,15 @@ vec_sro(vector short a, vector unsigned char b)
 static vector unsigned short __ATTRS_o_ai
 vec_sro(vector unsigned short a, vector signed char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_sro(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector pixel __ATTRS_o_ai
@@ -6424,13 +6567,15 @@ vec_sro(vector int a, vector unsigned char b)
 static vector unsigned int __ATTRS_o_ai
 vec_sro(vector unsigned int a, vector signed char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_sro(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector float __ATTRS_o_ai
@@ -6450,25 +6595,29 @@ vec_sro(vector float a, vector unsigned char b)
 static vector signed char __ATTRS_o_ai
 vec_vsro(vector signed char a, vector signed char b)
 {
-  return (vector signed char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector signed char __ATTRS_o_ai
 vec_vsro(vector signed char a, vector unsigned char b)
 {
-  return (vector signed char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector signed char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsro(vector unsigned char a, vector signed char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vsro(vector unsigned char a, vector unsigned char b)
 {
-  return (vector unsigned char)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned char)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector short __ATTRS_o_ai
@@ -6486,13 +6635,15 @@ vec_vsro(vector short a, vector unsigned char b)
 static vector unsigned short __ATTRS_o_ai
 vec_vsro(vector unsigned short a, vector signed char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned short __ATTRS_o_ai
 vec_vsro(vector unsigned short a, vector unsigned char b)
 {
-  return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned short)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector pixel __ATTRS_o_ai
@@ -6522,13 +6673,15 @@ vec_vsro(vector int a, vector unsigned char b)
 static vector unsigned int __ATTRS_o_ai
 vec_vsro(vector unsigned int a, vector signed char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector unsigned int __ATTRS_o_ai
 vec_vsro(vector unsigned int a, vector unsigned char b)
 {
-  return (vector unsigned int)__builtin_altivec_vsro((vector int)a, (vector int)b);
+  return (vector unsigned int)
+           __builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
 static vector float __ATTRS_o_ai
@@ -8379,7 +8532,1410 @@ vec_vxor(vector float a, vector bool int b)
   return (vector float)res;
 }
 
-/* ------------------------------ predicates ------------------------------------ */
+/* ------------------------ extensions for CBEA ----------------------------- */
+
+/* vec_extract */
+
+static signed char __ATTRS_o_ai
+vec_extract(vector signed char a, int b)
+{
+  return a[b];
+}
+
+static unsigned char __ATTRS_o_ai
+vec_extract(vector unsigned char a, int b)
+{
+  return a[b];
+}
+
+static short __ATTRS_o_ai
+vec_extract(vector short a, int b)
+{
+  return a[b];
+}
+
+static unsigned short __ATTRS_o_ai
+vec_extract(vector unsigned short a, int b)
+{
+  return a[b];
+}
+
+static int __ATTRS_o_ai
+vec_extract(vector int a, int b)
+{
+  return a[b];
+}
+
+static unsigned int __ATTRS_o_ai
+vec_extract(vector unsigned int a, int b)
+{
+  return a[b];
+}
+
+static float __ATTRS_o_ai
+vec_extract(vector float a, int b)
+{
+  return a[b];
+}
+
+/* vec_insert */
+
+static vector signed char __ATTRS_o_ai
+vec_insert(signed char a, vector signed char b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_insert(unsigned char a, vector unsigned char b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector short __ATTRS_o_ai
+vec_insert(short a, vector short b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_insert(unsigned short a, vector unsigned short b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector int __ATTRS_o_ai
+vec_insert(int a, vector int b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_insert(unsigned int a, vector unsigned int b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+static vector float __ATTRS_o_ai
+vec_insert(float a, vector float b, int c)
+{
+  b[c] = a;
+  return b;
+}
+
+/* vec_lvlx */
+
+static vector signed char __ATTRS_o_ai
+vec_lvlx(int a, const signed char *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector signed char)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector signed char __ATTRS_o_ai
+vec_lvlx(int a, const vector signed char *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector signed char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvlx(int a, const unsigned char *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector unsigned char)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvlx(int a, const vector unsigned char *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector unsigned char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_lvlx(int a, const vector bool char *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector bool char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvlx(int a, const short *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector short)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvlx(int a, const vector short *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvlx(int a, const unsigned short *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector unsigned short)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvlx(int a, const vector unsigned short *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector unsigned short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_lvlx(int a, const vector bool short *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector bool short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvlx(int a, const vector pixel *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector pixel)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvlx(int a, const int *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector int)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvlx(int a, const vector int *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvlx(int a, const unsigned int *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector unsigned int)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvlx(int a, const vector unsigned int *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector unsigned int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_lvlx(int a, const vector bool int *b)
+{
+  return vec_perm(vec_ld(a, b), 
+                  (vector bool int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvlx(int a, const float *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector float)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvlx(int a, const vector float *b)
+{
+  return vec_perm(vec_ld(a, b),
+                  (vector float)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+/* vec_lvlxl */
+
+static vector signed char __ATTRS_o_ai
+vec_lvlxl(int a, const signed char *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector signed char)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector signed char __ATTRS_o_ai
+vec_lvlxl(int a, const vector signed char *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector signed char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvlxl(int a, const unsigned char *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector unsigned char)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvlxl(int a, const vector unsigned char *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector unsigned char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_lvlxl(int a, const vector bool char *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector bool char)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvlxl(int a, const short *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector short)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvlxl(int a, const vector short *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvlxl(int a, const unsigned short *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector unsigned short)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvlxl(int a, const vector unsigned short *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector unsigned short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_lvlxl(int a, const vector bool short *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector bool short)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvlxl(int a, const vector pixel *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector pixel)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvlxl(int a, const int *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector int)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvlxl(int a, const vector int *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvlxl(int a, const unsigned int *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector unsigned int)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvlxl(int a, const vector unsigned int *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector unsigned int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_lvlxl(int a, const vector bool int *b)
+{
+  return vec_perm(vec_ldl(a, b), 
+                  (vector bool int)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvlxl(int a, const float *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector float)(0),
+                  vec_lvsl(a, b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvlxl(int a, vector float *b)
+{
+  return vec_perm(vec_ldl(a, b),
+                  (vector float)(0),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+/* vec_lvrx */
+
+static vector signed char __ATTRS_o_ai
+vec_lvrx(int a, const signed char *b)
+{
+  return vec_perm((vector signed char)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector signed char __ATTRS_o_ai
+vec_lvrx(int a, const vector signed char *b)
+{
+  return vec_perm((vector signed char)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvrx(int a, const unsigned char *b)
+{
+  return vec_perm((vector unsigned char)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvrx(int a, const vector unsigned char *b)
+{
+  return vec_perm((vector unsigned char)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_lvrx(int a, const vector bool char *b)
+{
+  return vec_perm((vector bool char)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvrx(int a, const short *b)
+{
+  return vec_perm((vector short)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvrx(int a, const vector short *b)
+{
+  return vec_perm((vector short)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvrx(int a, const unsigned short *b)
+{
+  return vec_perm((vector unsigned short)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvrx(int a, const vector unsigned short *b)
+{
+  return vec_perm((vector unsigned short)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_lvrx(int a, const vector bool short *b)
+{
+  return vec_perm((vector bool short)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvrx(int a, const vector pixel *b)
+{
+  return vec_perm((vector pixel)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvrx(int a, const int *b)
+{
+  return vec_perm((vector int)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvrx(int a, const vector int *b)
+{
+  return vec_perm((vector int)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvrx(int a, const unsigned int *b)
+{
+  return vec_perm((vector unsigned int)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvrx(int a, const vector unsigned int *b)
+{
+  return vec_perm((vector unsigned int)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_lvrx(int a, const vector bool int *b)
+{
+  return vec_perm((vector bool int)(0),
+                  vec_ld(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvrx(int a, const float *b)
+{
+  return vec_perm((vector float)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvrx(int a, const vector float *b)
+{
+  return vec_perm((vector float)(0),
+                  vec_ld(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+/* vec_lvrxl */
+
+static vector signed char __ATTRS_o_ai
+vec_lvrxl(int a, const signed char *b)
+{
+  return vec_perm((vector signed char)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector signed char __ATTRS_o_ai
+vec_lvrxl(int a, const vector signed char *b)
+{
+  return vec_perm((vector signed char)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvrxl(int a, const unsigned char *b)
+{
+  return vec_perm((vector unsigned char)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_lvrxl(int a, const vector unsigned char *b)
+{
+  return vec_perm((vector unsigned char)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_lvrxl(int a, const vector bool char *b)
+{
+  return vec_perm((vector bool char)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvrxl(int a, const short *b)
+{
+  return vec_perm((vector short)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector short __ATTRS_o_ai
+vec_lvrxl(int a, const vector short *b)
+{
+  return vec_perm((vector short)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvrxl(int a, const unsigned short *b)
+{
+  return vec_perm((vector unsigned short)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_lvrxl(int a, const vector unsigned short *b)
+{
+  return vec_perm((vector unsigned short)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_lvrxl(int a, const vector bool short *b)
+{
+  return vec_perm((vector bool short)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvrxl(int a, const vector pixel *b)
+{
+  return vec_perm((vector pixel)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvrxl(int a, const int *b)
+{
+  return vec_perm((vector int)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector int __ATTRS_o_ai
+vec_lvrxl(int a, const vector int *b)
+{
+  return vec_perm((vector int)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvrxl(int a, const unsigned int *b)
+{
+  return vec_perm((vector unsigned int)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_lvrxl(int a, const vector unsigned int *b)
+{
+  return vec_perm((vector unsigned int)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_lvrxl(int a, const vector bool int *b)
+{
+  return vec_perm((vector bool int)(0),
+                  vec_ldl(a, b), 
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvrxl(int a, const float *b)
+{
+  return vec_perm((vector float)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, b));
+}
+
+static vector float __ATTRS_o_ai
+vec_lvrxl(int a, const vector float *b)
+{
+  return vec_perm((vector float)(0),
+                  vec_ldl(a, b),
+                  vec_lvsl(a, (unsigned char *)b));
+}
+
+/* vec_stvlx */
+
+static void __ATTRS_o_ai
+vec_stvlx(vector signed char a, int b, signed char *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector signed char a, int b, vector signed char *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned char a, int b, unsigned char *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned char a, int b, vector unsigned char *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector bool char a, int b, vector bool char *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector short a, int b, short *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector short a, int b, vector short *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned short a, int b, unsigned short *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned short a, int b, vector unsigned short *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector bool short a, int b, vector bool short *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector pixel a, int b, vector pixel *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector int a, int b, int *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector int a, int b, vector int *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned int a, int b, unsigned int *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector unsigned int a, int b, vector unsigned int *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector bool int a, int b, vector bool int *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlx(vector float a, int b, vector float *c)
+{
+  return vec_st(vec_perm(vec_lvrx(b, c),
+                         a,
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+/* vec_stvlxl */
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector signed char a, int b, signed char *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector signed char a, int b, vector signed char *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned char a, int b, unsigned char *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned char a, int b, vector unsigned char *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector bool char a, int b, vector bool char *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector short a, int b, short *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector short a, int b, vector short *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned short a, int b, unsigned short *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned short a, int b, vector unsigned short *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector bool short a, int b, vector bool short *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector pixel a, int b, vector pixel *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector int a, int b, int *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector int a, int b, vector int *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned int a, int b, unsigned int *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector unsigned int a, int b, vector unsigned int *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector bool int a, int b, vector bool int *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvlxl(vector float a, int b, vector float *c)
+{
+  return vec_stl(vec_perm(vec_lvrx(b, c),
+                          a,
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+/* vec_stvrx */
+
+static void __ATTRS_o_ai
+vec_stvrx(vector signed char a, int b, signed char *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector signed char a, int b, vector signed char *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned char a, int b, unsigned char *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned char a, int b, vector unsigned char *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector bool char a, int b, vector bool char *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector short a, int b, short *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector short a, int b, vector short *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned short a, int b, unsigned short *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned short a, int b, vector unsigned short *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector bool short a, int b, vector bool short *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector pixel a, int b, vector pixel *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector int a, int b, int *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector int a, int b, vector int *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned int a, int b, unsigned int *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector unsigned int a, int b, vector unsigned int *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector bool int a, int b, vector bool int *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrx(vector float a, int b, vector float *c)
+{
+  return vec_st(vec_perm(a,
+                         vec_lvlx(b, c),
+                         vec_lvsr(b, (unsigned char *)c)),
+                b, c);
+}
+
+/* vec_stvrxl */
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector signed char a, int b, signed char *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector signed char a, int b, vector signed char *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned char a, int b, unsigned char *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned char a, int b, vector unsigned char *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector bool char a, int b, vector bool char *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector short a, int b, short *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector short a, int b, vector short *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned short a, int b, unsigned short *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned short a, int b, vector unsigned short *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector bool short a, int b, vector bool short *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector pixel a, int b, vector pixel *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector int a, int b, int *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector int a, int b, vector int *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned int a, int b, unsigned int *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector unsigned int a, int b, vector unsigned int *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector bool int a, int b, vector bool int *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvrxl(vector float a, int b, vector float *c)
+{
+  return vec_stl(vec_perm(a,
+                          vec_lvlx(b, c),
+                          vec_lvsr(b, (unsigned char *)c)),
+                 b, c);
+}
+
+/* vec_promote */
+
+static vector signed char __ATTRS_o_ai
+vec_promote(signed char a, int b)
+{
+  vector signed char res = (vector signed char)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_promote(unsigned char a, int b)
+{
+  vector unsigned char res = (vector unsigned char)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector short __ATTRS_o_ai
+vec_promote(short a, int b)
+{
+  vector short res = (vector short)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_promote(unsigned short a, int b)
+{
+  vector unsigned short res = (vector unsigned short)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector int __ATTRS_o_ai
+vec_promote(int a, int b)
+{
+  vector int res = (vector int)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_promote(unsigned int a, int b)
+{
+  vector unsigned int res = (vector unsigned int)(0);
+  res[b] = a;
+  return res;
+}
+
+static vector float __ATTRS_o_ai
+vec_promote(float a, int b)
+{
+  vector float res = (vector float)(0);
+  res[b] = a;
+  return res;
+}
+
+/* vec_splats */
+
+static vector signed char __ATTRS_o_ai
+vec_splats(signed char a)
+{
+  return (vector signed char)(a);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_splats(unsigned char a)
+{
+  return (vector unsigned char)(a);
+}
+
+static vector short __ATTRS_o_ai
+vec_splats(short a)
+{
+  return (vector short)(a);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_splats(unsigned short a)
+{
+  return (vector unsigned short)(a);
+}
+
+static vector int __ATTRS_o_ai
+vec_splats(int a)
+{
+  return (vector int)(a);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_splats(unsigned int a)
+{
+  return (vector unsigned int)(a);
+}
+
+static vector float __ATTRS_o_ai
+vec_splats(float a)
+{
+  return (vector float)(a);
+}
+
+/* ----------------------------- predicates --------------------------------- */
 
 /* vec_all_eq */
 
@@ -8440,37 +9996,43 @@ vec_all_eq(vector short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_eq(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_eq(vector unsigned short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_eq(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_eq(vector bool short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_eq(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_eq(vector pixel a, vector pixel b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
@@ -8550,8 +10112,9 @@ vec_all_ge(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)b, 
-                                                (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -8563,8 +10126,9 @@ vec_all_ge(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)b, 
-                                                (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -8594,8 +10158,9 @@ vec_all_ge(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)b, 
-                                                (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -8607,8 +10172,9 @@ vec_all_ge(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)b, 
-                                                (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -8638,8 +10204,9 @@ vec_all_ge(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)b, 
-                                                (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -8651,8 +10218,9 @@ vec_all_ge(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_all_ge(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)b, 
-                                                (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -8690,8 +10258,9 @@ vec_all_gt(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)a, 
-                                                (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -8703,8 +10272,9 @@ vec_all_gt(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)a, 
-                                                (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -8734,8 +10304,9 @@ vec_all_gt(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)a, 
-                                                (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -8747,8 +10318,9 @@ vec_all_gt(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)a, 
-                                                (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -8778,8 +10350,9 @@ vec_all_gt(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)a, 
-                                                (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -8791,8 +10364,9 @@ vec_all_gt(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_all_gt(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)a, 
-                                                (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -8838,8 +10412,9 @@ vec_all_le(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)a, 
-                                                (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -8851,8 +10426,9 @@ vec_all_le(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)a, 
-                                                (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -8882,8 +10458,9 @@ vec_all_le(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)a, 
-                                                (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -8895,8 +10472,9 @@ vec_all_le(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)a, 
-                                                (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -8926,8 +10504,9 @@ vec_all_le(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)a, 
-                                                (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -8939,8 +10518,9 @@ vec_all_le(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_all_le(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)a, 
-                                                (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -8978,8 +10558,9 @@ vec_all_lt(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)b, 
-                                                (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -8991,8 +10572,9 @@ vec_all_lt(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)b, 
-                                                (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -9022,8 +10604,9 @@ vec_all_lt(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)b, 
-                                                (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9035,8 +10618,9 @@ vec_all_lt(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)b, 
-                                                (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9066,8 +10650,9 @@ vec_all_lt(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)b, 
-                                                (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -9079,8 +10664,9 @@ vec_all_lt(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_all_lt(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)b, 
-                                                (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -9156,37 +10742,43 @@ vec_all_ne(vector short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_all_ne(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_ne(vector unsigned short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_ne(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_ne(vector bool short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_ne(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_all_ne(vector pixel a, vector pixel b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+  return
+    __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
@@ -9282,43 +10874,50 @@ vec_all_numeric(vector float a)
 static int __ATTRS_o_ai
 vec_any_eq(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector signed char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
@@ -9336,37 +10935,49 @@ vec_any_eq(vector short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, 
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, 
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector pixel a, vector pixel b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, 
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
@@ -9384,31 +10995,36 @@ vec_any_eq(vector int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector unsigned int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_eq(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
@@ -9446,8 +11062,9 @@ vec_any_ge(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_any_ge(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)b, 
-                                                    (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -9459,8 +11076,9 @@ vec_any_ge(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_any_ge(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)b, 
-                                                    (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -9484,14 +11102,16 @@ vec_any_ge(vector unsigned short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_any_ge(vector unsigned short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, a);
+  return
+    __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, a);
 }
 
 static int __ATTRS_o_ai
 vec_any_ge(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, 
-                                                    (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9504,8 +11124,9 @@ vec_any_ge(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_any_ge(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, 
-                                                    (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9535,8 +11156,9 @@ vec_any_ge(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_ge(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)b, 
-                                                    (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -9548,8 +11170,9 @@ vec_any_ge(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_any_ge(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)b, 
-                                                    (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -9588,8 +11211,9 @@ vec_any_gt(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_any_gt(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)a,
-                                                    (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -9602,8 +11226,9 @@ vec_any_gt(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_any_gt(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)a,
-                                                    (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -9634,21 +11259,24 @@ vec_any_gt(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_any_gt(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a,
-                                                    (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_gt(vector bool short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a, b);
+  return
+    __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a, b);
 }
 
 static int __ATTRS_o_ai
 vec_any_gt(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a, 
-                                                    (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -9678,8 +11306,9 @@ vec_any_gt(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_gt(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)a,
-                                                    (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -9691,8 +11320,9 @@ vec_any_gt(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_any_gt(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)a,
-                                                    (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -9731,8 +11361,9 @@ vec_any_le(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)a,
-                                                    (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -9745,8 +11376,9 @@ vec_any_le(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)a,
-                                                    (vector unsigned char)b);
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV,
+                                      (vector unsigned char)a,
+                                      (vector unsigned char)b);
 }
 
 static int __ATTRS_o_ai
@@ -9777,8 +11409,9 @@ vec_any_le(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)a,
-                                                    (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -9791,8 +11424,9 @@ vec_any_le(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)a,
-                                                    (vector unsigned short)b);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV,
+                                      (vector unsigned short)a,
+                                      (vector unsigned short)b);
 }
 
 static int __ATTRS_o_ai
@@ -9822,8 +11456,9 @@ vec_any_le(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)a,
-                                                    (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -9835,8 +11470,9 @@ vec_any_le(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_any_le(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)a,
-                                                    (vector unsigned int)b);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV,
+                                      (vector unsigned int)a,
+                                      (vector unsigned int)b);
 }
 
 static int __ATTRS_o_ai
@@ -9875,8 +11511,9 @@ vec_any_lt(vector unsigned char a, vector bool char b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)b,
-                                                    (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -9889,8 +11526,9 @@ vec_any_lt(vector bool char a, vector unsigned char b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)b,
-                                                    (vector unsigned char)a);
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV,
+                                      (vector unsigned char)b,
+                                      (vector unsigned char)a);
 }
 
 static int __ATTRS_o_ai
@@ -9921,8 +11559,9 @@ vec_any_lt(vector unsigned short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)b,
-                                                    (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9935,8 +11574,9 @@ vec_any_lt(vector bool short a, vector unsigned short b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)b,
-                                                    (vector unsigned short)a);
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV,
+                                      (vector unsigned short)b,
+                                      (vector unsigned short)a);
 }
 
 static int __ATTRS_o_ai
@@ -9966,8 +11606,9 @@ vec_any_lt(vector unsigned int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)b,
-                                                    (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -9979,8 +11620,9 @@ vec_any_lt(vector bool int a, vector unsigned int b)
 static int __ATTRS_o_ai
 vec_any_lt(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)b,
-                                                    (vector unsigned int)a);
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV,
+                                      (vector unsigned int)b,
+                                      (vector unsigned int)a);
 }
 
 static int __ATTRS_o_ai
@@ -10002,43 +11644,50 @@ vec_any_nan(vector float a)
 static int __ATTRS_o_ai
 vec_any_ne(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector signed char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool char a, vector bool char b)
 {
-  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+  return
+    __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
@@ -10056,37 +11705,49 @@ vec_any_ne(vector short a, vector bool short b)
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, 
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool short a, vector short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool short a, vector bool short b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector pixel a, vector pixel b)
 {
-  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV,
+                                      (vector short)a,
+                                      (vector short)b);
 }
 
 static int __ATTRS_o_ai
@@ -10104,31 +11765,36 @@ vec_any_ne(vector int a, vector bool int b)
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector unsigned int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool int a, vector int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
 vec_any_ne(vector bool int a, vector bool int b)
 {
-  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+  return
+    __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
diff --git a/lib/Headers/emmintrin.h b/lib/Headers/emmintrin.h
index e5dfe26b49a6..11b258178103 100644
--- a/lib/Headers/emmintrin.h
+++ b/lib/Headers/emmintrin.h
@@ -777,11 +777,8 @@ _mm_xor_si128(__m128i a, __m128i b)
   return a ^ b;
 }
 
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_slli_si128(__m128i a, int imm)
-{
-  return __builtin_ia32_pslldqi128(a, imm * 8);
-}
+#define _mm_slli_si128(VEC, IMM) \
+  ((__m128i)__builtin_ia32_pslldqi128((__m128i)(VEC), (IMM)*8))
 
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
 _mm_slli_epi16(__m128i a, int count)
@@ -843,11 +840,9 @@ _mm_sra_epi32(__m128i a, __m128i count)
   return (__m128i)__builtin_ia32_psrad128((__v4si)a, (__v4si)count);
 }
 
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_srli_si128(__m128i a, int imm)
-{
-  return __builtin_ia32_psrldqi128(a, imm * 8);
-}
+
+#define _mm_srli_si128(VEC, IMM) \
+  ((__m128i)__builtin_ia32_psrldqi128((__m128i)(VEC), (IMM)*8))
 
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
 _mm_srli_epi16(__m128i a, int count)
diff --git a/lib/Headers/limits.h b/lib/Headers/limits.h
index 26275337d583..ecd09a4a246d 100644
--- a/lib/Headers/limits.h
+++ b/lib/Headers/limits.h
@@ -31,8 +31,12 @@
 #define _GCC_LIMITS_H_
 #endif
 
-/* System headers include a number of constants from POSIX in <limits.h>. */
+/* System headers include a number of constants from POSIX in <limits.h>.
+   Include it if we're hosted. */
+#if __STDC_HOSTED__ && \
+    defined(__has_include_next) && __has_include_next(<limits.h>)
 #include_next <limits.h>
+#endif
 
 /* Many system headers try to "help us out" by defining these.  No really, we
    know how big each datatype is. */
diff --git a/lib/Headers/mm_malloc.h b/lib/Headers/mm_malloc.h
index fba865120b29..e7da5434587a 100644
--- a/lib/Headers/mm_malloc.h
+++ b/lib/Headers/mm_malloc.h
@@ -24,39 +24,48 @@
 #ifndef __MM_MALLOC_H
 #define __MM_MALLOC_H
 
-#include <errno.h>
 #include <stdlib.h>
 
-static __inline__ void *__attribute__((__always_inline__, __nodebug__))
+#ifdef _WIN32
+#include <malloc.h>
+#else
+#ifndef __cplusplus
+extern int posix_memalign(void **memptr, size_t alignment, size_t size);
+#else
+// Some systems (e.g. those with GNU libc) declare posix_memalign with an
+// exception specifier. Via an "egregious workaround" in
+// Sema::CheckEquivalentExceptionSpec, Clang accepts the following as a valid
+// redeclaration of glibc's declaration.
+extern "C" int posix_memalign(void **memptr, size_t alignment, size_t size);
+#endif
+#endif
+
+static __inline__ void *__attribute__((__always_inline__, __nodebug__,
+                                       __malloc__))
 _mm_malloc(size_t size, size_t align)
 {
-  if (align & (align - 1)) {
-    errno = EINVAL;
-    return 0;
+  if (align == 1) {
+    return malloc(size);
   }
 
-  if (!size)
-    return 0;
+  if (!(align & (align - 1)) && align < sizeof(void *))
+    align = sizeof(void *);
 
-  if (align < 2 * sizeof(void *))
-    align = 2 * sizeof(void *);
-
-  void *mallocedMemory = malloc(size + align);
-  if (!mallocedMemory)
+  void *mallocedMemory;
+#ifdef _WIN32
+  mallocedMemory = _aligned_malloc(size, align);
+#else
+  if (posix_memalign(&mallocedMemory, align, size))
     return 0;
+#endif
 
-  void *alignedMemory =
-    (void *)(((size_t)mallocedMemory + align) & ~((size_t)align - 1));
-  ((void **)alignedMemory)[-1] = mallocedMemory;
-
-  return alignedMemory;
+  return mallocedMemory;
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_free(void *p)
 {
-  if (p)
-    free(((void **)p)[-1]);
+  free(p);
 }
 
 #endif /* __MM_MALLOC_H */
diff --git a/lib/Headers/mmintrin.h b/lib/Headers/mmintrin.h
index bad9e1c059dc..fefb42fd74e3 100644
--- a/lib/Headers/mmintrin.h
+++ b/lib/Headers/mmintrin.h
@@ -43,14 +43,13 @@ _mm_empty(void)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cvtsi32_si64(int __i)
 {
-    return (__m64)(__v2si){__i, 0};
+    return (__m64)__builtin_ia32_vec_init_v2si(__i, 0);
 }
 
 static __inline__ int __attribute__((__always_inline__, __nodebug__))
 _mm_cvtsi64_si32(__m64 __m)
 {
-    __v2si __mmx_var2 = (__v2si)__m;
-    return __mmx_var2[0];
+    return __builtin_ia32_vec_ext_v2si((__v2si)__m, 0);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -86,59 +85,55 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 4, 8+4, 5,
-                                          8+5, 6, 8+6, 7, 8+7);
+    return (__m64)__builtin_ia32_punpckhbw((__v8qi)__m1, (__v8qi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 2, 4+2, 3,
-                                          4+3);
+    return (__m64)__builtin_ia32_punpckhwd((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 2+1);
+    return (__m64)__builtin_ia32_punpckhdq((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 0, 8+0, 1,
-                                          8+1, 2, 8+2, 3, 8+3);
+    return (__m64)__builtin_ia32_punpcklbw((__v8qi)__m1, (__v8qi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 0, 4+0, 1,
-                                          4+1);
+    return (__m64)__builtin_ia32_punpcklwd((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2+0);
+    return (__m64)__builtin_ia32_punpckldq((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_add_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v8qi)__m1 + (__v8qi)__m2);
+    return (__m64)__builtin_ia32_paddb((__v8qi)__m1, (__v8qi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_add_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v4hi)__m1 + (__v4hi)__m2);
+    return (__m64)__builtin_ia32_paddw((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_add_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v2si)__m1 + (__v2si)__m2);
+    return (__m64)__builtin_ia32_paddd((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -168,19 +163,19 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_sub_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v8qi)__m1 - (__v8qi)__m2);
+    return (__m64)__builtin_ia32_psubb((__v8qi)__m1, (__v8qi)__m2);
 }
  
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_sub_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v4hi)__m1 - (__v4hi)__m2);
+    return (__m64)__builtin_ia32_psubw((__v4hi)__m1, (__v4hi)__m2);
 }
  
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_sub_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v2si)__m1 - (__v2si)__m2);
+    return (__m64)__builtin_ia32_psubd((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -222,7 +217,7 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_mullo_pi16(__m64 __m1, __m64 __m2) 
 {
-    return (__m64)((__v4hi)__m1 * (__v4hi)__m2);
+    return (__m64)__builtin_ia32_pmullw((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -252,13 +247,13 @@ _mm_slli_pi32(__m64 __m, int __count)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_sll_si64(__m64 __m, __m64 __count)
 {
-    return __builtin_ia32_psllq(__m, __count);
+    return (__m64)__builtin_ia32_psllq(__m, __count);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_slli_si64(__m64 __m, int __count)
 {
-    return __builtin_ia32_psllqi(__m, __count);    
+    return (__m64)__builtin_ia32_psllqi(__m, __count);    
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -318,67 +313,67 @@ _mm_srl_si64(__m64 __m, __m64 __count)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_srli_si64(__m64 __m, int __count)
 {
-    return __builtin_ia32_psrlqi(__m, __count);    
+    return (__m64)__builtin_ia32_psrlqi(__m, __count);    
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_and_si64(__m64 __m1, __m64 __m2)
 {
-    return __m1 & __m2;
+    return __builtin_ia32_pand(__m1, __m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_andnot_si64(__m64 __m1, __m64 __m2)
 {
-    return ~__m1 & __m2;
+    return __builtin_ia32_pandn(__m1, __m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_or_si64(__m64 __m1, __m64 __m2)
 {
-    return __m1 | __m2;
+    return __builtin_ia32_por(__m1, __m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_xor_si64(__m64 __m1, __m64 __m2)
 {
-    return __m1 ^ __m2;
+    return __builtin_ia32_pxor(__m1, __m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v8qi)__m1 == (__v8qi)__m2);
+    return (__m64)__builtin_ia32_pcmpeqb((__v8qi)__m1, (__v8qi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v4hi)__m1 == (__v4hi)__m2);
+    return (__m64)__builtin_ia32_pcmpeqw((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v2si)__m1 == (__v2si)__m2);
+    return (__m64)__builtin_ia32_pcmpeqd((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v8qi)__m1 > (__v8qi)__m2);
+    return (__m64)__builtin_ia32_pcmpgtb((__v8qi)__m1, (__v8qi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
+    return (__m64)__builtin_ia32_pcmpgtw((__v4hi)__m1, (__v4hi)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
 {
-    return (__m64)((__v2si)__m1 > (__v2si)__m2);
+    return (__m64)__builtin_ia32_pcmpgtd((__v2si)__m1, (__v2si)__m2);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
@@ -390,57 +385,58 @@ _mm_setzero_si64(void)
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_set_pi32(int __i1, int __i0)
 {
-    return (__m64)(__v2si){ __i0, __i1 };
+    return (__m64)__builtin_ia32_vec_init_v2si(__i0, __i1);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_set_pi16(short __s3, short __s2, short __s1, short __s0)
 {
-    return (__m64)(__v4hi){ __s0, __s1, __s2, __s3 };    
+    return (__m64)__builtin_ia32_vec_init_v4hi(__s0, __s1, __s2, __s3);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
             char __b1, char __b0)
 {
-    return (__m64)(__v8qi){ __b0, __b1, __b2, __b3, __b4, __b5, __b6, __b7 };
+    return (__m64)__builtin_ia32_vec_init_v8qi(__b0, __b1, __b2, __b3,
+                                               __b4, __b5, __b6, __b7);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_set1_pi32(int __i)
 {
-    return (__m64)(__v2si){ __i, __i };
+    return _mm_set_pi32(__i, __i);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
-_mm_set1_pi16(short __s)
+_mm_set1_pi16(short __w)
 {
-    return (__m64)(__v4hi){ __s, __s, __s, __s };
+    return _mm_set_pi16(__w, __w, __w, __w);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_set1_pi8(char __b)
 {
-    return (__m64)(__v8qi){ __b, __b, __b, __b, __b, __b, __b, __b };
+    return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_setr_pi32(int __i1, int __i0)
 {
-    return (__m64)(__v2si){ __i1, __i0 };
+    return _mm_set_pi32(__i1, __i0);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
-_mm_setr_pi16(short __s3, short __s2, short __s1, short __s0)
+_mm_setr_pi16(short __w3, short __w2, short __w1, short __w0)
 {
-    return (__m64)(__v4hi){ __s3, __s2, __s1, __s0 };
+    return _mm_set_pi16(__w3, __w2, __w1, __w0);
 }
 
 static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
 _mm_setr_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
              char __b1, char __b0)
 {
-    return (__m64)(__v8qi){ __b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0 };
+    return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
 }
 
 
diff --git a/lib/Headers/stdbool.h b/lib/Headers/stdbool.h
index e44a1f9a979f..0467893f3403 100644
--- a/lib/Headers/stdbool.h
+++ b/lib/Headers/stdbool.h
@@ -26,11 +26,17 @@
 #ifndef __STDBOOL_H
 #define __STDBOOL_H
 
-/* Don't define bool, true, and false in C++ */
+/* Don't define bool, true, and false in C++, except as a GNU extension. */
 #ifndef __cplusplus
 #define bool _Bool
 #define true 1
 #define false 0
+#elif defined(__GNUC__) && !defined(__STRICT_ANSI__)
+/* Define _Bool, bool, false, true as a GNU extension. */
+#define _Bool bool
+#define bool  bool
+#define false false
+#define true  true
 #endif
 
 #define __bool_true_false_are_defined 1
diff --git a/lib/Headers/stddef.h b/lib/Headers/stddef.h
index 84ec1a7b4e24..7cc0bc1a75fd 100644
--- a/lib/Headers/stddef.h
+++ b/lib/Headers/stddef.h
@@ -46,13 +46,16 @@ typedef __WCHAR_TYPE__ wchar_t;
 #define NULL ((void*)0)
 #endif
 
-// Some C libraries expect to see a wint_t here. Others (notably MinGW) will use
-// __WINT_TYPE__ directly; accomodate both by requiring __need_wint_t
-#if defined(__need_wint_t) && !defined(_WINT_T)
-#define _WINT_T
-typedef __WINT_TYPE__ wint_t;
-#endif
-
 #define offsetof(t, d) __builtin_offsetof(t, d)
 
 #endif /* __STDDEF_H */
+
+/* Some C libraries expect to see a wint_t here. Others (notably MinGW) will use
+__WINT_TYPE__ directly; accomodate both by requiring __need_wint_t */
+#if defined(__need_wint_t)
+#if !defined(_WINT_T)
+#define _WINT_T
+typedef __WINT_TYPE__ wint_t;
+#endif /* _WINT_T */
+#undef __need_wint_t
+#endif /* __need_wint_t */
diff --git a/lib/Headers/xmmintrin.h b/lib/Headers/xmmintrin.h
index 8363b4594b1f..42dd3e8d3b87 100644
--- a/lib/Headers/xmmintrin.h
+++ b/lib/Headers/xmmintrin.h
@@ -34,7 +34,11 @@ typedef int __v4si __attribute__((__vector_size__(16)));
 typedef float __v4sf __attribute__((__vector_size__(16)));
 typedef float __m128 __attribute__((__vector_size__(16)));
 
+// This header should only be included in a hosted environment as it depends on
+// a standard library to provide allocation routines.
+#if __STDC_HOSTED__
 #include <mm_malloc.h>
+#endif
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_add_ss(__m128 a, __m128 b)
@@ -712,9 +716,7 @@ _mm_mulhi_pu16(__m64 a, __m64 b)
 }
 
 #define _mm_shuffle_pi16(a, n) \
-  ((__m64)__builtin_shufflevector((__v4hi)(a), (__v4hi) {0}, \
-                                  (n) & 0x3, ((n) & 0xc) >> 2, \
-                                  ((n) & 0x30) >> 4, ((n) & 0xc0) >> 6))
+  ((__m64)__builtin_ia32_pshufw(a, n))
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_maskmove_si64(__m64 d, __m64 n, char *p)
diff --git a/lib/Index/ASTVisitor.h b/lib/Index/ASTVisitor.h
index 943c72025312..0b8425b2f337 100644
--- a/lib/Index/ASTVisitor.h
+++ b/lib/Index/ASTVisitor.h
@@ -108,8 +108,7 @@ public:
   }
 
   void VisitStmt(Stmt *Node) {
-    for (Stmt::child_iterator
-           I = Node->child_begin(), E = Node->child_end(); I != E; ++I)
+    for (Stmt::child_range I = Node->children(); I; ++I)
       if (*I)
         Visit(*I);
   }
diff --git a/lib/Index/CMakeLists.txt b/lib/Index/CMakeLists.txt
index 61f69b218e26..0a3f7cb14d85 100644
--- a/lib/Index/CMakeLists.txt
+++ b/lib/Index/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST)
 
 add_clang_library(clangIndex
   ASTLocation.cpp
@@ -13,3 +13,6 @@ add_clang_library(clangIndex
   Program.cpp
   SelectorMap.cpp
   )
+
+add_dependencies(clangIndex ClangAttrClasses ClangAttrList
+                 ClangDeclNodes ClangStmtNodes)
diff --git a/lib/Index/CallGraph.cpp b/lib/Index/CallGraph.cpp
index dedcc0e80816..bf3f5a8a8d24 100644
--- a/lib/Index/CallGraph.cpp
+++ b/lib/Index/CallGraph.cpp
@@ -40,7 +40,7 @@ public:
   void VisitCallExpr(CallExpr *CE);
 
   void VisitChildren(Stmt *S) {
-    for (Stmt::child_iterator I=S->child_begin(), E=S->child_end(); I != E;++I)
+    for (Stmt::child_range I = S->children(); I; ++I)
       if (*I)
         static_cast<CGBuilder*>(this)->Visit(*I);
   }
diff --git a/lib/Lex/CMakeLists.txt b/lib/Lex/CMakeLists.txt
index 632fbc6340cc..80e2820101fe 100644
--- a/lib/Lex/CMakeLists.txt
+++ b/lib/Lex/CMakeLists.txt
@@ -1,7 +1,9 @@
-set(LLVM_NO_RTTI 1)
-
 # TODO: Add -maltivec when ARCH is PowerPC.
 
+set(LLVM_LINK_COMPONENTS support)
+
+set(LLVM_USED_LIBS clangBasic)
+
 add_clang_library(clangLex
   HeaderMap.cpp
   HeaderSearch.cpp
@@ -24,4 +26,4 @@ add_clang_library(clangLex
   TokenLexer.cpp
   )
 
-add_dependencies(clangLex ClangDiagnosticLex)
+add_dependencies(clangLex ClangDiagnosticLex ClangAttrSpellings)
diff --git a/lib/Lex/HeaderMap.cpp b/lib/Lex/HeaderMap.cpp
index 4010d613ac24..e424f9165515 100644
--- a/lib/Lex/HeaderMap.cpp
+++ b/lib/Lex/HeaderMap.cpp
@@ -15,9 +15,10 @@
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/System/DataTypes.h"
+#include "llvm/Support/DataTypes.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include <cctype>
 #include <cstdio>
 using namespace clang;
 
@@ -75,13 +76,12 @@ static inline unsigned HashHMapKey(llvm::StringRef Str) {
 /// map.  If it doesn't look like a HeaderMap, it gives up and returns null.
 /// If it looks like a HeaderMap but is obviously corrupted, it puts a reason
 /// into the string error argument and returns null.
-const HeaderMap *HeaderMap::Create(const FileEntry *FE) {
+const HeaderMap *HeaderMap::Create(const FileEntry *FE, FileManager &FM) {
   // If the file is too small to be a header map, ignore it.
   unsigned FileSize = FE->getSize();
   if (FileSize <= sizeof(HMapHeader)) return 0;
 
-  llvm::OwningPtr<const llvm::MemoryBuffer> FileBuffer(
-    llvm::MemoryBuffer::getFile(FE->getName(), 0, FE->getSize()));
+  llvm::OwningPtr<const llvm::MemoryBuffer> FileBuffer(FM.getBufferForFile(FE));
   if (FileBuffer == 0) return 0;  // Unreadable file?
   const char *FileStart = FileBuffer->getBufferStart();
 
@@ -223,6 +223,6 @@ const FileEntry *HeaderMap::LookupFile(llvm::StringRef Filename,
     llvm::SmallString<1024> DestPath;
     DestPath += getString(B.Prefix);
     DestPath += getString(B.Suffix);
-    return FM.getFile(DestPath.begin(), DestPath.end());
+    return FM.getFile(DestPath.str());
   }
 }
diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp
index 4554ababf76d..b028e339ae91 100644
--- a/lib/Lex/HeaderSearch.cpp
+++ b/lib/Lex/HeaderSearch.cpp
@@ -15,7 +15,8 @@
 #include "clang/Lex/HeaderMap.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/IdentifierTable.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
 #include "llvm/ADT/SmallString.h"
 #include <cstdio>
 using namespace clang;
@@ -32,11 +33,15 @@ HeaderFileInfo::getControllingMacro(ExternalIdentifierLookup *External) {
   return ControllingMacro;
 }
 
-HeaderSearch::HeaderSearch(FileManager &FM) : FileMgr(FM), FrameworkMap(64) {
+ExternalHeaderFileInfoSource::~ExternalHeaderFileInfoSource() {}
+
+HeaderSearch::HeaderSearch(FileManager &FM)
+    : FileMgr(FM), FrameworkMap(64) {
   SystemDirIdx = 0;
   NoCurDirSearch = false;
 
   ExternalLookup = 0;
+  ExternalSource = 0;
   NumIncluded = 0;
   NumMultiIncludeFileOptzn = 0;
   NumFrameworkLookups = NumSubFrameworkLookups = 0;
@@ -83,7 +88,7 @@ const HeaderMap *HeaderSearch::CreateHeaderMap(const FileEntry *FE) {
         return HeaderMaps[i].second;
   }
 
-  if (const HeaderMap *HM = HeaderMap::Create(FE)) {
+  if (const HeaderMap *HM = HeaderMap::Create(FE, FileMgr)) {
     HeaderMaps.push_back(std::make_pair(FE, HM));
     return HM;
   }
@@ -118,7 +123,7 @@ const FileEntry *DirectoryLookup::LookupFile(llvm::StringRef Filename,
     TmpDir += getDir()->getName();
     TmpDir.push_back('/');
     TmpDir.append(Filename.begin(), Filename.end());
-    return HS.getFileMgr().getFile(TmpDir.begin(), TmpDir.end());
+    return HS.getFileMgr().getFile(TmpDir.str());
   }
 
   if (isFramework())
@@ -169,8 +174,8 @@ const FileEntry *DirectoryLookup::DoFrameworkLookup(llvm::StringRef Filename,
 
     // If the framework dir doesn't exist, we fail.
     // FIXME: It's probably more efficient to query this with FileMgr.getDir.
-    if (!llvm::sys::Path(std::string(FrameworkName.begin(),
-                                     FrameworkName.end())).exists())
+    bool Exists;
+    if (llvm::sys::fs::exists(FrameworkName.str(), Exists) || !Exists)
       return 0;
 
     // Otherwise, if it does, remember that this is the right direntry for this
@@ -183,16 +188,14 @@ const FileEntry *DirectoryLookup::DoFrameworkLookup(llvm::StringRef Filename,
 
   FrameworkName += "Headers/";
   FrameworkName.append(Filename.begin()+SlashPos+1, Filename.end());
-  if (const FileEntry *FE = FileMgr.getFile(FrameworkName.begin(),
-                                            FrameworkName.end())) {
+  if (const FileEntry *FE = FileMgr.getFile(FrameworkName.str()))
     return FE;
-  }
 
   // Check "/System/Library/Frameworks/Cocoa.framework/PrivateHeaders/file.h"
   const char *Private = "Private";
   FrameworkName.insert(FrameworkName.begin()+OrigSize, Private,
                        Private+strlen(Private));
-  return FileMgr.getFile(FrameworkName.begin(), FrameworkName.end());
+  return FileMgr.getFile(FrameworkName.str());
 }
 
 
@@ -212,7 +215,7 @@ const FileEntry *HeaderSearch::LookupFile(llvm::StringRef Filename,
                                           const DirectoryLookup *&CurDir,
                                           const FileEntry *CurFileEnt) {
   // If 'Filename' is absolute, check to see if it exists and no searching.
-  if (llvm::sys::Path::isAbsolute(Filename.begin(), Filename.size())) {
+  if (llvm::sys::path::is_absolute(Filename)) {
     CurDir = 0;
 
     // If this was an #include_next "/absolute/file", fail.
@@ -329,7 +332,7 @@ LookupSubframeworkHeader(llvm::StringRef Filename,
   FrameworkName += ".framework/";
 
   llvm::StringMapEntry<const DirectoryEntry *> &CacheLookup =
-    FrameworkMap.GetOrCreateValue(Filename.begin(), Filename.begin()+SlashPos);
+    FrameworkMap.GetOrCreateValue(Filename.substr(0, SlashPos));
 
   // Some other location?
   if (CacheLookup.getValue() &&
@@ -343,8 +346,7 @@ LookupSubframeworkHeader(llvm::StringRef Filename,
     ++NumSubFrameworkLookups;
 
     // If the framework dir doesn't exist, we fail.
-    const DirectoryEntry *Dir = FileMgr.getDirectory(FrameworkName.begin(),
-                                                     FrameworkName.end());
+    const DirectoryEntry *Dir = FileMgr.getDirectory(FrameworkName.str());
     if (Dir == 0) return 0;
 
     // Otherwise, if it does, remember that this is the right direntry for this
@@ -358,14 +360,13 @@ LookupSubframeworkHeader(llvm::StringRef Filename,
   llvm::SmallString<1024> HeadersFilename(FrameworkName);
   HeadersFilename += "Headers/";
   HeadersFilename.append(Filename.begin()+SlashPos+1, Filename.end());
-  if (!(FE = FileMgr.getFile(HeadersFilename.begin(),
-                             HeadersFilename.end()))) {
+  if (!(FE = FileMgr.getFile(HeadersFilename.str()))) {
 
     // Check ".../Frameworks/HIToolbox.framework/PrivateHeaders/HIToolbox.h"
     HeadersFilename = FrameworkName;
     HeadersFilename += "PrivateHeaders/";
     HeadersFilename.append(Filename.begin()+SlashPos+1, Filename.end());
-    if (!(FE = FileMgr.getFile(HeadersFilename.begin(), HeadersFilename.end())))
+    if (!(FE = FileMgr.getFile(HeadersFilename.str())))
       return 0;
   }
 
@@ -389,12 +390,19 @@ LookupSubframeworkHeader(llvm::StringRef Filename,
 HeaderFileInfo &HeaderSearch::getFileInfo(const FileEntry *FE) {
   if (FE->getUID() >= FileInfo.size())
     FileInfo.resize(FE->getUID()+1);
-  return FileInfo[FE->getUID()];
+  
+  HeaderFileInfo &HFI = FileInfo[FE->getUID()];
+  if (ExternalSource && !HFI.Resolved) {
+    HFI = ExternalSource->GetHeaderFileInfo(FE);
+    HFI.Resolved = true;
+  }
+  return HFI;
 }
 
 void HeaderSearch::setHeaderFileInfoForUID(HeaderFileInfo HFI, unsigned UID) {
   if (UID >= FileInfo.size())
     FileInfo.resize(UID+1);
+  HFI.Resolved = true;
   FileInfo[UID] = HFI;
 }
 
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 917829be47c6..b17198b21983 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -212,6 +212,109 @@ void Lexer::Stringify(llvm::SmallVectorImpl<char> &Str) {
   }
 }
 
+//===----------------------------------------------------------------------===//
+// Token Spelling
+//===----------------------------------------------------------------------===//
+
+/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
+/// token are the characters used to represent the token in the source file
+/// after trigraph expansion and escaped-newline folding.  In particular, this
+/// wants to get the true, uncanonicalized, spelling of things like digraphs
+/// UCNs, etc.
+std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
+                               const LangOptions &Features, bool *Invalid) {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+  
+  // If this token contains nothing interesting, return it directly.
+  bool CharDataInvalid = false;
+  const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 
+                                                    &CharDataInvalid);
+  if (Invalid)
+    *Invalid = CharDataInvalid;
+  if (CharDataInvalid)
+    return std::string();
+  
+  if (!Tok.needsCleaning())
+    return std::string(TokStart, TokStart+Tok.getLength());
+  
+  std::string Result;
+  Result.reserve(Tok.getLength());
+  
+  // Otherwise, hard case, relex the characters into the string.
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
+    Ptr += CharSize;
+  }
+  assert(Result.size() != unsigned(Tok.getLength()) &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+  return Result;
+}
+
+/// getSpelling - This method is used to get the spelling of a token into a
+/// preallocated buffer, instead of as an std::string.  The caller is required
+/// to allocate enough space for the token, which is guaranteed to be at least
+/// Tok.getLength() bytes long.  The actual length of the token is returned.
+///
+/// Note that this method may do two possible things: it may either fill in
+/// the buffer specified with characters, or it may *change the input pointer*
+/// to point to a constant buffer with the data already in it (avoiding a
+/// copy).  The caller is not allowed to modify the returned buffer pointer
+/// if an internal buffer is returned.
+unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 
+                            const SourceManager &SourceMgr,
+                            const LangOptions &Features, bool *Invalid) {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+
+  const char *TokStart = 0;
+  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
+  if (Tok.is(tok::raw_identifier))
+    TokStart = Tok.getRawIdentifierData();
+  else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
+    // Just return the string from the identifier table, which is very quick.
+    Buffer = II->getNameStart();
+    return II->getLength();
+  }
+
+  // NOTE: this can be checked even after testing for an IdentifierInfo.
+  if (Tok.isLiteral())
+    TokStart = Tok.getLiteralData();
+
+  if (TokStart == 0) {
+    // Compute the start of the token in the input lexer buffer.
+    bool CharDataInvalid = false;
+    TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
+    if (Invalid)
+      *Invalid = CharDataInvalid;
+    if (CharDataInvalid) {
+      Buffer = "";
+      return 0;
+    }
+  }
+
+  // If this token contains nothing interesting, return it directly.
+  if (!Tok.needsCleaning()) {
+    Buffer = TokStart;
+    return Tok.getLength();
+  }
+
+  // Otherwise, hard case, relex the characters into the string.
+  char *OutBuf = const_cast<char*>(Buffer);
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
+    Ptr += CharSize;
+  }
+  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+
+  return OutBuf-Buffer;
+}
+
+
+
 static bool isWhitespace(unsigned char c);
 
 /// MeasureTokenLength - Relex the token at the specified location and return
@@ -242,7 +345,8 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
     return 0;
 
   // Create a lexer starting at the beginning of this token.
-  Lexer TheLexer(Loc, LangOpts, Buffer.begin(), StrData, Buffer.end());
+  Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
+                 Buffer.begin(), StrData, Buffer.end());
   TheLexer.SetCommentRetentionState(true);
   Token TheTok;
   TheLexer.LexFromRawLexer(TheTok);
@@ -253,6 +357,9 @@ SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
                                           const SourceManager &SM,
                                           const LangOptions &LangOpts) {
   std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  if (LocInfo.first.isInvalid())
+    return Loc;
+  
   bool Invalid = false;
   llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
@@ -261,6 +368,9 @@ SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
   // Back up from the current location until we hit the beginning of a line
   // (or the buffer). We'll relex from that point.
   const char *BufStart = Buffer.data();
+  if (LocInfo.second >= Buffer.size())
+    return Loc;
+  
   const char *StrData = BufStart+LocInfo.second;
   if (StrData[0] == '\n' || StrData[0] == '\r')
     return Loc;
@@ -371,10 +481,9 @@ Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
       // we don't have an identifier table available. Instead, just look at
       // the raw identifier to recognize and categorize preprocessor directives.
       TheLexer.LexFromRawLexer(TheTok);
-      if (TheTok.getKind() == tok::identifier && !TheTok.needsCleaning()) {
-        const char *IdStart = Buffer->getBufferStart() 
-                            + TheTok.getLocation().getRawEncoding() - 1;
-        llvm::StringRef Keyword(IdStart, TheTok.getLength());
+      if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
+        llvm::StringRef Keyword(TheTok.getRawIdentifierData(),
+                                TheTok.getLength());
         PreambleDirectiveKind PDK
           = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
               .Case("include", PDK_Skipped)
@@ -443,6 +552,83 @@ Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
                                : TheTok.isAtStartOfLine());
 }
 
+
+/// AdvanceToTokenCharacter - Given a location that specifies the start of a
+/// token, return a new location that specifies a character within the token.
+SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
+                                              unsigned CharNo,
+                                              const SourceManager &SM,
+                                              const LangOptions &Features) {
+  // Figure out how many physical characters away the specified instantiation
+  // character is.  This needs to take into consideration newlines and
+  // trigraphs.
+  bool Invalid = false;
+  const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);
+  
+  // If they request the first char of the token, we're trivially done.
+  if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
+    return TokStart;
+  
+  unsigned PhysOffset = 0;
+  
+  // The usual case is that tokens don't contain anything interesting.  Skip
+  // over the uninteresting characters.  If a token only consists of simple
+  // chars, this method is extremely fast.
+  while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
+    if (CharNo == 0)
+      return TokStart.getFileLocWithOffset(PhysOffset);
+    ++TokPtr, --CharNo, ++PhysOffset;
+  }
+  
+  // If we have a character that may be a trigraph or escaped newline, use a
+  // lexer to parse it correctly.
+  for (; CharNo; --CharNo) {
+    unsigned Size;
+    Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
+    TokPtr += Size;
+    PhysOffset += Size;
+  }
+  
+  // Final detail: if we end up on an escaped newline, we want to return the
+  // location of the actual byte of the token.  For example foo\<newline>bar
+  // advanced by 3 should return the location of b, not of \\.  One compounding
+  // detail of this is that the escape may be made by a trigraph.
+  if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
+    PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
+  
+  return TokStart.getFileLocWithOffset(PhysOffset);
+}
+
+/// \brief Computes the source location just past the end of the
+/// token at this source location.
+///
+/// This routine can be used to produce a source location that
+/// points just past the end of the token referenced by \p Loc, and
+/// is generally used when a diagnostic needs to point just after a
+/// token where it expected something different that it received. If
+/// the returned source location would not be meaningful (e.g., if
+/// it points into a macro), this routine returns an invalid
+/// source location.
+///
+/// \param Offset an offset from the end of the token, where the source
+/// location should refer to. The default offset (0) produces a source
+/// location pointing just past the end of the token; an offset of 1 produces
+/// a source location pointing to the last character in the token, etc.
+SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
+                                          const SourceManager &SM,
+                                          const LangOptions &Features) {
+  if (Loc.isInvalid() || !Loc.isFileID())
+    return SourceLocation();
+  
+  unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
+  if (Len > Offset)
+    Len = Len - Offset;
+  else
+    return Loc;
+  
+  return AdvanceToTokenCharacter(Loc, Len, SM, Features);
+}
+
 //===----------------------------------------------------------------------===//
 // Character information.
 //===----------------------------------------------------------------------===//
@@ -584,10 +770,8 @@ static inline bool isNumberBody(unsigned char c) {
 /// lexer buffer was all instantiated at a single point, perform the mapping.
 /// This is currently only used for _Pragma implementation, so it is the slow
 /// path of the hot getSourceLocation method.  Do not allow it to be inlined.
-static DISABLE_INLINE SourceLocation GetMappedTokenLoc(Preprocessor &PP,
-                                                       SourceLocation FileLoc,
-                                                       unsigned CharNo,
-                                                       unsigned TokLen);
+static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
+    Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
 static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
                                         SourceLocation FileLoc,
                                         unsigned CharNo, unsigned TokLen) {
@@ -869,19 +1053,17 @@ void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
   if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) {
 FinishIdentifier:
     const char *IdStart = BufferPtr;
-    FormTokenWithChars(Result, CurPtr, tok::identifier);
+    FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
+    Result.setRawIdentifierData(IdStart);
 
     // If we are in raw mode, return this identifier raw.  There is no need to
     // look up identifier information or attempt to macro expand it.
-    if (LexingRawMode) return;
-
-    // Fill in Result.IdentifierInfo, looking up the identifier in the
-    // identifier table.
-    IdentifierInfo *II = PP->LookUpIdentifierInfo(Result, IdStart);
+    if (LexingRawMode)
+      return;
 
-    // Change the kind of this identifier to the appropriate token kind, e.g.
-    // turning "for" into a keyword.
-    Result.setKind(II->getTokenID());
+    // Fill in Result.IdentifierInfo and update the token kind,
+    // looking up the identifier in the identifier table.
+    IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
 
     // Finally, now that we know we have an identifier, pass this off to the
     // preprocessor, which may macro expand it or something.
@@ -980,7 +1162,7 @@ void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
       if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
         PP->CodeCompleteNaturalLanguage();
       else if (!isLexingRawMode() && !Features.AsmPreprocessor)
-        Diag(BufferPtr, diag::err_unterminated_string);
+        Diag(BufferPtr, diag::warn_unterminated_string);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     }
@@ -1059,7 +1241,7 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
       if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
         PP->CodeCompleteNaturalLanguage();
       else if (!isLexingRawMode() && !Features.AsmPreprocessor)
-        Diag(BufferPtr, diag::err_unterminated_char);
+        Diag(BufferPtr, diag::warn_unterminated_char);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
     } else if (C == 0) {
@@ -2141,6 +2323,10 @@ LexNextToken:
         // If this is actually a '<<<<<<<' version control conflict marker,
         // recognize it as such and recover nicely.
         goto LexNextToken;
+      } else if (Features.CUDA && After == '<') {
+        Kind = tok::lesslessless;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
       } else {
         CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
         Kind = tok::lessless;
@@ -2172,6 +2358,10 @@ LexNextToken:
       } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
         // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
         goto LexNextToken;
+      } else if (Features.CUDA && After == '>') {
+        Kind = tok::greatergreatergreater;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
       } else {
         CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
         Kind = tok::greatergreater;
diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp
index fb543d0f03b3..16d7b36f02ef 100644
--- a/lib/Lex/LiteralSupport.cpp
+++ b/lib/Lex/LiteralSupport.cpp
@@ -33,8 +33,8 @@ static int HexDigitValue(char C) {
 /// either a character or a string literal.
 static unsigned ProcessCharEscape(const char *&ThisTokBuf,
                                   const char *ThisTokEnd, bool &HadError,
-                                  SourceLocation Loc, bool IsWide,
-                                  Preprocessor &PP, bool Complain) {
+                                  FullSourceLoc Loc, bool IsWide,
+                                  Diagnostic *Diags, const TargetInfo &Target) {
   // Skip the '\' char.
   ++ThisTokBuf;
 
@@ -54,13 +54,13 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     ResultChar = 8;
     break;
   case 'e':
-    if (Complain)
-      PP.Diag(Loc, diag::ext_nonstandard_escape) << "e";
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape) << "e";
     ResultChar = 27;
     break;
   case 'E':
-    if (Complain)
-      PP.Diag(Loc, diag::ext_nonstandard_escape) << "E";
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape) << "E";
     ResultChar = 27;
     break;
   case 'f':
@@ -81,8 +81,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
   case 'x': { // Hex escape.
     ResultChar = 0;
     if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
-      if (Complain)
-        PP.Diag(Loc, diag::err_hex_escape_no_digits);
+      if (Diags)
+        Diags->Report(Loc, diag::err_hex_escape_no_digits);
       HadError = 1;
       break;
     }
@@ -99,9 +99,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     }
 
     // See if any bits will be truncated when evaluated as a character.
-    unsigned CharWidth = IsWide
-                       ? PP.getTargetInfo().getWCharWidth()
-                       : PP.getTargetInfo().getCharWidth();
+    unsigned CharWidth =
+      IsWide ? Target.getWCharWidth() : Target.getCharWidth();
 
     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
       Overflow = true;
@@ -109,8 +108,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     }
 
     // Check for overflow.
-    if (Overflow && Complain)   // Too many digits to fit in
-      PP.Diag(Loc, diag::warn_hex_escape_too_large);
+    if (Overflow && Diags)   // Too many digits to fit in
+      Diags->Report(Loc, diag::warn_hex_escape_too_large);
     break;
   }
   case '0': case '1': case '2': case '3':
@@ -130,13 +129,12 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
              ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
 
     // Check for overflow.  Reject '\777', but not L'\777'.
-    unsigned CharWidth = IsWide
-                       ? PP.getTargetInfo().getWCharWidth()
-                       : PP.getTargetInfo().getCharWidth();
+    unsigned CharWidth =
+      IsWide ? Target.getWCharWidth() : Target.getCharWidth();
 
     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
-      if (Complain)
-        PP.Diag(Loc, diag::warn_octal_escape_too_large);
+      if (Diags)
+        Diags->Report(Loc, diag::warn_octal_escape_too_large);
       ResultChar &= ~0U >> (32-CharWidth);
     }
     break;
@@ -145,18 +143,20 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     // Otherwise, these are not valid escapes.
   case '(': case '{': case '[': case '%':
     // GCC accepts these as extensions.  We warn about them as such though.
-    if (Complain)
-      PP.Diag(Loc, diag::ext_nonstandard_escape)
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape)
         << std::string()+(char)ResultChar;
     break;
   default:
-    if (!Complain)
+    if (Diags == 0)
       break;
       
-    if (isgraph(ThisTokBuf[0]))
-      PP.Diag(Loc, diag::ext_unknown_escape) << std::string()+(char)ResultChar;
+    if (isgraph(ResultChar))
+      Diags->Report(Loc, diag::ext_unknown_escape)
+        << std::string()+(char)ResultChar;
     else
-      PP.Diag(Loc, diag::ext_unknown_escape) << "x"+llvm::utohexstr(ResultChar);
+      Diags->Report(Loc, diag::ext_unknown_escape)
+        << "x"+llvm::utohexstr(ResultChar);
     break;
   }
 
@@ -164,16 +164,13 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
 }
 
 /// ProcessUCNEscape - Read the Universal Character Name, check constraints and
-/// convert the UTF32 to UTF8. This is a subroutine of StringLiteralParser.
-/// When we decide to implement UCN's for character constants and identifiers,
-/// we will likely rework our support for UCN's.
-static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
-                             char *&ResultBuf, bool &HadError,
-                             SourceLocation Loc, Preprocessor &PP,
-                             bool wide,
-                             bool Complain) {
-  // FIXME: Add a warning - UCN's are only valid in C++ & C99.
-  // FIXME: Handle wide strings.
+/// return the UTF32.
+static bool ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
+                             uint32_t &UcnVal, unsigned short &UcnLen,
+                             FullSourceLoc Loc, Diagnostic *Diags, 
+                             const LangOptions &Features) {
+  if (!Features.CPlusPlus && !Features.C99 && Diags)
+    Diags->Report(Loc, diag::warn_ucn_not_valid_in_c89);
 
   // Save the beginning of the string (for error diagnostics).
   const char *ThisTokBegin = ThisTokBuf;
@@ -182,49 +179,87 @@ static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
   ThisTokBuf += 2;
 
   if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
-    if (Complain)
-      PP.Diag(Loc, diag::err_ucn_escape_no_digits);
-    HadError = 1;
-    return;
+    if (Diags)
+      Diags->Report(Loc, diag::err_ucn_escape_no_digits);
+    return false;
   }
-  typedef uint32_t UTF32;
-
-  UTF32 UcnVal = 0;
-  unsigned short UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
+  UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
   unsigned short UcnLenSave = UcnLen;
-  for (; ThisTokBuf != ThisTokEnd && UcnLen; ++ThisTokBuf, UcnLen--) {
+  for (; ThisTokBuf != ThisTokEnd && UcnLenSave; ++ThisTokBuf, UcnLenSave--) {
     int CharVal = HexDigitValue(ThisTokBuf[0]);
     if (CharVal == -1) break;
     UcnVal <<= 4;
     UcnVal |= CharVal;
   }
   // If we didn't consume the proper number of digits, there is a problem.
-  if (UcnLen) {
-    if (Complain)
-      PP.Diag(PP.AdvanceToTokenCharacter(Loc, ThisTokBuf-ThisTokBegin),
-              diag::err_ucn_escape_incomplete);
-    HadError = 1;
-    return;
+  if (UcnLenSave) {
+    if (Diags) {
+      SourceLocation L =
+        Lexer::AdvanceToTokenCharacter(Loc, ThisTokBuf-ThisTokBegin,
+                                       Loc.getManager(), Features);
+      Diags->Report(FullSourceLoc(L, Loc.getManager()),
+                    diag::err_ucn_escape_incomplete);
+    }
+    return false;
   }
   // Check UCN constraints (C99 6.4.3p2).
   if ((UcnVal < 0xa0 &&
       (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60 )) // $, @, `
       || (UcnVal >= 0xD800 && UcnVal <= 0xDFFF)
       || (UcnVal > 0x10FFFF)) /* the maximum legal UTF32 value */ {
-    if (Complain)
-      PP.Diag(Loc, diag::err_ucn_escape_invalid);
+    if (Diags)
+      Diags->Report(Loc, diag::err_ucn_escape_invalid);
+    return false;
+  }
+  return true;
+}
+
+/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
+/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
+/// StringLiteralParser. When we decide to implement UCN's for identifiers,
+/// we will likely rework our support for UCN's.
+static void EncodeUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
+                            char *&ResultBuf, bool &HadError,
+                            FullSourceLoc Loc, bool wide, Diagnostic *Diags, 
+                            const LangOptions &Features) {
+  typedef uint32_t UTF32;
+  UTF32 UcnVal = 0;
+  unsigned short UcnLen = 0;
+  if (!ProcessUCNEscape(ThisTokBuf, ThisTokEnd, UcnVal, UcnLen, Loc, Diags,
+                        Features)) {
     HadError = 1;
     return;
   }
+
   if (wide) {
-    (void)UcnLenSave;
-    assert(UcnLenSave == 4 && 
-           "ProcessUCNEscape - only ucn length of 4 supported");
-    // little endian assumed.
-    *ResultBuf++ = (UcnVal & 0x000000FF);
-    *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
-    *ResultBuf++ = (UcnVal & 0x00FF0000) >> 16;
-    *ResultBuf++ = (UcnVal & 0xFF000000) >> 24;
+    (void)UcnLen;
+    assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
+
+    if (!Features.ShortWChar) {
+      // Note: our internal rep of wide char tokens is always little-endian.
+      *ResultBuf++ = (UcnVal & 0x000000FF);
+      *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
+      *ResultBuf++ = (UcnVal & 0x00FF0000) >> 16;
+      *ResultBuf++ = (UcnVal & 0xFF000000) >> 24;
+      return;
+    }
+
+    // Convert to UTF16.
+    if (UcnVal < (UTF32)0xFFFF) {
+      *ResultBuf++ = (UcnVal & 0x000000FF);
+      *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
+      return;
+    }
+    if (Diags) Diags->Report(Loc, diag::warn_ucn_escape_too_large);
+
+    typedef uint16_t UTF16;
+    UcnVal -= 0x10000;
+    UTF16 surrogate1 = 0xD800 + (UcnVal >> 10);
+    UTF16 surrogate2 = 0xDC00 + (UcnVal & 0x3FF);
+    *ResultBuf++ = (surrogate1 & 0x000000FF);
+    *ResultBuf++ = (surrogate1 & 0x0000FF00) >> 8;
+    *ResultBuf++ = (surrogate2 & 0x000000FF);
+    *ResultBuf++ = (surrogate2 & 0x0000FF00) >> 8;
     return;
   }
   // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
@@ -398,6 +433,7 @@ NumericLiteralParser(const char *begin, const char *end,
       }
       continue;  // Success.
     case 'i':
+    case 'I':
       if (PP.getLangOptions().Microsoft) {
         if (isFPConstant || isLong || isLongLong) break;
 
@@ -410,22 +446,33 @@ NumericLiteralParser(const char *begin, const char *end,
               break;
             case '1':
               if (s + 2 == ThisTokEnd) break;
-              if (s[2] == '6') s += 3; // i16 suffix
+              if (s[2] == '6') {
+                s += 3; // i16 suffix
+                isMicrosoftInteger = true;
+              }
               else if (s[2] == '2') {
                 if (s + 3 == ThisTokEnd) break;
-                if (s[3] == '8') s += 4; // i128 suffix
+                if (s[3] == '8') {
+                  s += 4; // i128 suffix
+                  isMicrosoftInteger = true;
+                }
               }
-              isMicrosoftInteger = true;
               break;
             case '3':
               if (s + 2 == ThisTokEnd) break;
-              if (s[2] == '2') s += 3; // i32 suffix
-              isMicrosoftInteger = true;
+              if (s[2] == '2') {
+                s += 3; // i32 suffix
+                isLong = true;
+                isMicrosoftInteger = true;
+              }
               break;
             case '6':
               if (s + 2 == ThisTokEnd) break;
-              if (s[2] == '4') s += 3; // i64 suffix
-              isMicrosoftInteger = true;
+              if (s[2] == '4') {
+                s += 3; // i64 suffix
+                isLongLong = true;
+                isMicrosoftInteger = true;
+              }
               break;
             default:
               break;
@@ -434,7 +481,6 @@ NumericLiteralParser(const char *begin, const char *end,
         }
       }
       // fall through.
-    case 'I':
     case 'j':
     case 'J':
       if (isImaginary) break;   // Cannot be repeated.
@@ -681,11 +727,29 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
   bool Warned = false;
   while (begin[0] != '\'') {
     uint64_t ResultChar;
+
+      // Is this a Universal Character Name escape?
     if (begin[0] != '\\')     // If this is a normal character, consume it.
       ResultChar = *begin++;
-    else                      // Otherwise, this is an escape character.
-      ResultChar = ProcessCharEscape(begin, end, HadError, Loc, IsWide, PP,
-                                     /*Complain=*/true);
+    else {                    // Otherwise, this is an escape character.
+      // Check for UCN.
+      if (begin[1] == 'u' || begin[1] == 'U') {
+        uint32_t utf32 = 0;
+        unsigned short UcnLen = 0;
+        if (!ProcessUCNEscape(begin, end, utf32, UcnLen,
+                              FullSourceLoc(Loc, PP.getSourceManager()),
+                              &PP.getDiagnostics(), PP.getLangOptions())) {
+          HadError = 1;
+        }
+        ResultChar = utf32;
+      } else {
+        // Otherwise, this is a non-UCN escape character.  Process it.
+        ResultChar = ProcessCharEscape(begin, end, HadError,
+                                       FullSourceLoc(Loc,PP.getSourceManager()),
+                                       IsWide,
+                                       &PP.getDiagnostics(), PP.getTargetInfo());
+      }
+    }
 
     // If this is a multi-character constant (e.g. 'abc'), handle it.  These are
     // implementation defined (C99 6.4.4.4p10).
@@ -725,6 +789,9 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
   // Transfer the value from APInt to uint64_t
   Value = LitVal.getZExtValue();
 
+  if (IsWide && PP.getLangOptions().ShortWChar && Value > 0xFFFF)
+    PP.Diag(Loc, diag::warn_ucn_escape_too_large);
+
   // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
   // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
   // character constants are not sign extended in the this implementation:
@@ -771,7 +838,13 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
 ///
 StringLiteralParser::
 StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
-                    Preprocessor &pp, bool Complain) : PP(pp) {
+                    Preprocessor &PP, bool Complain)
+  : SM(PP.getSourceManager()), Features(PP.getLangOptions()),
+    Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() : 0) {
+  init(StringToks, NumStringToks);
+}
+
+void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
   // Scan all of the string portions, remember the max individual token length,
   // computing a bound on the concatenated string length, and see whether any
   // piece is a wide-string.  If any of the string portions is a wide-string
@@ -806,7 +879,7 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
   // query the target.  As such, wchar_tByteWidth is only valid if AnyWide=true.
   wchar_tByteWidth = ~0U;
   if (AnyWide) {
-    wchar_tByteWidth = PP.getTargetInfo().getWCharWidth();
+    wchar_tByteWidth = Target.getWCharWidth();
     assert((wchar_tByteWidth & 7) == 0 && "Assumes wchar_t is byte multiple!");
     wchar_tByteWidth /= 8;
   }
@@ -835,8 +908,9 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
     // that ThisTokBuf points to a buffer that is big enough for the whole token
     // and 'spelled' tokens can only shrink.
     bool StringInvalid = false;
-    unsigned ThisTokLen = PP.getSpelling(StringToks[i], ThisTokBuf, 
-                                         &StringInvalid);
+    unsigned ThisTokLen = 
+      Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
+                         &StringInvalid);
     if (StringInvalid) {
       hadError = 1;
       continue;
@@ -856,7 +930,7 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
     ++ThisTokBuf;
 
     // Check if this is a pascal string
-    if (pp.getLangOptions().PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
+    if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
         ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
 
       // If the \p sequence is found in the first token, we have a pascal string
@@ -894,15 +968,16 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
       }
       // Is this a Universal Character Name escape?
       if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
-        ProcessUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
-                         hadError, StringToks[i].getLocation(), PP, wide, 
-                         Complain);
+        EncodeUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
+                        hadError, FullSourceLoc(StringToks[i].getLocation(),SM),
+                        wide, Diags, Features);
         continue;
       }
       // Otherwise, this is a non-UCN escape character.  Process it.
-      unsigned ResultChar = ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
-                                              StringToks[i].getLocation(),
-                                              AnyWide, PP, Complain);
+      unsigned ResultChar =
+        ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
+                          FullSourceLoc(StringToks[i].getLocation(), SM),
+                          AnyWide, Diags, Target);
 
       // Note: our internal rep of wide char tokens is always little-endian.
       *ResultPtr++ = ResultChar & 0xFF;
@@ -920,25 +995,24 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
       ResultBuf[0] /= wchar_tByteWidth;
 
     // Verify that pascal strings aren't too large.
-    if (GetStringLength() > 256 && Complain) {
-      PP.Diag(StringToks[0].getLocation(), diag::err_pascal_string_too_long)
-        << SourceRange(StringToks[0].getLocation(),
-                       StringToks[NumStringToks-1].getLocation());
+    if (GetStringLength() > 256) {
+      if (Diags) 
+        Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+                      diag::err_pascal_string_too_long)
+          << SourceRange(StringToks[0].getLocation(),
+                         StringToks[NumStringToks-1].getLocation());
       hadError = 1;
       return;
     }
-  } else if (Complain) {
+  } else if (Diags) {
     // Complain if this string literal has too many characters.
-    unsigned MaxChars = PP.getLangOptions().CPlusPlus? 65536
-                      : PP.getLangOptions().C99 ? 4095
-                      : 509;
+    unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
     
     if (GetNumStringChars() > MaxChars)
-      PP.Diag(StringToks[0].getLocation(), diag::ext_string_too_long)
+      Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+                    diag::ext_string_too_long)
         << GetNumStringChars() << MaxChars
-        << (PP.getLangOptions().CPlusPlus? 2
-            : PP.getLangOptions().C99 ? 1
-            : 0)
+        << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
         << SourceRange(StringToks[0].getLocation(),
                        StringToks[NumStringToks-1].getLocation());
   }
@@ -949,19 +1023,17 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
 /// specified byte of the string data represented by Token.  This handles
 /// advancing over escape sequences in the string.
 unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
-                                                    unsigned ByteNo,
-                                                    Preprocessor &PP, 
-                                                    bool Complain) {
+                                                    unsigned ByteNo) const {
   // Get the spelling of the token.
-  llvm::SmallString<16> SpellingBuffer;
+  llvm::SmallString<32> SpellingBuffer;
   SpellingBuffer.resize(Tok.getLength());
 
   bool StringInvalid = false;
   const char *SpellingPtr = &SpellingBuffer[0];
-  unsigned TokLen = PP.getSpelling(Tok, SpellingPtr, &StringInvalid);
-  if (StringInvalid) {
+  unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
+                                       &StringInvalid);
+  if (StringInvalid)
     return 0;
-  }
 
   assert(SpellingPtr[0] != 'L' && "Doesn't handle wide strings yet");
 
@@ -987,7 +1059,8 @@ unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
     // Otherwise, this is an escape character.  Advance over it.
     bool HadError = false;
     ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
-                      Tok.getLocation(), false, PP, Complain);
+                      FullSourceLoc(Tok.getLocation(), SM),
+                      false, Diags, Target);
     assert(!HadError && "This method isn't valid on erroneous strings");
     --ByteNo;
   }
diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp
index c6d09349b5ef..c819011338e8 100644
--- a/lib/Lex/MacroInfo.cpp
+++ b/lib/Lex/MacroInfo.cpp
@@ -22,8 +22,9 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
   IsBuiltinMacro = false;
   IsFromAST = false;
   IsDisabled = false;
-  IsUsed = true;
+  IsUsed = false;
   IsAllowRedefinitionsWithoutWarning = false;
+  IsWarnIfUnused = false;
 
   ArgumentList = 0;
   NumArguments = 0;
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 8da7def9ed4d..0f0d25b887af 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -17,6 +17,7 @@
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/Pragma.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/APInt.h"
@@ -27,14 +28,23 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 MacroInfo *Preprocessor::AllocateMacroInfo() {
-  MacroInfo *MI;
+  MacroInfoChain *MIChain;
 
-  if (!MICache.empty()) {
-    MI = MICache.back();
-    MICache.pop_back();
-  } else
-    MI = (MacroInfo*) BP.Allocate<MacroInfo>();
-  return MI;
+  if (MICache) {
+    MIChain = MICache;
+    MICache = MICache->Next;
+  }
+  else {
+    MIChain = BP.Allocate<MacroInfoChain>();
+  }
+
+  MIChain->Next = MIChainHead;
+  MIChain->Prev = 0;
+  if (MIChainHead)
+    MIChainHead->Prev = MIChain;
+  MIChainHead = MIChain;
+
+  return &(MIChain->MI);
 }
 
 MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
@@ -52,10 +62,23 @@ MacroInfo *Preprocessor::CloneMacroInfo(const MacroInfo &MacroToClone) {
 /// ReleaseMacroInfo - Release the specified MacroInfo.  This memory will
 ///  be reused for allocating new MacroInfo objects.
 void Preprocessor::ReleaseMacroInfo(MacroInfo *MI) {
-  MICache.push_back(MI);
-  MI->FreeArgumentList();
-}
+  MacroInfoChain *MIChain = (MacroInfoChain*) MI;
+  if (MacroInfoChain *Prev = MIChain->Prev) {
+    MacroInfoChain *Next = MIChain->Next;
+    Prev->Next = Next;
+    if (Next)
+      Next->Prev = Prev;
+  }
+  else {
+    assert(MIChainHead == MIChain);
+    MIChainHead = MIChain->Next;
+    MIChainHead->Prev = 0;
+  }
+  MIChain->Next = MICache;
+  MICache = MIChain;
 
+  MI->Destroy();
+}
 
 /// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
 /// current line until the tok::eom token is found.
@@ -222,7 +245,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
 
     // If this isn't an identifier directive (e.g. is "# 1\n" or "#\n", or
     // something bogus), skip it.
-    if (Tok.isNot(tok::identifier)) {
+    if (Tok.isNot(tok::raw_identifier)) {
       CurPPLexer->ParsingPreprocessorDirective = false;
       // Restore comment saving mode.
       if (CurLexer) CurLexer->SetCommentRetentionState(KeepComments);
@@ -234,12 +257,8 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
     // to spell an i/e in a strange way that is another letter.  Skipping this
     // allows us to avoid looking up the identifier info for #define/#undef and
     // other common directives.
-    bool Invalid = false;
-    const char *RawCharData = SourceMgr.getCharacterData(Tok.getLocation(),
-                                                         &Invalid);
-    if (Invalid)
-      return;
-    
+    const char *RawCharData = Tok.getRawIdentifierData();
+
     char FirstChar = RawCharData[0];
     if (FirstChar >= 'a' && FirstChar <= 'z' &&
         FirstChar != 'i' && FirstChar != 'e') {
@@ -279,7 +298,10 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
         DiscardUntilEndOfDirective();
         CurPPLexer->pushConditionalLevel(Tok.getLocation(), /*wasskipping*/true,
                                        /*foundnonskip*/false,
-                                       /*fnddelse*/false);
+                                       /*foundelse*/false);
+
+        if (Callbacks)
+          Callbacks->Endif();
       }
     } else if (Directive[0] == 'e') {
       llvm::StringRef Sub = Directive.substr(1);
@@ -288,7 +310,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
         PPConditionalInfo CondInfo;
         CondInfo.WasSkipping = true; // Silence bogus warning.
         bool InCond = CurPPLexer->popConditionalLevel(CondInfo);
-        InCond = InCond;  // Silence warning in no-asserts mode.
+        (void)InCond;  // Silence warning in no-asserts mode.
         assert(!InCond && "Can't be skipping if not in a conditional!");
 
         // If we popped the outermost skipping block, we're done skipping!
@@ -307,6 +329,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
         // Note that we've seen a #else in this conditional.
         CondInfo.FoundElse = true;
 
+        if (Callbacks)
+          Callbacks->Else();
+
         // If the conditional is at the top level, and the #if block wasn't
         // entered, enter the #else block now.
         if (!CondInfo.WasSkipping && !CondInfo.FoundNonSkip) {
@@ -317,6 +342,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
 
         bool ShouldEnter;
+        const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
         // If this is in a skipping block or if we're already handled this #if
         // block, don't bother parsing the condition.
         if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
@@ -331,10 +357,14 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
           ShouldEnter = EvaluateDirectiveExpression(IfNDefMacro);
           CurPPLexer->LexingRawMode = true;
         }
+        const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
 
         // If this is a #elif with a #else before it, report the error.
         if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_elif_after_else);
 
+        if (Callbacks)
+          Callbacks->Elif(SourceRange(ConditionalBegin, ConditionalEnd));
+
         // If this condition is true, enter it!
         if (ShouldEnter) {
           CondInfo.FoundNonSkip = true;
@@ -366,7 +396,7 @@ void Preprocessor::PTHSkipExcludedConditionalBlock() {
       // have been consumed by the PTHLexer.  Just pop off the condition level.
       PPConditionalInfo CondInfo;
       bool InCond = CurPTHLexer->popConditionalLevel(CondInfo);
-      InCond = InCond;  // Silence warning in no-asserts mode.
+      (void)InCond;  // Silence warning in no-asserts mode.
       assert(!InCond && "Can't be skipping if not in a conditional!");
       break;
     }
@@ -568,9 +598,11 @@ TryAgain:
 
     // C99 6.10.2 - Source File Inclusion.
     case tok::pp_include:
-      return HandleIncludeDirective(Result);       // Handle #include.
+      // Handle #include.
+      return HandleIncludeDirective(SavedHash.getLocation(), Result);
     case tok::pp___include_macros:
-      return HandleIncludeMacrosDirective(Result); // Handle -imacros.
+      // Handle -imacros.
+      return HandleIncludeMacrosDirective(SavedHash.getLocation(), Result); 
 
     // C99 6.10.3 - Macro Replacement.
     case tok::pp_define:
@@ -588,13 +620,13 @@ TryAgain:
 
     // C99 6.10.6 - Pragma Directive.
     case tok::pp_pragma:
-      return HandlePragmaDirective();
+      return HandlePragmaDirective(PIK_HashPragma);
 
     // GNU Extensions.
     case tok::pp_import:
-      return HandleImportDirective(Result);
+      return HandleImportDirective(SavedHash.getLocation(), Result);
     case tok::pp_include_next:
-      return HandleIncludeNextDirective(Result);
+      return HandleIncludeNextDirective(SavedHash.getLocation(), Result);
 
     case tok::pp_warning:
       Diag(Result, diag::ext_pp_warning_directive);
@@ -622,6 +654,12 @@ TryAgain:
     // Return the # and the token after it.
     Toks[0] = SavedHash;
     Toks[1] = Result;
+    
+    // If the second token is a hashhash token, then we need to translate it to
+    // unknown so the token lexer doesn't try to perform token pasting.
+    if (Result.is(tok::hashhash))
+      Toks[1].setKind(tok::unknown);
+    
     // Enter this token stream so that we re-lex the tokens.  Make sure to
     // enable macro expansion, in case the token after the # is an identifier
     // that is expanded.
@@ -779,7 +817,9 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
     FileID CurFileID =
       SM.getDecomposedInstantiationLoc(FlagTok.getLocation()).first;
     PresumedLoc PLoc = SM.getPresumedLoc(FlagTok.getLocation());
-
+    if (PLoc.isInvalid())
+      return true;
+    
     // If there is no include loc (main file) or if the include loc is in a
     // different physical file, then we aren't in a "1" line marker flag region.
     SourceLocation IncLoc = PLoc.getIncludeLoc();
@@ -1011,11 +1051,20 @@ bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
 /// false if the > was found, otherwise it returns true if it finds and consumes
 /// the EOM marker.
 bool Preprocessor::ConcatenateIncludeName(
-  llvm::SmallString<128> &FilenameBuffer) {
+                                        llvm::SmallString<128> &FilenameBuffer,
+                                          SourceLocation &End) {
   Token CurTok;
 
   Lex(CurTok);
   while (CurTok.isNot(tok::eom)) {
+    End = CurTok.getLocation();
+    
+    // FIXME: Provide code completion for #includes.
+    if (CurTok.is(tok::code_completion)) {
+      Lex(CurTok);
+      continue;
+    }
+
     // Append the spelling of this token to the buffer. If there was a space
     // before it, add it now.
     if (CurTok.hasLeadingSpace())
@@ -1054,7 +1103,8 @@ bool Preprocessor::ConcatenateIncludeName(
 /// routine with functionality shared between #include, #include_next and
 /// #import.  LookupFrom is set when this is a #include_next directive, it
 /// specifies the file to start searching from.
-void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
+void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, 
+                                          Token &IncludeTok,
                                           const DirectoryLookup *LookupFrom,
                                           bool isImport) {
 
@@ -1064,7 +1114,8 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
   // Reserve a buffer to get the spelling.
   llvm::SmallString<128> FilenameBuffer;
   llvm::StringRef Filename;
-
+  SourceLocation End;
+  
   switch (FilenameTok.getKind()) {
   case tok::eom:
     // If the token kind is EOM, the error has already been diagnosed.
@@ -1073,13 +1124,14 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
   case tok::angle_string_literal:
   case tok::string_literal:
     Filename = getSpelling(FilenameTok, FilenameBuffer);
+    End = FilenameTok.getLocation();
     break;
 
   case tok::less:
     // This could be a <foo/bar.h> file coming from a macro expansion.  In this
     // case, glue the tokens together into FilenameBuffer and interpret those.
     FilenameBuffer.push_back('<');
-    if (ConcatenateIncludeName(FilenameBuffer))
+    if (ConcatenateIncludeName(FilenameBuffer, End))
       return;   // Found <eom> but no ">"?  Diagnostic already emitted.
     Filename = FilenameBuffer.str();
     break;
@@ -1118,6 +1170,11 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
     return;
   }
 
+  // Notify the callback object that we've seen an inclusion directive.
+  if (Callbacks)
+    Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled, File, 
+                                  End);
+  
   // The #included file will be considered to be a system header if either it is
   // in a system include directory, or if the #includer is a system include
   // header.
@@ -1147,7 +1204,8 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
 
 /// HandleIncludeNextDirective - Implements #include_next.
 ///
-void Preprocessor::HandleIncludeNextDirective(Token &IncludeNextTok) {
+void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
+                                              Token &IncludeNextTok) {
   Diag(IncludeNextTok, diag::ext_pp_include_next_directive);
 
   // #include_next is like #include, except that we start searching after
@@ -1164,23 +1222,25 @@ void Preprocessor::HandleIncludeNextDirective(Token &IncludeNextTok) {
     ++Lookup;
   }
 
-  return HandleIncludeDirective(IncludeNextTok, Lookup);
+  return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup);
 }
 
 /// HandleImportDirective - Implements #import.
 ///
-void Preprocessor::HandleImportDirective(Token &ImportTok) {
+void Preprocessor::HandleImportDirective(SourceLocation HashLoc,
+                                         Token &ImportTok) {
   if (!Features.ObjC1)  // #import is standard for ObjC.
     Diag(ImportTok, diag::ext_pp_import_directive);
 
-  return HandleIncludeDirective(ImportTok, 0, true);
+  return HandleIncludeDirective(HashLoc, ImportTok, 0, true);
 }
 
 /// HandleIncludeMacrosDirective - The -imacros command line option turns into a
 /// pseudo directive in the predefines buffer.  This handles it by sucking all
 /// tokens through the preprocessor and discarding them (only keeping the side
 /// effects on the preprocessor).
-void Preprocessor::HandleIncludeMacrosDirective(Token &IncludeMacrosTok) {
+void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
+                                                Token &IncludeMacrosTok) {
   // This directive should only occur in the predefines buffer.  If not, emit an
   // error and reject it.
   SourceLocation Loc = IncludeMacrosTok.getLocation();
@@ -1193,7 +1253,7 @@ void Preprocessor::HandleIncludeMacrosDirective(Token &IncludeMacrosTok) {
 
   // Treat this as a normal #include for checking purposes.  If this is
   // successful, it will push a new lexer onto the include stack.
-  HandleIncludeDirective(IncludeMacrosTok, 0, false);
+  HandleIncludeDirective(HashLoc, IncludeMacrosTok, 0, false);
 
   Token TmpTok;
   do {
@@ -1457,11 +1517,6 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
     }
   }
 
-  // If this is the primary source file, remember that this macro hasn't been
-  // used yet.
-  if (isInPrimaryFile())
-    MI->setIsUsed(false);
-
   MI->setDefinitionEndLoc(LastTok.getLocation());
 
   // Finally, if this identifier already had a macro defined for it, verify that
@@ -1472,7 +1527,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
     // then don't bother calling MacroInfo::isIdenticalTo.
     if (!getDiagnostics().getSuppressSystemWarnings() ||
         !SourceMgr.isInSystemHeader(DefineTok.getLocation())) {
-      if (!OtherMI->isUsed())
+      if (!OtherMI->isUsed() && OtherMI->isWarnIfUnused())
         Diag(OtherMI->getDefinitionLoc(), diag::pp_macro_not_used);
 
       // Macros must be identical.  This means all tokens and whitespace
@@ -1484,14 +1539,26 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
         Diag(OtherMI->getDefinitionLoc(), diag::note_previous_definition);
       }
     }
+    if (OtherMI->isWarnIfUnused())
+      WarnUnusedMacroLocs.erase(OtherMI->getDefinitionLoc());
     ReleaseMacroInfo(OtherMI);
   }
 
   setMacroInfo(MacroNameTok.getIdentifierInfo(), MI);
 
+  assert(!MI->isUsed());
+  // If we need warning for not using the macro, add its location in the
+  // warn-because-unused-macro set. If it gets used it will be removed from set.
+  if (isInPrimaryFile() && // don't warn for include'd macros.
+      Diags->getDiagnosticLevel(diag::pp_macro_not_used,
+                               MI->getDefinitionLoc()) != Diagnostic::Ignored) {
+    MI->setIsWarnIfUnused(true);
+    WarnUnusedMacroLocs.insert(MI->getDefinitionLoc());
+  }
+
   // If the callbacks want to know, tell them about the macro definition.
   if (Callbacks)
-    Callbacks->MacroDefined(MacroNameTok.getIdentifierInfo(), MI);
+    Callbacks->MacroDefined(MacroNameTok, MI);
 }
 
 /// HandleUndefDirective - Implements #undef.
@@ -1520,8 +1587,10 @@ void Preprocessor::HandleUndefDirective(Token &UndefTok) {
 
   // If the callbacks want to know, tell them about the macro #undef.
   if (Callbacks)
-    Callbacks->MacroUndefined(MacroNameTok.getLocation(),
-                              MacroNameTok.getIdentifierInfo(), MI);
+    Callbacks->MacroUndefined(MacroNameTok, MI);
+
+  if (MI->isWarnIfUnused())
+    WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
 
   // Free macro definition.
   ReleaseMacroInfo(MI);
@@ -1575,7 +1644,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef,
 
   // If there is a macro, process it.
   if (MI)  // Mark it used.
-    MI->setIsUsed(true);
+    markMacroAsUsed(MI);
 
   // Should we include the stuff contained by this directive?
   if (!MI == isIfndef) {
@@ -1584,11 +1653,18 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef,
                                      /*wasskip*/false, /*foundnonskip*/true,
                                      /*foundelse*/false);
   } else {
-    // No, skip the contents of this block and return the first token after it.
+    // No, skip the contents of this block.
     SkipExcludedConditionalBlock(DirectiveTok.getLocation(),
                                  /*Foundnonskip*/false,
                                  /*FoundElse*/false);
   }
+
+  if (Callbacks) {
+    if (isIfndef)
+      Callbacks->Ifndef(MacroNameTok);
+    else
+      Callbacks->Ifdef(MacroNameTok);
+  }
 }
 
 /// HandleIfDirective - Implements the #if directive.
@@ -1597,10 +1673,11 @@ void Preprocessor::HandleIfDirective(Token &IfToken,
                                      bool ReadAnyTokensBeforeDirective) {
   ++NumIf;
 
-  // Parse and evaluation the conditional expression.
+  // Parse and evaluate the conditional expression.
   IdentifierInfo *IfNDefMacro = 0;
-  bool ConditionalTrue = EvaluateDirectiveExpression(IfNDefMacro);
-
+  const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
+  const bool ConditionalTrue = EvaluateDirectiveExpression(IfNDefMacro);
+  const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
 
   // If this condition is equivalent to #ifndef X, and if this is the first
   // directive seen, handle it for the multiple-include optimization.
@@ -1617,10 +1694,13 @@ void Preprocessor::HandleIfDirective(Token &IfToken,
     CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false,
                                    /*foundnonskip*/true, /*foundelse*/false);
   } else {
-    // No, skip the contents of this block and return the first token after it.
+    // No, skip the contents of this block.
     SkipExcludedConditionalBlock(IfToken.getLocation(), /*Foundnonskip*/false,
                                  /*FoundElse*/false);
   }
+
+  if (Callbacks)
+    Callbacks->If(SourceRange(ConditionalBegin, ConditionalEnd));
 }
 
 /// HandleEndifDirective - Implements the #endif directive.
@@ -1644,9 +1724,13 @@ void Preprocessor::HandleEndifDirective(Token &EndifToken) {
 
   assert(!CondInfo.WasSkipping && !CurPPLexer->LexingRawMode &&
          "This code should only be reachable in the non-skipping case!");
-}
 
+  if (Callbacks)
+    Callbacks->Endif();
+}
 
+/// HandleElseDirective - Implements the #else directive.
+///
 void Preprocessor::HandleElseDirective(Token &Result) {
   ++NumElse;
 
@@ -1666,19 +1750,25 @@ void Preprocessor::HandleElseDirective(Token &Result) {
   // If this is a #else with a #else before it, report the error.
   if (CI.FoundElse) Diag(Result, diag::pp_err_else_after_else);
 
-  // Finally, skip the rest of the contents of this block and return the first
-  // token after it.
-  return SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
-                                      /*FoundElse*/true);
+  // Finally, skip the rest of the contents of this block.
+  SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
+                               /*FoundElse*/true);
+
+  if (Callbacks)
+    Callbacks->Else();
 }
 
+/// HandleElifDirective - Implements the #elif directive.
+///
 void Preprocessor::HandleElifDirective(Token &ElifToken) {
   ++NumElse;
 
   // #elif directive in a non-skipping conditional... start skipping.
   // We don't care what the condition is, because we will always skip it (since
   // the block immediately before it was included).
+  const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
   DiscardUntilEndOfDirective();
+  const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
 
   PPConditionalInfo CI;
   if (CurPPLexer->popConditionalLevel(CI)) {
@@ -1693,8 +1783,10 @@ void Preprocessor::HandleElifDirective(Token &ElifToken) {
   // If this is a #elif with a #else before it, report the error.
   if (CI.FoundElse) Diag(ElifToken, diag::pp_err_elif_after_else);
 
-  // Finally, skip the rest of the contents of this block and return the first
-  // token after it.
-  return SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
-                                      /*FoundElse*/CI.FoundElse);
+  // Finally, skip the rest of the contents of this block.
+  SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
+                               /*FoundElse*/CI.FoundElse);
+
+  if (Callbacks)
+    Callbacks->Elif(SourceRange(ConditionalBegin, ConditionalEnd));
 }
diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp
index 163e869400aa..1451c5a1ef5f 100644
--- a/lib/Lex/PPExpressions.cpp
+++ b/lib/Lex/PPExpressions.cpp
@@ -112,7 +112,7 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   // If there is a macro, mark it used.
   if (Result.Val != 0 && ValueLive) {
     MacroInfo *Macro = PP.getMacroInfo(II);
-    Macro->setIsUsed(true);
+    PP.markMacroAsUsed(Macro);
   }
 
   // Consume identifier.
@@ -519,7 +519,6 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
       RHS.Val.setIsUnsigned(Res.isUnsigned());
     }
 
-    // FIXME: All of these should detect and report overflow??
     bool Overflow = false;
     switch (Operator) {
     default: assert(0 && "Unknown operator token!");
@@ -534,9 +533,10 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
       break;
     case tok::slash:
       if (RHS.Val != 0) {
-        Res = LHS.Val / RHS.Val;
-        if (LHS.Val.isSigned())   // MININT/-1  -->  overflow.
-          Overflow = LHS.Val.isMinSignedValue() && RHS.Val.isAllOnesValue();
+        if (LHS.Val.isSigned())
+          Res = llvm::APSInt(LHS.Val.sdiv_ov(RHS.Val, Overflow), false);
+        else
+          Res = LHS.Val / RHS.Val;
       } else if (ValueLive) {
         PP.Diag(OpLoc, diag::err_pp_division_by_zero)
           << LHS.getRange() << RHS.getRange();
@@ -545,23 +545,22 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
       break;
 
     case tok::star:
-      Res = LHS.Val * RHS.Val;
-      if (Res.isSigned() && LHS.Val != 0 && RHS.Val != 0)
-        Overflow = Res/RHS.Val != LHS.Val || Res/LHS.Val != RHS.Val;
+      if (Res.isSigned())
+        Res = llvm::APSInt(LHS.Val.smul_ov(RHS.Val, Overflow), false);
+      else
+        Res = LHS.Val * RHS.Val;
       break;
     case tok::lessless: {
       // Determine whether overflow is about to happen.
       unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue());
-      if (ShAmt >= LHS.Val.getBitWidth())
-        Overflow = true, ShAmt = LHS.Val.getBitWidth()-1;
-      else if (LHS.isUnsigned())
-        Overflow = false;
-      else if (LHS.Val.isNonNegative()) // Don't allow sign change.
-        Overflow = ShAmt >= LHS.Val.countLeadingZeros();
-      else
-        Overflow = ShAmt >= LHS.Val.countLeadingOnes();
-
-      Res = LHS.Val << ShAmt;
+      if (LHS.isUnsigned()) {
+        Overflow = ShAmt >= LHS.Val.getBitWidth();
+        if (Overflow)
+          ShAmt = LHS.Val.getBitWidth()-1;
+        Res = LHS.Val << ShAmt;
+      } else {
+        Res = llvm::APSInt(LHS.Val.sshl_ov(ShAmt, Overflow), false);
+      }
       break;
     }
     case tok::greatergreater: {
@@ -573,20 +572,16 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
       break;
     }
     case tok::plus:
-      Res = LHS.Val + RHS.Val;
       if (LHS.isUnsigned())
-        Overflow = false;
-      else if (LHS.Val.isNonNegative() == RHS.Val.isNonNegative() &&
-               Res.isNonNegative() != LHS.Val.isNonNegative())
-        Overflow = true;  // Overflow for signed addition.
+        Res = LHS.Val + RHS.Val;
+      else
+        Res = llvm::APSInt(LHS.Val.sadd_ov(RHS.Val, Overflow), false);
       break;
     case tok::minus:
-      Res = LHS.Val - RHS.Val;
       if (LHS.isUnsigned())
-        Overflow = false;
-      else if (LHS.Val.isNonNegative() != RHS.Val.isNonNegative() &&
-               Res.isNonNegative() != LHS.Val.isNonNegative())
-        Overflow = true;  // Overflow for signed subtraction.
+        Res = LHS.Val - RHS.Val;
+      else
+        Res = llvm::APSInt(LHS.Val.ssub_ov(RHS.Val, Overflow), false);
       break;
     case tok::lessequal:
       Res = LHS.Val <= RHS.Val;
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 4a4040599263..eef42b69d87f 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -250,15 +250,11 @@ bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
 
   CurPPLexer = 0;
 
-  // This is the end of the top-level file.  If the diag::pp_macro_not_used
-  // diagnostic is enabled, look for macros that have not been used.
-  if (getDiagnostics().getDiagnosticLevel(diag::pp_macro_not_used) !=
-        Diagnostic::Ignored) {
-    for (macro_iterator I = macro_begin(false), E = macro_end(false); 
-         I != E; ++I)
-      if (!I->second->isUsed())
-        Diag(I->second->getDefinitionLoc(), diag::pp_macro_not_used);
-  }
+  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has collected
+  // all macro locations that we need to warn because they are not used.
+  for (WarnUnusedMacroLocsTy::iterator
+         I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end(); I!=E; ++I)
+    Diag(*I, diag::pp_macro_not_used);
 
   return true;
 }
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index 9015c278fc92..ba9261491030 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -20,12 +20,28 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ExternalPreprocessorSource.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
 #include <ctime>
 using namespace clang;
 
+MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
+  assert(II->hasMacroDefinition() && "Identifier is not a macro!");
+  
+  llvm::DenseMap<IdentifierInfo*, MacroInfo*>::const_iterator Pos
+    = Macros.find(II);
+  if (Pos == Macros.end()) {
+    // Load this macro from the external source.
+    getExternalSource()->LoadMacroDefinition(II);
+    Pos = Macros.find(II);
+  }
+  assert(Pos != Macros.end() && "Identifier macro info is missing!");
+  return Pos->second;
+}
+
 /// setMacroInfo - Specify a macro for this identifier.
 ///
 void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI) {
@@ -70,6 +86,7 @@ void Preprocessor::RegisterBuiltinMacros() {
   // Clang Extensions.
   Ident__has_feature      = RegisterBuiltinMacro(*this, "__has_feature");
   Ident__has_builtin      = RegisterBuiltinMacro(*this, "__has_builtin");
+  Ident__has_attribute    = RegisterBuiltinMacro(*this, "__has_attribute");
   Ident__has_include      = RegisterBuiltinMacro(*this, "__has_include");
   Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");
 
@@ -206,7 +223,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   }
 
   // Notice that this macro has been used.
-  MI->setIsUsed(true);
+  markMacroAsUsed(MI);
 
   // If we started lexing a macro, enter the macro expansion body.
 
@@ -231,6 +248,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
       if (IsAtStartOfLine) Identifier.setFlag(Token::StartOfLine);
       if (HadLeadingSpace) Identifier.setFlag(Token::LeadingSpace);
     }
+    Identifier.setFlag(Token::LeadingEmptyMacro);
     ++NumFastMacroExpanded;
     return false;
 
@@ -481,16 +499,25 @@ static void ComputeDATE_TIME(SourceLocation &DATELoc, SourceLocation &TIMELoc,
     "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"
   };
 
-  char TmpBuffer[100];
+  char TmpBuffer[32];
+#ifdef LLVM_ON_WIN32
   sprintf(TmpBuffer, "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
           TM->tm_year+1900);
+#else
+  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
+          TM->tm_year+1900);
+#endif
 
   Token TmpTok;
   TmpTok.startToken();
   PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
   DATELoc = TmpTok.getLocation();
 
+#ifdef LLVM_ON_WIN32
   sprintf(TmpBuffer, "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
+#else
+  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
+#endif
   PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
   TIMELoc = TmpTok.getLocation();
 }
@@ -505,40 +532,77 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            .Case("attribute_analyzer_noreturn", true)
            .Case("attribute_cf_returns_not_retained", true)
            .Case("attribute_cf_returns_retained", true)
+           .Case("attribute_deprecated_with_message", true)
            .Case("attribute_ext_vector_type", true)
            .Case("attribute_ns_returns_not_retained", true)
            .Case("attribute_ns_returns_retained", true)
+           .Case("attribute_ns_consumes_self", true)
+           .Case("attribute_ns_consumed", true)
+           .Case("attribute_cf_consumed", true)
            .Case("attribute_objc_ivar_unused", true)
            .Case("attribute_overloadable", true)
+           .Case("attribute_unavailable_with_message", true)
            .Case("blocks", LangOpts.Blocks)
-           .Case("cxx_attributes", LangOpts.CPlusPlus0x)
-           .Case("cxx_auto_type", LangOpts.CPlusPlus0x)
-           .Case("cxx_decltype", LangOpts.CPlusPlus0x)
-           .Case("cxx_deleted_functions", LangOpts.CPlusPlus0x)
            .Case("cxx_exceptions", LangOpts.Exceptions)
            .Case("cxx_rtti", LangOpts.RTTI)
-           .Case("cxx_static_assert", LangOpts.CPlusPlus0x)
+           .Case("enumerator_attributes", true)
            .Case("objc_nonfragile_abi", LangOpts.ObjCNonFragileABI)
            .Case("objc_weak_class", LangOpts.ObjCNonFragileABI)
            .Case("ownership_holds", true)
            .Case("ownership_returns", true)
            .Case("ownership_takes", true)
-           .Case("cxx_inline_namespaces", true)
-         //.Case("cxx_concepts", false)
+           // C++0x features
+           .Case("cxx_attributes", LangOpts.CPlusPlus0x)
+           .Case("cxx_auto_type", LangOpts.CPlusPlus0x)
+           .Case("cxx_decltype", LangOpts.CPlusPlus0x)
+           .Case("cxx_default_function_template_args", LangOpts.CPlusPlus0x)
+           .Case("cxx_deleted_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_inline_namespaces", LangOpts.CPlusPlus0x)
          //.Case("cxx_lambdas", false)
          //.Case("cxx_nullptr", false)
-         //.Case("cxx_rvalue_references", false)
-         //.Case("cxx_variadic_templates", false)
+           .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_rvalue_references", LangOpts.CPlusPlus0x)
+           .Case("cxx_strong_enums", LangOpts.CPlusPlus0x)
+           .Case("cxx_static_assert", LangOpts.CPlusPlus0x)
+           .Case("cxx_trailing_return", LangOpts.CPlusPlus0x)
+           .Case("cxx_variadic_templates", LangOpts.CPlusPlus0x)
+           // Type traits
+           .Case("has_nothrow_assign", LangOpts.CPlusPlus)
+           .Case("has_nothrow_copy", LangOpts.CPlusPlus)
+           .Case("has_nothrow_constructor", LangOpts.CPlusPlus)
+           .Case("has_trivial_assign", LangOpts.CPlusPlus)
+           .Case("has_trivial_copy", LangOpts.CPlusPlus)
+           .Case("has_trivial_constructor", LangOpts.CPlusPlus)
+           .Case("has_trivial_destructor", LangOpts.CPlusPlus)
+           .Case("has_virtual_destructor", LangOpts.CPlusPlus)
+           .Case("is_abstract", LangOpts.CPlusPlus)
+           .Case("is_base_of", LangOpts.CPlusPlus)
+           .Case("is_class", LangOpts.CPlusPlus)
+           .Case("is_convertible_to", LangOpts.CPlusPlus)
+           .Case("is_empty", LangOpts.CPlusPlus)
+           .Case("is_enum", LangOpts.CPlusPlus)
+           .Case("is_pod", LangOpts.CPlusPlus)
+           .Case("is_polymorphic", LangOpts.CPlusPlus)
+           .Case("is_union", LangOpts.CPlusPlus)
+           .Case("is_literal", LangOpts.CPlusPlus)
            .Case("tls", PP.getTargetInfo().isTLSSupported())
            .Default(false);
 }
 
+/// HasAttribute -  Return true if we recognize and implement the attribute
+/// specified by the given identifier.
+static bool HasAttribute(const IdentifierInfo *II) {
+    return llvm::StringSwitch<bool>(II->getName())
+#include "clang/Lex/AttrSpellings.inc"
+        .Default(false);
+}
+
 /// EvaluateHasIncludeCommon - Process a '__has_include("path")'
 /// or '__has_include_next("path")' expression.
 /// Returns true if successful.
-static bool EvaluateHasIncludeCommon(bool &Result, Token &Tok,
-        IdentifierInfo *II, Preprocessor &PP,
-        const DirectoryLookup *LookupFrom) {
+static bool EvaluateHasIncludeCommon(Token &Tok,
+                                     IdentifierInfo *II, Preprocessor &PP,
+                                     const DirectoryLookup *LookupFrom) {
   SourceLocation LParenLoc;
 
   // Get '('.
@@ -559,7 +623,8 @@ static bool EvaluateHasIncludeCommon(bool &Result, Token &Tok,
   // Reserve a buffer to get the spelling.
   llvm::SmallString<128> FilenameBuffer;
   llvm::StringRef Filename;
-
+  SourceLocation EndLoc;
+  
   switch (Tok.getKind()) {
   case tok::eom:
     // If the token kind is EOM, the error has already been diagnosed.
@@ -578,7 +643,7 @@ static bool EvaluateHasIncludeCommon(bool &Result, Token &Tok,
     // This could be a <foo/bar.h> file coming from a macro expansion.  In this
     // case, glue the tokens together into FilenameBuffer and interpret those.
     FilenameBuffer.push_back('<');
-    if (PP.ConcatenateIncludeName(FilenameBuffer))
+    if (PP.ConcatenateIncludeName(FilenameBuffer, EndLoc))
       return false;   // Found <eom> but no ">"?  Diagnostic already emitted.
     Filename = FilenameBuffer.str();
     break;
@@ -598,7 +663,7 @@ static bool EvaluateHasIncludeCommon(bool &Result, Token &Tok,
   const FileEntry *File = PP.LookupFile(Filename, isAngled, LookupFrom, CurDir);
 
   // Get the result value.  Result = true means the file exists.
-  Result = File != 0;
+  bool Result = File != 0;
 
   // Get ')'.
   PP.LexNonComment(Tok);
@@ -610,19 +675,19 @@ static bool EvaluateHasIncludeCommon(bool &Result, Token &Tok,
     return false;
   }
 
-  return true;
+  return Result;
 }
 
 /// EvaluateHasInclude - Process a '__has_include("path")' expression.
 /// Returns true if successful.
-static bool EvaluateHasInclude(bool &Result, Token &Tok, IdentifierInfo *II,
+static bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II,
                                Preprocessor &PP) {
-  return(EvaluateHasIncludeCommon(Result, Tok, II, PP, NULL));
+  return EvaluateHasIncludeCommon(Tok, II, PP, NULL);
 }
 
 /// EvaluateHasIncludeNext - Process '__has_include_next("path")' expression.
 /// Returns true if successful.
-static bool EvaluateHasIncludeNext(bool &Result, Token &Tok,
+static bool EvaluateHasIncludeNext(Token &Tok,
                                    IdentifierInfo *II, Preprocessor &PP) {
   // __has_include_next is like __has_include, except that we start
   // searching after the current found directory.  If we can't do this,
@@ -638,7 +703,7 @@ static bool EvaluateHasIncludeNext(bool &Result, Token &Tok,
     ++Lookup;
   }
 
-  return(EvaluateHasIncludeCommon(Result, Tok, II, PP, Lookup));
+  return EvaluateHasIncludeCommon(Tok, II, PP, Lookup);
 }
 
 /// ExpandBuiltinMacro - If an identifier token is read that is to be expanded
@@ -683,7 +748,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
 
     // __LINE__ expands to a simple numeric value.
-    OS << PLoc.getLine();
+    OS << (PLoc.isValid()? PLoc.getLine() : 1);
     Tok.setKind(tok::numeric_constant);
   } else if (II == Ident__FILE__ || II == Ident__BASE_FILE__) {
     // C99 6.10.8: "__FILE__: The presumed name of the current source file (a
@@ -692,19 +757,24 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
 
     // __BASE_FILE__ is a GNU extension that returns the top of the presumed
     // #include stack instead of the current file.
-    if (II == Ident__BASE_FILE__) {
+    if (II == Ident__BASE_FILE__ && PLoc.isValid()) {
       SourceLocation NextLoc = PLoc.getIncludeLoc();
       while (NextLoc.isValid()) {
         PLoc = SourceMgr.getPresumedLoc(NextLoc);
+        if (PLoc.isInvalid())
+          break;
+        
         NextLoc = PLoc.getIncludeLoc();
       }
     }
 
     // Escape this filename.  Turn '\' -> '\\' '"' -> '\"'
     llvm::SmallString<128> FN;
-    FN += PLoc.getFilename();
-    Lexer::Stringify(FN);
-    OS << '"' << FN.str() << '"';
+    if (PLoc.isValid()) {
+      FN += PLoc.getFilename();
+      Lexer::Stringify(FN);
+      OS << '"' << FN.str() << '"';
+    }
     Tok.setKind(tok::string_literal);
   } else if (II == Ident__DATE__) {
     if (!DATELoc.isValid())
@@ -730,9 +800,11 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     unsigned Depth = 0;
 
     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());
-    PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
-    for (; PLoc.isValid(); ++Depth)
+    if (PLoc.isValid()) {
       PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
+      for (; PLoc.isValid(); ++Depth)
+        PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
+    }
 
     // __INCLUDE_LEVEL__ expands to a simple numeric value.
     OS << Depth;
@@ -765,7 +837,8 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     OS << CounterValue++;
     Tok.setKind(tok::numeric_constant);
   } else if (II == Ident__has_feature ||
-             II == Ident__has_builtin) {
+             II == Ident__has_builtin ||
+             II == Ident__has_attribute) {
     // The argument to these two builtins should be a parenthesized identifier.
     SourceLocation StartLoc = Tok.getLocation();
 
@@ -793,7 +866,9 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     else if (II == Ident__has_builtin) {
       // Check for a builtin is trivial.
       Value = FeatureII->getBuiltinID() != 0;
-    } else {
+    } else if (II == Ident__has_attribute)
+      Value = HasAttribute(FeatureII);
+    else {
       assert(II == Ident__has_feature && "Must be feature check");
       Value = HasFeature(*this, FeatureII);
     }
@@ -805,12 +880,11 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     // The argument to these two builtins should be a parenthesized
     // file name string literal using angle brackets (<>) or
     // double-quotes ("").
-    bool Value = false;
-    bool IsValid;
+    bool Value;
     if (II == Ident__has_include)
-      IsValid = EvaluateHasInclude(Value, Tok, II, *this);
+      Value = EvaluateHasInclude(Tok, II, *this);
     else
-      IsValid = EvaluateHasIncludeNext(Value, Tok, II, *this);
+      Value = EvaluateHasIncludeNext(Tok, II, *this);
     OS << (int)Value;
     Tok.setKind(tok::numeric_constant);
   } else {
@@ -818,3 +892,11 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   }
   CreateString(OS.str().data(), OS.str().size(), Tok, Tok.getLocation());
 }
+
+void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
+  // If the 'used' status changed, and the macro requires 'unused' warning,
+  // remove its SourceLocation from the warn-for-unused-macro locations.
+  if (MI->isWarnIfUnused() && !MI->isUsed())
+    WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
+  MI->setIsUsed(true);
+}
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
index 63b4823cf19e..975753bc23bc 100644
--- a/lib/Lex/PTHLexer.cpp
+++ b/lib/Lex/PTHLexer.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/OnDiskHashTable.h"
 #include "clang/Lex/LexDiagnostic.h"
@@ -25,7 +26,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include <sys/stat.h>
+#include "llvm/Support/system_error.h"
 using namespace clang;
 using namespace clang::io;
 
@@ -434,23 +435,23 @@ static void InvalidPTH(Diagnostic &Diags, const char *Msg) {
   Diags.Report(Diags.getCustomDiagID(Diagnostic::Error, Msg));
 }
 
-PTHManager* PTHManager::Create(const std::string& file, Diagnostic &Diags) {
+PTHManager *PTHManager::Create(const std::string &file, Diagnostic &Diags) {
   // Memory map the PTH file.
-  llvm::OwningPtr<llvm::MemoryBuffer>
-  File(llvm::MemoryBuffer::getFile(file.c_str()));
+  llvm::OwningPtr<llvm::MemoryBuffer> File;
 
-  if (!File) {
+  if (llvm::MemoryBuffer::getFile(file, File)) {
+    // FIXME: Add ec.message() to this diag.
     Diags.Report(diag::err_invalid_pth_file) << file;
     return 0;
   }
 
   // Get the buffer ranges and check if there are at least three 32-bit
   // words at the end of the file.
-  const unsigned char* BufBeg = (unsigned char*)File->getBufferStart();
-  const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd();
+  const unsigned char *BufBeg = (unsigned char*)File->getBufferStart();
+  const unsigned char *BufEnd = (unsigned char*)File->getBufferEnd();
 
   // Check the prologue of the file.
-  if ((BufEnd - BufBeg) < (signed) (sizeof("cfe-pth") + 3 + 4) ||
+  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 3 + 4) ||
       memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) {
     Diags.Report(diag::err_invalid_pth_file) << file;
     return 0;
@@ -668,7 +669,7 @@ public:
   }
 };
 
-class PTHStatCache : public StatSysCallCache {
+class PTHStatCache : public FileSystemStatCache {
   typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
   CacheTy Cache;
 
@@ -679,29 +680,30 @@ public:
 
   ~PTHStatCache() {}
 
-  int stat(const char *path, struct stat *buf) {
+  LookupResult getStat(const char *Path, struct stat &StatBuf,
+                       int *FileDescriptor) {
     // Do the lookup for the file's data in the PTH file.
-    CacheTy::iterator I = Cache.find(path);
+    CacheTy::iterator I = Cache.find(Path);
 
     // If we don't get a hit in the PTH file just forward to 'stat'.
     if (I == Cache.end())
-      return StatSysCallCache::stat(path, buf);
+      return statChained(Path, StatBuf, FileDescriptor);
 
-    const PTHStatData& Data = *I;
+    const PTHStatData &Data = *I;
 
     if (!Data.hasStat)
-      return 1;
+      return CacheMissing;
 
-    buf->st_ino = Data.ino;
-    buf->st_dev = Data.dev;
-    buf->st_mtime = Data.mtime;
-    buf->st_mode = Data.mode;
-    buf->st_size = Data.size;
-    return 0;
+    StatBuf.st_ino = Data.ino;
+    StatBuf.st_dev = Data.dev;
+    StatBuf.st_mtime = Data.mtime;
+    StatBuf.st_mode = Data.mode;
+    StatBuf.st_size = Data.size;
+    return CacheExists;
   }
 };
 } // end anonymous namespace
 
-StatSysCallCache *PTHManager::createStatCache() {
+FileSystemStatCache *PTHManager::createStatCache() {
   return new PTHStatCache(*((PTHFileLookup*) FileLookup));
 }
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index a7b289e137eb..f0475bc0cb20 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -35,7 +35,9 @@ PragmaHandler::~PragmaHandler() {
 
 EmptyPragmaHandler::EmptyPragmaHandler() {}
 
-void EmptyPragmaHandler::HandlePragma(Preprocessor &PP, Token &FirstToken) {}
+void EmptyPragmaHandler::HandlePragma(Preprocessor &PP, 
+                                      PragmaIntroducerKind Introducer,
+                                      Token &FirstToken) {}
 
 //===----------------------------------------------------------------------===//
 // PragmaNamespace Implementation.
@@ -73,7 +75,9 @@ void PragmaNamespace::RemovePragmaHandler(PragmaHandler *Handler) {
   Handlers.erase(Handler->getName());
 }
 
-void PragmaNamespace::HandlePragma(Preprocessor &PP, Token &Tok) {
+void PragmaNamespace::HandlePragma(Preprocessor &PP, 
+                                   PragmaIntroducerKind Introducer,
+                                   Token &Tok) {
   // Read the 'namespace' that the directive is in, e.g. STDC.  Do not macro
   // expand it, the user can have a STDC #define, that should not affect this.
   PP.LexUnexpandedToken(Tok);
@@ -89,7 +93,7 @@ void PragmaNamespace::HandlePragma(Preprocessor &PP, Token &Tok) {
   }
 
   // Otherwise, pass it down.
-  Handler->HandlePragma(PP, Tok);
+  Handler->HandlePragma(PP, Introducer, Tok);
 }
 
 //===----------------------------------------------------------------------===//
@@ -98,12 +102,12 @@ void PragmaNamespace::HandlePragma(Preprocessor &PP, Token &Tok) {
 
 /// HandlePragmaDirective - The "#pragma" directive has been parsed.  Lex the
 /// rest of the pragma, passing it to the registered pragma handlers.
-void Preprocessor::HandlePragmaDirective() {
+void Preprocessor::HandlePragmaDirective(unsigned Introducer) {
   ++NumPragma;
 
   // Invoke the first level of pragma handlers which reads the namespace id.
   Token Tok;
-  PragmaHandlers->HandlePragma(*this, Tok);
+  PragmaHandlers->HandlePragma(*this, PragmaIntroducerKind(Introducer), Tok);
 
   // If the pragma handler didn't read the rest of the line, consume it now.
   if (CurPPLexer && CurPPLexer->ParsingPreprocessorDirective)
@@ -170,7 +174,7 @@ void Preprocessor::Handle_Pragma(Token &Tok) {
     }
   }
   
-  Handle_Pragma(StrVal, PragmaLoc, RParenLoc);
+  Handle_Pragma(PIK__Pragma, StrVal, PragmaLoc, RParenLoc);
 
   // Finally, return whatever came after the pragma directive.
   return Lex(Tok);
@@ -216,13 +220,14 @@ void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
   
   SourceLocation RParenLoc = Tok.getLocation();
 
-  Handle_Pragma(StrVal, PragmaLoc, RParenLoc);
+  Handle_Pragma(PIK___pragma, StrVal, PragmaLoc, RParenLoc);
 
   // Finally, return whatever came after the pragma directive.
   return Lex(Tok);
 }
 
-void Preprocessor::Handle_Pragma(const std::string &StrVal,
+void Preprocessor::Handle_Pragma(unsigned Introducer,
+                                 const std::string &StrVal,
                                  SourceLocation PragmaLoc,
                                  SourceLocation RParenLoc) {
 
@@ -241,7 +246,7 @@ void Preprocessor::Handle_Pragma(const std::string &StrVal,
   EnterSourceFileWithLexer(TL, 0);
 
   // With everything set up, lex this as a #pragma directive.
-  HandlePragmaDirective();
+  HandlePragmaDirective(Introducer);
 }
 
 
@@ -287,7 +292,7 @@ void Preprocessor::HandlePragmaPoison(Token &PoisonTok) {
     if (Tok.is(tok::eom)) return;
 
     // Can only poison identifiers.
-    if (Tok.isNot(tok::identifier)) {
+    if (Tok.isNot(tok::raw_identifier)) {
       Diag(Tok, diag::err_pp_invalid_poison);
       return;
     }
@@ -324,6 +329,9 @@ void Preprocessor::HandlePragmaSystemHeader(Token &SysHeaderTok) {
 
 
   PresumedLoc PLoc = SourceMgr.getPresumedLoc(SysHeaderTok.getLocation());
+  if (PLoc.isInvalid())
+    return;
+  
   unsigned FilenameLen = strlen(PLoc.getFilename());
   unsigned FilenameID = SourceMgr.getLineTableFilenameID(PLoc.getFilename(),
                                                          FilenameLen);
@@ -478,23 +486,32 @@ void Preprocessor::HandlePragmaComment(Token &Tok) {
     Callbacks->PragmaComment(CommentLoc, II, ArgumentString);
 }
 
-/// HandlePragmaMessage - Handle the microsoft #pragma message extension.  The
-/// syntax is:
-///   #pragma message(messagestring)
-/// messagestring is a string, which is fully macro expanded, and permits string
-/// concatenation, embedded escape characters etc.  See MSDN for more details.
+/// HandlePragmaMessage - Handle the microsoft and gcc #pragma message
+/// extension.  The syntax is:
+///   #pragma message(string)
+/// OR, in GCC mode:
+///   #pragma message string
+/// string is a string, which is fully macro expanded, and permits string
+/// concatenation, embedded escape characters, etc... See MSDN for more details.
 void Preprocessor::HandlePragmaMessage(Token &Tok) {
   SourceLocation MessageLoc = Tok.getLocation();
   Lex(Tok);
-  if (Tok.isNot(tok::l_paren)) {
+  bool ExpectClosingParen = false;
+  switch (Tok.getKind()) {
+  case tok::l_paren:
+    // We have a MSVC style pragma message.
+    ExpectClosingParen = true;
+    // Read the string.
+    Lex(Tok);
+    break;
+  case tok::string_literal:
+    // We have a GCC style pragma message, and we just read the string.
+    break;
+  default:
     Diag(MessageLoc, diag::err_pragma_message_malformed);
     return;
   }
 
-  // Read the string.
-  Lex(Tok);
-  
-
   // We need at least one string.
   if (Tok.isNot(tok::string_literal)) {
     Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
@@ -522,11 +539,13 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
 
   llvm::StringRef MessageString(Literal.GetString(), Literal.GetStringLength());
 
-  if (Tok.isNot(tok::r_paren)) {
-    Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
-    return;
+  if (ExpectClosingParen) {
+    if (Tok.isNot(tok::r_paren)) {
+      Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
+      return;
+    }
+    Lex(Tok);  // eat the r_paren.
   }
-  Lex(Tok);  // eat the r_paren.
 
   if (Tok.isNot(tok::eom)) {
     Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
@@ -580,7 +599,7 @@ IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) {
   // Create a Token from the string.
   Token MacroTok;
   MacroTok.startToken();
-  MacroTok.setKind(tok::identifier);
+  MacroTok.setKind(tok::raw_identifier);
   CreateString(&StrVal[1], StrVal.size() - 2, MacroTok);
 
   // Get the IdentifierInfo of MacroToPushTok.
@@ -611,7 +630,7 @@ void Preprocessor::HandlePragmaPushMacro(Token &PushMacroTok) {
   PragmaPushMacroInfo[IdentInfo].push_back(MacroCopyToPush);
 }
 
-/// HandlePragmaPopMacro - Handle #pragma push_macro.  
+/// HandlePragmaPopMacro - Handle #pragma pop_macro.  
 /// The syntax is:
 ///   #pragma pop_macro("macro")
 void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
@@ -627,7 +646,11 @@ void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
   if (iter != PragmaPushMacroInfo.end()) {
     // Release the MacroInfo currently associated with IdentInfo.
     MacroInfo *CurrentMI = getMacroInfo(IdentInfo);
-    if (CurrentMI) ReleaseMacroInfo(CurrentMI);
+    if (CurrentMI) {
+      if (CurrentMI->isWarnIfUnused())
+        WarnUnusedMacroLocs.erase(CurrentMI->getDefinitionLoc());
+      ReleaseMacroInfo(CurrentMI);
+    }
 
     // Get the MacroInfo we want to reinstall.
     MacroInfo *MacroToReInstall = iter->second.back();
@@ -700,11 +723,39 @@ void Preprocessor::RemovePragmaHandler(llvm::StringRef Namespace,
     PragmaHandlers->RemovePragmaHandler(NS);
 }
 
+bool Preprocessor::LexOnOffSwitch(tok::OnOffSwitch &Result) {
+  Token Tok;
+  LexUnexpandedToken(Tok);
+
+  if (Tok.isNot(tok::identifier)) {
+    Diag(Tok, diag::ext_on_off_switch_syntax);
+    return true;
+  }
+  IdentifierInfo *II = Tok.getIdentifierInfo();
+  if (II->isStr("ON"))
+    Result = tok::OOS_ON;
+  else if (II->isStr("OFF"))
+    Result = tok::OOS_OFF;
+  else if (II->isStr("DEFAULT"))
+    Result = tok::OOS_DEFAULT;
+  else {
+    Diag(Tok, diag::ext_on_off_switch_syntax);
+    return true;
+  }
+
+  // Verify that this is followed by EOM.
+  LexUnexpandedToken(Tok);
+  if (Tok.isNot(tok::eom))
+    Diag(Tok, diag::ext_pragma_syntax_eom);
+  return false;
+}
+
 namespace {
 /// PragmaOnceHandler - "#pragma once" marks the file as atomically included.
 struct PragmaOnceHandler : public PragmaHandler {
   PragmaOnceHandler() : PragmaHandler("once") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &OnceTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &OnceTok) {
     PP.CheckEndOfDirective("pragma once");
     PP.HandlePragmaOnce(OnceTok);
   }
@@ -714,7 +765,8 @@ struct PragmaOnceHandler : public PragmaHandler {
 /// rest of the line is not lexed.
 struct PragmaMarkHandler : public PragmaHandler {
   PragmaMarkHandler() : PragmaHandler("mark") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &MarkTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &MarkTok) {
     PP.HandlePragmaMark();
   }
 };
@@ -722,7 +774,8 @@ struct PragmaMarkHandler : public PragmaHandler {
 /// PragmaPoisonHandler - "#pragma poison x" marks x as not usable.
 struct PragmaPoisonHandler : public PragmaHandler {
   PragmaPoisonHandler() : PragmaHandler("poison") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &PoisonTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PoisonTok) {
     PP.HandlePragmaPoison(PoisonTok);
   }
 };
@@ -731,21 +784,24 @@ struct PragmaPoisonHandler : public PragmaHandler {
 /// as a system header, which silences warnings in it.
 struct PragmaSystemHeaderHandler : public PragmaHandler {
   PragmaSystemHeaderHandler() : PragmaHandler("system_header") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &SHToken) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &SHToken) {
     PP.HandlePragmaSystemHeader(SHToken);
     PP.CheckEndOfDirective("pragma");
   }
 };
 struct PragmaDependencyHandler : public PragmaHandler {
   PragmaDependencyHandler() : PragmaHandler("dependency") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &DepToken) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DepToken) {
     PP.HandlePragmaDependency(DepToken);
   }
 };
 
 struct PragmaDebugHandler : public PragmaHandler {
   PragmaDebugHandler() : PragmaHandler("__debug") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &DepToken) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DepToken) {
     Token Tok;
     PP.LexUnexpandedToken(Tok);
     if (Tok.isNot(tok::identifier)) {
@@ -783,7 +839,9 @@ struct PragmaDebugHandler : public PragmaHandler {
 struct PragmaDiagnosticHandler : public PragmaHandler {
 public:
   explicit PragmaDiagnosticHandler() : PragmaHandler("diagnostic") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &DiagToken) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DiagToken) {
+    SourceLocation DiagLoc = DiagToken.getLocation();
     Token Tok;
     PP.LexUnexpandedToken(Tok);
     if (Tok.isNot(tok::identifier)) {
@@ -802,12 +860,12 @@ public:
     else if (II->isStr("fatal"))
       Map = diag::MAP_FATAL;
     else if (II->isStr("pop")) {
-      if (!PP.getDiagnostics().popMappings())
+      if (!PP.getDiagnostics().popMappings(DiagLoc))
         PP.Diag(Tok, diag::warn_pragma_diagnostic_cannot_pop);
 
       return;
     } else if (II->isStr("push")) {
-      PP.getDiagnostics().pushMappings();
+      PP.getDiagnostics().pushMappings(DiagLoc);
       return;
     } else {
       PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
@@ -857,7 +915,7 @@ public:
     }
 
     if (PP.getDiagnostics().setDiagnosticGroupMapping(WarningName.c_str()+2,
-                                                      Map))
+                                                      Map, DiagLoc))
       PP.Diag(StrToks[0].getLocation(),
               diag::warn_pragma_diagnostic_unknown_warning) << WarningName;
   }
@@ -866,7 +924,8 @@ public:
 /// PragmaCommentHandler - "#pragma comment ...".
 struct PragmaCommentHandler : public PragmaHandler {
   PragmaCommentHandler() : PragmaHandler("comment") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &CommentTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &CommentTok) {
     PP.HandlePragmaComment(CommentTok);
   }
 };
@@ -874,7 +933,8 @@ struct PragmaCommentHandler : public PragmaHandler {
 /// PragmaMessageHandler - "#pragma message("...")".
 struct PragmaMessageHandler : public PragmaHandler {
   PragmaMessageHandler() : PragmaHandler("message") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &CommentTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &CommentTok) {
     PP.HandlePragmaMessage(CommentTok);
   }
 };
@@ -883,7 +943,8 @@ struct PragmaMessageHandler : public PragmaHandler {
 /// macro on the top of the stack.
 struct PragmaPushMacroHandler : public PragmaHandler {
   PragmaPushMacroHandler() : PragmaHandler("push_macro") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &PushMacroTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PushMacroTok) {
     PP.HandlePragmaPushMacro(PushMacroTok);
   }
 };
@@ -893,62 +954,23 @@ struct PragmaPushMacroHandler : public PragmaHandler {
 /// macro to the value on the top of the stack.
 struct PragmaPopMacroHandler : public PragmaHandler {
   PragmaPopMacroHandler() : PragmaHandler("pop_macro") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &PopMacroTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PopMacroTok) {
     PP.HandlePragmaPopMacro(PopMacroTok);
   }
 };
 
 // Pragma STDC implementations.
 
-enum STDCSetting {
-  STDC_ON, STDC_OFF, STDC_DEFAULT, STDC_INVALID
-};
-
-static STDCSetting LexOnOffSwitch(Preprocessor &PP) {
-  Token Tok;
-  PP.LexUnexpandedToken(Tok);
-
-  if (Tok.isNot(tok::identifier)) {
-    PP.Diag(Tok, diag::ext_stdc_pragma_syntax);
-    return STDC_INVALID;
-  }
-  IdentifierInfo *II = Tok.getIdentifierInfo();
-  STDCSetting Result;
-  if (II->isStr("ON"))
-    Result = STDC_ON;
-  else if (II->isStr("OFF"))
-    Result = STDC_OFF;
-  else if (II->isStr("DEFAULT"))
-    Result = STDC_DEFAULT;
-  else {
-    PP.Diag(Tok, diag::ext_stdc_pragma_syntax);
-    return STDC_INVALID;
-  }
-
-  // Verify that this is followed by EOM.
-  PP.LexUnexpandedToken(Tok);
-  if (Tok.isNot(tok::eom))
-    PP.Diag(Tok, diag::ext_stdc_pragma_syntax_eom);
-  return Result;
-}
-
-/// PragmaSTDC_FP_CONTRACTHandler - "#pragma STDC FP_CONTRACT ...".
-struct PragmaSTDC_FP_CONTRACTHandler : public PragmaHandler {
-  PragmaSTDC_FP_CONTRACTHandler() : PragmaHandler("FP_CONTRACT") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &Tok) {
-    // We just ignore the setting of FP_CONTRACT. Since we don't do contractions
-    // at all, our default is OFF and setting it to ON is an optimization hint
-    // we can safely ignore.  When we support -ffma or something, we would need
-    // to diagnose that we are ignoring FMA.
-    LexOnOffSwitch(PP);
-  }
-};
-
 /// PragmaSTDC_FENV_ACCESSHandler - "#pragma STDC FENV_ACCESS ...".
 struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler {
   PragmaSTDC_FENV_ACCESSHandler() : PragmaHandler("FENV_ACCESS") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &Tok) {
-    if (LexOnOffSwitch(PP) == STDC_ON)
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &Tok) {
+    tok::OnOffSwitch OOS;
+    if (PP.LexOnOffSwitch(OOS))
+     return;
+    if (OOS == tok::OOS_ON)
       PP.Diag(Tok, diag::warn_stdc_fenv_access_not_supported);
   }
 };
@@ -957,15 +979,18 @@ struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler {
 struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler {
   PragmaSTDC_CX_LIMITED_RANGEHandler()
     : PragmaHandler("CX_LIMITED_RANGE") {}
-  virtual void HandlePragma(Preprocessor &PP, Token &Tok) {
-    LexOnOffSwitch(PP);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &Tok) {
+    tok::OnOffSwitch OOS;
+    PP.LexOnOffSwitch(OOS);
   }
 };
 
 /// PragmaSTDC_UnknownHandler - "#pragma STDC ...".
 struct PragmaSTDC_UnknownHandler : public PragmaHandler {
   PragmaSTDC_UnknownHandler() {}
-  virtual void HandlePragma(Preprocessor &PP, Token &UnknownTok) {
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &UnknownTok) {
     // C99 6.10.6p2, unknown forms are not allowed.
     PP.Diag(UnknownTok, diag::ext_stdc_pragma_ignored);
   }
@@ -981,6 +1006,7 @@ void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler(new PragmaMarkHandler());
   AddPragmaHandler(new PragmaPushMacroHandler());
   AddPragmaHandler(new PragmaPopMacroHandler());
+  AddPragmaHandler(new PragmaMessageHandler());
 
   // #pragma GCC ...
   AddPragmaHandler("GCC", new PragmaPoisonHandler());
@@ -994,7 +1020,6 @@ void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler("clang", new PragmaDependencyHandler());
   AddPragmaHandler("clang", new PragmaDiagnosticHandler());
 
-  AddPragmaHandler("STDC", new PragmaSTDC_FP_CONTRACTHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_UnknownHandler());
@@ -1002,6 +1027,5 @@ void Preprocessor::RegisterBuiltinPragmas() {
   // MS extensions.
   if (Features.Microsoft) {
     AddPragmaHandler(new PragmaCommentHandler());
-    AddPragmaHandler(new PragmaMessageHandler());
   }
 }
diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp
index c446d96b4527..3a43ac11e4e0 100644
--- a/lib/Lex/PreprocessingRecord.cpp
+++ b/lib/Lex/PreprocessingRecord.cpp
@@ -14,11 +14,29 @@
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Token.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "llvm/Support/ErrorHandling.h"
 
 using namespace clang;
 
 ExternalPreprocessingRecordSource::~ExternalPreprocessingRecordSource() { }
 
+
+InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec,
+                                       InclusionKind Kind, 
+                                       llvm::StringRef FileName, 
+                                       bool InQuotes, const FileEntry *File, 
+                                       SourceRange Range)
+  : PreprocessingDirective(InclusionDirectiveKind, Range), 
+    InQuotes(InQuotes), Kind(Kind), File(File) 
+{ 
+  char *Memory 
+    = (char*)PPRec.Allocate(FileName.size() + 1, llvm::alignOf<char>());
+  memcpy(Memory, FileName.data(), FileName.size());
+  Memory[FileName.size()] = 0;
+  this->FileName = llvm::StringRef(Memory, FileName.size());
+}
+
 void PreprocessingRecord::MaybeLoadPreallocatedEntities() const {
   if (!ExternalSource || LoadedPreallocatedEntities)
     return;
@@ -109,17 +127,18 @@ void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI) {
                                                       Def));
 }
 
-void PreprocessingRecord::MacroDefined(const IdentifierInfo *II, 
+void PreprocessingRecord::MacroDefined(const Token &Id,
                                        const MacroInfo *MI) {
   SourceRange R(MI->getDefinitionLoc(), MI->getDefinitionEndLoc());
   MacroDefinition *Def
-    = new (*this) MacroDefinition(II, MI->getDefinitionLoc(), R);
+      = new (*this) MacroDefinition(Id.getIdentifierInfo(),
+                                    MI->getDefinitionLoc(),
+                                    R);
   MacroDefinitions[MI] = Def;
   PreprocessedEntities.push_back(Def);
 }
 
-void PreprocessingRecord::MacroUndefined(SourceLocation Loc,
-                                         const IdentifierInfo *II,
+void PreprocessingRecord::MacroUndefined(const Token &Id,
                                          const MacroInfo *MI) {
   llvm::DenseMap<const MacroInfo *, MacroDefinition *>::iterator Pos
     = MacroDefinitions.find(MI);
@@ -127,3 +146,38 @@ void PreprocessingRecord::MacroUndefined(SourceLocation Loc,
     MacroDefinitions.erase(Pos);
 }
 
+void PreprocessingRecord::InclusionDirective(SourceLocation HashLoc,
+                                             const clang::Token &IncludeTok, 
+                                             llvm::StringRef FileName, 
+                                             bool IsAngled, 
+                                             const FileEntry *File,
+                                           clang::SourceLocation EndLoc) {
+  InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
+  
+  switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
+  case tok::pp_include: 
+    Kind = InclusionDirective::Include; 
+    break;
+    
+  case tok::pp_import: 
+    Kind = InclusionDirective::Import; 
+    break;
+    
+  case tok::pp_include_next: 
+    Kind = InclusionDirective::IncludeNext; 
+    break;
+    
+  case tok::pp___include_macros: 
+    Kind = InclusionDirective::IncludeMacros;
+    break;
+    
+  default:
+    llvm_unreachable("Unknown include directive kind");
+    return;
+  }
+  
+  clang::InclusionDirective *ID
+    = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, 
+                                            File, SourceRange(HashLoc, EndLoc));
+  PreprocessedEntities.push_back(ID);
+}
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 5160acf19e1f..6fe414b66414 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -53,10 +53,12 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            IdentifierInfoLookup* IILookup,
                            bool OwnsHeaders)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
-    SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0),
+    SourceMgr(SM),
+    HeaderInfo(Headers), ExternalSource(0),
     Identifiers(opts, IILookup), BuiltinInfo(Target), CodeComplete(0),
     CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0), 
-    CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0) {
+    CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0), MIChainHead(0),
+    MICache(0) {
   ScratchBuf = new ScratchBuffer(SourceMgr);
   CounterValue = 0; // __COUNTER__ starts at 0.
   OwnsHeaderSearch = OwnsHeaders;
@@ -106,23 +108,8 @@ Preprocessor::~Preprocessor() {
   }
 
   // Free any macro definitions.
-  for (llvm::DenseMap<IdentifierInfo*, MacroInfo*>::iterator I =
-       Macros.begin(), E = Macros.end(); I != E; ++I) {
-    // We don't need to free the MacroInfo objects directly.  These
-    // will be released when the BumpPtrAllocator 'BP' object gets
-    // destroyed.  We still need to run the dtor, however, to free
-    // memory alocated by MacroInfo.
-    I->second->Destroy();
-    I->first->setHasMacroDefinition(false);
-  }
-  for (std::vector<MacroInfo*>::iterator I = MICache.begin(),
-                                         E = MICache.end(); I != E; ++I) {
-    // We don't need to free the MacroInfo objects directly.  These
-    // will be released when the BumpPtrAllocator 'BP' object gets
-    // destroyed.  We still need to run the dtor, however, to free
-    // memory alocated by MacroInfo.
-    (*I)->Destroy();
-  }
+  for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
+    I->MI.Destroy();
 
   // Free any cached macro expanders.
   for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
@@ -291,114 +278,6 @@ void Preprocessor::CodeCompleteNaturalLanguage() {
     CodeComplete->CodeCompleteNaturalLanguage();
 }
 
-//===----------------------------------------------------------------------===//
-// Token Spelling
-//===----------------------------------------------------------------------===//
-
-/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
-/// token are the characters used to represent the token in the source file
-/// after trigraph expansion and escaped-newline folding.  In particular, this
-/// wants to get the true, uncanonicalized, spelling of things like digraphs
-/// UCNs, etc.
-std::string Preprocessor::getSpelling(const Token &Tok,
-                                      const SourceManager &SourceMgr,
-                                      const LangOptions &Features, 
-                                      bool *Invalid) {
-  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
-
-  // If this token contains nothing interesting, return it directly.
-  bool CharDataInvalid = false;
-  const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 
-                                                    &CharDataInvalid);
-  if (Invalid)
-    *Invalid = CharDataInvalid;
-  if (CharDataInvalid)
-    return std::string();
-
-  if (!Tok.needsCleaning())
-    return std::string(TokStart, TokStart+Tok.getLength());
-
-  std::string Result;
-  Result.reserve(Tok.getLength());
-
-  // Otherwise, hard case, relex the characters into the string.
-  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
-       Ptr != End; ) {
-    unsigned CharSize;
-    Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features));
-    Ptr += CharSize;
-  }
-  assert(Result.size() != unsigned(Tok.getLength()) &&
-         "NeedsCleaning flag set on something that didn't need cleaning!");
-  return Result;
-}
-
-/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
-/// token are the characters used to represent the token in the source file
-/// after trigraph expansion and escaped-newline folding.  In particular, this
-/// wants to get the true, uncanonicalized, spelling of things like digraphs
-/// UCNs, etc.
-std::string Preprocessor::getSpelling(const Token &Tok, bool *Invalid) const {
-  return getSpelling(Tok, SourceMgr, Features, Invalid);
-}
-
-/// getSpelling - This method is used to get the spelling of a token into a
-/// preallocated buffer, instead of as an std::string.  The caller is required
-/// to allocate enough space for the token, which is guaranteed to be at least
-/// Tok.getLength() bytes long.  The actual length of the token is returned.
-///
-/// Note that this method may do two possible things: it may either fill in
-/// the buffer specified with characters, or it may *change the input pointer*
-/// to point to a constant buffer with the data already in it (avoiding a
-/// copy).  The caller is not allowed to modify the returned buffer pointer
-/// if an internal buffer is returned.
-unsigned Preprocessor::getSpelling(const Token &Tok,
-                                   const char *&Buffer, bool *Invalid) const {
-  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
-
-  // If this token is an identifier, just return the string from the identifier
-  // table, which is very quick.
-  if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
-    Buffer = II->getNameStart();
-    return II->getLength();
-  }
-
-  // Otherwise, compute the start of the token in the input lexer buffer.
-  const char *TokStart = 0;
-
-  if (Tok.isLiteral())
-    TokStart = Tok.getLiteralData();
-
-  if (TokStart == 0) {
-    bool CharDataInvalid = false;
-    TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
-    if (Invalid)
-      *Invalid = CharDataInvalid;
-    if (CharDataInvalid) {
-      Buffer = "";
-      return 0;
-    }
-  }
-
-  // If this token contains nothing interesting, return it directly.
-  if (!Tok.needsCleaning()) {
-    Buffer = TokStart;
-    return Tok.getLength();
-  }
-
-  // Otherwise, hard case, relex the characters into the string.
-  char *OutBuf = const_cast<char*>(Buffer);
-  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
-       Ptr != End; ) {
-    unsigned CharSize;
-    *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features);
-    Ptr += CharSize;
-  }
-  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
-         "NeedsCleaning flag set on something that didn't need cleaning!");
-
-  return OutBuf-Buffer;
-}
 
 /// getSpelling - This method is used to get the spelling of a token into a
 /// SmallVector. Note that the returned StringRef may not point to the
@@ -406,9 +285,12 @@ unsigned Preprocessor::getSpelling(const Token &Tok,
 llvm::StringRef Preprocessor::getSpelling(const Token &Tok,
                                           llvm::SmallVectorImpl<char> &Buffer,
                                           bool *Invalid) const {
-  // Try the fast path.
-  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
-    return II->getName();
+  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
+  if (Tok.isNot(tok::raw_identifier)) {
+    // Try the fast path.
+    if (const IdentifierInfo *II = Tok.getIdentifierInfo())
+      return II->getName();
+  }
 
   // Resize the buffer if we need to copy into it.
   if (Tok.needsCleaning())
@@ -434,71 +316,14 @@ void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
                                            InstantiationLoc, Len);
   Tok.setLocation(Loc);
 
-  // If this is a literal token, set the pointer data.
-  if (Tok.isLiteral())
+  // If this is a raw identifier or a literal token, set the pointer data.
+  if (Tok.is(tok::raw_identifier))
+    Tok.setRawIdentifierData(DestPtr);
+  else if (Tok.isLiteral())
     Tok.setLiteralData(DestPtr);
 }
 
 
-/// AdvanceToTokenCharacter - Given a location that specifies the start of a
-/// token, return a new location that specifies a character within the token.
-SourceLocation Preprocessor::AdvanceToTokenCharacter(SourceLocation TokStart,
-                                                     unsigned CharNo) {
-  // Figure out how many physical characters away the specified instantiation
-  // character is.  This needs to take into consideration newlines and
-  // trigraphs.
-  bool Invalid = false;
-  const char *TokPtr = SourceMgr.getCharacterData(TokStart, &Invalid);
-
-  // If they request the first char of the token, we're trivially done.
-  if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
-    return TokStart;
-
-  unsigned PhysOffset = 0;
-
-  // The usual case is that tokens don't contain anything interesting.  Skip
-  // over the uninteresting characters.  If a token only consists of simple
-  // chars, this method is extremely fast.
-  while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
-    if (CharNo == 0)
-      return TokStart.getFileLocWithOffset(PhysOffset);
-    ++TokPtr, --CharNo, ++PhysOffset;
-  }
-
-  // If we have a character that may be a trigraph or escaped newline, use a
-  // lexer to parse it correctly.
-  for (; CharNo; --CharNo) {
-    unsigned Size;
-    Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features);
-    TokPtr += Size;
-    PhysOffset += Size;
-  }
-
-  // Final detail: if we end up on an escaped newline, we want to return the
-  // location of the actual byte of the token.  For example foo\<newline>bar
-  // advanced by 3 should return the location of b, not of \\.  One compounding
-  // detail of this is that the escape may be made by a trigraph.
-  if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
-    PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
-
-  return TokStart.getFileLocWithOffset(PhysOffset);
-}
-
-SourceLocation Preprocessor::getLocForEndOfToken(SourceLocation Loc,
-                                                 unsigned Offset) {
-  if (Loc.isInvalid() || !Loc.isFileID())
-    return SourceLocation();
-
-  unsigned Len = Lexer::MeasureTokenLength(Loc, getSourceManager(), Features);
-  if (Len > Offset)
-    Len = Len - Offset;
-  else
-    return Loc;
-
-  return AdvanceToTokenCharacter(Loc, Len);
-}
-
-
 
 //===----------------------------------------------------------------------===//
 // Preprocessor Initialization Methods
@@ -549,25 +374,29 @@ void Preprocessor::EndSourceFile() {
 // Lexer Event Handling.
 //===----------------------------------------------------------------------===//
 
-/// LookUpIdentifierInfo - Given a tok::identifier token, look up the
-/// identifier information for the token and install it into the token.
-IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier,
-                                                   const char *BufPtr) const {
-  assert(Identifier.is(tok::identifier) && "Not an identifier!");
-  assert(Identifier.getIdentifierInfo() == 0 && "Identinfo already exists!");
+/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
+/// identifier information for the token and install it into the token,
+/// updating the token kind accordingly.
+IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
+  assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
 
   // Look up this token, see if it is a macro, or if it is a language keyword.
   IdentifierInfo *II;
-  if (BufPtr && !Identifier.needsCleaning()) {
+  if (!Identifier.needsCleaning()) {
     // No cleaning needed, just use the characters from the lexed buffer.
-    II = getIdentifierInfo(llvm::StringRef(BufPtr, Identifier.getLength()));
+    II = getIdentifierInfo(llvm::StringRef(Identifier.getRawIdentifierData(),
+                                           Identifier.getLength()));
   } else {
     // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
     llvm::SmallString<64> IdentifierBuffer;
     llvm::StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
     II = getIdentifierInfo(CleanedStr);
   }
+
+  // Update the token info (identifier info and appropriate token kind).
   Identifier.setIdentifierInfo(II);
+  Identifier.setKind(II->getTokenID());
+
   return II;
 }
 
diff --git a/lib/Lex/TokenConcatenation.cpp b/lib/Lex/TokenConcatenation.cpp
index fc6db2151a3a..3e9e8550313c 100644
--- a/lib/Lex/TokenConcatenation.cpp
+++ b/lib/Lex/TokenConcatenation.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Lex/TokenConcatenation.h"
 #include "clang/Lex/Preprocessor.h"
+#include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
 
@@ -165,7 +166,14 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
   }
 
   switch (PrevKind) {
-  default: assert(0 && "InitAvoidConcatTokenInfo built wrong");
+  default:
+    llvm_unreachable("InitAvoidConcatTokenInfo built wrong");
+    return true;
+
+  case tok::raw_identifier:
+    llvm_unreachable("tok::raw_identifier in non-raw lexing mode!");
+    return true;
+
   case tok::identifier:   // id+id or id+number or id+L"foo".
     // id+'.'... will not append.
     if (Tok.is(tok::numeric_constant))
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 94719b0baa3c..ea39b47904b0 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -287,7 +287,7 @@ void TokenLexer::ExpandFunctionArguments() {
     llvm::BumpPtrAllocator &Alloc = PP.getPreprocessorAllocator();
     Token *Res =
       static_cast<Token *>(Alloc.Allocate(sizeof(Token)*ResultToks.size(),
-                                          llvm::alignof<Token>()));
+                                          llvm::alignOf<Token>()));
     if (NumTokens)
       memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token));
     Tokens = Res;
@@ -435,12 +435,13 @@ bool TokenLexer::PasteTokens(Token &Tok) {
     // Lex the resultant pasted token into Result.
     Token Result;
 
-    if (Tok.is(tok::identifier) && RHS.is(tok::identifier)) {
+    if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
       // Common paste case: identifier+identifier = identifier.  Avoid creating
       // a lexer and other overhead.
       PP.IncrementPasteCounter(true);
       Result.startToken();
-      Result.setKind(tok::identifier);
+      Result.setKind(tok::raw_identifier);
+      Result.setRawIdentifierData(ResultTokStrPtr);
       Result.setLocation(ResultTokLoc);
       Result.setLength(LHSLen+RHSLen);
     } else {
@@ -524,10 +525,10 @@ bool TokenLexer::PasteTokens(Token &Tok) {
   // Now that we got the result token, it will be subject to expansion.  Since
   // token pasting re-lexes the result token in raw mode, identifier information
   // isn't looked up.  As such, if the result is an identifier, look up id info.
-  if (Tok.is(tok::identifier)) {
+  if (Tok.is(tok::raw_identifier)) {
     // Look up the identifier info for the token.  We disabled identifier lookup
     // by saying we're skipping contents, so we need to do this manually.
-    PP.LookUpIdentifierInfo(Tok, ResultTokStrPtr);
+    PP.LookUpIdentifierInfo(Tok);
   }
   return false;
 }
diff --git a/lib/Makefile b/lib/Makefile
index dbd0eb699ee1..bf357fc7d569 100755
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -9,7 +9,7 @@
 CLANG_LEVEL := ..
 
 PARALLEL_DIRS = Headers Basic Lex Parse AST Sema CodeGen Analysis \
-                Checker Rewrite Serialization Frontend FrontendTool Index Driver
+                StaticAnalyzer Rewrite Serialization Frontend FrontendTool Index Driver
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Parse/CMakeLists.txt b/lib/Parse/CMakeLists.txt
index 189af3dc882f..6bf5e64cc6f8 100644
--- a/lib/Parse/CMakeLists.txt
+++ b/lib/Parse/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST clangLex clangSema)
 
 add_clang_library(clangParse
   ParseAST.cpp
diff --git a/lib/Parse/ParseAST.cpp b/lib/Parse/ParseAST.cpp
index d02787941b0d..edb1675b99e5 100644
--- a/lib/Parse/ParseAST.cpp
+++ b/lib/Parse/ParseAST.cpp
@@ -25,26 +25,6 @@
 
 using namespace clang;
 
-static void DumpRecordLayouts(ASTContext &C) {
-  for (ASTContext::type_iterator I = C.types_begin(), E = C.types_end();
-       I != E; ++I) {
-    const RecordType *RT = dyn_cast<RecordType>(*I);
-    if (!RT)
-      continue;
-
-    const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-    if (!RD || RD->isImplicit() || RD->isDependentType() ||
-        RD->isInvalidDecl() || !RD->getDefinition())
-      continue;
-
-    // FIXME: Do we really need to hard code this?
-    if (RD->getQualifiedNameAsString() == "__va_list_tag")
-      continue;
-
-    C.DumpRecordLayout(RD, llvm::errs());
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Public interface to the file
 //===----------------------------------------------------------------------===//
@@ -97,10 +77,6 @@ void clang::ParseAST(Sema &S, bool PrintStats) {
        E = S.WeakTopLevelDecls().end(); I != E; ++I)
     Consumer->HandleTopLevelDecl(DeclGroupRef(*I));
   
-  // Dump record layouts, if requested.
-  if (S.getLangOptions().DumpRecordLayouts)
-    DumpRecordLayouts(S.getASTContext());
-  
   Consumer->HandleTranslationUnit(S.getASTContext());
   
   if (PrintStats) {
diff --git a/lib/Parse/ParseCXXInlineMethods.cpp b/lib/Parse/ParseCXXInlineMethods.cpp
index d327db485c09..399473840a94 100644
--- a/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/lib/Parse/ParseCXXInlineMethods.cpp
@@ -20,10 +20,10 @@ using namespace clang;
 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
 /// Declarator is a well formed C++ inline method definition. Now lex its body
 /// and store its tokens for parsing after the C++ class is complete.
-Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
-                                const ParsedTemplateInfo &TemplateInfo) {
-  assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
-         "This isn't a function declarator!");
+Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, ParsingDeclarator &D,
+                                const ParsedTemplateInfo &TemplateInfo,
+                                const VirtSpecifiers& VS) {
+  assert(D.isFunctionDeclarator() && "This isn't a function declarator!");
   assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try)) &&
          "Current token not a '{', ':' or 'try'!");
 
@@ -36,19 +36,29 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
     // FIXME: Friend templates
     FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D, true,
                                           move(TemplateParams));
-  else // FIXME: pass template information through
+  else { // FIXME: pass template information through
+    if (VS.isOverrideSpecified())
+      Diag(VS.getOverrideLoc(), diag::ext_override_inline) << "override";
+    if (VS.isFinalSpecified())
+      Diag(VS.getFinalLoc(), diag::ext_override_inline) << "final";
+    if (VS.isNewSpecified())
+      Diag(VS.getNewLoc(), diag::ext_override_inline) << "new";
+
     FnD = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS, D,
-                                           move(TemplateParams), 0, 0,
-                                           /*IsDefinition*/true);
+                                           move(TemplateParams), 0, 
+                                           VS, 0, /*IsDefinition*/true);
+  }
 
   HandleMemberFunctionDefaultArgs(D, FnD);
 
+  D.complete(FnD);
+
   // Consume the tokens and store them for later parsing.
 
-  getCurrentClass().MethodDefs.push_back(LexedMethod(FnD));
-  getCurrentClass().MethodDefs.back().TemplateScope
-    = getCurScope()->isTemplateParamScope();
-  CachedTokens &Toks = getCurrentClass().MethodDefs.back().Toks;
+  LexedMethod* LM = new LexedMethod(this, FnD);
+  getCurrentClass().LateParsedDeclarations.push_back(LM);
+  LM->TemplateScope = getCurScope()->isTemplateParamScope();
+  CachedTokens &Toks = LM->Toks;
 
   tok::TokenKind kind = Tok.getKind();
   // We may have a constructor initializer or function-try-block here.
@@ -62,7 +72,8 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
         // don't try to parse this method later.
         Diag(Tok.getLocation(), diag::err_expected_lbrace);
         ConsumeAnyToken();
-        getCurrentClass().MethodDefs.pop_back();
+        delete getCurrentClass().LateParsedDeclarations.back();
+        getCurrentClass().LateParsedDeclarations.pop_back();
         return FnD;
       }
     }
@@ -86,13 +97,40 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
   return FnD;
 }
 
+Parser::LateParsedDeclaration::~LateParsedDeclaration() {}
+void Parser::LateParsedDeclaration::ParseLexedMethodDeclarations() {}
+void Parser::LateParsedDeclaration::ParseLexedMethodDefs() {}
+
+Parser::LateParsedClass::LateParsedClass(Parser *P, ParsingClass *C)
+  : Self(P), Class(C) {}
+
+Parser::LateParsedClass::~LateParsedClass() {
+  Self->DeallocateParsedClasses(Class);
+}
+
+void Parser::LateParsedClass::ParseLexedMethodDeclarations() {
+  Self->ParseLexedMethodDeclarations(*Class);
+}
+
+void Parser::LateParsedClass::ParseLexedMethodDefs() {
+  Self->ParseLexedMethodDefs(*Class);
+}
+
+void Parser::LateParsedMethodDeclaration::ParseLexedMethodDeclarations() {
+  Self->ParseLexedMethodDeclaration(*this);
+}
+
+void Parser::LexedMethod::ParseLexedMethodDefs() {
+  Self->ParseLexedMethodDef(*this);
+}
+
 /// ParseLexedMethodDeclarations - We finished parsing the member
 /// specification of a top (non-nested) C++ class. Now go over the
 /// stack of method declarations with some parts for which parsing was
 /// delayed (such as default arguments) and parse them.
 void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
   bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope TemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
+  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
   if (HasTemplateScope)
     Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
 
@@ -104,75 +142,79 @@ void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
   if (HasClassScope)
     Actions.ActOnStartDelayedMemberDeclarations(getCurScope(), Class.TagOrTemplate);
 
-  for (; !Class.MethodDecls.empty(); Class.MethodDecls.pop_front()) {
-    LateParsedMethodDeclaration &LM = Class.MethodDecls.front();
-
-    // If this is a member template, introduce the template parameter scope.
-    ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
-    if (LM.TemplateScope)
-      Actions.ActOnReenterTemplateScope(getCurScope(), LM.Method);
-
-    // Start the delayed C++ method declaration
-    Actions.ActOnStartDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
-
-    // Introduce the parameters into scope and parse their default
-    // arguments.
-    ParseScope PrototypeScope(this,
-                              Scope::FunctionPrototypeScope|Scope::DeclScope);
-    for (unsigned I = 0, N = LM.DefaultArgs.size(); I != N; ++I) {
-      // Introduce the parameter into scope.
-      Actions.ActOnDelayedCXXMethodParameter(getCurScope(), LM.DefaultArgs[I].Param);
+  for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
+    Class.LateParsedDeclarations[i]->ParseLexedMethodDeclarations();
+  }
 
-      if (CachedTokens *Toks = LM.DefaultArgs[I].Toks) {
-        // Save the current token position.
-        SourceLocation origLoc = Tok.getLocation();
+  if (HasClassScope)
+    Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(), Class.TagOrTemplate);
+}
 
-        // Parse the default argument from its saved token stream.
-        Toks->push_back(Tok); // So that the current token doesn't get lost
-        PP.EnterTokenStream(&Toks->front(), Toks->size(), true, false);
+void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
+  // If this is a member template, introduce the template parameter scope.
+  ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
+  if (LM.TemplateScope)
+    Actions.ActOnReenterTemplateScope(getCurScope(), LM.Method);
+
+  // Start the delayed C++ method declaration
+  Actions.ActOnStartDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
+
+  // Introduce the parameters into scope and parse their default
+  // arguments.
+  ParseScope PrototypeScope(this,
+                            Scope::FunctionPrototypeScope|Scope::DeclScope);
+  for (unsigned I = 0, N = LM.DefaultArgs.size(); I != N; ++I) {
+    // Introduce the parameter into scope.
+    Actions.ActOnDelayedCXXMethodParameter(getCurScope(), LM.DefaultArgs[I].Param);
+
+    if (CachedTokens *Toks = LM.DefaultArgs[I].Toks) {
+      // Save the current token position.
+      SourceLocation origLoc = Tok.getLocation();
+
+      // Parse the default argument from its saved token stream.
+      Toks->push_back(Tok); // So that the current token doesn't get lost
+      PP.EnterTokenStream(&Toks->front(), Toks->size(), true, false);
+
+      // Consume the previously-pushed token.
+      ConsumeAnyToken();
+
+      // Consume the '='.
+      assert(Tok.is(tok::equal) && "Default argument not starting with '='");
+      SourceLocation EqualLoc = ConsumeToken();
+
+      // The argument isn't actually potentially evaluated unless it is
+      // used.
+      EnterExpressionEvaluationContext Eval(Actions,
+                                            Sema::PotentiallyEvaluatedIfUsed);
+
+      ExprResult DefArgResult(ParseAssignmentExpression());
+      if (DefArgResult.isInvalid())
+        Actions.ActOnParamDefaultArgumentError(LM.DefaultArgs[I].Param);
+      else {
+        if (Tok.is(tok::cxx_defaultarg_end))
+          ConsumeToken();
+        else
+          Diag(Tok.getLocation(), diag::err_default_arg_unparsed);
+        Actions.ActOnParamDefaultArgument(LM.DefaultArgs[I].Param, EqualLoc,
+                                          DefArgResult.take());
+      }
 
-        // Consume the previously-pushed token.
+      assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
+                                                         Tok.getLocation()) &&
+             "ParseAssignmentExpression went over the default arg tokens!");
+      // There could be leftover tokens (e.g. because of an error).
+      // Skip through until we reach the original token position.
+      while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
         ConsumeAnyToken();
 
-        // Consume the '='.
-        assert(Tok.is(tok::equal) && "Default argument not starting with '='");
-        SourceLocation EqualLoc = ConsumeToken();
-
-        ExprResult DefArgResult(ParseAssignmentExpression());
-        if (DefArgResult.isInvalid())
-          Actions.ActOnParamDefaultArgumentError(LM.DefaultArgs[I].Param);
-        else {
-          if (Tok.is(tok::cxx_defaultarg_end))
-            ConsumeToken();
-          else
-            Diag(Tok.getLocation(), diag::err_default_arg_unparsed);
-          Actions.ActOnParamDefaultArgument(LM.DefaultArgs[I].Param, EqualLoc,
-                                            DefArgResult.take());
-        }
-
-        assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
-                                                           Tok.getLocation()) &&
-               "ParseAssignmentExpression went over the default arg tokens!");
-        // There could be leftover tokens (e.g. because of an error).
-        // Skip through until we reach the original token position.
-        while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
-          ConsumeAnyToken();
-
-        delete Toks;
-        LM.DefaultArgs[I].Toks = 0;
-      }
+      delete Toks;
+      LM.DefaultArgs[I].Toks = 0;
     }
-    PrototypeScope.Exit();
-
-    // Finish the delayed C++ method declaration.
-    Actions.ActOnFinishDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
   }
+  PrototypeScope.Exit();
 
-  for (unsigned I = 0, N = Class.NestedClasses.size(); I != N; ++I)
-    ParseLexedMethodDeclarations(*Class.NestedClasses[I]);
-
-  if (HasClassScope)
-    Actions.ActOnFinishDelayedMemberDeclarations(getCurScope(), Class.TagOrTemplate);
+  // Finish the delayed C++ method declaration.
+  Actions.ActOnFinishDelayedCXXMethodDeclaration(getCurScope(), LM.Method);
 }
 
 /// ParseLexedMethodDefs - We finished parsing the member specification of a top
@@ -180,7 +222,7 @@ void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
 /// collected during its parsing and parse them all.
 void Parser::ParseLexedMethodDefs(ParsingClass &Class) {
   bool HasTemplateScope = !Class.TopLevelClass && Class.TemplateScope;
-  ParseScope TemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
+  ParseScope ClassTemplateScope(this, Scope::TemplateParamScope, HasTemplateScope);
   if (HasTemplateScope)
     Actions.ActOnReenterTemplateScope(getCurScope(), Class.TagOrTemplate);
 
@@ -188,73 +230,72 @@ void Parser::ParseLexedMethodDefs(ParsingClass &Class) {
   ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope,
                         HasClassScope);
 
-  for (; !Class.MethodDefs.empty(); Class.MethodDefs.pop_front()) {
-    LexedMethod &LM = Class.MethodDefs.front();
-
-    // If this is a member template, introduce the template parameter scope.
-    ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
-    if (LM.TemplateScope)
-      Actions.ActOnReenterTemplateScope(getCurScope(), LM.D);
-
-    // Save the current token position.
-    SourceLocation origLoc = Tok.getLocation();
-
-    assert(!LM.Toks.empty() && "Empty body!");
-    // Append the current token at the end of the new token stream so that it
-    // doesn't get lost.
-    LM.Toks.push_back(Tok);
-    PP.EnterTokenStream(LM.Toks.data(), LM.Toks.size(), true, false);
-
-    // Consume the previously pushed token.
-    ConsumeAnyToken();
-    assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try))
-           && "Inline method not starting with '{', ':' or 'try'");
+  for (size_t i = 0; i < Class.LateParsedDeclarations.size(); ++i) {
+    Class.LateParsedDeclarations[i]->ParseLexedMethodDefs();
+  }
+}
 
-    // Parse the method body. Function body parsing code is similar enough
-    // to be re-used for method bodies as well.
-    ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope);
-    Actions.ActOnStartOfFunctionDef(getCurScope(), LM.D);
+void Parser::ParseLexedMethodDef(LexedMethod &LM) {
+  // If this is a member template, introduce the template parameter scope.
+  ParseScope TemplateScope(this, Scope::TemplateParamScope, LM.TemplateScope);
+  if (LM.TemplateScope)
+    Actions.ActOnReenterTemplateScope(getCurScope(), LM.D);
+
+  // Save the current token position.
+  SourceLocation origLoc = Tok.getLocation();
+
+  assert(!LM.Toks.empty() && "Empty body!");
+  // Append the current token at the end of the new token stream so that it
+  // doesn't get lost.
+  LM.Toks.push_back(Tok);
+  PP.EnterTokenStream(LM.Toks.data(), LM.Toks.size(), true, false);
+
+  // Consume the previously pushed token.
+  ConsumeAnyToken();
+  assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try))
+         && "Inline method not starting with '{', ':' or 'try'");
+
+  // Parse the method body. Function body parsing code is similar enough
+  // to be re-used for method bodies as well.
+  ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope);
+  Actions.ActOnStartOfFunctionDef(getCurScope(), LM.D);
+
+  if (Tok.is(tok::kw_try)) {
+    ParseFunctionTryBlock(LM.D);
+    assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
+                                                         Tok.getLocation()) &&
+           "ParseFunctionTryBlock went over the cached tokens!");
+    // There could be leftover tokens (e.g. because of an error).
+    // Skip through until we reach the original token position.
+    while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
+      ConsumeAnyToken();
+    return;
+  }
+  if (Tok.is(tok::colon)) {
+    ParseConstructorInitializer(LM.D);
 
-    if (Tok.is(tok::kw_try)) {
-      ParseFunctionTryBlock(LM.D);
-      assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
-                                                           Tok.getLocation()) &&
-             "ParseFunctionTryBlock went over the cached tokens!");
-      // There could be leftover tokens (e.g. because of an error).
-      // Skip through until we reach the original token position.
+    // Error recovery.
+    if (!Tok.is(tok::l_brace)) {
+      Actions.ActOnFinishFunctionBody(LM.D, 0);
+      return;
+    }
+  } else
+    Actions.ActOnDefaultCtorInitializers(LM.D);
+
+  ParseFunctionStatementBody(LM.D);
+
+  if (Tok.getLocation() != origLoc) {
+    // Due to parsing error, we either went over the cached tokens or
+    // there are still cached tokens left. If it's the latter case skip the
+    // leftover tokens.
+    // Since this is an uncommon situation that should be avoided, use the
+    // expensive isBeforeInTranslationUnit call.
+    if (PP.getSourceManager().isBeforeInTranslationUnit(Tok.getLocation(),
+                                                        origLoc))
       while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
         ConsumeAnyToken();
-      continue;
-    }
-    if (Tok.is(tok::colon)) {
-      ParseConstructorInitializer(LM.D);
-
-      // Error recovery.
-      if (!Tok.is(tok::l_brace)) {
-        Actions.ActOnFinishFunctionBody(LM.D, 0);
-        continue;
-      }
-    } else
-      Actions.ActOnDefaultCtorInitializers(LM.D);
-
-    ParseFunctionStatementBody(LM.D);
-
-    if (Tok.getLocation() != origLoc) {
-      // Due to parsing error, we either went over the cached tokens or
-      // there are still cached tokens left. If it's the latter case skip the
-      // leftover tokens.
-      // Since this is an uncommon situation that should be avoided, use the
-      // expensive isBeforeInTranslationUnit call.
-      if (PP.getSourceManager().isBeforeInTranslationUnit(Tok.getLocation(),
-                                                          origLoc))
-        while (Tok.getLocation() != origLoc && Tok.isNot(tok::eof))
-          ConsumeAnyToken();
 
-    }
   }
-
-  for (unsigned I = 0, N = Class.NestedClasses.size(); I != N; ++I)
-    ParseLexedMethodDefs(*Class.NestedClasses[I]);
 }
 
 /// ConsumeAndStoreUntil - Consume and store the token at the passed token
diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp
index 555fcf0dec55..5a7fc7e72d06 100644
--- a/lib/Parse/ParseDecl.cpp
+++ b/lib/Parse/ParseDecl.cpp
@@ -29,13 +29,14 @@ using namespace clang;
 ///         specifier-qualifier-list abstract-declarator[opt]
 ///
 /// Called type-id in C++.
-TypeResult Parser::ParseTypeName(SourceRange *Range) {
+TypeResult Parser::ParseTypeName(SourceRange *Range,
+                                 Declarator::TheContext Context) {
   // Parse the common declaration-specifiers piece.
   DeclSpec DS;
   ParseSpecifierQualifierList(DS);
 
   // Parse the abstract-declarator, if present.
-  Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
+  Declarator DeclaratorInfo(DS, Context);
   ParseDeclarator(DeclaratorInfo);
   if (Range)
     *Range = DeclaratorInfo.getSourceRange();
@@ -82,21 +83,20 @@ TypeResult Parser::ParseTypeName(SourceRange *Range) {
 /// attributes are very simple in practice. Until we find a bug, I don't see
 /// a pressing need to implement the 2 token lookahead.
 
-AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
+void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
+                                SourceLocation *endLoc) {
   assert(Tok.is(tok::kw___attribute) && "Not a GNU attribute list!");
 
-  AttributeList *CurrAttr = 0;
-
   while (Tok.is(tok::kw___attribute)) {
     ConsumeToken();
     if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
                          "attribute")) {
       SkipUntil(tok::r_paren, true); // skip until ) or ;
-      return CurrAttr;
+      return;
     }
     if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after, "(")) {
       SkipUntil(tok::r_paren, true); // skip until ) or ;
-      return CurrAttr;
+      return;
     }
     // Parse the attribute-list. e.g. __attribute__(( weak, alias("__f") ))
     while (Tok.is(tok::identifier) || isDeclarationSpecifier() ||
@@ -122,8 +122,8 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
           if (Tok.is(tok::r_paren)) {
             // __attribute__(( mode(byte) ))
             ConsumeParen(); // ignore the right paren loc for now
-            CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                         ParmName, ParmLoc, 0, 0, CurrAttr);
+            attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                         ParmName, ParmLoc, 0, 0));
           } else if (Tok.is(tok::comma)) {
             ConsumeToken();
             // __attribute__(( format(printf, 1, 2) ))
@@ -146,10 +146,9 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
             }
             if (ArgExprsOk && Tok.is(tok::r_paren)) {
               ConsumeParen(); // ignore the right paren loc for now
-              CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0,
+              attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0,
                                            AttrNameLoc, ParmName, ParmLoc,
-                                           ArgExprs.take(), ArgExprs.size(),
-                                           CurrAttr);
+                                           ArgExprs.take(), ArgExprs.size()));
             }
           }
         } else { // not an identifier
@@ -158,8 +157,8 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
           // parse a possibly empty comma separated list of expressions
             // __attribute__(( nonnull() ))
             ConsumeParen(); // ignore the right paren loc for now
-            CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                         0, SourceLocation(), 0, 0, CurrAttr);
+            attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                         0, SourceLocation(), 0, 0));
             break;
           case tok::kw_char:
           case tok::kw_wchar_t:
@@ -174,10 +173,12 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
           case tok::kw_float:
           case tok::kw_double:
           case tok::kw_void:
-          case tok::kw_typeof:
-            CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                         0, SourceLocation(), 0, 0, CurrAttr);
-            if (CurrAttr->getKind() == AttributeList::AT_IBOutletCollection)
+          case tok::kw_typeof: {
+            AttributeList *attr
+                     = AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                          0, SourceLocation(), 0, 0);
+            attrs.add(attr);
+            if (attr->getKind() == AttributeList::AT_IBOutletCollection)
               Diag(Tok, diag::err_iboutletcollection_builtintype);
             // If it's a builtin type name, eat it and expect a rparen
             // __attribute__(( vec_type_hint(char) ))
@@ -185,6 +186,7 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
             if (Tok.is(tok::r_paren))
               ConsumeParen();
             break;
+          }
           default:
             // __attribute__(( aligned(16) ))
             ExprVector ArgExprs(Actions);
@@ -207,17 +209,16 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
             // Match the ')'.
             if (ArgExprsOk && Tok.is(tok::r_paren)) {
               ConsumeParen(); // ignore the right paren loc for now
-              CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0,
-                           AttrNameLoc, 0, SourceLocation(), ArgExprs.take(),
-                           ArgExprs.size(),
-                           CurrAttr);
+              attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0,
+                                           AttrNameLoc, 0, SourceLocation(),
+                                           ArgExprs.take(), ArgExprs.size()));
             }
             break;
           }
         }
       } else {
-        CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                     0, SourceLocation(), 0, 0, CurrAttr);
+        attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                     0, SourceLocation(), 0, 0));
       }
     }
     if (ExpectAndConsume(tok::r_paren, diag::err_expected_rparen))
@@ -226,10 +227,9 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
     if (ExpectAndConsume(tok::r_paren, diag::err_expected_rparen)) {
       SkipUntil(tok::r_paren, false);
     }
-    if (EndLoc)
-      *EndLoc = Loc;
+    if (endLoc)
+      *endLoc = Loc;
   }
-  return CurrAttr;
 }
 
 /// ParseMicrosoftDeclSpec - Parse an __declspec construct
@@ -241,14 +241,14 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
 ///             extended-decl-modifier[opt]
 ///             extended-decl-modifier extended-decl-modifier-seq
 
-AttributeList* Parser::ParseMicrosoftDeclSpec(AttributeList *CurrAttr) {
+void Parser::ParseMicrosoftDeclSpec(ParsedAttributes &attrs) {
   assert(Tok.is(tok::kw___declspec) && "Not a declspec!");
 
   ConsumeToken();
   if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
                        "declspec")) {
     SkipUntil(tok::r_paren, true); // skip until ) or ;
-    return CurrAttr;
+    return;
   }
   while (Tok.getIdentifierInfo()) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
@@ -260,23 +260,22 @@ AttributeList* Parser::ParseMicrosoftDeclSpec(AttributeList *CurrAttr) {
       ExprResult ArgExpr(ParseAssignmentExpression());
       if (!ArgExpr.isInvalid()) {
         Expr *ExprList = ArgExpr.take();
-        CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
-                                     SourceLocation(), &ExprList, 1,
-                                     CurrAttr, true);
+        attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
+                                     SourceLocation(), &ExprList, 1, true));
       }
       if (ExpectAndConsume(tok::r_paren, diag::err_expected_rparen))
         SkipUntil(tok::r_paren, false);
     } else {
-      CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                   0, SourceLocation(), 0, 0, CurrAttr, true);
+      attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                   0, SourceLocation(), 0, 0, true));
     }
   }
   if (ExpectAndConsume(tok::r_paren, diag::err_expected_rparen))
     SkipUntil(tok::r_paren, false);
-  return CurrAttr;
+  return;
 }
 
-AttributeList* Parser::ParseMicrosoftTypeAttributes(AttributeList *CurrAttr) {
+void Parser::ParseMicrosoftTypeAttributes(ParsedAttributes &attrs) {
   // Treat these like attributes
   // FIXME: Allow Sema to distinguish between these and real attributes!
   while (Tok.is(tok::kw___fastcall) || Tok.is(tok::kw___stdcall) ||
@@ -287,21 +286,34 @@ AttributeList* Parser::ParseMicrosoftTypeAttributes(AttributeList *CurrAttr) {
     if (Tok.is(tok::kw___ptr64) || Tok.is(tok::kw___w64))
       // FIXME: Support these properly!
       continue;
-    CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
-                                 SourceLocation(), 0, 0, CurrAttr, true);
+    attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
+                                 SourceLocation(), 0, 0, true));
   }
-  return CurrAttr;
 }
 
-AttributeList* Parser::ParseBorlandTypeAttributes(AttributeList *CurrAttr) {
+void Parser::ParseBorlandTypeAttributes(ParsedAttributes &attrs) {
   // Treat these like attributes
   while (Tok.is(tok::kw___pascal)) {
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
     SourceLocation AttrNameLoc = ConsumeToken();
-    CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
-                                 SourceLocation(), 0, 0, CurrAttr, true);
+    attrs.add(AttrFactory.Create(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
+                                 SourceLocation(), 0, 0, true));
+  }
+}
+
+void Parser::ParseOpenCLAttributes(ParsedAttributes &attrs) {
+  // Treat these like attributes
+  while (Tok.is(tok::kw___kernel)) {
+    SourceLocation AttrNameLoc = ConsumeToken();
+    attrs.add(AttrFactory.Create(PP.getIdentifierInfo("opencl_kernel_function"),
+                                 AttrNameLoc, 0, AttrNameLoc, 0,
+                                 SourceLocation(), 0, 0, false));
   }
-  return CurrAttr;
+}
+
+void Parser::DiagnoseProhibitedAttributes(ParsedAttributesWithRange &attrs) {
+  Diag(attrs.Range.getBegin(), diag::err_attributes_not_allowed)
+    << attrs.Range;
 }
 
 /// ParseDeclaration - Parse a full 'declaration', which consists of
@@ -320,48 +332,43 @@ AttributeList* Parser::ParseBorlandTypeAttributes(AttributeList *CurrAttr) {
 /// [C++0x] static_assert-declaration
 ///         others... [FIXME]
 ///
-Parser::DeclGroupPtrTy Parser::ParseDeclaration(unsigned Context,
+Parser::DeclGroupPtrTy Parser::ParseDeclaration(StmtVector &Stmts,
+                                                unsigned Context,
                                                 SourceLocation &DeclEnd,
-                                                CXX0XAttributeList Attr) {
+                                          ParsedAttributesWithRange &attrs) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   
   Decl *SingleDecl = 0;
   switch (Tok.getKind()) {
   case tok::kw_template:
   case tok::kw_export:
-    if (Attr.HasAttr)
-      Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-        << Attr.Range;
+    ProhibitAttributes(attrs);
     SingleDecl = ParseDeclarationStartingWithTemplate(Context, DeclEnd);
     break;
   case tok::kw_inline:
     // Could be the start of an inline namespace. Allowed as an ext in C++03.
     if (getLang().CPlusPlus && NextToken().is(tok::kw_namespace)) {
-      if (Attr.HasAttr)
-        Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-          << Attr.Range;
+      ProhibitAttributes(attrs);
       SourceLocation InlineLoc = ConsumeToken();
       SingleDecl = ParseNamespace(Context, DeclEnd, InlineLoc);
       break;
     }
-    return ParseSimpleDeclaration(Context, DeclEnd, Attr.AttrList, true);
+    return ParseSimpleDeclaration(Stmts, Context, DeclEnd, attrs, 
+                                  true);
   case tok::kw_namespace:
-    if (Attr.HasAttr)
-      Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-        << Attr.Range;
+    ProhibitAttributes(attrs);
     SingleDecl = ParseNamespace(Context, DeclEnd);
     break;
   case tok::kw_using:
-    SingleDecl = ParseUsingDirectiveOrDeclaration(Context, DeclEnd, Attr);
+    SingleDecl = ParseUsingDirectiveOrDeclaration(Context, ParsedTemplateInfo(),
+                                                  DeclEnd, attrs);
     break;
   case tok::kw_static_assert:
-    if (Attr.HasAttr)
-      Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-        << Attr.Range;
+    ProhibitAttributes(attrs);
     SingleDecl = ParseStaticAssertDeclaration(DeclEnd);
     break;
   default:
-    return ParseSimpleDeclaration(Context, DeclEnd, Attr.AttrList, true);
+    return ParseSimpleDeclaration(Stmts, Context, DeclEnd, attrs, true);
   }
   
   // This routine returns a DeclGroup, if the thing we parsed only contains a
@@ -376,16 +383,19 @@ Parser::DeclGroupPtrTy Parser::ParseDeclaration(unsigned Context,
 ///
 /// If RequireSemi is false, this does not check for a ';' at the end of the
 /// declaration.  If it is true, it checks for and eats it.
-Parser::DeclGroupPtrTy Parser::ParseSimpleDeclaration(unsigned Context,
+Parser::DeclGroupPtrTy Parser::ParseSimpleDeclaration(StmtVector &Stmts, 
+                                                      unsigned Context,
                                                       SourceLocation &DeclEnd,
-                                                      AttributeList *Attr,
+                                                      ParsedAttributes &attrs,
                                                       bool RequireSemi) {
   // Parse the common declaration-specifiers piece.
   ParsingDeclSpec DS(*this);
-  if (Attr)
-    DS.AddAttributes(Attr);
+  DS.takeAttributesFrom(attrs);
   ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS_none,
-                            getDeclSpecContextFromDeclaratorContext(Context));
+                             getDeclSpecContextFromDeclaratorContext(Context));
+  StmtResult R = Actions.ActOnVlaStmt(DS);
+  if (R.isUsable())
+    Stmts.push_back(R.release());
 
   // C99 6.7.2.3p6: Handle "struct-or-union identifier;", "enum { X };"
   // declaration-specifiers init-declarator-list[opt] ';'
@@ -474,11 +484,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
     //    short __attribute__((common)) var;    -> declspec
     //    short var __attribute__((common));    -> declarator
     //    short x, __attribute__((common)) var;    -> declarator
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      D.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(D);
 
     ParseDeclarator(D);
 
@@ -547,12 +553,7 @@ Decl *Parser::ParseDeclarationAfterDeclarator(Declarator &D,
     D.SetRangeEnd(Loc);
   }
 
-  // If attributes are present, parse them.
-  if (Tok.is(tok::kw___attribute)) {
-    SourceLocation Loc;
-    AttributeList *AttrList = ParseGNUAttributes(&Loc);
-    D.AddAttributes(AttrList, Loc);
-  }
+  MaybeParseGNUAttributes(D);
 
   // Inform the current actions module that we just parsed this declarator.
   Decl *ThisDecl = 0;
@@ -586,11 +587,19 @@ Decl *Parser::ParseDeclarationAfterDeclarator(Declarator &D,
     }
   }
 
+  bool TypeContainsAuto =
+    D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto;
+
   // Parse declarator '=' initializer.
-  if (Tok.is(tok::equal)) {
+  if (isTokenEqualOrMistypedEqualEqual(
+                               diag::err_invalid_equalequal_after_declarator)) {
     ConsumeToken();
-    if (getLang().CPlusPlus0x && Tok.is(tok::kw_delete)) {
+    if (Tok.is(tok::kw_delete)) {
       SourceLocation DelLoc = ConsumeToken();
+      
+      if (!getLang().CPlusPlus0x)
+        Diag(DelLoc, diag::warn_deleted_function_accepted_as_extension);
+
       Actions.SetDeclDeleted(ThisDecl, DelLoc);
     } else {
       if (getLang().CPlusPlus && D.getCXXScopeSpec().isSet()) {
@@ -616,7 +625,8 @@ Decl *Parser::ParseDeclarationAfterDeclarator(Declarator &D,
         SkipUntil(tok::comma, true, true);
         Actions.ActOnInitializerError(ThisDecl);
       } else
-        Actions.AddInitializerToDecl(ThisDecl, Init.take());
+        Actions.AddInitializerToDecl(ThisDecl, Init.take(),
+                                     /*DirectInit=*/false, TypeContainsAuto);
     }
   } else if (Tok.is(tok::l_paren)) {
     // Parse C++ direct initializer: '(' expression-list ')'
@@ -650,12 +660,11 @@ Decl *Parser::ParseDeclarationAfterDeclarator(Declarator &D,
 
       Actions.AddCXXDirectInitializerToDecl(ThisDecl, LParenLoc,
                                             move_arg(Exprs),
-                                            CommaLocs.data(), RParenLoc);
+                                            RParenLoc,
+                                            TypeContainsAuto);
     }
   } else {
-    bool TypeContainsUndeducedAuto =
-      D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto;
-    Actions.ActOnUninitializedDecl(ThisDecl, TypeContainsUndeducedAuto);
+    Actions.ActOnUninitializedDecl(ThisDecl, TypeContainsAuto);
   }
 
   return ThisDecl;
@@ -675,7 +684,7 @@ void Parser::ParseSpecifierQualifierList(DeclSpec &DS) {
   // Validate declspec for type-name.
   unsigned Specs = DS.getParsedSpecifiers();
   if (Specs == DeclSpec::PQ_None && !DS.getNumProtocolQualifiers() &&
-      !DS.getAttributes())
+      !DS.hasAttributes())
     Diag(Tok, diag::err_typename_requires_specqual);
 
   // Issue diagnostic and remove storage class if present.
@@ -868,6 +877,7 @@ Parser::getDeclSpecContextFromDeclaratorContext(unsigned Context) {
 /// [C99]   'inline'
 /// [C++]   'virtual'
 /// [C++]   'explicit'
+/// [OpenCL] '__kernel'
 ///       'friend': [C++ dcl.friend]
 ///       'constexpr': [C++0x dcl.constexpr]
 
@@ -877,6 +887,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
                                         AccessSpecifier AS,
                                         DeclSpecContext DSContext) {  
   DS.SetRangeStart(Tok.getLocation());
+  DS.SetRangeEnd(Tok.getLocation());
   while (1) {
     bool isInvalid = false;
     const char *PrevSpec = 0;
@@ -905,13 +916,16 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
           = DSContext == DSC_top_level || 
             (DSContext == DSC_class && DS.isFriendSpecified());
 
-        Actions.CodeCompleteDeclarator(getCurScope(), AllowNonIdentifiers, 
-                                       AllowNestedNameSpecifiers);
+        Actions.CodeCompleteDeclSpec(getCurScope(), DS,
+                                     AllowNonIdentifiers, 
+                                     AllowNestedNameSpecifiers);
         ConsumeCodeCompletionToken();
         return;
       } 
       
-      if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate)
+      if (getCurScope()->getFnParent() || getCurScope()->getBlockParent())
+        CCC = Sema::PCC_LocalDeclarationSpecifiers;
+      else if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate)
         CCC = DSContext == DSC_class? Sema::PCC_MemberTemplate 
                                     : Sema::PCC_Template;
       else if (DSContext == DSC_class)
@@ -1005,7 +1019,8 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         ConsumeToken(); // The C++ scope.
         if (Tok.getAnnotationValue()) {
           ParsedType T = getTypeAnnotation(Tok);
-          isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, 
+          isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename,
+                                         Tok.getAnnotationEndLoc(), 
                                          PrevSpec, DiagID, T);
         }
         else
@@ -1080,20 +1095,10 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
       // Objective-C supports syntax of the form 'id<proto1,proto2>' where 'id'
       // is a specific typedef and 'itf<proto1,proto2>' where 'itf' is an
-      // Objective-C interface.  If we don't have Objective-C or a '<', this is
-      // just a normal reference to a typedef name.
-      if (!Tok.is(tok::less) || !getLang().ObjC1)
-        continue;
-
-      SourceLocation LAngleLoc, EndProtoLoc;
-      llvm::SmallVector<Decl *, 8> ProtocolDecl;
-      llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
-      ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
-                                  LAngleLoc, EndProtoLoc);
-      DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
-                               ProtocolLocs.data(), LAngleLoc);
-
-      DS.SetRangeEnd(EndProtoLoc);
+      // Objective-C interface. 
+      if (Tok.is(tok::less) && getLang().ObjC1)
+        ParseObjCProtocolQualifiers(DS);
+      
       continue;
     }
 
@@ -1150,21 +1155,10 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
       // Objective-C supports syntax of the form 'id<proto1,proto2>' where 'id'
       // is a specific typedef and 'itf<proto1,proto2>' where 'itf' is an
-      // Objective-C interface.  If we don't have Objective-C or a '<', this is
-      // just a normal reference to a typedef name.
-      if (!Tok.is(tok::less) || !getLang().ObjC1)
-        continue;
-
-      SourceLocation LAngleLoc, EndProtoLoc;
-      llvm::SmallVector<Decl *, 8> ProtocolDecl;
-      llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
-      ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
-                                  LAngleLoc, EndProtoLoc);
-      DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
-                               ProtocolLocs.data(), LAngleLoc);
-
-      DS.SetRangeEnd(EndProtoLoc);
-
+      // Objective-C interface. 
+      if (Tok.is(tok::less) && getLang().ObjC1)
+        ParseObjCProtocolQualifiers(DS);
+      
       // Need to support trailing type qualifiers (e.g. "id<p> const").
       // If a type specifier follows, it will be diagnosed elsewhere.
       continue;
@@ -1196,12 +1190,12 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
     // GNU attributes support.
     case tok::kw___attribute:
-      DS.AddAttributes(ParseGNUAttributes());
+      ParseGNUAttributes(DS.getAttributes());
       continue;
 
     // Microsoft declspec support.
     case tok::kw___declspec:
-      DS.AddAttributes(ParseMicrosoftDeclSpec());
+      ParseMicrosoftDeclSpec(DS.getAttributes());
       continue;
 
     // Microsoft single token adornments.
@@ -1215,34 +1209,39 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     case tok::kw___stdcall:
     case tok::kw___fastcall:
     case tok::kw___thiscall:
-      DS.AddAttributes(ParseMicrosoftTypeAttributes());
+      ParseMicrosoftTypeAttributes(DS.getAttributes());
       continue;
 
     // Borland single token adornments.
     case tok::kw___pascal:
-      DS.AddAttributes(ParseBorlandTypeAttributes());
+      ParseBorlandTypeAttributes(DS.getAttributes());
+      continue;
+
+    // OpenCL single token adornments.
+    case tok::kw___kernel:
+      ParseOpenCLAttributes(DS.getAttributes());
       continue;
 
     // storage-class-specifier
     case tok::kw_typedef:
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_typedef, Loc, PrevSpec,
-                                         DiagID);
+                                         DiagID, getLang());
       break;
     case tok::kw_extern:
       if (DS.isThreadSpecified())
         Diag(Tok, diag::ext_thread_before) << "extern";
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_extern, Loc, PrevSpec,
-                                         DiagID);
+                                         DiagID, getLang());
       break;
     case tok::kw___private_extern__:
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_private_extern, Loc,
-                                         PrevSpec, DiagID);
+                                         PrevSpec, DiagID, getLang());
       break;
     case tok::kw_static:
       if (DS.isThreadSpecified())
         Diag(Tok, diag::ext_thread_before) << "static";
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_static, Loc, PrevSpec,
-                                         DiagID);
+                                         DiagID, getLang());
       break;
     case tok::kw_auto:
       if (getLang().CPlusPlus0x)
@@ -1250,15 +1249,15 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
                                        DiagID);
       else
         isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_auto, Loc, PrevSpec,
-                                           DiagID);
+                                           DiagID, getLang());
       break;
     case tok::kw_register:
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_register, Loc, PrevSpec,
-                                         DiagID);
+                                         DiagID, getLang());
       break;
     case tok::kw_mutable:
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_mutable, Loc, PrevSpec,
-                                         DiagID);
+                                         DiagID, getLang());
       break;
     case tok::kw___thread:
       isInvalid = DS.SetStorageClassSpecThread(Loc, PrevSpec, DiagID);
@@ -1354,8 +1353,16 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       break;
     case tok::kw_bool:
     case tok::kw__Bool:
-      isInvalid = DS.SetTypeSpecType(DeclSpec::TST_bool, Loc, PrevSpec,
-                                     DiagID);
+      if (Tok.is(tok::kw_bool) &&
+          DS.getTypeSpecType() != DeclSpec::TST_unspecified &&
+          DS.getStorageClassSpec() == DeclSpec::SCS_typedef) {
+        PrevSpec = ""; // Not used by the diagnostic.
+        DiagID = diag::err_bool_redeclaration;
+        isInvalid = true;
+      } else {
+        isInvalid = DS.SetTypeSpecType(DeclSpec::TST_bool, Loc, PrevSpec,
+                                       DiagID);
+      }
       break;
     case tok::kw__Decimal32:
       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_decimal32, Loc, PrevSpec,
@@ -1432,23 +1439,14 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       if (DS.hasTypeSpecifier() || !getLang().ObjC1)
         goto DoneWithDeclSpec;
 
-      {
-        SourceLocation LAngleLoc, EndProtoLoc;
-        llvm::SmallVector<Decl *, 8> ProtocolDecl;
-        llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
-        ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
-                                    LAngleLoc, EndProtoLoc);
-        DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
-                                 ProtocolLocs.data(), LAngleLoc);
-        DS.SetRangeEnd(EndProtoLoc);
-
+      if (!ParseObjCProtocolQualifiers(DS))
         Diag(Loc, diag::warn_objc_protocol_qualifier_missing_id)
           << FixItHint::CreateInsertion(Loc, "id")
-          << SourceRange(Loc, EndProtoLoc);
-        // Need to support trailing type qualifiers (e.g. "id<p> const").
-        // If a type specifier follows, it will be diagnosed elsewhere.
-        continue;
-      }
+          << SourceRange(Loc, DS.getSourceRange().getEnd());
+      
+      // Need to support trailing type qualifiers (e.g. "id<p> const").
+      // If a type specifier follows, it will be diagnosed elsewhere.
+      continue;
     }
     // If the specifier wasn't legal, issue a diagnostic.
     if (isInvalid) {
@@ -1552,7 +1550,8 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
   // simple-type-specifier:
   case tok::annot_typename: {
     if (ParsedType T = getTypeAnnotation(Tok)) {
-      isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
+      isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename,
+                                     Tok.getAnnotationEndLoc(), PrevSpec,
                                      DiagID, T);
     } else
       DS.SetTypeSpecError();
@@ -1563,18 +1562,9 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
     // is a specific typedef and 'itf<proto1,proto2>' where 'itf' is an
     // Objective-C interface.  If we don't have Objective-C or a '<', this is
     // just a normal reference to a typedef name.
-    if (!Tok.is(tok::less) || !getLang().ObjC1)
-      return true;
-
-    SourceLocation LAngleLoc, EndProtoLoc;
-    llvm::SmallVector<Decl *, 8> ProtocolDecl;
-    llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
-    ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
-                                LAngleLoc, EndProtoLoc);
-    DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
-                             ProtocolLocs.data(), LAngleLoc);
-
-    DS.SetRangeEnd(EndProtoLoc);
+    if (Tok.is(tok::less) && getLang().ObjC1)
+      ParseObjCProtocolQualifiers(DS);
+    
     return true;
   }
 
@@ -1706,11 +1696,11 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
   case tok::kw___stdcall:
   case tok::kw___fastcall:
   case tok::kw___thiscall:
-    DS.AddAttributes(ParseMicrosoftTypeAttributes());
+    ParseMicrosoftTypeAttributes(DS.getAttributes());
     return true;
 
   case tok::kw___pascal:
-    DS.AddAttributes(ParseBorlandTypeAttributes());
+    ParseBorlandTypeAttributes(DS.getAttributes());
     return true;
 
   default:
@@ -1756,7 +1746,6 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
   }
 
   // Parse the common specifier-qualifiers-list piece.
-  SourceLocation DSStart = Tok.getLocation();
   ParseSpecifierQualifierList(DS);
 
   // If there are no declarators, this is a free-standing declaration
@@ -1773,11 +1762,8 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
     FieldDeclarator DeclaratorInfo(DS);
 
     // Attributes are only allowed here on successive declarators.
-    if (!FirstDeclarator && Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      DeclaratorInfo.D.AddAttributes(AttrList, Loc);
-    }
+    if (!FirstDeclarator)
+      MaybeParseGNUAttributes(DeclaratorInfo.D);
 
     /// struct-declarator: declarator
     /// struct-declarator: declarator[opt] ':' constant-expression
@@ -1797,11 +1783,7 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
     }
 
     // If attributes exist after the declarator, parse them.
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      DeclaratorInfo.D.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(DeclaratorInfo.D);
 
     // We're done with this declarator;  invoke the callback.
     Decl *D = Fields.invoke(DeclaratorInfo);
@@ -1922,20 +1904,18 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
 
   SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
 
-  llvm::OwningPtr<AttributeList> AttrList;
+  ParsedAttributes attrs;
   // If attributes exist after struct contents, parse them.
-  if (Tok.is(tok::kw___attribute))
-    AttrList.reset(ParseGNUAttributes());
+  MaybeParseGNUAttributes(attrs);
 
   Actions.ActOnFields(getCurScope(),
                       RecordLoc, TagDecl, FieldDecls.data(), FieldDecls.size(),
                       LBraceLoc, RBraceLoc,
-                      AttrList.get());
+                      attrs.getList());
   StructScope.Exit();
   Actions.ActOnTagFinishDefinition(getCurScope(), TagDecl, RBraceLoc);
 }
 
-
 /// ParseEnumSpecifier
 ///       enum-specifier: [C99 6.7.2.2]
 ///         'enum' identifier[opt] '{' enumerator-list '}'
@@ -1945,6 +1925,21 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
 ///         'enum' identifier
 /// [GNU]   'enum' attributes[opt] identifier
 ///
+/// [C++0x] enum-head '{' enumerator-list[opt] '}'
+/// [C++0x] enum-head '{' enumerator-list ','  '}'
+///
+///       enum-head: [C++0x]
+///         enum-key attributes[opt] identifier[opt] enum-base[opt]
+///         enum-key attributes[opt] nested-name-specifier identifier enum-base[opt]
+///
+///       enum-key: [C++0x]
+///         'enum'
+///         'enum' 'class'
+///         'enum' 'struct'
+///
+///       enum-base: [C++0x]
+///         ':' type-specifier-seq
+///
 /// [C++] elaborated-type-specifier:
 /// [C++]   'enum' '::'[opt] nested-name-specifier[opt] identifier
 ///
@@ -1958,10 +1953,9 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
     ConsumeCodeCompletionToken();
   }
   
-  llvm::OwningPtr<AttributeList> Attr;
   // If attributes exist after tag, parse them.
-  if (Tok.is(tok::kw___attribute))
-    Attr.reset(ParseGNUAttributes());
+  ParsedAttributes attrs;
+  MaybeParseGNUAttributes(attrs);
 
   CXXScopeSpec &SS = DS.getTypeSpecScope();
   if (getLang().CPlusPlus) {
@@ -1979,6 +1973,16 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
     }
   }
 
+  bool IsScopedEnum = false;
+  bool IsScopedUsingClassTag = false;
+
+  if (getLang().CPlusPlus0x &&
+      (Tok.is(tok::kw_class) || Tok.is(tok::kw_struct))) {
+    IsScopedEnum = true;
+    IsScopedUsingClassTag = Tok.is(tok::kw_class);
+    ConsumeToken();
+  }
+
   // Must have either 'enum name' or 'enum {...}'.
   if (Tok.isNot(tok::identifier) && Tok.isNot(tok::l_brace)) {
     Diag(Tok, diag::err_expected_ident_lbrace);
@@ -1996,6 +2000,69 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
     NameLoc = ConsumeToken();
   }
 
+  if (!Name && IsScopedEnum) {
+    // C++0x 7.2p2: The optional identifier shall not be omitted in the
+    // declaration of a scoped enumeration.
+    Diag(Tok, diag::err_scoped_enum_missing_identifier);
+    IsScopedEnum = false;
+    IsScopedUsingClassTag = false;
+  }
+
+  TypeResult BaseType;
+
+  // Parse the fixed underlying type.
+  if (getLang().CPlusPlus0x && Tok.is(tok::colon)) {
+    bool PossibleBitfield = false;
+    if (getCurScope()->getFlags() & Scope::ClassScope) {
+      // If we're in class scope, this can either be an enum declaration with
+      // an underlying type, or a declaration of a bitfield member. We try to
+      // use a simple disambiguation scheme first to catch the common cases
+      // (integer literal, sizeof); if it's still ambiguous, we then consider 
+      // anything that's a simple-type-specifier followed by '(' as an 
+      // expression. This suffices because function types are not valid 
+      // underlying types anyway.
+      TPResult TPR = isExpressionOrTypeSpecifierSimple(NextToken().getKind());
+      // If the next token starts an expression, we know we're parsing a 
+      // bit-field. This is the common case.
+      if (TPR == TPResult::True())
+        PossibleBitfield = true;
+      // If the next token starts a type-specifier-seq, it may be either a
+      // a fixed underlying type or the start of a function-style cast in C++;
+      // lookahead one more token to see if it's obvious that we have a 
+      // fixed underlying type.
+      else if (TPR == TPResult::False() && 
+               GetLookAheadToken(2).getKind() == tok::semi) {
+        // Consume the ':'.
+        ConsumeToken();
+      } else {
+        // We have the start of a type-specifier-seq, so we have to perform
+        // tentative parsing to determine whether we have an expression or a
+        // type.
+        TentativeParsingAction TPA(*this);
+
+        // Consume the ':'.
+        ConsumeToken();
+      
+        if (isCXXDeclarationSpecifier() != TPResult::True()) {
+          // We'll parse this as a bitfield later.
+          PossibleBitfield = true;
+          TPA.Revert();
+        } else {
+          // We have a type-specifier-seq.
+          TPA.Commit();
+        }
+      }
+    } else {
+      // Consume the ':'.
+      ConsumeToken();
+    }
+
+    if (!PossibleBitfield) {
+      SourceRange Range;
+      BaseType = ParseTypeName(&Range);
+    }
+  }
+
   // There are three options here.  If we have 'enum foo;', then this is a
   // forward declaration.  If we have 'enum foo {...' then this is a
   // definition. Otherwise we have something like 'enum foo xyz', a reference.
@@ -2029,10 +2096,12 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   const char *PrevSpec = 0;
   unsigned DiagID;
   Decl *TagDecl = Actions.ActOnTag(getCurScope(), DeclSpec::TST_enum, TUK,
-                                   StartLoc, SS, Name, NameLoc, Attr.get(),
+                                   StartLoc, SS, Name, NameLoc, attrs.getList(),
                                    AS,
                                    MultiTemplateParamsArg(Actions),
-                                   Owned, IsDependent);
+                                   Owned, IsDependent, IsScopedEnum,
+                                   IsScopedUsingClassTag, BaseType);
+
   if (IsDependent) {
     // This enum has a dependent nested-name-specifier. Handle it as a 
     // dependent tag.
@@ -2109,6 +2178,10 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
     IdentifierInfo *Ident = Tok.getIdentifierInfo();
     SourceLocation IdentLoc = ConsumeToken();
 
+    // If attributes exist after the enumerator, parse them.
+    ParsedAttributes attrs;
+    MaybeParseGNUAttributes(attrs);
+
     SourceLocation EqualLoc;
     ExprResult AssignedVal;
     if (Tok.is(tok::equal)) {
@@ -2122,11 +2195,19 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
     Decl *EnumConstDecl = Actions.ActOnEnumConstant(getCurScope(), EnumDecl,
                                                     LastEnumConstDecl,
                                                     IdentLoc, Ident,
-                                                    EqualLoc,
+                                                    attrs.getList(), EqualLoc,
                                                     AssignedVal.release());
     EnumConstantDecls.push_back(EnumConstDecl);
     LastEnumConstDecl = EnumConstDecl;
 
+    if (Tok.is(tok::identifier)) {
+      // We're missing a comma between enumerators.
+      SourceLocation Loc = PP.getLocForEndOfToken(PrevTokLocation);
+      Diag(Loc, diag::err_enumerator_list_missing_comma)      
+        << FixItHint::CreateInsertion(Loc, ", ");
+      continue;
+    }
+    
     if (Tok.isNot(tok::comma))
       break;
     SourceLocation CommaLoc = ConsumeToken();
@@ -2141,14 +2222,13 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
   // Eat the }.
   SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
 
-  llvm::OwningPtr<AttributeList> Attr;
   // If attributes exist after the identifier list, parse them.
-  if (Tok.is(tok::kw___attribute))
-    Attr.reset(ParseGNUAttributes()); // FIXME: where do they do?
+  ParsedAttributes attrs;
+  MaybeParseGNUAttributes(attrs);
 
   Actions.ActOnEnumBody(StartLoc, LBraceLoc, RBraceLoc, EnumDecl,
                         EnumConstantDecls.data(), EnumConstantDecls.size(),
-                        getCurScope(), Attr.get());
+                        getCurScope(), attrs.getList());
 
   EnumScope.Exit();
   Actions.ActOnTagFinishDefinition(getCurScope(), EnumDecl, RBraceLoc);
@@ -2296,7 +2376,10 @@ bool Parser::isTypeSpecifierQualifier() {
 
 /// isDeclarationSpecifier() - Return true if the current token is part of a
 /// declaration specifier.
-bool Parser::isDeclarationSpecifier() {
+///
+/// \param DisambiguatingWithExpression True to indicate that the purpose of
+/// this check is to disambiguate between an expression and a declaration.
+bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   switch (Tok.getKind()) {
   default: return false;
 
@@ -2314,6 +2397,16 @@ bool Parser::isDeclarationSpecifier() {
       return true;
     if (Tok.is(tok::identifier))
       return false;
+      
+    // If we're in Objective-C and we have an Objective-C class type followed
+    // by an identifier and then either ':' or ']', in a place where an 
+    // expression is permitted, then this is probably a class message send
+    // missing the initial '['. In this case, we won't consider this to be
+    // the start of a declaration.
+    if (DisambiguatingWithExpression && 
+        isStartOfObjCClassMessageMissingOpenBracket())
+      return false;
+      
     return isDeclarationSpecifier();
 
   case tok::coloncolon:   // ::foo::bar
@@ -2441,6 +2534,10 @@ bool Parser::isConstructorDeclarator() {
   if (SS.isSet() && Actions.ShouldEnterDeclaratorScope(getCurScope(), SS))
     DeclScopeObj.EnterDeclaratorScope();
 
+  // Optionally skip Microsoft attributes.
+  ParsedAttributes Attrs;
+  MaybeParseMicrosoftAttributes(Attrs);
+
   // Check whether the next token(s) are part of a declaration
   // specifier, in which case we have the start of a parameter and,
   // therefore, we know that this is a constructor.
@@ -2466,9 +2563,10 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
                                        bool CXX0XAttributesAllowed) {
   if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
     SourceLocation Loc = Tok.getLocation();
-    CXX0XAttributeList Attr = ParseCXX0XAttributes();
+    ParsedAttributesWithRange attrs;
+    ParseCXX0XAttributes(attrs);
     if (CXX0XAttributesAllowed)
-      DS.AddAttributes(Attr.AttrList);
+      DS.takeAttributesFrom(attrs);
     else
       Diag(Loc, diag::err_attributes_not_allowed);
   }
@@ -2504,19 +2602,19 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
     case tok::kw___fastcall:
     case tok::kw___thiscall:
       if (VendorAttributesAllowed) {
-        DS.AddAttributes(ParseMicrosoftTypeAttributes());
+        ParseMicrosoftTypeAttributes(DS.getAttributes());
         continue;
       }
       goto DoneWithTypeQuals;
     case tok::kw___pascal:
       if (VendorAttributesAllowed) {
-        DS.AddAttributes(ParseBorlandTypeAttributes());
+        ParseBorlandTypeAttributes(DS.getAttributes());
         continue;
       }
       goto DoneWithTypeQuals;
     case tok::kw___attribute:
       if (VendorAttributesAllowed) {
-        DS.AddAttributes(ParseGNUAttributes());
+        ParseGNUAttributes(DS.getAttributes());
         continue; // do *not* consume the next token!
       }
       // otherwise, FALL THROUGH!
@@ -2602,7 +2700,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
       // Sema will have to catch (syntactically invalid) pointers into global
       // scope. It has to catch pointers into namespace scope anyway.
       D.AddTypeInfo(DeclaratorChunk::getMemberPointer(SS,DS.getTypeQualifiers(),
-                                                      Loc, DS.TakeAttributes()),
+                                                      Loc, DS.takeAttributes()),
                     /* Don't replace range end. */SourceLocation());
       return;
     }
@@ -2636,12 +2734,12 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
     if (Kind == tok::star)
       // Remember that we parsed a pointer type, and remember the type-quals.
       D.AddTypeInfo(DeclaratorChunk::getPointer(DS.getTypeQualifiers(), Loc,
-                                                DS.TakeAttributes()),
+                                                DS.takeAttributes()),
                     SourceLocation());
     else
       // Remember that we parsed a Block type, and remember the type-quals.
       D.AddTypeInfo(DeclaratorChunk::getBlockPointer(DS.getTypeQualifiers(),
-                                                     Loc, DS.TakeAttributes()),
+                                                     Loc, DS.takeAttributes()),
                     SourceLocation());
   } else {
     // Is a reference
@@ -2650,7 +2748,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
     // Complain about rvalue references in C++03, but then go on and build
     // the declarator.
     if (Kind == tok::ampamp && !getLang().CPlusPlus0x)
-      Diag(Loc, diag::err_rvalue_reference);
+      Diag(Loc, diag::ext_rvalue_reference);
 
     // C++ 8.3.2p1: cv-qualified references are ill-formed except when the
     // cv-qualifiers are introduced through the use of a typedef or of a
@@ -2693,7 +2791,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
 
     // Remember that we parsed a reference type. It doesn't have type-quals.
     D.AddTypeInfo(DeclaratorChunk::getReference(DS.getTypeQualifiers(), Loc,
-                                                DS.TakeAttributes(),
+                                                DS.takeAttributes(),
                                                 Kind == tok::amp),
                   SourceLocation());
   }
@@ -2718,7 +2816,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
 /// [C++]   declarator-id
 ///
 ///       declarator-id: [C++ 8]
-///         id-expression
+///         '...'[opt] id-expression
 ///         '::'[opt] nested-name-specifier[opt] type-name
 ///
 ///       id-expression: [C++ 5.1]
@@ -2748,6 +2846,20 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
         DeclScopeObj.EnterDeclaratorScope();
     }
 
+    // C++0x [dcl.fct]p14:
+    //   There is a syntactic ambiguity when an ellipsis occurs at the end
+    //   of a parameter-declaration-clause without a preceding comma. In 
+    //   this case, the ellipsis is parsed as part of the 
+    //   abstract-declarator if the type of the parameter names a template 
+    //   parameter pack that has not been expanded; otherwise, it is parsed
+    //   as part of the parameter-declaration-clause.
+    if (Tok.is(tok::ellipsis) &&
+        !((D.getContext() == Declarator::PrototypeContext ||
+           D.getContext() == Declarator::BlockLiteralContext) &&
+          NextToken().is(tok::r_paren) &&
+          !Actions.containsUnexpandedParameterPacks(D)))
+      D.setEllipsisLoc(ConsumeToken());
+    
     if (Tok.is(tok::identifier) || Tok.is(tok::kw_operator) ||
         Tok.is(tok::annot_template_id) || Tok.is(tok::tilde)) {
       // We found something that indicates the start of an unqualified-id.
@@ -2830,12 +2942,8 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
          "Haven't past the location of the identifier yet?");
 
   // Don't parse attributes unless we have an identifier.
-  if (D.getIdentifier() && getLang().CPlusPlus0x
-   && isCXX0XAttributeSpecifier(true)) {
-    SourceLocation AttrEndLoc;
-    CXX0XAttributeList Attr = ParseCXX0XAttributes();
-    D.AddAttributes(Attr.AttrList, AttrEndLoc);
-  }
+  if (D.getIdentifier())
+    MaybeParseCXX0XAttributes(D);
 
   while (1) {
     if (Tok.is(tok::l_paren)) {
@@ -2849,7 +2957,8 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
         if (!isCXXFunctionDeclarator(warnIfAmbiguous))
           break;
       }
-      ParseFunctionDeclarator(ConsumeParen(), D);
+      ParsedAttributes attrs;
+      ParseFunctionDeclarator(ConsumeParen(), D, attrs);
     } else if (Tok.is(tok::l_square)) {
       ParseBracketDeclarator(D);
     } else {
@@ -2885,10 +2994,10 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   // In either case, we need to eat any attributes to be able to determine what
   // sort of paren this is.
   //
-  llvm::OwningPtr<AttributeList> AttrList;
+  ParsedAttributes attrs;
   bool RequiresArg = false;
   if (Tok.is(tok::kw___attribute)) {
-    AttrList.reset(ParseGNUAttributes());
+    ParseGNUAttributes(attrs);
 
     // We require that the argument list (if this is a non-grouping paren) be
     // present even if the attribute list was empty.
@@ -2898,12 +3007,11 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   if  (Tok.is(tok::kw___cdecl) || Tok.is(tok::kw___stdcall) ||
        Tok.is(tok::kw___thiscall) || Tok.is(tok::kw___fastcall) ||
        Tok.is(tok::kw___w64) || Tok.is(tok::kw___ptr64)) {
-    AttrList.reset(ParseMicrosoftTypeAttributes(AttrList.take()));
+    ParseMicrosoftTypeAttributes(attrs);
   }
   // Eat any Borland extensions.
-  if  (Tok.is(tok::kw___pascal)) {
-    AttrList.reset(ParseBorlandTypeAttributes(AttrList.take()));
-  }
+  if  (Tok.is(tok::kw___pascal))
+    ParseBorlandTypeAttributes(attrs);
 
   // If we haven't past the identifier yet (or where the identifier would be
   // stored, if this is an abstract declarator), then this is probably just
@@ -2932,15 +3040,15 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   if (isGrouping) {
     bool hadGroupingParens = D.hasGroupingParens();
     D.setGroupingParens(true);
-    if (AttrList)
-      D.AddAttributes(AttrList.take(), SourceLocation());
+    if (!attrs.empty())
+      D.addAttributes(attrs.getList(), SourceLocation());
 
     ParseDeclaratorInternal(D, &Parser::ParseDirectDeclarator);
     // Match the ')'.
-    SourceLocation Loc = MatchRHSPunctuation(tok::r_paren, StartLoc);
+    SourceLocation EndLoc = MatchRHSPunctuation(tok::r_paren, StartLoc);
+    D.AddTypeInfo(DeclaratorChunk::getParen(StartLoc, EndLoc), EndLoc);
 
     D.setGroupingParens(hadGroupingParens);
-    D.SetRangeEnd(Loc);
     return;
   }
 
@@ -2950,7 +3058,7 @@ void Parser::ParseParenDeclarator(Declarator &D) {
   // ParseFunctionDeclarator to handle of argument list.
   D.SetIdentifier(0, Tok.getLocation());
 
-  ParseFunctionDeclarator(StartLoc, D, AttrList.take(), RequiresArg);
+  ParseFunctionDeclarator(StartLoc, D, attrs, RequiresArg);
 }
 
 /// ParseFunctionDeclarator - We are after the identifier and have parsed the
@@ -2982,37 +3090,51 @@ void Parser::ParseParenDeclarator(Declarator &D) {
 ///           '=' assignment-expression
 /// [GNU]   declaration-specifiers abstract-declarator[opt] attributes
 ///
-/// For C++, after the parameter-list, it also parses "cv-qualifier-seq[opt]"
-/// and "exception-specification[opt]".
+/// For C++, after the parameter-list, it also parses "cv-qualifier-seq[opt]",
+/// C++0x "ref-qualifier[opt]" and "exception-specification[opt]".
 ///
 void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
-                                     AttributeList *AttrList,
+                                     ParsedAttributes &attrs,
                                      bool RequiresArg) {
   // lparen is already consumed!
   assert(D.isPastIdentifier() && "Should not call before identifier!");
 
+  ParsedType TrailingReturnType;
+
   // This parameter list may be empty.
   if (Tok.is(tok::r_paren)) {
-    if (RequiresArg) {
+    if (RequiresArg)
       Diag(Tok, diag::err_argument_required_after_attribute);
-      delete AttrList;
-    }
 
     SourceLocation RParenLoc = ConsumeParen();  // Eat the closing ')'.
     SourceLocation EndLoc = RParenLoc;
 
     // cv-qualifier-seq[opt].
     DeclSpec DS;
+    SourceLocation RefQualifierLoc;
+    bool RefQualifierIsLValueRef = true;
     bool hasExceptionSpec = false;
     SourceLocation ThrowLoc;
     bool hasAnyExceptionSpec = false;
     llvm::SmallVector<ParsedType, 2> Exceptions;
     llvm::SmallVector<SourceRange, 2> ExceptionRanges;
     if (getLang().CPlusPlus) {
+      MaybeParseCXX0XAttributes(attrs);
+
       ParseTypeQualifierListOpt(DS, false /*no attributes*/);
       if (!DS.getSourceRange().getEnd().isInvalid())
         EndLoc = DS.getSourceRange().getEnd();
 
+      // Parse ref-qualifier[opt]
+      if (Tok.is(tok::amp) || Tok.is(tok::ampamp)) {
+        if (!getLang().CPlusPlus0x)
+          Diag(Tok, diag::ext_ref_qualifier);
+        
+        RefQualifierIsLValueRef = Tok.is(tok::amp);
+        RefQualifierLoc = ConsumeToken();
+        EndLoc = RefQualifierLoc;
+      }
+      
       // Parse exception-specification[opt].
       if (Tok.is(tok::kw_throw)) {
         hasExceptionSpec = true;
@@ -3022,21 +3144,30 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
         assert(Exceptions.size() == ExceptionRanges.size() &&
                "Produced different number of exception types and ranges.");
       }
+
+      // Parse trailing-return-type.
+      if (getLang().CPlusPlus0x && Tok.is(tok::arrow)) {
+        TrailingReturnType = ParseTrailingReturnType().get();
+      }
     }
 
     // Remember that we parsed a function type, and remember the attributes.
     // int() -> no prototype, no '...'.
-    D.AddTypeInfo(DeclaratorChunk::getFunction(/*prototype*/getLang().CPlusPlus,
+    D.AddTypeInfo(DeclaratorChunk::getFunction(attrs,
+                                               /*prototype*/getLang().CPlusPlus,
                                                /*variadic*/ false,
                                                SourceLocation(),
                                                /*arglist*/ 0, 0,
                                                DS.getTypeQualifiers(),
+                                               RefQualifierIsLValueRef,
+                                               RefQualifierLoc,
                                                hasExceptionSpec, ThrowLoc,
                                                hasAnyExceptionSpec,
                                                Exceptions.data(),
                                                ExceptionRanges.data(),
                                                Exceptions.size(),
-                                               LParenLoc, RParenLoc, D),
+                                               LParenLoc, RParenLoc, D,
+                                               TrailingReturnType),
                   EndLoc);
     return;
   }
@@ -3048,10 +3179,8 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
     if (TryAnnotateTypeOrScopeToken() || !Tok.is(tok::annot_typename)) {
       // K&R identifier lists can't have typedefs as identifiers, per
       // C99 6.7.5.3p11.
-      if (RequiresArg) {
+      if (RequiresArg)
         Diag(Tok, diag::err_argument_required_after_attribute);
-        delete AttrList;
-      }
       
       // Identifier list.  Note that '(' identifier-list ')' is only allowed for
       // normal declarators, not for abstract-declarators.  Get the first
@@ -3101,17 +3230,20 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
       break;
     }
 
-    SourceLocation DSStart = Tok.getLocation();
-
     // Parse the declaration-specifiers.
     // Just use the ParsingDeclaration "scope" of the declarator.
     DeclSpec DS;
+	
+    // Skip any Microsoft attributes before a param.
+    if (getLang().Microsoft && Tok.is(tok::l_square))
+      ParseMicrosoftAttributes(DS.getAttributes());
+
+    SourceLocation DSStart = Tok.getLocation();
 
     // If the caller parsed attributes for the first argument, add them now.
-    if (AttrList) {
-      DS.AddAttributes(AttrList);
-      AttrList = 0;  // Only apply the attributes to the first parameter.
-    }
+    // Take them so that we only apply the attributes to the first parameter.
+    DS.takeAttributesFrom(attrs);
+
     ParseDeclarationSpecifiers(DS);
 
     // Parse the declarator.  This is "PrototypeContext", because we must
@@ -3120,11 +3252,7 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
     ParseDeclarator(ParmDecl);
 
     // Parse GNU attributes, if present.
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      ParmDecl.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(ParmDecl);
 
     // Remember this parsed parameter in ParamInfo.
     IdentifierInfo *ParmII = ParmDecl.getIdentifier();
@@ -3184,6 +3312,11 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
           // Consume the '='.
           ConsumeToken();
 
+          // The argument isn't actually potentially evaluated unless it is 
+          // used.
+          EnterExpressionEvaluationContext Eval(Actions,
+                                              Sema::PotentiallyEvaluatedIfUsed);
+
           ExprResult DefArgResult(ParseAssignmentExpression());
           if (DefArgResult.isInvalid()) {
             Actions.ActOnParamDefaultArgumentError(Param);
@@ -3222,14 +3355,13 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
     ConsumeToken();
   }
 
-  // Leave prototype scope.
-  PrototypeScope.Exit();
-
   // If we have the closing ')', eat it.
   SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
   SourceLocation EndLoc = RParenLoc;
 
   DeclSpec DS;
+  SourceLocation RefQualifierLoc;
+  bool RefQualifierIsLValueRef = true;
   bool hasExceptionSpec = false;
   SourceLocation ThrowLoc;
   bool hasAnyExceptionSpec = false;
@@ -3237,11 +3369,23 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
   llvm::SmallVector<SourceRange, 2> ExceptionRanges;
   
   if (getLang().CPlusPlus) {
+    MaybeParseCXX0XAttributes(attrs);
+
     // Parse cv-qualifier-seq[opt].
     ParseTypeQualifierListOpt(DS, false /*no attributes*/);
       if (!DS.getSourceRange().getEnd().isInvalid())
         EndLoc = DS.getSourceRange().getEnd();
 
+    // Parse ref-qualifier[opt]
+    if (Tok.is(tok::amp) || Tok.is(tok::ampamp)) {
+      if (!getLang().CPlusPlus0x)
+        Diag(Tok, diag::ext_ref_qualifier);
+      
+      RefQualifierIsLValueRef = Tok.is(tok::amp);
+      RefQualifierLoc = ConsumeToken();
+      EndLoc = RefQualifierLoc;
+    }
+
     // Parse exception-specification[opt].
     if (Tok.is(tok::kw_throw)) {
       hasExceptionSpec = true;
@@ -3251,19 +3395,34 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
       assert(Exceptions.size() == ExceptionRanges.size() &&
              "Produced different number of exception types and ranges.");
     }
+
+    // Parse trailing-return-type.
+    if (getLang().CPlusPlus0x && Tok.is(tok::arrow)) {
+      TrailingReturnType = ParseTrailingReturnType().get();
+    }
   }
 
+  // FIXME: We should leave the prototype scope before parsing the exception
+  // specification, and then reenter it when parsing the trailing return type.
+
+  // Leave prototype scope.
+  PrototypeScope.Exit();
+
   // Remember that we parsed a function type, and remember the attributes.
-  D.AddTypeInfo(DeclaratorChunk::getFunction(/*proto*/true, IsVariadic,
+  D.AddTypeInfo(DeclaratorChunk::getFunction(attrs,
+                                             /*proto*/true, IsVariadic,
                                              EllipsisLoc,
                                              ParamInfo.data(), ParamInfo.size(),
                                              DS.getTypeQualifiers(),
+                                             RefQualifierIsLValueRef,
+                                             RefQualifierLoc,
                                              hasExceptionSpec, ThrowLoc,
                                              hasAnyExceptionSpec,
                                              Exceptions.data(),
                                              ExceptionRanges.data(),
                                              Exceptions.size(),
-                                             LParenLoc, RParenLoc, D),
+                                             LParenLoc, RParenLoc, D,
+                                             TrailingReturnType),
                 EndLoc);
 }
 
@@ -3333,10 +3492,12 @@ void Parser::ParseFunctionDeclaratorIdentifierList(SourceLocation LParenLoc,
   // Remember that we parsed a function type, and remember the attributes.  This
   // function type is always a K&R style function type, which is not varargs and
   // has no prototype.
-  D.AddTypeInfo(DeclaratorChunk::getFunction(/*proto*/false, /*varargs*/false,
+  D.AddTypeInfo(DeclaratorChunk::getFunction(ParsedAttributes(),
+                                             /*proto*/false, /*varargs*/false,
                                              SourceLocation(),
                                              &ParamInfo[0], ParamInfo.size(),
                                              /*TypeQuals*/0,
+                                             true, SourceLocation(),
                                              /*exception*/false,
                                              SourceLocation(), false, 0, 0, 0,
                                              LParenLoc, RLoc, D),
@@ -3355,15 +3516,12 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
   // This code does a fast path to handle some of the most obvious cases.
   if (Tok.getKind() == tok::r_square) {
     SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
-    //FIXME: Use these
-    CXX0XAttributeList Attr;
-    if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier(true)) {
-      Attr = ParseCXX0XAttributes();
-    }
+    ParsedAttributes attrs;
+    MaybeParseCXX0XAttributes(attrs);
     
     // Remember that we parsed the empty array type.
     ExprResult NumElements;
-    D.AddTypeInfo(DeclaratorChunk::getArray(0, false, false, 0,
+    D.AddTypeInfo(DeclaratorChunk::getArray(0, attrs, false, false, 0,
                                             StartLoc, EndLoc),
                   EndLoc);
     return;
@@ -3374,18 +3532,12 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     ConsumeToken();
 
     SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
-    //FIXME: Use these
-    CXX0XAttributeList Attr;
-    if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
-      Attr = ParseCXX0XAttributes();
-    }
-
-    // If there was an error parsing the assignment-expression, recover.
-    if (ExprRes.isInvalid())
-      ExprRes.release();  // Deallocate expr, just use [].
+    ParsedAttributes attrs;
+    MaybeParseCXX0XAttributes(attrs);
 
     // Remember that we parsed a array type, and remember its features.
-    D.AddTypeInfo(DeclaratorChunk::getArray(0, false, 0, ExprRes.release(),
+    D.AddTypeInfo(DeclaratorChunk::getArray(0, attrs, false, 0,
+                                            ExprRes.release(),
                                             StartLoc, EndLoc),
                   EndLoc);
     return;
@@ -3446,14 +3598,11 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
 
   SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
 
-  //FIXME: Use these
-  CXX0XAttributeList Attr;
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
-    Attr = ParseCXX0XAttributes();
-  }
+  ParsedAttributes attrs;
+  MaybeParseCXX0XAttributes(attrs);
 
   // Remember that we parsed a array type, and remember its features.
-  D.AddTypeInfo(DeclaratorChunk::getArray(DS.getTypeQualifiers(),
+  D.AddTypeInfo(DeclaratorChunk::getArray(DS.getTypeQualifiers(), attrs,
                                           StaticLoc.isValid(), isStar,
                                           NumElements.release(),
                                           StartLoc, EndLoc),
diff --git a/lib/Parse/ParseDeclCXX.cpp b/lib/Parse/ParseDeclCXX.cpp
index b277156a0d06..b3ad25b024fc 100644
--- a/lib/Parse/ParseDeclCXX.cpp
+++ b/lib/Parse/ParseDeclCXX.cpp
@@ -69,16 +69,16 @@ Decl *Parser::ParseNamespace(unsigned Context,
   }
 
   // Read label attributes, if present.
-  llvm::OwningPtr<AttributeList> AttrList;
+  ParsedAttributes attrs;
   if (Tok.is(tok::kw___attribute)) {
     attrTok = Tok;
 
     // FIXME: save these somewhere.
-    AttrList.reset(ParseGNUAttributes());
+    ParseGNUAttributes(attrs);
   }
 
   if (Tok.is(tok::equal)) {
-    if (AttrList)
+    if (!attrs.empty())
       Diag(attrTok, diag::err_unexpected_namespace_attributes_alias);
     if (InlineLoc.isValid())
       Diag(InlineLoc, diag::err_inline_namespace_alias)
@@ -112,16 +112,16 @@ Decl *Parser::ParseNamespace(unsigned Context,
 
   Decl *NamespcDecl =
     Actions.ActOnStartNamespaceDef(getCurScope(), InlineLoc, IdentLoc, Ident,
-                                   LBrace, AttrList.get());
+                                   LBrace, attrs.getList());
 
   PrettyDeclStackTraceEntry CrashInfo(Actions, NamespcDecl, NamespaceLoc,
                                       "parsing namespace");
 
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
-    CXX0XAttributeList Attr;
-    if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-      Attr = ParseCXX0XAttributes();
-    ParseExternalDeclaration(Attr);
+    ParsedAttributesWithRange attrs;
+    MaybeParseCXX0XAttributes(attrs);
+    MaybeParseMicrosoftAttributes(attrs);
+    ParseExternalDeclaration(attrs);
   }
 
   // Leave the namespace scope.
@@ -181,11 +181,9 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 ///         'extern' string-literal '{' declaration-seq[opt] '}'
 ///         'extern' string-literal declaration
 ///
-Decl *Parser::ParseLinkage(ParsingDeclSpec &DS,
-                                       unsigned Context) {
+Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, unsigned Context) {
   assert(Tok.is(tok::string_literal) && "Not a string literal!");
   llvm::SmallString<8> LangBuffer;
-  // LangBuffer is guaranteed to be big enough.
   bool Invalid = false;
   llvm::StringRef Lang = PP.getSpelling(Tok, LangBuffer, &Invalid);
   if (Invalid)
@@ -201,41 +199,40 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS,
                                        Tok.is(tok::l_brace)? Tok.getLocation()
                                                            : SourceLocation());
 
-  CXX0XAttributeList Attr;
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
-    Attr = ParseCXX0XAttributes();
-  }
+  ParsedAttributesWithRange attrs;
+  MaybeParseCXX0XAttributes(attrs);
+  MaybeParseMicrosoftAttributes(attrs);
 
   if (Tok.isNot(tok::l_brace)) {
     DS.setExternInLinkageSpec(true);
-    ParseExternalDeclaration(Attr, &DS);
+    ParseExternalDeclaration(attrs, &DS);
     return Actions.ActOnFinishLinkageSpecification(getCurScope(), LinkageSpec,
                                                    SourceLocation());
   }
 
   DS.abort();
 
-  if (Attr.HasAttr)
-    Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-      << Attr.Range;
+  ProhibitAttributes(attrs);
 
   SourceLocation LBrace = ConsumeBrace();
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
-    CXX0XAttributeList Attr;
-    if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-      Attr = ParseCXX0XAttributes();
-    ParseExternalDeclaration(Attr);
+    ParsedAttributesWithRange attrs;
+    MaybeParseCXX0XAttributes(attrs);
+    MaybeParseMicrosoftAttributes(attrs);
+    ParseExternalDeclaration(attrs);
   }
 
   SourceLocation RBrace = MatchRHSPunctuation(tok::r_brace, LBrace);
-  return Actions.ActOnFinishLinkageSpecification(getCurScope(), LinkageSpec, RBrace);
+  return Actions.ActOnFinishLinkageSpecification(getCurScope(), LinkageSpec,
+                                                 RBrace);
 }
 
 /// ParseUsingDirectiveOrDeclaration - Parse C++ using using-declaration or
 /// using-directive. Assumes that current token is 'using'.
 Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
-                                                     SourceLocation &DeclEnd,
-                                                     CXX0XAttributeList Attr) {
+                                         const ParsedTemplateInfo &TemplateInfo,
+                                               SourceLocation &DeclEnd,
+                                             ParsedAttributesWithRange &attrs) {
   assert(Tok.is(tok::kw_using) && "Not using token");
 
   // Eat 'using'.
@@ -246,17 +243,24 @@ Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
     ConsumeCodeCompletionToken();
   }
 
-  if (Tok.is(tok::kw_namespace))
-    // Next token after 'using' is 'namespace' so it must be using-directive
-    return ParseUsingDirective(Context, UsingLoc, DeclEnd, Attr.AttrList);
+  // 'using namespace' means this is a using-directive.
+  if (Tok.is(tok::kw_namespace)) {
+    // Template parameters are always an error here.
+    if (TemplateInfo.Kind) {
+      SourceRange R = TemplateInfo.getSourceRange();
+      Diag(UsingLoc, diag::err_templated_using_directive)
+        << R << FixItHint::CreateRemoval(R);
+    }
+
+    return ParseUsingDirective(Context, UsingLoc, DeclEnd, attrs);
+  }
+
+  // Otherwise, it must be a using-declaration.
 
-  if (Attr.HasAttr)
-    Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-      << Attr.Range;
+  // Using declarations can't have attributes.
+  ProhibitAttributes(attrs);
 
-  // Otherwise, it must be using-declaration.
-  // Ignore illegal attributes (the caller should already have issued an error.
-  return ParseUsingDeclaration(Context, UsingLoc, DeclEnd);
+  return ParseUsingDeclaration(Context, TemplateInfo, UsingLoc, DeclEnd);
 }
 
 /// ParseUsingDirective - Parse C++ using-directive, assumes
@@ -270,9 +274,9 @@ Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
 ///                 namespace-name attributes[opt] ;
 ///
 Decl *Parser::ParseUsingDirective(unsigned Context,
-                                              SourceLocation UsingLoc,
-                                              SourceLocation &DeclEnd,
-                                              AttributeList *Attr) {
+                                  SourceLocation UsingLoc,
+                                  SourceLocation &DeclEnd,
+                                  ParsedAttributes &attrs) {
   assert(Tok.is(tok::kw_namespace) && "Not 'namespace' token");
 
   // Eat 'namespace'.
@@ -307,17 +311,18 @@ Decl *Parser::ParseUsingDirective(unsigned Context,
   bool GNUAttr = false;
   if (Tok.is(tok::kw___attribute)) {
     GNUAttr = true;
-    Attr = addAttributeLists(Attr, ParseGNUAttributes());
+    ParseGNUAttributes(attrs);
   }
 
   // Eat ';'.
   DeclEnd = Tok.getLocation();
   ExpectAndConsume(tok::semi,
-                   GNUAttr ? diag::err_expected_semi_after_attribute_list :
-                   diag::err_expected_semi_after_namespace_name, "", tok::semi);
+                   GNUAttr ? diag::err_expected_semi_after_attribute_list
+                           : diag::err_expected_semi_after_namespace_name, 
+                   "", tok::semi);
 
   return Actions.ActOnUsingDirective(getCurScope(), UsingLoc, NamespcLoc, SS,
-                                      IdentLoc, NamespcName, Attr);
+                                     IdentLoc, NamespcName, attrs.getList());
 }
 
 /// ParseUsingDeclaration - Parse C++ using-declaration. Assumes that
@@ -329,13 +334,18 @@ Decl *Parser::ParseUsingDirective(unsigned Context,
 ///       'using' :: unqualified-id
 ///
 Decl *Parser::ParseUsingDeclaration(unsigned Context,
-                                                SourceLocation UsingLoc,
-                                                SourceLocation &DeclEnd,
-                                                AccessSpecifier AS) {
+                                    const ParsedTemplateInfo &TemplateInfo,
+                                    SourceLocation UsingLoc,
+                                    SourceLocation &DeclEnd,
+                                    AccessSpecifier AS) {
   CXXScopeSpec SS;
   SourceLocation TypenameLoc;
   bool IsTypeName;
 
+  // TODO: in C++0x, if we have template parameters this must be a
+  // template alias:
+  //   template <...> using id = type;
+
   // Ignore optional 'typename'.
   // FIXME: This is wrong; we should parse this as a typename-specifier.
   if (Tok.is(tok::kw_typename)) {
@@ -370,18 +380,30 @@ Decl *Parser::ParseUsingDeclaration(unsigned Context,
   }
 
   // Parse (optional) attributes (most likely GNU strong-using extension).
-  llvm::OwningPtr<AttributeList> AttrList;
-  if (Tok.is(tok::kw___attribute))
-    AttrList.reset(ParseGNUAttributes());
+  ParsedAttributes attrs;
+  MaybeParseGNUAttributes(attrs);
 
   // Eat ';'.
   DeclEnd = Tok.getLocation();
   ExpectAndConsume(tok::semi, diag::err_expected_semi_after,
-                   AttrList ? "attributes list" : "using declaration",
+                   !attrs.empty() ? "attributes list" : "using declaration",
                    tok::semi);
 
-  return Actions.ActOnUsingDeclaration(getCurScope(), AS, true, UsingLoc, SS, Name,
-                                       AttrList.get(), IsTypeName, TypenameLoc);
+  // Diagnose an attempt to declare a templated using-declaration.
+  if (TemplateInfo.Kind) {
+    SourceRange R = TemplateInfo.getSourceRange();
+    Diag(UsingLoc, diag::err_templated_using_declaration)
+      << R << FixItHint::CreateRemoval(R);
+
+    // Unfortunately, we have to bail out instead of recovering by
+    // ignoring the parameters, just in case the nested name specifier
+    // depends on the parameters.
+    return 0;
+  }
+
+  return Actions.ActOnUsingDeclaration(getCurScope(), AS, true, UsingLoc, SS,
+                                       Name, attrs.getList(),
+                                       IsTypeName, TypenameLoc);
 }
 
 /// ParseStaticAssertDeclaration - Parse C++0x static_assert-declaratoion.
@@ -422,7 +444,7 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
   MatchRHSPunctuation(tok::r_paren, LParenLoc);
 
   DeclEnd = Tok.getLocation();
-  ExpectAndConsume(tok::semi, diag::err_expected_semi_after_static_assert);
+  ExpectAndConsumeSemi(diag::err_expected_semi_after_static_assert);
 
   return Actions.ActOnStaticAssertDeclaration(StaticAssertLoc,
                                               AssertExpr.take(),
@@ -652,20 +674,19 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     SuppressingAccessChecks = true;
   }
 
-  AttributeList *AttrList = 0;
+  ParsedAttributes attrs;
   // If attributes exist after tag, parse them.
   if (Tok.is(tok::kw___attribute))
-    AttrList = ParseGNUAttributes();
+    ParseGNUAttributes(attrs);
 
   // If declspecs exist after tag, parse them.
   while (Tok.is(tok::kw___declspec))
-    AttrList = ParseMicrosoftDeclSpec(AttrList);
+    ParseMicrosoftDeclSpec(attrs);
 
   // If C++0x attributes exist here, parse them.
   // FIXME: Are we consistent with the ordering of parsing of different
   // styles of attributes?
-  if (isCXX0XAttributeSpecifier())
-    AttrList = addAttributeLists(AttrList, ParseCXX0XAttributes().AttrList);
+  MaybeParseCXX0XAttributes(attrs);
 
   if (TagType == DeclSpec::TST_struct && Tok.is(tok::kw___is_pod)) {
     // GNU libstdc++ 4.2 uses __is_pod as the name of a struct template, but
@@ -786,9 +807,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 
   // There are four options here.  If we have 'struct foo;', then this
   // is either a forward declaration or a friend declaration, which
-  // have to be treated differently.  If we have 'struct foo {...' or
-  // 'struct foo :...' then this is a definition. Otherwise we have
-  // something like 'struct foo xyz', a reference.
+  // have to be treated differently.  If we have 'struct foo {...',
+  // 'struct foo :...' or 'struct foo <class-virt-specifier>' then this is a
+  // definition. Otherwise we have something like 'struct foo xyz', a reference.
   // However, in some contexts, things look like declarations but are just
   // references, e.g.
   // new struct s;
@@ -798,7 +819,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   Sema::TagUseKind TUK;
   if (SuppressDeclarations)
     TUK = Sema::TUK_Reference;
-  else if (Tok.is(tok::l_brace) || (getLang().CPlusPlus && Tok.is(tok::colon))){
+  else if (Tok.is(tok::l_brace) || 
+           (getLang().CPlusPlus && Tok.is(tok::colon)) ||
+           isCXX0XClassVirtSpecifier() != ClassVirtSpecifiers::CVS_None) {
     if (DS.isFriendSpecified()) {
       // C++ [class.friend]p2:
       //   A class shall not be defined in a friend declaration.
@@ -859,7 +882,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                              TemplateId->LAngleLoc,
                                              TemplateArgsPtr,
                                              TemplateId->RAngleLoc,
-                                             AttrList);
+                                             attrs.getList());
 
     // Friend template-ids are treated as references unless
     // they have template headers, in which case they're ill-formed
@@ -875,7 +898,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                       TemplateArgsPtr,
                                       TemplateId->RAngleLoc);
 
-      TypeResult = Actions.ActOnTagTemplateIdType(TypeResult, TUK,
+      TypeResult = Actions.ActOnTagTemplateIdType(SS, TypeResult, TUK,
                                                   TagType, StartLoc);
     } else {
       // This is an explicit specialization or a class template
@@ -921,7 +944,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                        TemplateId->LAngleLoc,
                        TemplateArgsPtr,
                        TemplateId->RAngleLoc,
-                       AttrList,
+                       attrs.getList(),
                        MultiTemplateParamsArg(Actions,
                                     TemplateParams? &(*TemplateParams)[0] : 0,
                                  TemplateParams? TemplateParams->size() : 0));
@@ -939,7 +962,16 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                            TemplateInfo.ExternLoc,
                                            TemplateInfo.TemplateLoc,
                                            TagType, StartLoc, SS, Name,
-                                           NameLoc, AttrList);
+                                           NameLoc, attrs.getList());
+  } else if (TUK == Sema::TUK_Friend &&
+             TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate) {
+    TagOrTempResult =
+      Actions.ActOnTemplatedFriendTag(getCurScope(), DS.getFriendSpecLoc(),
+                                      TagType, StartLoc, SS,
+                                      Name, NameLoc, attrs.getList(),
+                                      MultiTemplateParamsArg(Actions,
+                                    TemplateParams? &(*TemplateParams)[0] : 0,
+                                 TemplateParams? TemplateParams->size() : 0));
   } else {
     if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
         TUK == Sema::TUK_Definition) {
@@ -948,25 +980,34 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 
     bool IsDependent = false;
 
+    // Don't pass down template parameter lists if this is just a tag
+    // reference.  For example, we don't need the template parameters here:
+    //   template <class T> class A *makeA(T t);
+    MultiTemplateParamsArg TParams;
+    if (TUK != Sema::TUK_Reference && TemplateParams)
+      TParams =
+        MultiTemplateParamsArg(&(*TemplateParams)[0], TemplateParams->size());
+
     // Declaration or definition of a class type
-    TagOrTempResult = Actions.ActOnTag(getCurScope(), TagType, TUK, StartLoc, SS,
-                                       Name, NameLoc, AttrList, AS,
-                                       MultiTemplateParamsArg(Actions,
-                                    TemplateParams? &(*TemplateParams)[0] : 0,
-                                    TemplateParams? TemplateParams->size() : 0),
-                                       Owned, IsDependent);
+    TagOrTempResult = Actions.ActOnTag(getCurScope(), TagType, TUK, StartLoc,
+                                       SS, Name, NameLoc, attrs.getList(), AS,
+                                       TParams, Owned, IsDependent, false,
+                                       false, clang::TypeResult());
 
     // If ActOnTag said the type was dependent, try again with the
     // less common call.
-    if (IsDependent)
+    if (IsDependent) {
+      assert(TUK == Sema::TUK_Reference || TUK == Sema::TUK_Friend);
       TypeResult = Actions.ActOnDependentTag(getCurScope(), TagType, TUK,
                                              SS, Name, StartLoc, NameLoc);
+    }
   }
 
   // If there is a body, parse it and inform the actions module.
   if (TUK == Sema::TUK_Definition) {
     assert(Tok.is(tok::l_brace) ||
-           (getLang().CPlusPlus && Tok.is(tok::colon)));
+           (getLang().CPlusPlus && Tok.is(tok::colon)) ||
+           isCXX0XClassVirtSpecifier() != ClassVirtSpecifiers::CVS_None);
     if (getLang().CPlusPlus)
       ParseCXXMemberSpecification(StartLoc, TagType, TagOrTempResult.get());
     else
@@ -1158,13 +1199,20 @@ Parser::BaseResult Parser::ParseBaseSpecifier(Decl *ClassDecl) {
   if (BaseType.isInvalid())
     return true;
 
+  // Parse the optional ellipsis (for a pack expansion). The ellipsis is 
+  // actually part of the base-specifier-list grammar productions, but we
+  // parse it here for convenience.
+  SourceLocation EllipsisLoc;
+  if (Tok.is(tok::ellipsis))
+    EllipsisLoc = ConsumeToken();
+  
   // Find the complete source range for the base-specifier.
   SourceRange Range(StartLoc, EndLocation);
 
   // Notify semantic analysis that we have parsed a complete
   // base-specifier.
   return Actions.ActOnBaseSpecifier(ClassDecl, Range, IsVirtual, Access,
-                                    BaseType.get(), BaseLoc);
+                                    BaseType.get(), BaseLoc, EllipsisLoc);
 }
 
 /// getAccessSpecifierIfPresent - Determine whether the next token is
@@ -1189,15 +1237,14 @@ void Parser::HandleMemberFunctionDefaultArgs(Declarator& DeclaratorInfo,
   // has any default arguments, we'll need to parse them later.
   LateParsedMethodDeclaration *LateMethod = 0;
   DeclaratorChunk::FunctionTypeInfo &FTI
-    = DeclaratorInfo.getTypeObject(0).Fun;
+    = DeclaratorInfo.getFunctionTypeInfo();
   for (unsigned ParamIdx = 0; ParamIdx < FTI.NumArgs; ++ParamIdx) {
     if (LateMethod || FTI.ArgInfo[ParamIdx].DefaultArgTokens) {
       if (!LateMethod) {
         // Push this method onto the stack of late-parsed method
         // declarations.
-        getCurrentClass().MethodDecls.push_back(
-                                LateParsedMethodDeclaration(ThisDecl));
-        LateMethod = &getCurrentClass().MethodDecls.back();
+        LateMethod = new LateParsedMethodDeclaration(this, ThisDecl);
+        getCurrentClass().LateParsedDeclarations.push_back(LateMethod);
         LateMethod->TemplateScope = getCurScope()->isTemplateParamScope();
 
         // Add all of the parameters prior to this one (they don't
@@ -1217,6 +1264,122 @@ void Parser::HandleMemberFunctionDefaultArgs(Declarator& DeclaratorInfo,
   }
 }
 
+/// isCXX0XVirtSpecifier - Determine whether the next token is a C++0x
+/// virt-specifier.
+///
+///       virt-specifier:
+///         override
+///         final
+///         new
+VirtSpecifiers::Specifier Parser::isCXX0XVirtSpecifier() const {
+  if (!getLang().CPlusPlus)
+    return VirtSpecifiers::VS_None;
+
+  if (Tok.is(tok::kw_new))
+    return VirtSpecifiers::VS_New;
+
+  if (Tok.is(tok::identifier)) {
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+
+    // Initialize the contextual keywords.
+    if (!Ident_final) {
+      Ident_final = &PP.getIdentifierTable().get("final");
+      Ident_override = &PP.getIdentifierTable().get("override");
+    }
+
+    if (II == Ident_override)
+      return VirtSpecifiers::VS_Override;
+
+    if (II == Ident_final)
+      return VirtSpecifiers::VS_Final;
+  }
+
+  return VirtSpecifiers::VS_None;
+}
+
+/// ParseOptionalCXX0XVirtSpecifierSeq - Parse a virt-specifier-seq.
+///
+///       virt-specifier-seq:
+///         virt-specifier
+///         virt-specifier-seq virt-specifier
+void Parser::ParseOptionalCXX0XVirtSpecifierSeq(VirtSpecifiers &VS) {
+  while (true) {
+    VirtSpecifiers::Specifier Specifier = isCXX0XVirtSpecifier();
+    if (Specifier == VirtSpecifiers::VS_None)
+      return;
+
+    // C++ [class.mem]p8:
+    //   A virt-specifier-seq shall contain at most one of each virt-specifier.
+    const char *PrevSpec = 0;
+    if (VS.SetSpecifier(Specifier, Tok.getLocation(), PrevSpec))
+      Diag(Tok.getLocation(), diag::err_duplicate_virt_specifier)
+        << PrevSpec
+        << FixItHint::CreateRemoval(Tok.getLocation());
+
+    if (!getLang().CPlusPlus0x)
+      Diag(Tok.getLocation(), diag::ext_override_control_keyword)
+        << VirtSpecifiers::getSpecifierName(Specifier);
+    ConsumeToken();
+  }
+}
+
+/// isCXX0XClassVirtSpecifier - Determine whether the next token is a C++0x
+/// class-virt-specifier.
+///
+///       class-virt-specifier:
+///         final
+///         explicit
+ClassVirtSpecifiers::Specifier Parser::isCXX0XClassVirtSpecifier() const {
+  if (!getLang().CPlusPlus)
+    return ClassVirtSpecifiers::CVS_None;
+
+  if (Tok.is(tok::kw_explicit))
+    return ClassVirtSpecifiers::CVS_Explicit;
+
+  if (Tok.is(tok::identifier)) {
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+  
+    // Initialize the contextual keywords.
+    if (!Ident_final) {
+      Ident_final = &PP.getIdentifierTable().get("final");
+      Ident_override = &PP.getIdentifierTable().get("override");
+    }
+
+    if (II == Ident_final)
+      return ClassVirtSpecifiers::CVS_Final;
+  }
+
+  return ClassVirtSpecifiers::CVS_None;
+}
+
+/// ParseOptionalCXX0XClassVirtSpecifierSeq - Parse a class-virt-specifier-seq.
+///
+///       class-virt-specifier-seq:
+///         class-virt-specifier
+///         class-virt-specifier-seq class-virt-specifier
+void Parser::ParseOptionalCXX0XClassVirtSpecifierSeq(ClassVirtSpecifiers &CVS) {
+  while (true) {
+    ClassVirtSpecifiers::Specifier Specifier = isCXX0XClassVirtSpecifier();
+    if (Specifier == ClassVirtSpecifiers::CVS_None)
+      return;
+
+    // C++ [class]p1:
+    // A class-virt-specifier-seq shall contain at most one of each 
+    // class-virt-specifier.
+    const char *PrevSpec = 0;
+    if (CVS.SetSpecifier(Specifier, Tok.getLocation(), PrevSpec))
+      Diag(Tok.getLocation(), diag::err_duplicate_class_virt_specifier)
+       << PrevSpec
+       << FixItHint::CreateRemoval(Tok.getLocation());
+
+    if (!getLang().CPlusPlus0x)
+      Diag(Tok.getLocation(), diag::ext_override_control_keyword)
+      << ClassVirtSpecifiers::getSpecifierName(Specifier);
+
+    ConsumeToken();
+  }
+}
+
 /// ParseCXXClassMemberDeclaration - Parse a C++ class member declaration.
 ///
 ///       member-declaration:
@@ -1233,10 +1396,19 @@ void Parser::HandleMemberFunctionDefaultArgs(Declarator& DeclaratorInfo,
 ///         member-declarator-list ',' member-declarator
 ///
 ///       member-declarator:
-///         declarator pure-specifier[opt]
+///         declarator virt-specifier-seq[opt] pure-specifier[opt]
 ///         declarator constant-initializer[opt]
 ///         identifier[opt] ':' constant-expression
 ///
+///       virt-specifier-seq:
+///         virt-specifier
+///         virt-specifier-seq virt-specifier
+///
+///       virt-specifier:
+///         override
+///         final
+///         new
+/// 
 ///       pure-specifier:
 ///         '= 0'
 ///
@@ -1315,17 +1487,15 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   // is a bitfield.
   ColonProtectionRAIIObject X(*this);
 
-  CXX0XAttributeList AttrList;
+  ParsedAttributesWithRange attrs;
   // Optional C++0x attribute-specifier
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-    AttrList = ParseCXX0XAttributes();
+  MaybeParseCXX0XAttributes(attrs);
+  MaybeParseMicrosoftAttributes(attrs);
 
   if (Tok.is(tok::kw_using)) {
     // FIXME: Check for template aliases
 
-    if (AttrList.HasAttr)
-      Diag(AttrList.Range.getBegin(), diag::err_attributes_not_allowed)
-        << AttrList.Range;
+    ProhibitAttributes(attrs);
 
     // Eat 'using'.
     SourceLocation UsingLoc = ConsumeToken();
@@ -1336,16 +1506,16 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     } else {
       SourceLocation DeclEnd;
       // Otherwise, it must be using-declaration.
-      ParseUsingDeclaration(Declarator::MemberContext, UsingLoc, DeclEnd, AS);
+      ParseUsingDeclaration(Declarator::MemberContext, TemplateInfo,
+                            UsingLoc, DeclEnd, AS);
     }
     return;
   }
 
-  SourceLocation DSStart = Tok.getLocation();
   // decl-specifier-seq:
   // Parse the common declaration-specifiers piece.
   ParsingDeclSpec DS(*this, TemplateDiags);
-  DS.AddAttributes(AttrList.AttrList);
+  DS.takeAttributesFrom(attrs);
   ParseDeclarationSpecifiers(DS, TemplateInfo, AS, DSC_class);
 
   MultiTemplateParamsArg TemplateParams(Actions,
@@ -1361,6 +1531,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   }
 
   ParsingDeclarator DeclaratorInfo(*this, DS, Declarator::MemberContext);
+  VirtSpecifiers VS;
 
   if (Tok.isNot(tok::colon)) {
     // Don't parse FOO:BAR as if it were a typo for FOO::BAR.
@@ -1377,12 +1548,10 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
       return;
     }
 
+    ParseOptionalCXX0XVirtSpecifierSeq(VS);
+
     // If attributes exist after the declarator, but before an '{', parse them.
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      DeclaratorInfo.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(DeclaratorInfo);
 
     // function-definition:
     if (Tok.is(tok::l_brace)
@@ -1392,6 +1561,10 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         Diag(Tok, diag::err_func_def_no_params);
         ConsumeBrace();
         SkipUntil(tok::r_brace, true);
+        
+        // Consume the optional ';'
+        if (Tok.is(tok::semi))
+          ConsumeToken();
         return;
       }
 
@@ -1402,10 +1575,18 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         // assumes the declarator represents a function, not a typedef.
         ConsumeBrace();
         SkipUntil(tok::r_brace, true);
+
+        // Consume the optional ';'
+        if (Tok.is(tok::semi))
+          ConsumeToken();
         return;
       }
 
-      ParseCXXInlineMethodDef(AS, DeclaratorInfo, TemplateInfo);
+      ParseCXXInlineMethodDef(AS, DeclaratorInfo, TemplateInfo, VS);
+      // Consume the optional ';'
+      if (Tok.is(tok::semi))
+        ConsumeToken();
+
       return;
     }
   }
@@ -1431,6 +1612,8 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
         SkipUntil(tok::comma, true, true);
     }
 
+    ParseOptionalCXX0XVirtSpecifierSeq(VS);
+
     // pure-specifier:
     //   '= 0'
     //
@@ -1442,7 +1625,9 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     //   '=' 'delete'
     if (Tok.is(tok::equal)) {
       ConsumeToken();
-      if (getLang().CPlusPlus0x && Tok.is(tok::kw_delete)) {
+      if (Tok.is(tok::kw_delete)) {
+        if (!getLang().CPlusPlus0x)
+          Diag(Tok, diag::warn_deleted_function_accepted_as_extension);
         ConsumeToken();
         Deleted = true;
       } else {
@@ -1464,11 +1649,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     }
 
     // If attributes exist after the declarator, parse them.
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      DeclaratorInfo.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(DeclaratorInfo);
 
     // NOTE: If Sema is the Action module and declarator is an instance field,
     // this call will *not* return the created decl; It will return null.
@@ -1485,7 +1666,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
                                                   DeclaratorInfo,
                                                   move(TemplateParams),
                                                   BitfieldSize.release(),
-                                                  Init.release(),
+                                                  VS, Init.release(),
                                                   /*IsDefinition*/Deleted,
                                                   Deleted);
     }
@@ -1510,16 +1691,13 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
 
     // Parse the next declarator.
     DeclaratorInfo.clear();
+    VS.clear();
     BitfieldSize = 0;
     Init = 0;
     Deleted = false;
 
     // Attributes are only allowed on the second declarator.
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      DeclaratorInfo.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(DeclaratorInfo);
 
     if (Tok.isNot(tok::colon))
       ParseDeclarator(DeclaratorInfo);
@@ -1585,6 +1763,9 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   if (TagDecl)
     Actions.ActOnTagStartDefinition(getCurScope(), TagDecl);
 
+  ClassVirtSpecifiers CVS;
+  ParseOptionalCXX0XClassVirtSpecifierSeq(CVS);
+
   if (Tok.is(tok::colon)) {
     ParseBaseClause(TagDecl);
 
@@ -1602,7 +1783,8 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   SourceLocation LBraceLoc = ConsumeBrace();
 
   if (TagDecl)
-    Actions.ActOnStartCXXMemberDeclarations(getCurScope(), TagDecl, LBraceLoc);
+    Actions.ActOnStartCXXMemberDeclarations(getCurScope(), TagDecl, CVS,
+                                            LBraceLoc);
 
   // C++ 11p3: Members of a class defined with the keyword class are private
   // by default. Members of a class defined with the keywords struct or union
@@ -1654,14 +1836,13 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   }
 
   // If attributes exist after class contents, parse them.
-  llvm::OwningPtr<AttributeList> AttrList;
-  if (Tok.is(tok::kw___attribute))
-    AttrList.reset(ParseGNUAttributes());
+  ParsedAttributes attrs;
+  MaybeParseGNUAttributes(attrs);
 
   if (TagDecl)
     Actions.ActOnFinishCXXMemberSpecification(getCurScope(), RecordLoc, TagDecl,
                                               LBraceLoc, RBraceLoc,
-                                              AttrList.get());
+                                              attrs.getList());
 
   // C++ 9.2p2: Within the class member-specification, the class is regarded as
   // complete within function bodies, default arguments,
@@ -1707,14 +1888,14 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
 ///          ':' mem-initializer-list
 ///
 /// [C++]  mem-initializer-list:
-///          mem-initializer
-///          mem-initializer , mem-initializer-list
+///          mem-initializer ...[opt]
+///          mem-initializer ...[opt] , mem-initializer-list
 void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
   assert(Tok.is(tok::colon) && "Constructor initializer always starts with ':'");
 
   SourceLocation ColonLoc = ConsumeToken();
 
-  llvm::SmallVector<CXXBaseOrMemberInitializer*, 4> MemInitializers;
+  llvm::SmallVector<CXXCtorInitializer*, 4> MemInitializers;
   bool AnyErrors = false;
 
   do {
@@ -1735,7 +1916,13 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
       ConsumeToken();
     else if (Tok.is(tok::l_brace))
       break;
-    else {
+    // If the next token looks like a base or member initializer, assume that
+    // we're just missing a comma.
+    else if (Tok.is(tok::identifier) || Tok.is(tok::coloncolon)) {
+      SourceLocation Loc = PP.getLocForEndOfToken(PrevTokLocation);
+      Diag(Loc, diag::err_ctor_init_missing_comma)
+        << FixItHint::CreateInsertion(Loc, ", ");
+    } else {
       // Skip over garbage, until we get to '{'.  Don't eat the '{'.
       Diag(Tok.getLocation(), diag::err_expected_lbrace_or_comma);
       SkipUntil(tok::l_brace, true, true);
@@ -1801,11 +1988,15 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 
   SourceLocation RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
 
+  SourceLocation EllipsisLoc;
+  if (Tok.is(tok::ellipsis))
+    EllipsisLoc = ConsumeToken();
+  
   return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
                                      TemplateTypeTy, IdLoc,
                                      LParenLoc, ArgExprs.take(),
-                                     ArgExprs.size(), CommaLocs.data(),
-                                     RParenLoc);
+                                     ArgExprs.size(), RParenLoc,
+                                     EllipsisLoc);
 }
 
 /// ParseExceptionSpecification - Parse a C++ exception-specification
@@ -1816,8 +2007,8 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 /// [MS]    'throw' '(' '...' ')'
 ///
 ///       type-id-list:
-///         type-id
-///         type-id-list ',' type-id
+///         type-id ... [opt]
+///         type-id-list ',' type-id ... [opt]
 ///
 bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
                                          llvm::SmallVectorImpl<ParsedType>
@@ -1827,7 +2018,7 @@ bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
                                          bool &hasAnyExceptionSpec) {
   assert(Tok.is(tok::kw_throw) && "expected throw");
 
-  SourceLocation ThrowLoc = ConsumeToken();
+  ConsumeToken();
 
   if (!Tok.is(tok::l_paren)) {
     return Diag(Tok, diag::err_expected_lparen_after) << "throw";
@@ -1849,10 +2040,21 @@ bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
   SourceRange Range;
   while (Tok.isNot(tok::r_paren)) {
     TypeResult Res(ParseTypeName(&Range));
+    
+    if (Tok.is(tok::ellipsis)) {
+      // C++0x [temp.variadic]p5:
+      //   - In a dynamic-exception-specification (15.4); the pattern is a 
+      //     type-id.
+      SourceLocation Ellipsis = ConsumeToken();
+      if (!Res.isInvalid())
+        Res = Actions.ActOnPackExpansion(Res.get(), Ellipsis);
+    }
+    
     if (!Res.isInvalid()) {
       Exceptions.push_back(Res.get());
       Ranges.push_back(Range);
     }
+    
     if (Tok.is(tok::comma))
       ConsumeToken();
     else
@@ -1863,20 +2065,41 @@ bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
   return false;
 }
 
+/// ParseTrailingReturnType - Parse a trailing return type on a new-style
+/// function declaration.
+TypeResult Parser::ParseTrailingReturnType() {
+  assert(Tok.is(tok::arrow) && "expected arrow");
+
+  ConsumeToken();
+
+  // FIXME: Need to suppress declarations when parsing this typename.
+  // Otherwise in this function definition:
+  //
+  //   auto f() -> struct X {}
+  //
+  // struct X is parsed as class definition because of the trailing
+  // brace.
+
+  SourceRange Range;
+  return ParseTypeName(&Range);
+}
+
 /// \brief We have just started parsing the definition of a new class,
 /// so push that class onto our stack of classes that is currently
 /// being parsed.
-void Parser::PushParsingClass(Decl *ClassDecl, bool NonNestedClass) {
+Sema::ParsingClassState
+Parser::PushParsingClass(Decl *ClassDecl, bool NonNestedClass) {
   assert((NonNestedClass || !ClassStack.empty()) &&
          "Nested class without outer class");
   ClassStack.push(new ParsingClass(ClassDecl, NonNestedClass));
+  return Actions.PushParsingClass();
 }
 
 /// \brief Deallocate the given parsed class and all of its nested
 /// classes.
 void Parser::DeallocateParsedClasses(Parser::ParsingClass *Class) {
-  for (unsigned I = 0, N = Class->NestedClasses.size(); I != N; ++I)
-    DeallocateParsedClasses(Class->NestedClasses[I]);
+  for (unsigned I = 0, N = Class->LateParsedDeclarations.size(); I != N; ++I)
+    delete Class->LateParsedDeclarations[I];
   delete Class;
 }
 
@@ -1889,9 +2112,11 @@ void Parser::DeallocateParsedClasses(Parser::ParsingClass *Class) {
 ///
 /// \returns true if the class we've popped is a top-level class,
 /// false otherwise.
-void Parser::PopParsingClass() {
+void Parser::PopParsingClass(Sema::ParsingClassState state) {
   assert(!ClassStack.empty() && "Mismatched push/pop for class parsing");
 
+  Actions.PopParsingClass(state);
+
   ParsingClass *Victim = ClassStack.top();
   ClassStack.pop();
   if (Victim->TopLevelClass) {
@@ -1902,13 +2127,12 @@ void Parser::PopParsingClass() {
   }
   assert(!ClassStack.empty() && "Missing top-level class?");
 
-  if (Victim->MethodDecls.empty() && Victim->MethodDefs.empty() &&
-      Victim->NestedClasses.empty()) {
+  if (Victim->LateParsedDeclarations.empty()) {
     // The victim is a nested class, but we will not need to perform
     // any processing after the definition of this class since it has
     // no members whose handling was delayed. Therefore, we can just
     // remove this nested class.
-    delete Victim;
+    DeallocateParsedClasses(Victim);
     return;
   }
 
@@ -1916,7 +2140,7 @@ void Parser::PopParsingClass() {
   // after the top-level class is completely defined. Therefore, add
   // it to the list of nested classes within its parent.
   assert(getCurScope()->isClassScope() && "Nested class outside of class scope?");
-  ClassStack.top()->NestedClasses.push_back(Victim);
+  ClassStack.top()->LateParsedDeclarations.push_back(new LateParsedClass(this, Victim));
   Victim->TemplateScope = getCurScope()->getParent()->isTemplateParamScope();
 }
 
@@ -1955,12 +2179,12 @@ void Parser::PopParsingClass() {
 ///         '[' balanced-token-seq ']'
 ///         '{' balanced-token-seq '}'
 ///         any token but '(', ')', '[', ']', '{', or '}'
-CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
+void Parser::ParseCXX0XAttributes(ParsedAttributesWithRange &attrs,
+                                  SourceLocation *endLoc) {
   assert(Tok.is(tok::l_square) && NextToken().is(tok::l_square)
       && "Not a C++0x attribute list");
 
   SourceLocation StartLoc = Tok.getLocation(), Loc;
-  AttributeList *CurrAttr = 0;
 
   ConsumeBracket();
   ConsumeBracket();
@@ -2004,21 +2228,16 @@ CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
       switch(AttributeList::getKind(AttrName))
       {
       // No arguments
-      case AttributeList::AT_base_check:
       case AttributeList::AT_carries_dependency:
-      case AttributeList::AT_final:
-      case AttributeList::AT_hiding:
-      case AttributeList::AT_noreturn:
-      case AttributeList::AT_override: {
+      case AttributeList::AT_noreturn: {
         if (Tok.is(tok::l_paren)) {
           Diag(Tok.getLocation(), diag::err_cxx0x_attribute_forbids_arguments)
             << AttrName->getName();
           break;
         }
 
-        CurrAttr = new AttributeList(AttrName, AttrLoc, 0, AttrLoc, 0,
-                                     SourceLocation(), 0, 0, CurrAttr, false,
-                                     true);
+        attrs.add(AttrFactory.Create(AttrName, AttrLoc, 0, AttrLoc, 0,
+                                     SourceLocation(), 0, 0, false, true));
         AttrParsed = true;
         break;
       }
@@ -2038,9 +2257,9 @@ CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
 
         ExprVector ArgExprs(Actions);
         ArgExprs.push_back(ArgExpr.release());
-        CurrAttr = new AttributeList(AttrName, AttrLoc, 0, AttrLoc,
-                                     0, ParamLoc, ArgExprs.take(), 1, CurrAttr,
-                                     false, true);
+        attrs.add(AttrFactory.Create(AttrName, AttrLoc, 0, AttrLoc,
+                                     0, ParamLoc, ArgExprs.take(), 1,
+                                     false, true));
 
         AttrParsed = true;
         break;
@@ -2065,8 +2284,7 @@ CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
   if (ExpectAndConsume(tok::r_square, diag::err_expected_rsquare))
     SkipUntil(tok::r_square, false);
 
-  CXX0XAttributeList Attr (CurrAttr, SourceRange(StartLoc, Loc), true);
-  return Attr;
+  attrs.Range = SourceRange(StartLoc, Loc);
 }
 
 /// ParseCXX0XAlignArgument - Parse the argument to C++0x's [[align]]
@@ -2088,3 +2306,23 @@ ExprResult Parser::ParseCXX0XAlignArgument(SourceLocation Start) {
   } else
     return ParseConstantExpression();
 }
+
+/// ParseMicrosoftAttributes - Parse a Microsoft attribute [Attr]
+///
+/// [MS] ms-attribute:
+///             '[' token-seq ']'
+///
+/// [MS] ms-attribute-seq:
+///             ms-attribute[opt]
+///             ms-attribute ms-attribute-seq
+void Parser::ParseMicrosoftAttributes(ParsedAttributes &attrs,
+                                      SourceLocation *endLoc) {
+  assert(Tok.is(tok::l_square) && "Not a Microsoft attribute list");
+
+  while (Tok.is(tok::l_square)) {
+    ConsumeBracket();
+    SkipUntil(tok::r_square, true, true);
+    if (endLoc) *endLoc = Tok.getLocation();
+    ExpectAndConsume(tok::r_square, diag::err_expected_rsquare);
+  }
+}
diff --git a/lib/Parse/ParseExpr.cpp b/lib/Parse/ParseExpr.cpp
index c4beab191d37..616c251583fb 100644
--- a/lib/Parse/ParseExpr.cpp
+++ b/lib/Parse/ParseExpr.cpp
@@ -172,13 +172,11 @@ static prec::Level getBinOpPrecedence(tok::TokenKind Kind,
 ///         = *= /= %= += -= <<= >>= &= ^= |=
 ///
 ///       expression: [C99 6.5.17]
-///         assignment-expression
-///         expression ',' assignment-expression
+///         assignment-expression ...[opt]
+///         expression ',' assignment-expression ...[opt]
 ///
 ExprResult Parser::ParseExpression() {
   ExprResult LHS(ParseAssignmentExpression());
-  if (LHS.isInvalid()) return move(LHS);
-
   return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
 }
 
@@ -190,8 +188,6 @@ ExprResult Parser::ParseExpression() {
 ExprResult
 Parser::ParseExpressionWithLeadingAt(SourceLocation AtLoc) {
   ExprResult LHS(ParseObjCAtExpression(AtLoc));
-  if (LHS.isInvalid()) return move(LHS);
-
   return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
 }
 
@@ -206,14 +202,13 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
     ExtensionRAIIObject O(Diags);
 
     LHS = ParseCastExpression(false);
-    if (LHS.isInvalid()) return move(LHS);
   }
 
-  LHS = Actions.ActOnUnaryOp(getCurScope(), ExtLoc, tok::kw___extension__,
-                             LHS.take());
-  if (LHS.isInvalid()) return move(LHS);
+  if (!LHS.isInvalid())
+    LHS = Actions.ActOnUnaryOp(getCurScope(), ExtLoc, tok::kw___extension__,
+                               LHS.take());
 
-  return ParseRHSOfBinaryExpression(LHS.take(), prec::Comma);
+  return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
 }
 
 /// ParseAssignmentExpression - Parse an expr that doesn't include commas.
@@ -228,9 +223,7 @@ ExprResult Parser::ParseAssignmentExpression() {
     return ParseThrowExpression();
 
   ExprResult LHS(ParseCastExpression(false));
-  if (LHS.isInvalid()) return move(LHS);
-
-  return ParseRHSOfBinaryExpression(LHS.take(), prec::Assignment);
+  return ParseRHSOfBinaryExpression(move(LHS), prec::Assignment);
 }
 
 /// ParseAssignmentExprWithObjCMessageExprStart - Parse an assignment expression
@@ -249,10 +242,8 @@ Parser::ParseAssignmentExprWithObjCMessageExprStart(SourceLocation LBracLoc,
   ExprResult R
     = ParseObjCMessageExpressionBody(LBracLoc, SuperLoc,
                                      ReceiverType, ReceiverExpr);
-  if (R.isInvalid()) return move(R);
-  R = ParsePostfixExpressionSuffix(R.take());
-  if (R.isInvalid()) return move(R);
-  return ParseRHSOfBinaryExpression(R.take(), prec::Assignment);
+  R = ParsePostfixExpressionSuffix(R);
+  return ParseRHSOfBinaryExpression(R, prec::Assignment);
 }
 
 
@@ -264,9 +255,7 @@ ExprResult Parser::ParseConstantExpression() {
                                                Sema::Unevaluated);
 
   ExprResult LHS(ParseCastExpression(false));
-  if (LHS.isInvalid()) return move(LHS);
-
-  return ParseRHSOfBinaryExpression(LHS.take(), prec::Conditional);
+  return ParseRHSOfBinaryExpression(LHS, prec::Conditional);
 }
 
 /// ParseRHSOfBinaryExpression - Parse a binary expression that starts with
@@ -301,8 +290,10 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
         // In particular, the RHS of the '?' is 'expression', not
         // 'logical-OR-expression' as we might expect.
         TernaryMiddle = ParseExpression();
-        if (TernaryMiddle.isInvalid())
-          return move(TernaryMiddle);
+        if (TernaryMiddle.isInvalid()) {
+          LHS = ExprError();
+          TernaryMiddle = 0;
+        }
       } else {
         // Special case handling of "X ? Y : Z" where Y is empty:
         //   logical-OR-expression '?' ':' conditional-expression   [GNU]
@@ -362,9 +353,10 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       RHS = ParseAssignmentExpression();
     else
       RHS = ParseCastExpression(false);
-    if (RHS.isInvalid())
-      return move(RHS);
 
+    if (RHS.isInvalid())
+      LHS = ExprError();
+    
     // Remember the precedence of this operator and get the precedence of the
     // operator immediately to the right of the RHS.
     prec::Level ThisPrec = NextTokPrec;
@@ -384,10 +376,11 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
       // is okay, to bind exactly as tightly.  For example, compile A=B=C=D as
       // A=(B=(C=D)), where each paren is a level of recursion here.
       // The function takes ownership of the RHS.
-      RHS = ParseRHSOfBinaryExpression(RHS.get(), 
+      RHS = ParseRHSOfBinaryExpression(RHS, 
                             static_cast<prec::Level>(ThisPrec + !isRightAssoc));
+
       if (RHS.isInvalid())
-        return move(RHS);
+        LHS = ExprError();
 
       NextTokPrec = getBinOpPrecedence(Tok.getKind(), GreaterThanIsOperator,
                                        getLang().CPlusPlus0x);
@@ -426,9 +419,9 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                                                      ParsedType TypeOfCast) {
   bool NotCastExpr;
   ExprResult Res = ParseCastExpression(isUnaryExpression,
-                                             isAddressOfOperand,
-                                             NotCastExpr,
-                                             TypeOfCast);
+                                       isAddressOfOperand,
+                                       NotCastExpr,
+                                       TypeOfCast);
   if (NotCastExpr)
     Diag(Tok, diag::err_expected_expression);
   return move(Res);
@@ -451,12 +444,14 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 ///         unary-operator cast-expression
 ///         'sizeof' unary-expression
 ///         'sizeof' '(' type-name ')'
+/// [C++0x] 'sizeof' '...' '(' identifier ')'
 /// [GNU]   '__alignof' unary-expression
 /// [GNU]   '__alignof' '(' type-name ')'
 /// [C++0x] 'alignof' '(' type-id ')'
 /// [GNU]   '&&' identifier
 /// [C++]   new-expression
 /// [C++]   delete-expression
+/// [C++0x] 'noexcept' '(' expression ')'
 ///
 ///       unary-operator: one of
 ///         '&'  '*'  '+'  '-'  '~'  '!'
@@ -542,13 +537,14 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 ///                   '__is_polymorphic'
 ///                   '__is_union'
 ///
-/// [GNU] binary-type-trait:
-///                   '__is_base_of'                          [TODO]
+///       binary-type-trait:
+/// [GNU]             '__is_base_of'       
+/// [MS]              '__is_convertible_to'
 ///
 ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
-                                                     bool isAddressOfOperand,
-                                                     bool &NotCastExpr,
-                                                     ParsedType TypeOfCast) {
+                                       bool isAddressOfOperand,
+                                       bool &NotCastExpr,
+                                       ParsedType TypeOfCast) {
   ExprResult Res;
   tok::TokenKind SavedKind = Tok.getKind();
   NotCastExpr = false;
@@ -571,17 +567,15 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     ParenParseOption ParenExprType =
       (isUnaryExpression && !getLang().CPlusPlus)? CompoundLiteral : CastExpr;
     ParsedType CastTy;
-    SourceLocation LParenLoc = Tok.getLocation();
     SourceLocation RParenLoc;
     
     {
-      // The inside of the parens don't need to be a colon protected scope.
+      // The inside of the parens don't need to be a colon protected scope, and
+      // isn't immediately a message send.
       ColonProtectionRAIIObject X(*this, false);
-    
+
       Res = ParseParenExpression(ParenExprType, false/*stopIfCastExr*/,
                                  TypeOfCast, CastTy, RParenLoc);
-      if (Res.isInvalid()) 
-        return move(Res);
     }
 
     switch (ParenExprType) {
@@ -647,7 +641,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
         (Actions.getTypeName(II, ILoc, getCurScope()) ||
          // Allow the base to be 'super' if in an objc-method.
          (&II == Ident_super && getCurScope()->isInObjcMethodScope()))) {
-      SourceLocation DotLoc = ConsumeToken();
+      ConsumeToken();
       
       if (Tok.isNot(tok::identifier)) {
         Diag(Tok, diag::err_expected_property_name);
@@ -661,6 +655,54 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
       break;
     }
 
+    // In an Objective-C method, if we have "super" followed by an identifier,
+    // the token sequence is ill-formed. However, if there's a ':' or ']' after
+    // that identifier, this is probably a message send with a missing open
+    // bracket. Treat it as such. 
+    if (getLang().ObjC1 && &II == Ident_super && !InMessageExpression &&
+        getCurScope()->isInObjcMethodScope() &&
+        ((Tok.is(tok::identifier) &&
+         (NextToken().is(tok::colon) || NextToken().is(tok::r_square))) ||
+         Tok.is(tok::code_completion))) {
+      Res = ParseObjCMessageExpressionBody(SourceLocation(), ILoc, ParsedType(), 
+                                           0);
+      break;
+    }
+    
+    // If we have an Objective-C class name followed by an identifier
+    // and either ':' or ']', this is an Objective-C class message
+    // send that's missing the opening '['. Recovery
+    // appropriately. Also take this path if we're performing code
+    // completion after an Objective-C class name.
+    if (getLang().ObjC1 && 
+        ((Tok.is(tok::identifier) && !InMessageExpression) || 
+         Tok.is(tok::code_completion))) {
+      const Token& Next = NextToken();
+      if (Tok.is(tok::code_completion) || 
+          Next.is(tok::colon) || Next.is(tok::r_square))
+        if (ParsedType Typ = Actions.getTypeName(II, ILoc, getCurScope()))
+          if (Typ.get()->isObjCObjectOrInterfaceType()) {
+            // Fake up a Declarator to use with ActOnTypeName.
+            DeclSpec DS;
+            DS.SetRangeStart(ILoc);
+            DS.SetRangeEnd(ILoc);
+            const char *PrevSpec = 0;
+            unsigned DiagID;
+            DS.SetTypeSpecType(TST_typename, ILoc, PrevSpec, DiagID, Typ);
+            
+            Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
+            TypeResult Ty = Actions.ActOnTypeName(getCurScope(), 
+                                                  DeclaratorInfo);
+            if (Ty.isInvalid())
+              break;
+            
+            Res = ParseObjCMessageExpressionBody(SourceLocation(), 
+                                                 SourceLocation(), 
+                                                 Ty.get(), 0);
+            break;
+          }
+    }
+    
     // Make sure to pass down the right value for isAddressOfOperand.
     if (isAddressOfOperand && isPostfixExpressionSuffixStart())
       isAddressOfOperand = false;
@@ -692,7 +734,6 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw___builtin_va_arg:
   case tok::kw___builtin_offsetof:
   case tok::kw___builtin_choose_expr:
-  case tok::kw___builtin_types_compatible_p:
     return ParseBuiltinPrimaryExpression();
   case tok::kw___null:
     return Actions.ActOnGNUNullExpr(ConsumeToken());
@@ -753,9 +794,13 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     if (Tok.isNot(tok::identifier))
       return ExprError(Diag(Tok, diag::err_expected_ident));
 
+    if (getCurScope()->getFnParent() == 0)
+      return ExprError(Diag(Tok, diag::err_address_of_label_outside_fn));
+    
     Diag(AmpAmpLoc, diag::ext_gnu_address_of_label);
-    Res = Actions.ActOnAddrLabel(AmpAmpLoc, Tok.getLocation(),
-                                 Tok.getIdentifierInfo());
+    LabelDecl *LD = Actions.LookupOrCreateLabel(Tok.getIdentifierInfo(),
+                                                Tok.getLocation());
+    Res = Actions.ActOnAddrLabel(AmpAmpLoc, Tok.getLocation(), LD);
     ConsumeToken();
     return move(Res);
   }
@@ -768,10 +813,39 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw_typeid:
     Res = ParseCXXTypeid();
     break;
+  case tok::kw___uuidof:
+    Res = ParseCXXUuidof();
+    break;
   case tok::kw_this:
     Res = ParseCXXThis();
     break;
 
+  case tok::annot_typename:
+    if (isStartOfObjCClassMessageMissingOpenBracket()) {
+      ParsedType Type = getTypeAnnotation(Tok);
+      
+      // Fake up a Declarator to use with ActOnTypeName.
+      DeclSpec DS;
+      DS.SetRangeStart(Tok.getLocation());
+      DS.SetRangeEnd(Tok.getLastLoc());
+
+      const char *PrevSpec = 0;
+      unsigned DiagID;
+      DS.SetTypeSpecType(TST_typename, Tok.getAnnotationEndLoc(),
+                         PrevSpec, DiagID, Type);
+      
+      Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
+      TypeResult Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+      if (Ty.isInvalid())
+        break;
+
+      ConsumeToken();
+      Res = ParseObjCMessageExpressionBody(SourceLocation(), SourceLocation(),
+                                           Ty.get(), 0);
+      break;
+    }
+    // Fall through
+      
   case tok::kw_char:
   case tok::kw_wchar_t:
   case tok::kw_char16_t:
@@ -787,8 +861,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw_void:
   case tok::kw_typename:
   case tok::kw_typeof:
-  case tok::kw___vector:
-  case tok::annot_typename: {
+  case tok::kw___vector: {
     if (!getLang().CPlusPlus) {
       Diag(Tok, diag::err_expected_expression);
       return ExprError();
@@ -888,6 +961,23 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw_delete: // [C++] delete-expression
     return ParseCXXDeleteExpression(false, Tok.getLocation());
 
+  case tok::kw_noexcept: { // [C++0x] 'noexcept' '(' expression ')'
+    SourceLocation KeyLoc = ConsumeToken();
+    SourceLocation LParen = Tok.getLocation();
+    if (ExpectAndConsume(tok::l_paren,
+                         diag::err_expected_lparen_after, "noexcept"))
+      return ExprError();
+    // C++ [expr.unary.noexcept]p1:
+    //   The noexcept operator determines whether the evaluation of its operand,
+    //   which is an unevaluated operand, can throw an exception.
+    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
+    ExprResult Result = ParseExpression();
+    SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+    if (!Result.isInvalid())
+      Result = Actions.ActOnNoexceptExpr(KeyLoc, LParen, Result.take(), RParen);
+    return move(Result);
+  }
+
   case tok::kw___is_pod: // [GNU] unary-type-trait
   case tok::kw___is_class:
   case tok::kw___is_enum:
@@ -906,6 +996,11 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw___has_virtual_destructor:
     return ParseUnaryTypeTrait();
 
+  case tok::kw___builtin_types_compatible_p:
+  case tok::kw___is_base_of:
+  case tok::kw___is_convertible_to:
+    return ParseBinaryTypeTrait();
+
   case tok::at: {
     SourceLocation AtLoc = ConsumeToken();
     return ParseObjCAtExpression(AtLoc);
@@ -928,8 +1023,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   }
 
   // These can be followed by postfix-expr pieces.
-  if (Res.isInvalid()) return move(Res);
-  return ParsePostfixExpressionSuffix(Res.get());
+  return ParsePostfixExpressionSuffix(Res);
 }
 
 /// ParsePostfixExpressionSuffix - Once the leading part of a postfix-expression
@@ -947,8 +1041,8 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 ///         '(' type-name ')' '{' initializer-list ',' '}'
 ///
 ///       argument-expression-list: [C99 6.5.2]
-///         argument-expression
-///         argument-expression-list ',' assignment-expression
+///         argument-expression ...[opt]
+///         argument-expression-list ',' assignment-expression ...[opt]
 ///
 ExprResult
 Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
@@ -957,6 +1051,28 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
   SourceLocation Loc;
   while (1) {
     switch (Tok.getKind()) {
+    case tok::code_completion:
+      if (InMessageExpression)
+        return move(LHS);
+        
+      Actions.CodeCompletePostfixExpression(getCurScope(), LHS);
+      ConsumeCodeCompletionToken();
+      LHS = ExprError();
+      break;
+        
+    case tok::identifier:
+      // If we see identifier: after an expression, and we're not already in a
+      // message send, then this is probably a message send with a missing
+      // opening bracket '['.
+      if (getLang().ObjC1 && !InMessageExpression && 
+          (NextToken().is(tok::colon) || NextToken().is(tok::r_square))) {
+        LHS = ParseObjCMessageExpressionBody(SourceLocation(), SourceLocation(),
+                                             ParsedType(), LHS.get());
+        break;
+      }
+        
+      // Fall through; this isn't a message send.
+                
     default:  // Not a postfix-expression suffix.
       return move(LHS);
     case tok::l_square: {  // postfix-expression: p-e '[' expression ']'
@@ -986,45 +1102,87 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       break;
     }
 
-    case tok::l_paren: {   // p-e: p-e '(' argument-expression-list[opt] ')'
-      ExprVector ArgExprs(Actions);
-      CommaLocsTy CommaLocs;
+    case tok::l_paren:         // p-e: p-e '(' argument-expression-list[opt] ')'
+    case tok::lesslessless: {  // p-e: p-e '<<<' argument-expression-list '>>>'
+                               //   '(' argument-expression-list[opt] ')'
+      tok::TokenKind OpKind = Tok.getKind();
+      InMessageExpressionRAIIObject InMessage(*this, false);
+      
+      Expr *ExecConfig = 0;
 
-      Loc = ConsumeParen();
+      if (OpKind == tok::lesslessless) {
+        ExprVector ExecConfigExprs(Actions);
+        CommaLocsTy ExecConfigCommaLocs;
+        SourceLocation LLLLoc, GGGLoc;
 
-      if (LHS.isInvalid()) {
-        SkipUntil(tok::r_paren);
-        return ExprError();
+        LLLLoc = ConsumeToken();
+
+        if (ParseExpressionList(ExecConfigExprs, ExecConfigCommaLocs)) {
+          LHS = ExprError();
+        }
+
+        if (LHS.isInvalid()) {
+          SkipUntil(tok::greatergreatergreater);
+        } else if (Tok.isNot(tok::greatergreatergreater)) {
+          MatchRHSPunctuation(tok::greatergreatergreater, LLLLoc);
+          LHS = ExprError();
+        } else {
+          GGGLoc = ConsumeToken();
+        }
+
+        if (!LHS.isInvalid()) {
+          if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen, ""))
+            LHS = ExprError();
+          else
+            Loc = PrevTokLocation;
+        }
+
+        if (!LHS.isInvalid()) {
+          ExprResult ECResult = Actions.ActOnCUDAExecConfigExpr(getCurScope(),
+                                     LLLLoc, move_arg(ExecConfigExprs), GGGLoc);
+          if (ECResult.isInvalid())
+            LHS = ExprError();
+          else
+            ExecConfig = ECResult.get();
+        }
+      } else {
+        Loc = ConsumeParen();
       }
 
+      ExprVector ArgExprs(Actions);
+      CommaLocsTy CommaLocs;
+      
       if (Tok.is(tok::code_completion)) {
         Actions.CodeCompleteCall(getCurScope(), LHS.get(), 0, 0);
         ConsumeCodeCompletionToken();
       }
-      
-      if (Tok.isNot(tok::r_paren)) {
-        if (ParseExpressionList(ArgExprs, CommaLocs, &Sema::CodeCompleteCall,
-                                LHS.get())) {
-          SkipUntil(tok::r_paren);
-          return ExprError();
+
+      if (OpKind == tok::l_paren || !LHS.isInvalid()) {
+        if (Tok.isNot(tok::r_paren)) {
+          if (ParseExpressionList(ArgExprs, CommaLocs, &Sema::CodeCompleteCall,
+                                  LHS.get())) {
+            SkipUntil(tok::r_paren);
+            LHS = ExprError();
+          }
         }
       }
 
       // Match the ')'.
-      if (Tok.isNot(tok::r_paren)) {
+      if (LHS.isInvalid()) {
+        SkipUntil(tok::r_paren);
+      } else if (Tok.isNot(tok::r_paren)) {
         MatchRHSPunctuation(tok::r_paren, Loc);
-        return ExprError();
-      }
-
-      if (!LHS.isInvalid()) {
-        assert((ArgExprs.size() == 0 || ArgExprs.size()-1 == CommaLocs.size())&&
+        LHS = ExprError();
+      } else {
+        assert((ArgExprs.size() == 0 || 
+                ArgExprs.size()-1 == CommaLocs.size())&&
                "Unexpected number of commas!");
         LHS = Actions.ActOnCallExpr(getCurScope(), LHS.take(), Loc,
-                                    move_arg(ArgExprs), CommaLocs.data(),
-                                    Tok.getLocation());
+                                    move_arg(ArgExprs), Tok.getLocation(),
+                                    ExecConfig);
+        ConsumeParen();
       }
 
-      ConsumeParen();
       break;
     }
     case tok::arrow:
@@ -1069,14 +1227,16 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       // expression), or we didn't see a '~' in the right place. We
       // can still parse a destructor name here, but in that case it
       // names a real destructor.
+      // Allow explicit constructor calls in Microsoft mode.
+      // FIXME: Add support for explicit call of template constructor.
       UnqualifiedId Name;
       if (ParseUnqualifiedId(SS, 
                              /*EnteringContext=*/false, 
                              /*AllowDestructorName=*/true,
-                             /*AllowConstructorName=*/false, 
+                             /*AllowConstructorName=*/ getLang().Microsoft, 
                              ObjectType,
                              Name))
-        return ExprError();
+        LHS = ExprError();
       
       if (!LHS.isInvalid())
         LHS = Actions.ActOnMemberAccessExpr(getCurScope(), LHS.take(), OpLoc, 
@@ -1189,6 +1349,7 @@ Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
 ///       unary-expression:  [C99 6.5.3]
 ///         'sizeof' unary-expression
 ///         'sizeof' '(' type-name ')'
+/// [C++0x] 'sizeof' '...' '(' identifier ')'
 /// [GNU]   '__alignof' unary-expression
 /// [GNU]   '__alignof' '(' type-name ')'
 /// [C++0x] 'alignof' '(' type-id ')'
@@ -1199,6 +1360,46 @@ ExprResult Parser::ParseSizeofAlignofExpression() {
   Token OpTok = Tok;
   ConsumeToken();
 
+  // [C++0x] 'sizeof' '...' '(' identifier ')'
+  if (Tok.is(tok::ellipsis) && OpTok.is(tok::kw_sizeof)) {
+    SourceLocation EllipsisLoc = ConsumeToken();
+    SourceLocation LParenLoc, RParenLoc;
+    IdentifierInfo *Name = 0;
+    SourceLocation NameLoc;
+    if (Tok.is(tok::l_paren)) {
+      LParenLoc = ConsumeParen();
+      if (Tok.is(tok::identifier)) {
+        Name = Tok.getIdentifierInfo();
+        NameLoc = ConsumeToken();
+        RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+        if (RParenLoc.isInvalid())
+          RParenLoc = PP.getLocForEndOfToken(NameLoc);
+      } else {
+        Diag(Tok, diag::err_expected_parameter_pack);
+        SkipUntil(tok::r_paren);
+      }
+    } else if (Tok.is(tok::identifier)) {
+      Name = Tok.getIdentifierInfo();
+      NameLoc = ConsumeToken();
+      LParenLoc = PP.getLocForEndOfToken(EllipsisLoc);
+      RParenLoc = PP.getLocForEndOfToken(NameLoc);
+      Diag(LParenLoc, diag::err_paren_sizeof_parameter_pack)
+        << Name
+        << FixItHint::CreateInsertion(LParenLoc, "(")
+        << FixItHint::CreateInsertion(RParenLoc, ")");
+    } else {
+      Diag(Tok, diag::err_sizeof_parameter_pack);
+    }
+    
+    if (!Name)
+      return ExprError();
+    
+    return Actions.ActOnSizeofParameterPackExpr(getCurScope(),
+                                                OpTok.getLocation(), 
+                                                *Name, NameLoc,
+                                                RParenLoc);
+  }
+  
   bool isCastExpr;
   ParsedType CastTy;
   SourceRange CastRange;
@@ -1256,21 +1457,18 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
   default: assert(0 && "Not a builtin primary expression!");
   case tok::kw___builtin_va_arg: {
     ExprResult Expr(ParseAssignmentExpression());
-    if (Expr.isInvalid()) {
-      SkipUntil(tok::r_paren);
-      return ExprError();
-    }
 
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
-      return ExprError();
+      Expr = ExprError();
 
     TypeResult Ty = ParseTypeName();
 
     if (Tok.isNot(tok::r_paren)) {
       Diag(Tok, diag::err_expected_rparen);
-      return ExprError();
+      Expr = ExprError();
     }
-    if (Ty.isInvalid())
+
+    if (Expr.isInvalid() || Ty.isInvalid())
       Res = ExprError();
     else
       Res = Actions.ActOnVAArg(StartLoc, Expr.take(), Ty.get(), ConsumeParen());
@@ -1378,25 +1576,6 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
                                   Expr2.take(), ConsumeParen());
     break;
   }
-  case tok::kw___builtin_types_compatible_p:
-    TypeResult Ty1 = ParseTypeName();
-
-    if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
-      return ExprError();
-
-    TypeResult Ty2 = ParseTypeName();
-
-    if (Tok.isNot(tok::r_paren)) {
-      Diag(Tok, diag::err_expected_rparen);
-      return ExprError();
-    }
-
-    if (Ty1.isInvalid() || Ty2.isInvalid())
-      Res = ExprError();
-    else
-      Res = Actions.ActOnTypesCompatibleExpr(StartLoc, Ty1.get(), Ty2.get(),
-                                             ConsumeParen());
-    break;
   }
 
   if (Res.isInvalid())
@@ -1432,9 +1611,18 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
   bool isAmbiguousTypeId;
   CastTy = ParsedType();
 
+  if (Tok.is(tok::code_completion)) {
+    Actions.CodeCompleteOrdinaryName(getCurScope(), 
+                 ExprType >= CompoundLiteral? Sema::PCC_ParenthesizedExpression
+                                            : Sema::PCC_Expression);
+    ConsumeCodeCompletionToken();
+    return ExprError();
+  }
+  
   if (ExprType >= CompoundStmt && Tok.is(tok::l_brace)) {
     Diag(Tok, diag::ext_gnu_statement_expr);
-    StmtResult Stmt(ParseCompoundStatement(0, true));
+    ParsedAttributes attrs;
+    StmtResult Stmt(ParseCompoundStatement(attrs, true));
     ExprType = CompoundStmt;
 
     // If the substmt parsed correctly, build the AST node.
@@ -1455,55 +1643,74 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       return ParseCXXAmbiguousParenExpression(ExprType, CastTy,
                                               OpenLoc, RParenLoc);
 
-    TypeResult Ty = ParseTypeName();
-
-    // Match the ')'.
-    if (Tok.is(tok::r_paren))
-      RParenLoc = ConsumeParen();
-    else
-      MatchRHSPunctuation(tok::r_paren, OpenLoc);
-
-    if (Tok.is(tok::l_brace)) {
-      ExprType = CompoundLiteral;
-      return ParseCompoundLiteralExpression(Ty.get(), OpenLoc, RParenLoc);
+    TypeResult Ty;
+    
+    {
+      InMessageExpressionRAIIObject InMessage(*this, false);
+      Ty = ParseTypeName();
     }
+    
+    // If our type is followed by an identifier and either ':' or ']', then 
+    // this is probably an Objective-C message send where the leading '[' is
+    // missing. Recover as if that were the case.
+    if (!Ty.isInvalid() && Tok.is(tok::identifier) && !InMessageExpression &&
+        getLang().ObjC1 && !Ty.get().get().isNull() &&
+        (NextToken().is(tok::colon) || NextToken().is(tok::r_square)) &&
+        Ty.get().get()->isObjCObjectOrInterfaceType()) {
+      Result = ParseObjCMessageExpressionBody(SourceLocation(), 
+                                              SourceLocation(), 
+                                              Ty.get(), 0);
+    } else {          
+      // Match the ')'.
+      if (Tok.is(tok::r_paren))
+        RParenLoc = ConsumeParen();
+      else
+        MatchRHSPunctuation(tok::r_paren, OpenLoc);
+
+      if (Tok.is(tok::l_brace)) {
+        ExprType = CompoundLiteral;
+        return ParseCompoundLiteralExpression(Ty.get(), OpenLoc, RParenLoc);
+      }
 
-    if (ExprType == CastExpr) {
-      // We parsed '(' type-name ')' and the thing after it wasn't a '{'.
+      if (ExprType == CastExpr) {
+        // We parsed '(' type-name ')' and the thing after it wasn't a '{'.
 
-      if (Ty.isInvalid())
-        return ExprError();
+        if (Ty.isInvalid())
+          return ExprError();
 
-      CastTy = Ty.get();
+        CastTy = Ty.get();
 
-      // Note that this doesn't parse the subsequent cast-expression, it just
-      // returns the parsed type to the callee.
-      if (stopIfCastExpr)
-        return ExprResult();
-      
-      // Reject the cast of super idiom in ObjC.
-      if (Tok.is(tok::identifier) && getLang().ObjC1 &&
-          Tok.getIdentifierInfo() == Ident_super && 
-          getCurScope()->isInObjcMethodScope() &&
-          GetLookAheadToken(1).isNot(tok::period)) {
-        Diag(Tok.getLocation(), diag::err_illegal_super_cast)
-          << SourceRange(OpenLoc, RParenLoc);
-        return ExprError();
+        // Note that this doesn't parse the subsequent cast-expression, it just
+        // returns the parsed type to the callee.
+        if (stopIfCastExpr)
+          return ExprResult();
+        
+        // Reject the cast of super idiom in ObjC.
+        if (Tok.is(tok::identifier) && getLang().ObjC1 &&
+            Tok.getIdentifierInfo() == Ident_super && 
+            getCurScope()->isInObjcMethodScope() &&
+            GetLookAheadToken(1).isNot(tok::period)) {
+          Diag(Tok.getLocation(), diag::err_illegal_super_cast)
+            << SourceRange(OpenLoc, RParenLoc);
+          return ExprError();
+        }
+
+        // Parse the cast-expression that follows it next.
+        // TODO: For cast expression with CastTy.
+        Result = ParseCastExpression(false, false, CastTy);
+        if (!Result.isInvalid())
+          Result = Actions.ActOnCastExpr(getCurScope(), OpenLoc, CastTy, 
+                                         RParenLoc, Result.take());
+        return move(Result);
       }
 
-      // Parse the cast-expression that follows it next.
-      // TODO: For cast expression with CastTy.
-      Result = ParseCastExpression(false, false, CastTy);
-      if (!Result.isInvalid())
-        Result = Actions.ActOnCastExpr(getCurScope(), OpenLoc, CastTy, RParenLoc,
-                                       Result.take());
-      return move(Result);
+      Diag(Tok, diag::err_expected_lbrace_in_compound_literal);
+      return ExprError();
     }
-
-    Diag(Tok, diag::err_expected_lbrace_in_compound_literal);
-    return ExprError();
   } else if (TypeOfCast) {
     // Parse the expression-list.
+    InMessageExpressionRAIIObject InMessage(*this, false);
+    
     ExprVector ArgExprs(Actions);
     CommaLocsTy CommaLocs;
 
@@ -1513,6 +1720,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
                                           move_arg(ArgExprs), TypeOfCast);
     }
   } else {
+    InMessageExpressionRAIIObject InMessage(*this, false);
+    
     Result = ParseExpression();
     ExprType = SimpleExpr;
     if (!Result.isInvalid() && Tok.is(tok::r_paren))
@@ -1582,8 +1791,8 @@ ExprResult Parser::ParseStringLiteralExpression() {
 ///         argument-expression-list , assignment-expression
 ///
 /// [C++] expression-list:
-/// [C++]   assignment-expression
-/// [C++]   expression-list , assignment-expression
+/// [C++]   assignment-expression ...[opt]
+/// [C++]   expression-list , assignment-expression ...[opt]
 ///
 bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
                             llvm::SmallVectorImpl<SourceLocation> &CommaLocs,
@@ -1596,10 +1805,14 @@ bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
     if (Tok.is(tok::code_completion)) {
       if (Completer)
         (Actions.*Completer)(getCurScope(), Data, Exprs.data(), Exprs.size());
+      else
+        Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
       ConsumeCodeCompletionToken();
     }
     
     ExprResult Expr(ParseAssignmentExpression());
+    if (Tok.is(tok::ellipsis))
+      Expr = Actions.ActOnPackExpansion(Expr.get(), ConsumeToken());    
     if (Expr.isInvalid())
       return true;
 
@@ -1618,6 +1831,11 @@ bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
 /// [clang]   specifier-qualifier-list block-declarator
 ///
 void Parser::ParseBlockId() {
+  if (Tok.is(tok::code_completion)) {
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type);
+    ConsumeCodeCompletionToken();
+  }
+  
   // Parse the specifier-qualifier-list piece.
   DeclSpec DS;
   ParseSpecifierQualifierList(DS);
@@ -1627,14 +1845,9 @@ void Parser::ParseBlockId() {
   ParseDeclarator(DeclaratorInfo);
 
   // We do this for: ^ __attribute__((noreturn)) {, as DS has the attributes.
-  DeclaratorInfo.AddAttributes(DS.TakeAttributes(),
-                               SourceLocation());
+  DeclaratorInfo.addAttributes(DS.takeAttributes());
 
-  if (Tok.is(tok::kw___attribute)) {
-    SourceLocation Loc;
-    AttributeList *AttrList = ParseGNUAttributes(&Loc);
-    DeclaratorInfo.AddAttributes(AttrList, Loc);
-  }
+  MaybeParseGNUAttributes(DeclaratorInfo);
 
   // Inform sema that we are starting a block.
   Actions.ActOnBlockArguments(DeclaratorInfo, getCurScope());
@@ -1692,11 +1905,7 @@ ExprResult Parser::ParseBlockLiteralExpression() {
       return ExprError();
     }
 
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      ParamInfo.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(ParamInfo);
 
     // Inform sema that we are starting a block.
     Actions.ActOnBlockArguments(ParamInfo, getCurScope());
@@ -1704,20 +1913,18 @@ ExprResult Parser::ParseBlockLiteralExpression() {
     ParseBlockId();
   } else {
     // Otherwise, pretend we saw (void).
-    ParamInfo.AddTypeInfo(DeclaratorChunk::getFunction(true, false,
+    ParamInfo.AddTypeInfo(DeclaratorChunk::getFunction(ParsedAttributes(),
+                                                       true, false,
                                                        SourceLocation(),
                                                        0, 0, 0,
+                                                       true, SourceLocation(),
                                                        false, SourceLocation(),
                                                        false, 0, 0, 0,
                                                        CaretLoc, CaretLoc,
                                                        ParamInfo),
                           CaretLoc);
 
-    if (Tok.is(tok::kw___attribute)) {
-      SourceLocation Loc;
-      AttributeList *AttrList = ParseGNUAttributes(&Loc);
-      ParamInfo.AddAttributes(AttrList, Loc);
-    }
+    MaybeParseGNUAttributes(ParamInfo);
 
     // Inform sema that we are starting a block.
     Actions.ActOnBlockArguments(ParamInfo, getCurScope());
diff --git a/lib/Parse/ParseExprCXX.cpp b/lib/Parse/ParseExprCXX.cpp
index 5041a212924a..e73578f23e36 100644
--- a/lib/Parse/ParseExprCXX.cpp
+++ b/lib/Parse/ParseExprCXX.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
+#include "RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -500,9 +501,9 @@ ExprResult Parser::ParseCXXTypeid() {
     TypeResult Ty = ParseTypeName();
 
     // Match the ')'.
-    MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
 
-    if (Ty.isInvalid())
+    if (Ty.isInvalid() || RParenLoc.isInvalid())
       return ExprError();
 
     Result = Actions.ActOnCXXTypeid(OpLoc, LParenLoc, /*isType=*/true,
@@ -524,8 +525,10 @@ ExprResult Parser::ParseCXXTypeid() {
     if (Result.isInvalid())
       SkipUntil(tok::r_paren);
     else {
-      MatchRHSPunctuation(tok::r_paren, LParenLoc);
-
+      RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+      if (RParenLoc.isInvalid())
+        return ExprError();
+      
       Result = Actions.ActOnCXXTypeid(OpLoc, LParenLoc, /*isType=*/false,
                                       Result.release(), RParenLoc);
     }
@@ -534,6 +537,54 @@ ExprResult Parser::ParseCXXTypeid() {
   return move(Result);
 }
 
+/// ParseCXXUuidof - This handles the Microsoft C++ __uuidof expression.
+///
+///         '__uuidof' '(' expression ')'
+///         '__uuidof' '(' type-id ')'
+///
+ExprResult Parser::ParseCXXUuidof() {
+  assert(Tok.is(tok::kw___uuidof) && "Not '__uuidof'!");
+
+  SourceLocation OpLoc = ConsumeToken();
+  SourceLocation LParenLoc = Tok.getLocation();
+  SourceLocation RParenLoc;
+
+  // __uuidof expressions are always parenthesized.
+  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after,
+      "__uuidof"))
+    return ExprError();
+
+  ExprResult Result;
+
+  if (isTypeIdInParens()) {
+    TypeResult Ty = ParseTypeName();
+
+    // Match the ')'.
+    RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+
+    if (Ty.isInvalid())
+      return ExprError();
+
+    Result = Actions.ActOnCXXUuidof(OpLoc, LParenLoc, /*isType=*/true,
+                                    Ty.get().getAsOpaquePtr(), RParenLoc);
+  } else {
+    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
+    Result = ParseExpression();
+
+    // Match the ')'.
+    if (Result.isInvalid())
+      SkipUntil(tok::r_paren);
+    else {
+      RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
+
+      Result = Actions.ActOnCXXUuidof(OpLoc, LParenLoc, /*isType=*/false,
+                                      Result.release(), RParenLoc);
+    }
+  }
+
+  return move(Result);
+}
+
 /// \brief Parse a C++ pseudo-destructor expression after the base,
 /// . or -> operator, and nested-name-specifier have already been
 /// parsed.
@@ -670,6 +721,8 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
   ParsedType TypeRep = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
 
   assert(Tok.is(tok::l_paren) && "Expected '('!");
+  GreaterThanIsOperatorScope G(GreaterThanIsOperator, true);
+
   SourceLocation LParenLoc = ConsumeParen();
 
   ExprVector Exprs(Actions);
@@ -691,9 +744,8 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
 
   assert((Exprs.size() == 0 || Exprs.size()-1 == CommaLocs.size())&&
          "Unexpected number of commas!");
-  return Actions.ActOnCXXTypeConstructExpr(DS.getSourceRange(), TypeRep,
-                                           LParenLoc, move_arg(Exprs),
-                                           CommaLocs.data(), RParenLoc);
+  return Actions.ActOnCXXTypeConstructExpr(TypeRep, LParenLoc, move_arg(Exprs),
+                                           RParenLoc);
 }
 
 /// ParseCXXCondition - if/switch/while condition expression.
@@ -761,24 +813,22 @@ bool Parser::ParseCXXCondition(ExprResult &ExprOut,
   }
 
   // If attributes are present, parse them.
-  if (Tok.is(tok::kw___attribute)) {
-    SourceLocation Loc;
-    AttributeList *AttrList = ParseGNUAttributes(&Loc);
-    DeclaratorInfo.AddAttributes(AttrList, Loc);
-  }
+  MaybeParseGNUAttributes(DeclaratorInfo);
 
   // Type-check the declaration itself.
   DeclResult Dcl = Actions.ActOnCXXConditionDeclaration(getCurScope(), 
-                                                                DeclaratorInfo);
+                                                        DeclaratorInfo);
   DeclOut = Dcl.get();
   ExprOut = ExprError();
-  
+
   // '=' assignment-expression
-  if (Tok.is(tok::equal)) {
-    SourceLocation EqualLoc = ConsumeToken();
+  if (isTokenEqualOrMistypedEqualEqual(
+                               diag::err_invalid_equalequal_after_declarator)) {
+    ConsumeToken();
     ExprResult AssignExpr(ParseAssignmentExpression());
     if (!AssignExpr.isInvalid()) 
-      Actions.AddInitializerToDecl(DeclOut, AssignExpr.take());
+      Actions.AddInitializerToDecl(DeclOut, AssignExpr.take(), false,
+                                   DS.getTypeSpecType() == DeclSpec::TST_auto);
   } else {
     // FIXME: C++0x allows a braced-init-list
     Diag(Tok, diag::err_expected_equal_after_declarator);
@@ -862,9 +912,24 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
 
   // type-name
   case tok::annot_typename: {
-    DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID,
-                       getTypeAnnotation(Tok));
-    break;
+    if (getTypeAnnotation(Tok))
+      DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID,
+                         getTypeAnnotation(Tok));
+    else
+      DS.SetTypeSpecError();
+    
+    DS.SetRangeEnd(Tok.getAnnotationEndLoc());
+    ConsumeToken();
+    
+    // Objective-C supports syntax of the form 'id<proto1,proto2>' where 'id'
+    // is a specific typedef and 'itf<proto1,proto2>' where 'itf' is an
+    // Objective-C interface.  If we don't have Objective-C or a '<', this is
+    // just a normal reference to a typedef name.
+    if (Tok.is(tok::less) && getLang().ObjC1)
+      ParseObjCProtocolQualifiers(DS);
+    
+    DS.Finish(Diags, PP);
+    return;
   }
 
   // builtin types
@@ -943,7 +1008,7 @@ bool Parser::ParseCXXTypeSpecifierSeq(DeclSpec &DS) {
   // Parse one or more of the type specifiers.
   if (!ParseOptionalTypeSpecifier(DS, isInvalid, PrevSpec, DiagID,
       ParsedTemplateInfo(), /*SuppressDeclarations*/true)) {
-    Diag(Tok, diag::err_operator_missing_type_specifier);
+    Diag(Tok, diag::err_expected_type);
     return true;
   }
 
@@ -1667,7 +1732,8 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
     first = false;
 
     SourceLocation RLoc = MatchRHSPunctuation(tok::r_square, LLoc);
-    D.AddTypeInfo(DeclaratorChunk::getArray(0, /*static=*/false, /*star=*/false,
+    D.AddTypeInfo(DeclaratorChunk::getArray(0, ParsedAttributes(),
+                                            /*static=*/false, /*star=*/false,
                                             Size.release(), LLoc, RLoc),
                   RLoc);
 
@@ -1738,7 +1804,7 @@ Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
 
 static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
   switch(kind) {
-  default: assert(false && "Not a known unary type trait.");
+  default: llvm_unreachable("Not a known unary type trait");
   case tok::kw___has_nothrow_assign:      return UTT_HasNothrowAssign;
   case tok::kw___has_nothrow_copy:        return UTT_HasNothrowCopy;
   case tok::kw___has_nothrow_constructor: return UTT_HasNothrowConstructor;
@@ -1758,6 +1824,15 @@ static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
   }
 }
 
+static BinaryTypeTrait BinaryTypeTraitFromTokKind(tok::TokenKind kind) {
+  switch(kind) {
+  default: llvm_unreachable("Not a known binary type trait");
+  case tok::kw___is_base_of:                 return BTT_IsBaseOf;
+  case tok::kw___builtin_types_compatible_p: return BTT_TypeCompatible;
+  case tok::kw___is_convertible_to:          return BTT_IsConvertibleTo;
+  }
+}
+
 /// ParseUnaryTypeTrait - Parse the built-in unary type-trait
 /// pseudo-functions that allow implementation of the TR1/C++0x type traits
 /// templates.
@@ -1783,7 +1858,44 @@ ExprResult Parser::ParseUnaryTypeTrait() {
   if (Ty.isInvalid())
     return ExprError();
 
-  return Actions.ActOnUnaryTypeTrait(UTT, Loc, LParen, Ty.get(), RParen);
+  return Actions.ActOnUnaryTypeTrait(UTT, Loc, Ty.get(), RParen);
+}
+
+/// ParseBinaryTypeTrait - Parse the built-in binary type-trait
+/// pseudo-functions that allow implementation of the TR1/C++0x type traits
+/// templates.
+///
+///       primary-expression:
+/// [GNU]             binary-type-trait '(' type-id ',' type-id ')'
+///
+ExprResult Parser::ParseBinaryTypeTrait() {
+  BinaryTypeTrait BTT = BinaryTypeTraitFromTokKind(Tok.getKind());
+  SourceLocation Loc = ConsumeToken();
+
+  SourceLocation LParen = Tok.getLocation();
+  if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen))
+    return ExprError();
+
+  TypeResult LhsTy = ParseTypeName();
+  if (LhsTy.isInvalid()) {
+    SkipUntil(tok::r_paren);
+    return ExprError();
+  }
+
+  if (ExpectAndConsume(tok::comma, diag::err_expected_comma)) {
+    SkipUntil(tok::r_paren);
+    return ExprError();
+  }
+
+  TypeResult RhsTy = ParseTypeName();
+  if (RhsTy.isInvalid()) {
+    SkipUntil(tok::r_paren);
+    return ExprError();
+  }
+
+  SourceLocation RParen = MatchRHSPunctuation(tok::r_paren, LParen);
+
+  return Actions.ActOnBinaryTypeTrait(BTT, Loc, LhsTy.get(), RhsTy.get(), RParen);
 }
 
 /// ParseCXXAmbiguousParenExpression - We have parsed the left paren of a
diff --git a/lib/Parse/ParseInit.cpp b/lib/Parse/ParseInit.cpp
index 4347294141ac..82dda2b793d3 100644
--- a/lib/Parse/ParseInit.cpp
+++ b/lib/Parse/ParseInit.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
+#include "RAIIObjectsForParser.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/SmallString.h"
@@ -136,6 +137,8 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
     //   [foo ... bar]     -> array designator
     //   [4][foo bar]      -> obsolete GNU designation with objc message send.
     //
+    InMessageExpressionRAIIObject InMessage(*this, true);
+    
     SourceLocation StartLoc = ConsumeBracket();
     ExprResult Idx;
 
@@ -146,7 +149,8 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
     if  (getLang().ObjC1 && getLang().CPlusPlus) {
       // Send to 'super'.
       if (Tok.is(tok::identifier) && Tok.getIdentifierInfo() == Ident_super &&
-          NextToken().isNot(tok::period) && getCurScope()->isInObjcMethodScope()) {
+          NextToken().isNot(tok::period) && 
+          getCurScope()->isInObjcMethodScope()) {
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
         return ParseAssignmentExprWithObjCMessageExprStart(StartLoc,
                                                            ConsumeToken(),
@@ -306,10 +310,12 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
 /// [GNU]   '{' '}'
 ///
 ///       initializer-list:
-///         designation[opt] initializer
-///         initializer-list ',' designation[opt] initializer
+///         designation[opt] initializer ...[opt]
+///         initializer-list ',' designation[opt] initializer ...[opt]
 ///
 ExprResult Parser::ParseBraceInitializer() {
+  InMessageExpressionRAIIObject InMessage(*this, false);
+  
   SourceLocation LBraceLoc = ConsumeBrace();
 
   /// InitExprs - This is the actual list of expressions contained in the
@@ -338,6 +344,9 @@ ExprResult Parser::ParseBraceInitializer() {
     else
       SubElt = ParseInitializer();
 
+    if (Tok.is(tok::ellipsis))
+      SubElt = Actions.ActOnPackExpansion(SubElt.get(), ConsumeToken());
+    
     // If we couldn't parse the subelement, bail out.
     if (!SubElt.isInvalid()) {
       InitExprs.push_back(SubElt.release());
diff --git a/lib/Parse/ParseObjc.cpp b/lib/Parse/ParseObjc.cpp
index 6861ce940f76..f32a322f024a 100644
--- a/lib/Parse/ParseObjc.cpp
+++ b/lib/Parse/ParseObjc.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
+#include "RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "clang/Sema/Scope.h"
@@ -39,10 +40,14 @@ Decl *Parser::ParseObjCAtDirectives() {
   switch (Tok.getObjCKeywordID()) {
   case tok::objc_class:
     return ParseObjCAtClassDeclaration(AtLoc);
-  case tok::objc_interface:
-    return ParseObjCAtInterfaceDeclaration(AtLoc);
-  case tok::objc_protocol:
-    return ParseObjCAtProtocolDeclaration(AtLoc);
+  case tok::objc_interface: {
+    ParsedAttributes attrs;
+    return ParseObjCAtInterfaceDeclaration(AtLoc, attrs);
+  }
+  case tok::objc_protocol: {
+    ParsedAttributes attrs;
+    return ParseObjCAtProtocolDeclaration(AtLoc, attrs);
+  }
   case tok::objc_implementation:
     return ParseObjCAtImplementationDeclaration(AtLoc);
   case tok::objc_end:
@@ -123,8 +128,8 @@ Decl *Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
 ///     __attribute__((unavailable))
 ///     __attribute__((objc_exception)) - used by NSException on 64-bit
 ///
-Decl *Parser::ParseObjCAtInterfaceDeclaration(
-  SourceLocation atLoc, AttributeList *attrList) {
+Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation atLoc,
+                                              ParsedAttributes &attrs) {
   assert(Tok.isObjCAtKeyword(tok::objc_interface) &&
          "ParseObjCAtInterfaceDeclaration(): Expected @interface");
   ConsumeToken(); // the "interface" identifier
@@ -145,7 +150,9 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(
   SourceLocation nameLoc = ConsumeToken();
   if (Tok.is(tok::l_paren) && 
       !isKnownToBeTypeSpecifier(GetLookAheadToken(1))) { // we have a category.
-    SourceLocation lparenLoc = ConsumeParen();
+    // TODO(dgregor): Use the return value from the next line to provide better
+    // recovery.
+    ConsumeParen();
     SourceLocation categoryLoc, rparenLoc;
     IdentifierInfo *categoryId = 0;
     if (Tok.is(tok::code_completion)) {
@@ -177,7 +184,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(
                                     LAngleLoc, EndProtoLoc))
       return 0;
 
-    if (attrList) // categories don't support attributes.
+    if (!attrs.empty()) // categories don't support attributes.
       Diag(Tok, diag::err_objc_no_attributes_on_category);
 
     Decl *CategoryType =
@@ -229,7 +236,7 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(
                                      superClassId, superClassLoc,
                                      ProtocolRefs.data(), ProtocolRefs.size(),
                                      ProtocolLocs.data(),
-                                     EndProtoLoc, attrList);
+                                     EndProtoLoc, attrs.getList());
 
   if (Tok.is(tok::l_brace))
     ParseObjCClassInstanceVariables(ClsType, tok::objc_protected, atLoc);
@@ -364,7 +371,8 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
 
       // FIXME: as the name implies, this rule allows function definitions.
       // We could pass a flag or check for functions during semantic analysis.
-      allTUVariables.push_back(ParseDeclarationOrFunctionDefinition(0));
+      ParsedAttributes attrs;
+      allTUVariables.push_back(ParseDeclarationOrFunctionDefinition(attrs));
       continue;
     }
 
@@ -401,7 +409,13 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
       // Skip until we see an '@' or '}' or ';'.
       SkipUntil(tok::r_brace, tok::at);
       break;
-
+        
+    case tok::objc_implementation:
+    case tok::objc_interface:
+      Diag(Tok, diag::err_objc_missing_end);
+      ConsumeToken();
+      break;
+        
     case tok::objc_required:
     case tok::objc_optional:
       // This is only valid on protocols.
@@ -414,13 +428,12 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
 
     case tok::objc_property:
       if (!getLang().ObjC2)
-        Diag(AtLoc, diag::err_objc_propertoes_require_objc2);
+        Diag(AtLoc, diag::err_objc_properties_require_objc2);
 
       ObjCDeclSpec OCDS;
       // Parse property attribute list, if any.
       if (Tok.is(tok::l_paren))
-        ParseObjCPropertyAttribute(OCDS, interfaceDecl,
-                                   allMethods.data(), allMethods.size());
+        ParseObjCPropertyAttribute(OCDS, interfaceDecl);
 
       ObjCPropertyCallback Callback(*this, interfaceDecl, allProperties,
                                     OCDS, AtLoc, MethodImplKind);
@@ -469,9 +482,7 @@ void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
 ///     copy
 ///     nonatomic
 ///
-void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl,
-                                        Decl **Methods, 
-                                        unsigned NumMethods) {
+void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl) {
   assert(Tok.getKind() == tok::l_paren);
   SourceLocation LHSLoc = ConsumeParen(); // consume '('
 
@@ -502,32 +513,40 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl,
       DS.setPropertyAttributes(ObjCDeclSpec::DQ_PR_copy);
     else if (II->isStr("nonatomic"))
       DS.setPropertyAttributes(ObjCDeclSpec::DQ_PR_nonatomic);
+    else if (II->isStr("atomic"))
+      DS.setPropertyAttributes(ObjCDeclSpec::DQ_PR_atomic);
     else if (II->isStr("getter") || II->isStr("setter")) {
+      bool IsSetter = II->getNameStart()[0] == 's';
+
       // getter/setter require extra treatment.
-      if (ExpectAndConsume(tok::equal, diag::err_objc_expected_equal, "",
-                           tok::r_paren))
+      unsigned DiagID = IsSetter ? diag::err_objc_expected_equal_for_setter :
+        diag::err_objc_expected_equal_for_getter;
+
+      if (ExpectAndConsume(tok::equal, DiagID, "", tok::r_paren))
         return;
 
       if (Tok.is(tok::code_completion)) {
-        if (II->getNameStart()[0] == 's')
-          Actions.CodeCompleteObjCPropertySetter(getCurScope(), ClassDecl,
-                                                 Methods, NumMethods);
+        if (IsSetter)
+          Actions.CodeCompleteObjCPropertySetter(getCurScope(), ClassDecl);
         else
-          Actions.CodeCompleteObjCPropertyGetter(getCurScope(), ClassDecl,
-                                                 Methods, NumMethods);
+          Actions.CodeCompleteObjCPropertyGetter(getCurScope(), ClassDecl);
         ConsumeCodeCompletionToken();
       }
 
-      if (Tok.isNot(tok::identifier)) {
-        Diag(Tok, diag::err_expected_ident);
+      
+      SourceLocation SelLoc;
+      IdentifierInfo *SelIdent = ParseObjCSelectorPiece(SelLoc);
+
+      if (!SelIdent) {
+        Diag(Tok, diag::err_objc_expected_selector_for_getter_setter)
+          << IsSetter;
         SkipUntil(tok::r_paren);
         return;
       }
 
-      if (II->getNameStart()[0] == 's') {
+      if (IsSetter) {
         DS.setPropertyAttributes(ObjCDeclSpec::DQ_PR_setter);
-        DS.setSetterName(Tok.getIdentifierInfo());
-        ConsumeToken();  // consume method name
+        DS.setSetterName(SelIdent);
 
         if (ExpectAndConsume(tok::colon, 
                              diag::err_expected_colon_after_setter_name, "",
@@ -535,8 +554,7 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl,
           return;
       } else {
         DS.setPropertyAttributes(ObjCDeclSpec::DQ_PR_getter);
-        DS.setGetterName(Tok.getIdentifierInfo());
-        ConsumeToken();  // consume method name
+        DS.setGetterName(SelIdent);
       }
     } else {
       Diag(AttrName, diag::err_objc_expected_property_attr) << II;
@@ -705,7 +723,7 @@ bool Parser::isTokIdentifier_in() const {
 void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS, bool IsParameter) {
   while (1) {
     if (Tok.is(tok::code_completion)) {
-      Actions.CodeCompleteObjCPassingType(getCurScope(), DS);
+      Actions.CodeCompleteObjCPassingType(getCurScope(), DS, IsParameter);
       ConsumeCodeCompletionToken();
     }
     
@@ -819,9 +837,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     ReturnType = ParseObjCTypeName(DSRet, false);
 
   // If attributes exist before the method, parse them.
-  llvm::OwningPtr<AttributeList> MethodAttrs;
-  if (getLang().ObjC2 && Tok.is(tok::kw___attribute))
-    MethodAttrs.reset(ParseGNUAttributes());
+  ParsedAttributes attrs;
+  if (getLang().ObjC2)
+    MaybeParseGNUAttributes(attrs);
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
@@ -845,25 +863,25 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   llvm::SmallVector<DeclaratorChunk::ParamInfo, 8> CParamInfo;
   if (Tok.isNot(tok::colon)) {
     // If attributes exist after the method, parse them.
-    if (getLang().ObjC2 && Tok.is(tok::kw___attribute))
-      MethodAttrs.reset(addAttributeLists(MethodAttrs.take(),
-                                          ParseGNUAttributes()));
+    if (getLang().ObjC2)
+      MaybeParseGNUAttributes(attrs);
 
     Selector Sel = PP.getSelectorTable().getNullarySelector(SelIdent);
     Decl *Result
-         = Actions.ActOnMethodDeclaration(mLoc, Tok.getLocation(),
+         = Actions.ActOnMethodDeclaration(getCurScope(), mLoc, Tok.getLocation(),
                                           mType, IDecl, DSRet, ReturnType, Sel,
                                           0, 
                                           CParamInfo.data(), CParamInfo.size(),
-                                          MethodAttrs.get(),
-                                          MethodImplKind);
+                                          attrs.getList(), MethodImplKind);
     PD.complete(Result);
     return Result;
   }
 
   llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
   llvm::SmallVector<Sema::ObjCArgInfo, 12> ArgInfos;
-
+  ParseScope PrototypeScope(this,
+                            Scope::FunctionPrototypeScope|Scope::DeclScope);
+  
   while (1) {
     Sema::ObjCArgInfo ArgInfo;
 
@@ -880,8 +898,11 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
     // If attributes exist before the argument name, parse them.
     ArgInfo.ArgAttrs = 0;
-    if (getLang().ObjC2 && Tok.is(tok::kw___attribute))
-      ArgInfo.ArgAttrs = ParseGNUAttributes();
+    if (getLang().ObjC2) {
+      ParsedAttributes attrs;
+      MaybeParseGNUAttributes(attrs);
+      ArgInfo.ArgAttrs = attrs.getList();
+    }
 
     // Code completion for the next piece of the selector.
     if (Tok.is(tok::code_completion)) {
@@ -953,30 +974,30 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 
   }
 
-  // FIXME: Add support for optional parmameter list...
+  // FIXME: Add support for optional parameter list...
   // If attributes exist after the method, parse them.
-  if (getLang().ObjC2 && Tok.is(tok::kw___attribute))
-    MethodAttrs.reset(addAttributeLists(MethodAttrs.take(),
-                                        ParseGNUAttributes()));
-
-  if (KeyIdents.size() == 0)
+  if (getLang().ObjC2)
+    MaybeParseGNUAttributes(attrs);
+  
+  if (KeyIdents.size() == 0) {
+    // Leave prototype scope.
+    PrototypeScope.Exit();
     return 0;
+  }
+  
   Selector Sel = PP.getSelectorTable().getSelector(KeyIdents.size(),
                                                    &KeyIdents[0]);
   Decl *Result
-       = Actions.ActOnMethodDeclaration(mLoc, Tok.getLocation(),
+       = Actions.ActOnMethodDeclaration(getCurScope(), mLoc, Tok.getLocation(),
                                         mType, IDecl, DSRet, ReturnType, Sel,
                                         &ArgInfos[0], 
                                         CParamInfo.data(), CParamInfo.size(),
-                                        MethodAttrs.get(),
+                                        attrs.getList(),
                                         MethodImplKind, isVariadic);
-  PD.complete(Result);
-  
-  // Delete referenced AttributeList objects.
-  for (llvm::SmallVectorImpl<Sema::ObjCArgInfo>::iterator
-       I = ArgInfos.begin(), E = ArgInfos.end(); I != E; ++I)
-    delete I->ArgAttrs;
+  // Leave prototype scope.
+  PrototypeScope.Exit();
   
+  PD.complete(Result);
   return Result;
 }
 
@@ -1031,6 +1052,24 @@ ParseObjCProtocolReferences(llvm::SmallVectorImpl<Decl *> &Protocols,
   return false;
 }
 
+/// \brief Parse the Objective-C protocol qualifiers that follow a typename
+/// in a decl-specifier-seq, starting at the '<'.
+bool Parser::ParseObjCProtocolQualifiers(DeclSpec &DS) {
+  assert(Tok.is(tok::less) && "Protocol qualifiers start with '<'");
+  assert(getLang().ObjC1 && "Protocol qualifiers only exist in Objective-C");
+  SourceLocation LAngleLoc, EndProtoLoc;
+  llvm::SmallVector<Decl *, 8> ProtocolDecl;
+  llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
+  bool Result = ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
+                                            LAngleLoc, EndProtoLoc);
+  DS.setProtocolQualifiers(ProtocolDecl.data(), ProtocolDecl.size(),
+                           ProtocolLocs.data(), LAngleLoc);
+  if (EndProtoLoc.isValid())
+    DS.SetRangeEnd(EndProtoLoc);
+  return Result;
+}
+
+
 ///   objc-class-instance-variables:
 ///     '{' objc-instance-variable-decl-list[opt] '}'
 ///
@@ -1164,7 +1203,7 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
 ///   identifier-list ;": objc-interface-decl-list may not start with a
 ///   semicolon in the first alternative if objc-protocol-refs are omitted.
 Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
-                                                      AttributeList *attrList) {
+                                             ParsedAttributes &attrs) {
   assert(Tok.isObjCAtKeyword(tok::objc_protocol) &&
          "ParseObjCAtProtocolDeclaration(): Expected @protocol");
   ConsumeToken(); // the "protocol" identifier
@@ -1186,7 +1225,7 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
     IdentifierLocPair ProtoInfo(protocolName, nameLoc);
     ConsumeToken();
     return Actions.ActOnForwardProtocolDeclaration(AtLoc, &ProtoInfo, 1,
-                                                   attrList);
+                                                   attrs.getList());
   }
 
   if (Tok.is(tok::comma)) { // list of forward declarations.
@@ -1215,7 +1254,7 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
     return Actions.ActOnForwardProtocolDeclaration(AtLoc,
                                                    &ProtocolRefs[0],
                                                    ProtocolRefs.size(),
-                                                   attrList);
+                                                   attrs.getList());
   }
 
   // Last, and definitely not least, parse a protocol declaration.
@@ -1233,7 +1272,7 @@ Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
                                         ProtocolRefs.data(),
                                         ProtocolRefs.size(),
                                         ProtocolLocs.data(),
-                                        EndProtoLoc, attrList);
+                                        EndProtoLoc, attrs.getList());
   ParseObjCInterfaceDeclList(ProtoType, tok::objc_protocol);
   return ProtoType;
 }
@@ -1270,7 +1309,7 @@ Decl *Parser::ParseObjCAtImplementationDeclaration(
 
   if (Tok.is(tok::l_paren)) {
     // we have a category implementation.
-    SourceLocation lparenLoc = ConsumeParen();
+    ConsumeParen();
     SourceLocation categoryLoc, rparenLoc;
     IdentifierInfo *categoryId = 0;
 
@@ -1370,10 +1409,8 @@ Decl *Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
   }
   IdentifierInfo *classId = Tok.getIdentifierInfo();
   SourceLocation classLoc = ConsumeToken(); // consume class-name;
-  if (Tok.isNot(tok::semi)) {
-    Diag(Tok, diag::err_expected_semi_after) << "@compatibility_alias";
-    return 0;
-  }
+  ExpectAndConsume(tok::semi, diag::err_expected_semi_after, 
+                   "@compatibility_alias");
   return Actions.ActOnCompatiblityAlias(atLoc, aliasId, aliasLoc,
                                         classId, classLoc);
 }
@@ -1392,7 +1429,7 @@ Decl *Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
 Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_synthesize) &&
          "ParseObjCPropertyDynamic(): Expected '@synthesize'");
-  SourceLocation loc = ConsumeToken(); // consume synthesize
+  ConsumeToken(); // consume synthesize
 
   while (true) {
     if (Tok.is(tok::code_completion)) {
@@ -1409,6 +1446,7 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
     IdentifierInfo *propertyIvar = 0;
     IdentifierInfo *propertyId = Tok.getIdentifierInfo();
     SourceLocation propertyLoc = ConsumeToken(); // consume property name
+    SourceLocation propertyIvarLoc;
     if (Tok.is(tok::equal)) {
       // property '=' ivar-name
       ConsumeToken(); // consume '='
@@ -1424,20 +1462,15 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
         break;
       }
       propertyIvar = Tok.getIdentifierInfo();
-      ConsumeToken(); // consume ivar-name
+      propertyIvarLoc = ConsumeToken(); // consume ivar-name
     }
     Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, true, ObjCImpDecl,
-                                  propertyId, propertyIvar);
+                                  propertyId, propertyIvar, propertyIvarLoc);
     if (Tok.isNot(tok::comma))
       break;
     ConsumeToken(); // consume ','
   }
-  if (Tok.isNot(tok::semi)) {
-    Diag(Tok, diag::err_expected_semi_after) << "@synthesize";
-    SkipUntil(tok::semi);
-  }
-  else
-    ConsumeToken(); // consume ';'
+  ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@synthesize");
   return 0;
 }
 
@@ -1451,7 +1484,7 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
 Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_dynamic) &&
          "ParseObjCPropertyDynamic(): Expected '@dynamic'");
-  SourceLocation loc = ConsumeToken(); // consume dynamic
+  ConsumeToken(); // consume dynamic
   while (true) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteObjCPropertyDefinition(getCurScope(), ObjCImpDecl);
@@ -1467,18 +1500,13 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
     IdentifierInfo *propertyId = Tok.getIdentifierInfo();
     SourceLocation propertyLoc = ConsumeToken(); // consume property name
     Actions.ActOnPropertyImplDecl(getCurScope(), atLoc, propertyLoc, false, ObjCImpDecl,
-                                  propertyId, 0);
+                                  propertyId, 0, SourceLocation());
 
     if (Tok.isNot(tok::comma))
       break;
     ConsumeToken(); // consume ','
   }
-  if (Tok.isNot(tok::semi)) {
-    Diag(Tok, diag::err_expected_semi_after) << "@dynamic";
-    SkipUntil(tok::semi);
-  }
-  else
-    ConsumeToken(); // consume ';'
+  ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@dynamic");
   return 0;
 }
 
@@ -1689,6 +1717,10 @@ Decl *Parser::ParseObjCMethodDefinition() {
   // specified Declarator for the method.
   Actions.ActOnStartOfObjCMethodDef(getCurScope(), MDecl);
 
+  if (PP.isCodeCompletionEnabled())
+    if (trySkippingFunctionBodyForCodeCompletion())
+      return Actions.ActOnFinishFunctionBody(MDecl, 0);
+
   StmtResult FnBody(ParseCompoundStatementBody());
 
   // If the function body could not be parsed, make a bogus compoundstmt.
@@ -1731,7 +1763,7 @@ StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   }
   
   // Otherwise, eat the semicolon.
-  ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
+  ExpectAndConsumeSemi(diag::err_expected_semi_after_expr);
   return Actions.ActOnExprStmt(Actions.MakeFullExpr(Res.take()));
 }
 
@@ -1785,6 +1817,8 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
 ///     simple-type-specifier
 ///     typename-specifier
 bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
+  InMessageExpressionRAIIObject InMessage(*this, true);
+
   if (Tok.is(tok::identifier) || Tok.is(tok::coloncolon) || 
       Tok.is(tok::kw_typename) || Tok.is(tok::annot_cxxscope))
     TryAnnotateTypeOrScopeToken();
@@ -1859,6 +1893,35 @@ bool Parser::isSimpleObjCMessageExpression() {
          GetLookAheadToken(2).is(tok::identifier);
 }
 
+bool Parser::isStartOfObjCClassMessageMissingOpenBracket() {
+  if (!getLang().ObjC1 || !NextToken().is(tok::identifier) || 
+      InMessageExpression)
+    return false;
+  
+  
+  ParsedType Type;
+
+  if (Tok.is(tok::annot_typename)) 
+    Type = getTypeAnnotation(Tok);
+  else if (Tok.is(tok::identifier))
+    Type = Actions.getTypeName(*Tok.getIdentifierInfo(), Tok.getLocation(), 
+                               getCurScope());
+  else
+    return false;
+  
+  if (!Type.get().isNull() && Type.get()->isObjCObjectOrInterfaceType()) {
+    const Token &AfterNext = GetLookAheadToken(2);
+    if (AfterNext.is(tok::colon) || AfterNext.is(tok::r_square)) {
+      if (Tok.is(tok::identifier))
+        TryAnnotateTypeOrScopeToken();
+      
+      return Tok.is(tok::annot_typename);
+    }
+  }
+
+  return false;
+}
+
 ///   objc-message-expr:
 ///     '[' objc-receiver objc-message-args ']'
 ///
@@ -1879,6 +1942,8 @@ ExprResult Parser::ParseObjCMessageExpression() {
     return ExprError();
   }
   
+  InMessageExpressionRAIIObject InMessage(*this, true);
+  
   if (getLang().CPlusPlus) {
     // We completely separate the C and C++ cases because C++ requires
     // more complicated (read: slower) parsing. 
@@ -1893,7 +1958,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
 
     // Parse the receiver, which is either a type or an expression.
     bool IsExpr;
-    void *TypeOrExpr;
+    void *TypeOrExpr = NULL;
     if (ParseObjCXXMessageReceiver(IsExpr, TypeOrExpr)) {
       SkipUntil(tok::r_square);
       return ExprError();
@@ -1992,14 +2057,18 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                        SourceLocation SuperLoc,
                                        ParsedType ReceiverType,
                                        ExprArg ReceiverExpr) {
+  InMessageExpressionRAIIObject InMessage(*this, true);
+
   if (Tok.is(tok::code_completion)) {
     if (SuperLoc.isValid())
-      Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, 0, 0);
+      Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, 0, 0,
+                                           false);
     else if (ReceiverType)
-      Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType, 0, 0);
+      Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType, 0, 0,
+                                           false);
     else
       Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
-                                              0, 0);
+                                              0, 0, false);
     ConsumeCodeCompletionToken();
   }
   
@@ -2028,6 +2097,29 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
 
       ConsumeToken(); // Eat the ':'.
       ///  Parse the expression after ':'
+      
+      if (Tok.is(tok::code_completion)) {
+        if (SuperLoc.isValid())
+          Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, 
+                                               KeyIdents.data(), 
+                                               KeyIdents.size(),
+                                               /*AtArgumentEpression=*/true);
+        else if (ReceiverType)
+          Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType,
+                                               KeyIdents.data(), 
+                                               KeyIdents.size(),
+                                               /*AtArgumentEpression=*/true);
+        else
+          Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
+                                                  KeyIdents.data(), 
+                                                  KeyIdents.size(),
+                                                  /*AtArgumentEpression=*/true);
+
+        ConsumeCodeCompletionToken();
+        SkipUntil(tok::r_square);
+        return ExprError();
+      }
+      
       ExprResult Res(ParseAssignmentExpression());
       if (Res.isInvalid()) {
         // We must manually skip to a ']', otherwise the expression skipper will
@@ -2045,16 +2137,21 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
         if (SuperLoc.isValid())
           Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, 
                                                KeyIdents.data(), 
-                                               KeyIdents.size());
+                                               KeyIdents.size(),
+                                               /*AtArgumentEpression=*/false);
         else if (ReceiverType)
           Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType,
                                                KeyIdents.data(), 
-                                               KeyIdents.size());
+                                               KeyIdents.size(),
+                                               /*AtArgumentEpression=*/false);
         else
           Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                                   KeyIdents.data(), 
-                                                  KeyIdents.size());
+                                                  KeyIdents.size(),
+                                                /*AtArgumentEpression=*/false);
         ConsumeCodeCompletionToken();
+        SkipUntil(tok::r_square);
+        return ExprError();
       }
             
       // Check for another keyword selector.
diff --git a/lib/Parse/ParsePragma.cpp b/lib/Parse/ParsePragma.cpp
index ddba09ae0f95..dfd0da079df5 100644
--- a/lib/Parse/ParsePragma.cpp
+++ b/lib/Parse/ParsePragma.cpp
@@ -17,11 +17,24 @@
 #include "clang/Lex/Preprocessor.h"
 using namespace clang;
 
+/// \brief Handle the annotation token produced for #pragma unused(...)
+///
+/// Each annot_pragma_unused is followed by the argument token so e.g.
+/// "#pragma unused(x,y)" becomes:
+/// annot_pragma_unused 'x' annot_pragma_unused 'y'
+void Parser::HandlePragmaUnused() {
+  assert(Tok.is(tok::annot_pragma_unused));
+  SourceLocation UnusedLoc = ConsumeToken();
+  Actions.ActOnPragmaUnused(Tok, getCurScope(), UnusedLoc);
+  ConsumeToken(); // The argument token.
+}
 
 // #pragma GCC visibility comes in two variants:
 //   'push' '(' [visibility] ')'
 //   'pop'
-void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, Token &VisTok) {
+void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, 
+                                              PragmaIntroducerKind Introducer,
+                                              Token &VisTok) {
   SourceLocation VisLoc = VisTok.getLocation();
 
   Token Tok;
@@ -74,7 +87,9 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, Token &VisTok) {
 //   pack '(' [integer] ')'
 //   pack '(' 'show' ')'
 //   pack '(' ('push' | 'pop') [',' identifier] [, integer] ')'
-void PragmaPackHandler::HandlePragma(Preprocessor &PP, Token &PackTok) {
+void PragmaPackHandler::HandlePragma(Preprocessor &PP, 
+                                     PragmaIntroducerKind Introducer,
+                                     Token &PackTok) {
   SourceLocation PackLoc = PackTok.getLocation();
 
   Token Tok;
@@ -222,16 +237,22 @@ static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok,
   Actions.ActOnPragmaOptionsAlign(Kind, FirstTok.getLocation(), KindLoc);
 }
 
-void PragmaAlignHandler::HandlePragma(Preprocessor &PP, Token &AlignTok) {
+void PragmaAlignHandler::HandlePragma(Preprocessor &PP, 
+                                      PragmaIntroducerKind Introducer,
+                                      Token &AlignTok) {
   ParseAlignPragma(Actions, PP, AlignTok, /*IsOptions=*/false);
 }
 
-void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, Token &OptionsTok) {
+void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, 
+                                        PragmaIntroducerKind Introducer,
+                                        Token &OptionsTok) {
   ParseAlignPragma(Actions, PP, OptionsTok, /*IsOptions=*/true);
 }
 
 // #pragma unused(identifier)
-void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, Token &UnusedTok) {
+void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, 
+                                       PragmaIntroducerKind Introducer,
+                                       Token &UnusedTok) {
   // FIXME: Should we be expanding macros here? My guess is no.
   SourceLocation UnusedLoc = UnusedTok.getLocation();
 
@@ -291,14 +312,27 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP, Token &UnusedTok) {
   assert(RParenLoc.isValid() && "Valid '#pragma unused' must have ')'");
   assert(!Identifiers.empty() && "Valid '#pragma unused' must have arguments");
 
-  // Perform the action to handle the pragma.
-  Actions.ActOnPragmaUnused(Identifiers.data(), Identifiers.size(),
-                            parser.getCurScope(), UnusedLoc, LParenLoc, RParenLoc);
+  // For each identifier token, insert into the token stream a
+  // annot_pragma_unused token followed by the identifier token.
+  // This allows us to cache a "#pragma unused" that occurs inside an inline
+  // C++ member function.
+
+  Token *Toks = new Token[2*Identifiers.size()];
+  for (unsigned i=0; i != Identifiers.size(); i++) {
+    Token &pragmaUnusedTok = Toks[2*i], &idTok = Toks[2*i+1];
+    pragmaUnusedTok.startToken();
+    pragmaUnusedTok.setKind(tok::annot_pragma_unused);
+    pragmaUnusedTok.setLocation(UnusedLoc);
+    idTok = Identifiers[i];
+  }
+  PP.EnterTokenStream(Toks, 2*Identifiers.size(), /*DisableMacroExpansion=*/true, /*OwnsTokens=*/true);
 }
 
 // #pragma weak identifier
 // #pragma weak identifier '=' identifier
-void PragmaWeakHandler::HandlePragma(Preprocessor &PP, Token &WeakTok) {
+void PragmaWeakHandler::HandlePragma(Preprocessor &PP, 
+                                     PragmaIntroducerKind Introducer,
+                                     Token &WeakTok) {
   // FIXME: Should we be expanding macros here? My guess is no.
   SourceLocation WeakLoc = WeakTok.getLocation();
 
@@ -337,3 +371,64 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP, Token &WeakTok) {
     Actions.ActOnPragmaWeakID(WeakName, WeakLoc, WeakNameLoc);
   }
 }
+
+void
+PragmaFPContractHandler::HandlePragma(Preprocessor &PP, 
+                                      PragmaIntroducerKind Introducer,
+                                      Token &Tok) {
+  tok::OnOffSwitch OOS;
+  if (PP.LexOnOffSwitch(OOS))
+    return;
+
+  Actions.ActOnPragmaFPContract(OOS);
+}
+
+void 
+PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP, 
+                                           PragmaIntroducerKind Introducer,
+                                           Token &Tok) {
+  PP.Lex(Tok);
+  if (Tok.isNot(tok::identifier)) {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier) <<
+      "OPENCL";
+    return;
+  }
+  IdentifierInfo *ename = Tok.getIdentifierInfo();
+  SourceLocation NameLoc = Tok.getLocation();
+
+  PP.Lex(Tok);
+  if (Tok.isNot(tok::colon)) {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_colon) << ename;
+    return;
+  }
+
+  PP.Lex(Tok);
+  if (Tok.isNot(tok::identifier)) {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable);
+    return;
+  }
+  IdentifierInfo *op = Tok.getIdentifierInfo();
+
+  unsigned state;
+  if (op->isStr("enable")) {
+    state = 1;
+  } else if (op->isStr("disable")) {
+    state = 0;
+  } else {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_enable_disable);
+    return;
+  }
+
+  OpenCLOptions &f = Actions.getOpenCLOptions();
+  if (ename->isStr("all")) {
+#define OPENCLEXT(nm)   f.nm = state;
+#include "clang/Basic/OpenCLExtensions.def"
+  }
+#define OPENCLEXT(nm) else if (ename->isStr(#nm)) { f.nm = state; }
+#include "clang/Basic/OpenCLExtensions.def"
+  else {
+    PP.Diag(NameLoc, diag::warn_pragma_unknown_extension) << ename;
+    return;
+  }
+}
+
diff --git a/lib/Parse/ParsePragma.h b/lib/Parse/ParsePragma.h
index 0feaa9919e8d..bee6af3f4cfd 100644
--- a/lib/Parse/ParsePragma.h
+++ b/lib/Parse/ParsePragma.h
@@ -25,7 +25,8 @@ class PragmaAlignHandler : public PragmaHandler {
 public:
   explicit PragmaAlignHandler(Sema &A) : PragmaHandler("align"), Actions(A) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
 class PragmaGCCVisibilityHandler : public PragmaHandler {
@@ -34,7 +35,8 @@ public:
   explicit PragmaGCCVisibilityHandler(Sema &A) : PragmaHandler("visibility"),
                                                  Actions(A) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
 class PragmaOptionsHandler : public PragmaHandler {
@@ -43,7 +45,8 @@ public:
   explicit PragmaOptionsHandler(Sema &A) : PragmaHandler("options"),
                                            Actions(A) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
 class PragmaPackHandler : public PragmaHandler {
@@ -52,7 +55,8 @@ public:
   explicit PragmaPackHandler(Sema &A) : PragmaHandler("pack"),
                                         Actions(A) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
 class PragmaUnusedHandler : public PragmaHandler {
@@ -62,7 +66,8 @@ public:
   PragmaUnusedHandler(Sema &A, Parser& p)
     : PragmaHandler("unused"), Actions(A), parser(p) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
 class PragmaWeakHandler : public PragmaHandler {
@@ -71,9 +76,32 @@ public:
   explicit PragmaWeakHandler(Sema &A)
     : PragmaHandler("weak"), Actions(A) {}
 
-  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
 };
 
+class PragmaOpenCLExtensionHandler : public PragmaHandler {
+  Sema &Actions;
+  Parser &parser;
+public:
+  PragmaOpenCLExtensionHandler(Sema &S, Parser& p) : 
+    PragmaHandler("EXTENSION"), Actions(S), parser(p) {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
+};
+  
+
+class PragmaFPContractHandler : public PragmaHandler {
+  Sema &Actions;
+  Parser &parser;
+public:
+  PragmaFPContractHandler(Sema &S, Parser& p) : 
+    PragmaHandler("FP_CONTRACT"), Actions(S), parser(p) {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &FirstToken);
+};
+  
+
 }  // end namespace clang
 
 #endif
diff --git a/lib/Parse/ParseStmt.cpp b/lib/Parse/ParseStmt.cpp
index 6c240e608c2e..2d9758333f0c 100644
--- a/lib/Parse/ParseStmt.cpp
+++ b/lib/Parse/ParseStmt.cpp
@@ -75,16 +75,14 @@ using namespace clang;
 /// [OBC]   '@' 'throw' ';'
 ///
 StmtResult
-Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
+Parser::ParseStatementOrDeclaration(StmtVector &Stmts, bool OnlyStatement) {
   const char *SemiError = 0;
   StmtResult Res;
   
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
 
-  CXX0XAttributeList Attr;
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-    Attr = ParseCXX0XAttributes();
-  llvm::OwningPtr<AttributeList> AttrList(Attr.AttrList);
+  ParsedAttributesWithRange attrs;
+  MaybeParseCXX0XAttributes(attrs);
 
   // Cases in this switch statement should fall through if the parser expects
   // the token to end in a semicolon (in which case SemiError should be set),
@@ -101,21 +99,20 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
   case tok::code_completion:
     Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
     ConsumeCodeCompletionToken();
-    return ParseStatementOrDeclaration(OnlyStatement);
+    return ParseStatementOrDeclaration(Stmts, OnlyStatement);
       
   case tok::identifier:
     if (NextToken().is(tok::colon)) { // C99 6.8.1: labeled-statement
       // identifier ':' statement
-      return ParseLabeledStatement(AttrList.take());
+      return ParseLabeledStatement(attrs);
     }
     // PASS THROUGH.
 
   default: {
     if ((getLang().CPlusPlus || !OnlyStatement) && isDeclarationStatement()) {
       SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
-      AttrList.take(); //Passing 'Attr' to ParseDeclaration transfers ownership.
-      DeclGroupPtrTy Decl = ParseDeclaration(Declarator::BlockContext, DeclEnd,
-                                             Attr);
+      DeclGroupPtrTy Decl = ParseDeclaration(Stmts, Declarator::BlockContext,
+                                             DeclEnd, attrs);
       return Actions.ActOnDeclStmt(Decl, DeclStart, DeclEnd);
     }
 
@@ -137,64 +134,65 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
       return StmtError();
     }
     // Otherwise, eat the semicolon.
-    ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
+    ExpectAndConsumeSemi(diag::err_expected_semi_after_expr);
     return Actions.ActOnExprStmt(Actions.MakeFullExpr(Expr.get()));
   }
 
   case tok::kw_case:                // C99 6.8.1: labeled-statement
-    return ParseCaseStatement(AttrList.take());
+    return ParseCaseStatement(attrs);
   case tok::kw_default:             // C99 6.8.1: labeled-statement
-    return ParseDefaultStatement(AttrList.take());
+    return ParseDefaultStatement(attrs);
 
   case tok::l_brace:                // C99 6.8.2: compound-statement
-    return ParseCompoundStatement(AttrList.take());
-  case tok::semi:                   // C99 6.8.3p3: expression[opt] ';'
-    return Actions.ActOnNullStmt(ConsumeToken());
+    return ParseCompoundStatement(attrs);
+  case tok::semi: {                 // C99 6.8.3p3: expression[opt] ';'
+    bool LeadingEmptyMacro = Tok.hasLeadingEmptyMacro();
+    return Actions.ActOnNullStmt(ConsumeToken(), LeadingEmptyMacro);
+  }
 
   case tok::kw_if:                  // C99 6.8.4.1: if-statement
-    return ParseIfStatement(AttrList.take());
+    return ParseIfStatement(attrs);
   case tok::kw_switch:              // C99 6.8.4.2: switch-statement
-    return ParseSwitchStatement(AttrList.take());
+    return ParseSwitchStatement(attrs);
 
   case tok::kw_while:               // C99 6.8.5.1: while-statement
-    return ParseWhileStatement(AttrList.take());
+    return ParseWhileStatement(attrs);
   case tok::kw_do:                  // C99 6.8.5.2: do-statement
-    Res = ParseDoStatement(AttrList.take());
+    Res = ParseDoStatement(attrs);
     SemiError = "do/while";
     break;
   case tok::kw_for:                 // C99 6.8.5.3: for-statement
-    return ParseForStatement(AttrList.take());
+    return ParseForStatement(attrs);
 
   case tok::kw_goto:                // C99 6.8.6.1: goto-statement
-    Res = ParseGotoStatement(AttrList.take());
+    Res = ParseGotoStatement(attrs);
     SemiError = "goto";
     break;
   case tok::kw_continue:            // C99 6.8.6.2: continue-statement
-    Res = ParseContinueStatement(AttrList.take());
+    Res = ParseContinueStatement(attrs);
     SemiError = "continue";
     break;
   case tok::kw_break:               // C99 6.8.6.3: break-statement
-    Res = ParseBreakStatement(AttrList.take());
+    Res = ParseBreakStatement(attrs);
     SemiError = "break";
     break;
   case tok::kw_return:              // C99 6.8.6.4: return-statement
-    Res = ParseReturnStatement(AttrList.take());
+    Res = ParseReturnStatement(attrs);
     SemiError = "return";
     break;
 
   case tok::kw_asm: {
-    if (Attr.HasAttr)
-      Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-        << Attr.Range;
+    ProhibitAttributes(attrs);
     bool msAsm = false;
     Res = ParseAsmStatement(msAsm);
+    Res = Actions.ActOnFinishFullStmt(Res.get());
     if (msAsm) return move(Res);
     SemiError = "asm";
     break;
   }
 
   case tok::kw_try:                 // C++ 15: try-block
-    return ParseCXXTryBlock(AttrList.take());
+    return ParseCXXTryBlock(attrs);
   }
 
   // If we reached this code, the statement must end in a semicolon.
@@ -218,11 +216,10 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
 ///         identifier ':' statement
 /// [GNU]   identifier ':' attributes[opt] statement
 ///
-StmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
+StmtResult Parser::ParseLabeledStatement(ParsedAttributes &attrs) {
   assert(Tok.is(tok::identifier) && Tok.getIdentifierInfo() &&
          "Not an identifier!");
 
-  llvm::OwningPtr<AttributeList> AttrList(Attr);
   Token IdentTok = Tok;  // Save the whole token.
   ConsumeToken();  // eat the identifier.
 
@@ -232,19 +229,21 @@ StmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
   SourceLocation ColonLoc = ConsumeToken();
 
   // Read label attributes, if present.
-  if (Tok.is(tok::kw___attribute))
-    AttrList.reset(addAttributeLists(AttrList.take(), ParseGNUAttributes()));
+  MaybeParseGNUAttributes(attrs);
 
   StmtResult SubStmt(ParseStatement());
 
   // Broken substmt shouldn't prevent the label from being added to the AST.
   if (SubStmt.isInvalid())
     SubStmt = Actions.ActOnNullStmt(ColonLoc);
-
-  // FIXME: use attributes?
-  return Actions.ActOnLabelStmt(IdentTok.getLocation(),
-                                IdentTok.getIdentifierInfo(),
-                                ColonLoc, SubStmt.get());
+  
+  LabelDecl *LD = Actions.LookupOrCreateLabel(IdentTok.getIdentifierInfo(),
+                                              IdentTok.getLocation());
+  if (AttributeList *Attrs = attrs.getList())
+    Actions.ProcessDeclAttributeList(Actions.CurScope, LD, Attrs);
+  
+  return Actions.ActOnLabelStmt(IdentTok.getLocation(), LD, ColonLoc,
+                                SubStmt.get());
 }
 
 /// ParseCaseStatement
@@ -252,10 +251,9 @@ StmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
 ///         'case' constant-expression ':' statement
 /// [GNU]   'case' constant-expression '...' constant-expression ':' statement
 ///
-StmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
+StmtResult Parser::ParseCaseStatement(ParsedAttributes &attrs) {
   assert(Tok.is(tok::kw_case) && "Not a case stmt!");
   // FIXME: Use attributes?
-  delete Attr;
 
   // It is very very common for code to contain many case statements recursively
   // nested, as in (but usually without indentation):
@@ -316,14 +314,22 @@ StmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
     
     ColonProtection.restore();
 
-    if (Tok.isNot(tok::colon)) {
-      Diag(Tok, diag::err_expected_colon_after) << "'case'";
-      SkipUntil(tok::colon);
-      return StmtError();
-    }
-
-    SourceLocation ColonLoc = ConsumeToken();
+    SourceLocation ColonLoc;
+    if (Tok.is(tok::colon)) {
+      ColonLoc = ConsumeToken();
 
+    // Treat "case blah;" as a typo for "case blah:".
+    } else if (Tok.is(tok::semi)) {
+      ColonLoc = ConsumeToken();
+      Diag(ColonLoc, diag::err_expected_colon_after) << "'case'"
+        << FixItHint::CreateReplacement(ColonLoc, ":");
+    } else {
+      SourceLocation ExpectedLoc = PP.getLocForEndOfToken(PrevTokLocation);
+      Diag(ExpectedLoc, diag::err_expected_colon_after) << "'case'"
+        << FixItHint::CreateInsertion(ExpectedLoc, ":");
+      ColonLoc = ExpectedLoc;
+    }
+    
     StmtResult Case =
       Actions.ActOnCaseStmt(CaseLoc, LHS.get(), DotDotDotLoc,
                             RHS.get(), ColonLoc);
@@ -379,21 +385,28 @@ StmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
 ///         'default' ':' statement
 /// Note that this does not parse the 'statement' at the end.
 ///
-StmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
+StmtResult Parser::ParseDefaultStatement(ParsedAttributes &attrs) {
   //FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_default) && "Not a default stmt!");
   SourceLocation DefaultLoc = ConsumeToken();  // eat the 'default'.
 
-  if (Tok.isNot(tok::colon)) {
-    Diag(Tok, diag::err_expected_colon_after) << "'default'";
-    SkipUntil(tok::colon);
-    return StmtError();
-  }
-
-  SourceLocation ColonLoc = ConsumeToken();
+  SourceLocation ColonLoc;
+  if (Tok.is(tok::colon)) {
+    ColonLoc = ConsumeToken();
 
+  // Treat "default;" as a typo for "default:".
+  } else if (Tok.is(tok::semi)) {
+    ColonLoc = ConsumeToken();
+    Diag(ColonLoc, diag::err_expected_colon_after) << "'default'"
+      << FixItHint::CreateReplacement(ColonLoc, ":");
+  } else {
+    SourceLocation ExpectedLoc = PP.getLocForEndOfToken(PrevTokLocation);
+    Diag(ExpectedLoc, diag::err_expected_colon_after) << "'default'"
+      << FixItHint::CreateInsertion(ExpectedLoc, ":");
+    ColonLoc = ExpectedLoc;
+  }
+  
   // Diagnose the common error "switch (X) {... default: }", which is not valid.
   if (Tok.is(tok::r_brace)) {
     Diag(Tok, diag::err_label_end_of_compound_statement);
@@ -436,10 +449,9 @@ StmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
 /// [OMP]   barrier-directive
 /// [OMP]   flush-directive
 ///
-StmtResult Parser::ParseCompoundStatement(AttributeList *Attr,
+StmtResult Parser::ParseCompoundStatement(ParsedAttributes &attrs,
                                                         bool isStmtExpr) {
   //FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::l_brace) && "Not a compount stmt!");
 
@@ -460,18 +472,53 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   PrettyStackTraceLoc CrashInfo(PP.getSourceManager(),
                                 Tok.getLocation(),
                                 "in compound statement ('{}')");
-
+  InMessageExpressionRAIIObject InMessage(*this, false);
+  
   SourceLocation LBraceLoc = ConsumeBrace();  // eat the '{'.
 
-  // TODO: "__label__ X, Y, Z;" is the GNU "Local Label" extension.  These are
-  // only allowed at the start of a compound stmt regardless of the language.
+  StmtVector Stmts(Actions);
 
-  typedef StmtVector StmtsTy;
-  StmtsTy Stmts(Actions);
+  // "__label__ X, Y, Z;" is the GNU "Local Label" extension.  These are
+  // only allowed at the start of a compound stmt regardless of the language.
+  while (Tok.is(tok::kw___label__)) {
+    SourceLocation LabelLoc = ConsumeToken();
+    Diag(LabelLoc, diag::ext_gnu_local_label);
+    
+    llvm::SmallVector<Decl *, 8> DeclsInGroup;
+    while (1) {
+      if (Tok.isNot(tok::identifier)) {
+        Diag(Tok, diag::err_expected_ident);
+        break;
+      }
+      
+      IdentifierInfo *II = Tok.getIdentifierInfo();
+      SourceLocation IdLoc = ConsumeToken();
+      DeclsInGroup.push_back(Actions.LookupOrCreateLabel(II, IdLoc, true));
+      
+      if (!Tok.is(tok::comma))
+        break;
+      ConsumeToken();
+    }
+    
+    DeclSpec DS;
+    DeclGroupPtrTy Res = Actions.FinalizeDeclaratorGroup(getCurScope(), DS,
+                                      DeclsInGroup.data(), DeclsInGroup.size());
+    StmtResult R = Actions.ActOnDeclStmt(Res, LabelLoc, Tok.getLocation());
+    
+    ExpectAndConsume(tok::semi, diag::err_expected_semi_declaration);
+    if (R.isUsable())
+      Stmts.push_back(R.release());
+  }
+  
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
+    if (Tok.is(tok::annot_pragma_unused)) {
+      HandlePragmaUnused();
+      continue;
+    }
+
     StmtResult R;
     if (Tok.isNot(tok::kw___extension__)) {
-      R = ParseStatementOrDeclaration(false);
+      R = ParseStatementOrDeclaration(Stmts, false);
     } else {
       // __extension__ can start declarations and it can also be a unary
       // operator for expressions.  Consume multiple __extension__ markers here
@@ -481,9 +528,8 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
       while (Tok.is(tok::kw___extension__))
         ConsumeToken();
 
-      CXX0XAttributeList Attr;
-      if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-        Attr = ParseCXX0XAttributes();
+      ParsedAttributesWithRange attrs;
+      MaybeParseCXX0XAttributes(attrs);
 
       // If this is the start of a declaration, parse it as such.
       if (isDeclarationStatement()) {
@@ -492,8 +538,9 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
         ExtensionRAIIObject O(Diags);
 
         SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
-        DeclGroupPtrTy Res = ParseDeclaration(Declarator::BlockContext, DeclEnd,
-                                              Attr);
+        DeclGroupPtrTy Res = ParseDeclaration(Stmts,
+                                              Declarator::BlockContext, DeclEnd,
+                                              attrs);
         R = Actions.ActOnDeclStmt(Res, DeclStart, DeclEnd);
       } else {
         // Otherwise this was a unary __extension__ marker.
@@ -507,7 +554,7 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
         // FIXME: Use attributes?
         // Eat the semicolon at the end of stmt and convert the expr into a
         // statement.
-        ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
+        ExpectAndConsumeSemi(diag::err_expected_semi_after_expr);
         R = Actions.ActOnExprStmt(Actions.MakeFullExpr(Res.get()));
       }
     }
@@ -543,12 +590,9 @@ bool Parser::ParseParenExprOrCondition(ExprResult &ExprResult,
                                        Decl *&DeclResult,
                                        SourceLocation Loc,
                                        bool ConvertToBoolean) {
-  bool ParseError = false;
-  
   SourceLocation LParenLoc = ConsumeParen();
   if (getLang().CPlusPlus) 
-    ParseError = ParseCXXCondition(ExprResult, DeclResult, Loc, 
-                                   ConvertToBoolean);
+    ParseCXXCondition(ExprResult, DeclResult, Loc, ConvertToBoolean);
   else {
     ExprResult = ParseExpression();
     DeclResult = 0;
@@ -583,9 +627,8 @@ bool Parser::ParseParenExprOrCondition(ExprResult &ExprResult,
 /// [C++]   'if' '(' condition ')' statement
 /// [C++]   'if' '(' condition ')' statement 'else' statement
 ///
-StmtResult Parser::ParseIfStatement(AttributeList *Attr) {
+StmtResult Parser::ParseIfStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_if) && "Not an if stmt!");
   SourceLocation IfLoc = ConsumeToken();  // eat the 'if'.
@@ -706,9 +749,8 @@ StmtResult Parser::ParseIfStatement(AttributeList *Attr) {
 ///       switch-statement:
 ///         'switch' '(' expression ')' statement
 /// [C++]   'switch' '(' condition ')' statement
-StmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
+StmtResult Parser::ParseSwitchStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_switch) && "Not a switch stmt!");
   SourceLocation SwitchLoc = ConsumeToken();  // eat the 'switch'.
@@ -792,9 +834,8 @@ StmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
 ///       while-statement: [C99 6.8.5.1]
 ///         'while' '(' expression ')' statement
 /// [C++]   'while' '(' condition ')' statement
-StmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
+StmtResult Parser::ParseWhileStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_while) && "Not a while stmt!");
   SourceLocation WhileLoc = Tok.getLocation();
@@ -867,9 +908,8 @@ StmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
 ///       do-statement: [C99 6.8.5.2]
 ///         'do' statement 'while' '(' expression ')' ';'
 /// Note: this lets the caller parse the end ';'.
-StmtResult Parser::ParseDoStatement(AttributeList *Attr) {
+StmtResult Parser::ParseDoStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_do) && "Not a do stmt!");
   SourceLocation DoLoc = ConsumeToken();  // eat the 'do'.
@@ -944,9 +984,8 @@ StmtResult Parser::ParseDoStatement(AttributeList *Attr) {
 /// [C++]   expression-statement
 /// [C++]   simple-declaration
 ///
-StmtResult Parser::ParseForStatement(AttributeList *Attr) {
+StmtResult Parser::ParseForStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_for) && "Not a for stmt!");
   SourceLocation ForLoc = ConsumeToken();  // eat the 'for'.
@@ -1010,13 +1049,13 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
     if (!C99orCXXorObjC)   // Use of C99-style for loops in C90 mode?
       Diag(Tok, diag::ext_c99_variable_decl_in_for_loop);
 
-    AttributeList *AttrList = 0;
-    if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-      AttrList = ParseCXX0XAttributes().AttrList;
+    ParsedAttributesWithRange attrs;
+    MaybeParseCXX0XAttributes(attrs);
 
     SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
-    DeclGroupPtrTy DG = ParseSimpleDeclaration(Declarator::ForContext, DeclEnd,
-                                               AttrList, false);
+    StmtVector Stmts(Actions);
+    DeclGroupPtrTy DG = ParseSimpleDeclaration(Stmts, Declarator::ForContext, 
+                                               DeclEnd, attrs, false);
     FirstPart = Actions.ActOnDeclStmt(DG, DeclStart, Tok.getLocation());
 
     if (Tok.is(tok::semi)) {  // for (int x = 4;
@@ -1033,18 +1072,23 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
       Collection = ParseExpression();
     } else {
       Diag(Tok, diag::err_expected_semi_for);
-      SkipUntil(tok::semi);
     }
   } else {
     Value = ParseExpression();
 
+    ForEach = isTokIdentifier_in();
+
     // Turn the expression into a stmt.
-    if (!Value.isInvalid())
-      FirstPart = Actions.ActOnExprStmt(Actions.MakeFullExpr(Value.get()));
+    if (!Value.isInvalid()) {
+      if (ForEach)
+        FirstPart = Actions.ActOnForEachLValueExpr(Value.get());
+      else
+        FirstPart = Actions.ActOnExprStmt(Actions.MakeFullExpr(Value.get()));
+    }
 
     if (Tok.is(tok::semi)) {
       ConsumeToken();
-    } else if ((ForEach = isTokIdentifier_in())) {
+    } else if (ForEach) {
       ConsumeToken(); // consume 'in'
       
       if (Tok.is(tok::code_completion)) {
@@ -1053,8 +1097,14 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
       }
       Collection = ParseExpression();
     } else {
-      if (!Value.isInvalid()) Diag(Tok, diag::err_expected_semi_for);
-      SkipUntil(tok::semi);
+      if (!Value.isInvalid()) {
+        Diag(Tok, diag::err_expected_semi_for);
+      } else {
+        // Skip until semicolon or rparen, don't consume it.
+        SkipUntil(tok::r_paren, true, true);
+        if (Tok.is(tok::semi))
+          ConsumeToken();
+      }
     }
   }
   if (!ForEach) {
@@ -1062,6 +1112,8 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
     // Parse the second part of the for specifier.
     if (Tok.is(tok::semi)) {  // for (...;;
       // no second part.
+    } else if (Tok.is(tok::r_paren)) {
+      // missing both semicolons.
     } else {
       ExprResult Second;
       if (getLang().CPlusPlus)
@@ -1076,12 +1128,16 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
       SecondPart = Actions.MakeFullExpr(Second.get());
     }
 
+    if (Tok.isNot(tok::semi)) {
+      if (!SecondPartIsInvalid || SecondVar)
+        Diag(Tok, diag::err_expected_semi_for);
+      else
+        // Skip until semicolon or rparen, don't consume it.
+        SkipUntil(tok::r_paren, true, true);
+    }
+
     if (Tok.is(tok::semi)) {
       ConsumeToken();
-    } else {
-      if (!SecondPartIsInvalid || SecondVar) 
-        Diag(Tok, diag::err_expected_semi_for);
-      SkipUntil(tok::semi);
     }
 
     // Parse the third part of the for specifier.
@@ -1137,17 +1193,17 @@ StmtResult Parser::ParseForStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-StmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
+StmtResult Parser::ParseGotoStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_goto) && "Not a goto stmt!");
   SourceLocation GotoLoc = ConsumeToken();  // eat the 'goto'.
 
   StmtResult Res;
   if (Tok.is(tok::identifier)) {
-    Res = Actions.ActOnGotoStmt(GotoLoc, Tok.getLocation(),
-                                Tok.getIdentifierInfo());
+    LabelDecl *LD = Actions.LookupOrCreateLabel(Tok.getIdentifierInfo(),
+                                                Tok.getLocation());
+    Res = Actions.ActOnGotoStmt(GotoLoc, Tok.getLocation(), LD);
     ConsumeToken();
   } else if (Tok.is(tok::star)) {
     // GNU indirect goto extension.
@@ -1173,9 +1229,8 @@ StmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-StmtResult Parser::ParseContinueStatement(AttributeList *Attr) {
+StmtResult Parser::ParseContinueStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   SourceLocation ContinueLoc = ConsumeToken();  // eat the 'continue'.
   return Actions.ActOnContinueStmt(ContinueLoc, getCurScope());
@@ -1187,9 +1242,8 @@ StmtResult Parser::ParseContinueStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-StmtResult Parser::ParseBreakStatement(AttributeList *Attr) {
+StmtResult Parser::ParseBreakStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   SourceLocation BreakLoc = ConsumeToken();  // eat the 'break'.
   return Actions.ActOnBreakStmt(BreakLoc, getCurScope());
@@ -1198,9 +1252,8 @@ StmtResult Parser::ParseBreakStatement(AttributeList *Attr) {
 /// ParseReturnStatement
 ///       jump-statement:
 ///         'return' expression[opt] ';'
-StmtResult Parser::ParseReturnStatement(AttributeList *Attr) {
+StmtResult Parser::ParseReturnStatement(ParsedAttributes &attrs) {
   // FIXME: Use attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_return) && "Not a return stmt!");
   SourceLocation ReturnLoc = ConsumeToken();  // eat the 'return'.
@@ -1225,10 +1278,12 @@ StmtResult Parser::ParseReturnStatement(AttributeList *Attr) {
 
 /// FuzzyParseMicrosoftAsmStatement. When -fms-extensions is enabled, this
 /// routine is called to skip/ignore tokens that comprise the MS asm statement.
-StmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
+StmtResult Parser::FuzzyParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
+  SourceLocation EndLoc;
   if (Tok.is(tok::l_brace)) {
     unsigned short savedBraceCount = BraceCount;
     do {
+      EndLoc = Tok.getLocation();
       ConsumeAnyToken();
     } while (BraceCount > savedBraceCount && Tok.isNot(tok::eof));
   } else {
@@ -1238,6 +1293,7 @@ StmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
     SourceLocation TokLoc = Tok.getLocation();
     unsigned LineNo = SrcMgr.getInstantiationLineNumber(TokLoc);
     do {
+      EndLoc = TokLoc;
       ConsumeAnyToken();
       TokLoc = Tok.getLocation();
     } while ((SrcMgr.getInstantiationLineNumber(TokLoc) == LineNo) &&
@@ -1253,10 +1309,10 @@ StmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
   ExprVector Constraints(Actions);
   ExprVector Exprs(Actions);
   ExprVector Clobbers(Actions);
-  return Actions.ActOnAsmStmt(Tok.getLocation(), true, true, 0, 0, 0,
+  return Actions.ActOnAsmStmt(AsmLoc, true, true, 0, 0, 0,
                               move_arg(Constraints), move_arg(Exprs),
                               AsmString.take(), move_arg(Clobbers),
-                              Tok.getLocation(), true);
+                              EndLoc, true);
 }
 
 /// ParseAsmStatement - Parse a GNU extended asm statement.
@@ -1292,7 +1348,7 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
 
   if (getLang().Microsoft && Tok.isNot(tok::l_paren) && !isTypeQualifier()) {
     msAsm = true;
-    return FuzzyParseMicrosoftAsmStatement();
+    return FuzzyParseMicrosoftAsmStatement(AsmLoc);
   }
   DeclSpec DS;
   SourceLocation Loc = Tok.getLocation();
@@ -1470,6 +1526,10 @@ Decl *Parser::ParseFunctionStatementBody(Decl *Decl) {
   assert(Tok.is(tok::l_brace));
   SourceLocation LBraceLoc = Tok.getLocation();
 
+  if (PP.isCodeCompletionEnabled())
+    if (trySkippingFunctionBodyForCodeCompletion())
+      return Actions.ActOnFinishFunctionBody(Decl, 0);
+
   PrettyDeclStackTraceEntry CrashInfo(Actions, Decl, LBraceLoc,
                                       "parsing function body");
 
@@ -1502,6 +1562,10 @@ Decl *Parser::ParseFunctionTryBlock(Decl *Decl) {
   if (Tok.is(tok::colon))
     ParseConstructorInitializer(Decl);
 
+  if (PP.isCodeCompletionEnabled())
+    if (trySkippingFunctionBodyForCodeCompletion())
+      return Actions.ActOnFinishFunctionBody(Decl, 0);
+
   SourceLocation LBraceLoc = Tok.getLocation();
   StmtResult FnBody(ParseCXXTryBlockCommon(TryLoc));
   // If we failed to parse the try-catch, we just give the function an empty
@@ -1513,14 +1577,32 @@ Decl *Parser::ParseFunctionTryBlock(Decl *Decl) {
   return Actions.ActOnFinishFunctionBody(Decl, FnBody.take());
 }
 
+bool Parser::trySkippingFunctionBodyForCodeCompletion() {
+  assert(Tok.is(tok::l_brace));
+  assert(PP.isCodeCompletionEnabled() &&
+         "Should only be called when in code-completion mode");
+
+  // We're in code-completion mode. Skip parsing for all function bodies unless
+  // the body contains the code-completion point.
+  TentativeParsingAction PA(*this);
+  ConsumeBrace();
+  if (SkipUntil(tok::r_brace, /*StopAtSemi=*/false, /*DontConsume=*/false,
+                /*StopAtCodeCompletion=*/true)) {
+    PA.Commit();
+    return true;
+  }
+
+  PA.Revert();
+  return false;
+}
+
 /// ParseCXXTryBlock - Parse a C++ try-block.
 ///
 ///       try-block:
 ///         'try' compound-statement handler-seq
 ///
-StmtResult Parser::ParseCXXTryBlock(AttributeList* Attr) {
+StmtResult Parser::ParseCXXTryBlock(ParsedAttributes &attrs) {
   // FIXME: Add attributes?
-  delete Attr;
 
   assert(Tok.is(tok::kw_try) && "Expected 'try'");
 
@@ -1544,16 +1626,15 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
   if (Tok.isNot(tok::l_brace))
     return StmtError(Diag(Tok, diag::err_expected_lbrace));
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
-  StmtResult TryBlock(ParseCompoundStatement(0));
+  ParsedAttributesWithRange attrs;
+  StmtResult TryBlock(ParseCompoundStatement(attrs));
   if (TryBlock.isInvalid())
     return move(TryBlock);
 
   StmtVector Handlers(Actions);
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
-    CXX0XAttributeList Attr = ParseCXX0XAttributes();
-    Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-      << Attr.Range;
-  }
+  MaybeParseCXX0XAttributes(attrs);
+  ProhibitAttributes(attrs);
+
   if (Tok.isNot(tok::kw_catch))
     return StmtError(Diag(Tok, diag::err_expected_catch));
   while (Tok.is(tok::kw_catch)) {
@@ -1614,7 +1695,8 @@ StmtResult Parser::ParseCXXCatchBlock() {
     return StmtError(Diag(Tok, diag::err_expected_lbrace));
 
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
-  StmtResult Block(ParseCompoundStatement(0));
+  ParsedAttributes attrs;
+  StmtResult Block(ParseCompoundStatement(attrs));
   if (Block.isInvalid())
     return move(Block);
 
diff --git a/lib/Parse/ParseTemplate.cpp b/lib/Parse/ParseTemplate.cpp
index dfb4785489d0..8387c8819525 100644
--- a/lib/Parse/ParseTemplate.cpp
+++ b/lib/Parse/ParseTemplate.cpp
@@ -196,12 +196,18 @@ Parser::ParseSingleDeclarationAfterTemplate(
     return 0;
   }
 
+  ParsedAttributesWithRange prefixAttrs;
+  MaybeParseCXX0XAttributes(prefixAttrs);
+
+  if (Tok.is(tok::kw_using))
+    return ParseUsingDirectiveOrDeclaration(Context, TemplateInfo, DeclEnd,
+                                            prefixAttrs);
+
   // Parse the declaration specifiers, stealing the accumulated
   // diagnostics from the template parameters.
   ParsingDeclSpec DS(DiagsFromTParams);
 
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-    DS.AddAttributes(ParseCXX0XAttributes().AttrList);
+  DS.takeAttributesFrom(prefixAttrs);
 
   ParseDeclarationSpecifiers(DS, TemplateInfo, AS,
                              getDeclSpecContextFromDeclaratorContext(Context));
@@ -240,7 +246,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
 
     // Eat the semi colon after the declaration.
     ExpectAndConsume(tok::semi, diag::err_expected_semi_declaration);
-    DS.complete(ThisDecl);
+    DeclaratorInfo.complete(ThisDecl);
     return ThisDecl;
   }
 
@@ -337,7 +343,7 @@ Parser::ParseTemplateParameterList(unsigned Depth,
       // subsumed by whatever goes on in ParseTemplateParameter.
       // TODO: This could match >>, and it would be nice to avoid those
       // silly errors with template <vec<T>>.
-      // Diag(Tok.getLocation(), diag::err_expected_comma_greater);
+      Diag(Tok.getLocation(), diag::err_expected_comma_greater);
       SkipUntil(tok::greater, true, true);
       return false;
     }
@@ -415,12 +421,14 @@ bool Parser::isStartOfTemplateTypeParameter() {
 ///         parameter-declaration
 ///
 ///       type-parameter: (see below)
-///         'class' ...[opt][C++0x] identifier[opt]
+///         'class' ...[opt] identifier[opt]
 ///         'class' identifier[opt] '=' type-id
-///         'typename' ...[opt][C++0x] identifier[opt]
+///         'typename' ...[opt] identifier[opt]
 ///         'typename' identifier[opt] '=' type-id
-///         'template' ...[opt][C++0x] '<' template-parameter-list '>' 'class' identifier[opt]
-///         'template' '<' template-parameter-list '>' 'class' identifier[opt] = id-expression
+///         'template' '<' template-parameter-list '>' 
+///               'class' ...[opt] identifier[opt]
+///         'template' '<' template-parameter-list '>' 'class' identifier[opt]
+///               = id-expression
 Decl *Parser::ParseTemplateParameter(unsigned Depth, unsigned Position) {
   if (isStartOfTemplateTypeParameter())
     return ParseTypeParameter(Depth, Position);
@@ -459,7 +467,7 @@ Decl *Parser::ParseTypeParameter(unsigned Depth, unsigned Position) {
     EllipsisLoc = ConsumeToken();
 
     if (!getLang().CPlusPlus0x)
-      Diag(EllipsisLoc, diag::err_variadic_templates);
+      Diag(EllipsisLoc, diag::ext_variadic_templates);
   }
 
   // Grab the template parameter name (if given)
@@ -496,8 +504,10 @@ Decl *Parser::ParseTypeParameter(unsigned Depth, unsigned Position) {
 /// template parameters.
 ///
 ///       type-parameter:    [C++ temp.param]
-///         'template' '<' template-parameter-list '>' 'class' identifier[opt]
-///         'template' '<' template-parameter-list '>' 'class' identifier[opt] = id-expression
+///         'template' '<' template-parameter-list '>' 'class' 
+///                  ...[opt] identifier[opt]
+///         'template' '<' template-parameter-list '>' 'class' identifier[opt] 
+///                  = id-expression
 Decl *
 Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   assert(Tok.is(tok::kw_template) && "Expected 'template' keyword");
@@ -521,8 +531,17 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
       << PP.getSpelling(Tok);
     return 0;
   }
-  SourceLocation ClassLoc = ConsumeToken();
+  ConsumeToken();
 
+  // Parse the ellipsis, if given.
+  SourceLocation EllipsisLoc;
+  if (Tok.is(tok::ellipsis)) {
+    EllipsisLoc = ConsumeToken();
+    
+    if (!getLang().CPlusPlus0x)
+      Diag(EllipsisLoc, diag::ext_variadic_templates);
+  }
+      
   // Get the identifier, if given.
   SourceLocation NameLoc;
   IdentifierInfo* ParamName = 0;
@@ -540,7 +559,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   TemplateParamsTy *ParamList =
     Actions.ActOnTemplateParameterList(Depth, SourceLocation(),
                                        TemplateLoc, LAngleLoc,
-                                       &TemplateParams[0],
+                                       TemplateParams.data(),
                                        TemplateParams.size(),
                                        RAngleLoc);
 
@@ -563,9 +582,9 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   }
   
   return Actions.ActOnTemplateTemplateParameter(getCurScope(), TemplateLoc,
-                                                ParamList, ParamName,
-                                                NameLoc, Depth, Position,
-                                                EqualLoc, DefaultArg);
+                                                ParamList, EllipsisLoc, 
+                                                ParamName, NameLoc, Depth, 
+                                                Position, EqualLoc, DefaultArg);
 }
 
 /// ParseNonTypeTemplateParameter - Handle the parsing of non-type
@@ -576,8 +595,6 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
 ///         parameter-declaration
 Decl *
 Parser::ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position) {
-  SourceLocation StartLoc = Tok.getLocation();
-
   // Parse the declaration-specifiers (i.e., the type).
   // FIXME: The type should probably be restricted in some way... Not all
   // declarators (parts of declarators?) are accepted for parameters.
@@ -658,7 +675,7 @@ Parser::ParseTemplateIdAfterTemplateName(TemplateTy Template,
   bool Invalid = false;
   {
     GreaterThanIsOperatorScope G(GreaterThanIsOperator, false);
-    if (Tok.isNot(tok::greater))
+    if (Tok.isNot(tok::greater) && Tok.isNot(tok::greatergreater))
       Invalid = ParseTemplateArgumentList(TemplateArgs);
 
     if (Invalid) {
@@ -888,7 +905,7 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
   // We parse an id-expression that refers to a class template or template
   // alias. The grammar we parse is:
   //
-  //   nested-name-specifier[opt] template[opt] identifier
+  //   nested-name-specifier[opt] template[opt] identifier ...[opt]
   //
   // followed by a token that terminates a template argument, such as ',', 
   // '>', or (in some cases) '>>'.
@@ -896,6 +913,8 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
   ParseOptionalCXXScopeSpecifier(SS, ParsedType(),
                                  /*EnteringContext=*/false);
   
+  ParsedTemplateArgument Result;
+  SourceLocation EllipsisLoc;
   if (SS.isSet() && Tok.is(tok::kw_template)) {
     // Parse the optional 'template' keyword following the 
     // nested-name-specifier.
@@ -907,6 +926,10 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
       Name.setIdentifier(Tok.getIdentifierInfo(), Tok.getLocation());
       ConsumeToken(); // the identifier
       
+      // Parse the ellipsis.
+      if (Tok.is(tok::ellipsis))
+        EllipsisLoc = ConsumeToken();
+      
       // If the next token signals the end of a template argument,
       // then we have a dependent template name that could be a template
       // template argument.
@@ -917,7 +940,7 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
                                              /*ObjectType=*/ ParsedType(),
                                              /*EnteringContext=*/false,
                                              Template))
-        return ParsedTemplateArgument(SS, Template, Name.StartLocation);
+        Result = ParsedTemplateArgument(SS, Template, Name.StartLocation);
     }
   } else if (Tok.is(tok::identifier)) {
     // We may have a (non-dependent) template name.
@@ -926,6 +949,10 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
     Name.setIdentifier(Tok.getIdentifierInfo(), Tok.getLocation());
     ConsumeToken(); // the identifier
     
+    // Parse the ellipsis.
+    if (Tok.is(tok::ellipsis))
+      EllipsisLoc = ConsumeToken();
+
     if (isEndOfTemplateArgument(Tok)) {
       bool MemberOfUnknownSpecialization;
       TemplateNameKind TNK = Actions.isTemplateName(getCurScope(), SS,
@@ -938,13 +965,16 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
       if (TNK == TNK_Dependent_template_name || TNK == TNK_Type_template) {
         // We have an id-expression that refers to a class template or
         // (C++0x) template alias. 
-        return ParsedTemplateArgument(SS, Template, Name.StartLocation);
+        Result = ParsedTemplateArgument(SS, Template, Name.StartLocation);
       }
     }
   }
   
-  // We don't have a template template argument.  
-  return ParsedTemplateArgument();
+  // If this is a pack expansion, build it as such.
+  if (EllipsisLoc.isValid() && !Result.isInvalid())
+    Result = Actions.ActOnPackExpansion(Result, EllipsisLoc);
+  
+  return Result;
 }
 
 /// ParseTemplateArgument - Parse a C++ template argument (C++ [temp.names]).
@@ -962,7 +992,8 @@ ParsedTemplateArgument Parser::ParseTemplateArgument() {
   // Therefore, we initially try to parse a type-id.  
   if (isCXXTypeId(TypeIdAsTemplateArgument)) {
     SourceLocation Loc = Tok.getLocation();
-    TypeResult TypeArg = ParseTypeName();
+    TypeResult TypeArg = ParseTypeName(/*Range=*/0, 
+                                       Declarator::TemplateTypeArgContext);
     if (TypeArg.isInvalid())
       return ParsedTemplateArgument();
     
@@ -1037,6 +1068,11 @@ bool
 Parser::ParseTemplateArgumentList(TemplateArgList &TemplateArgs) {
   while (true) {
     ParsedTemplateArgument Arg = ParseTemplateArgument();
+    if (Tok.is(tok::ellipsis)) {
+      SourceLocation EllipsisLoc  = ConsumeToken();
+      Arg = Actions.ActOnPackExpansion(Arg, EllipsisLoc);
+    }
+
     if (Arg.isInvalid()) {
       SkipUntil(tok::comma, tok::greater, true, true);
       return true;
@@ -1075,3 +1111,14 @@ Decl *Parser::ParseExplicitInstantiation(SourceLocation ExternLoc,
                                              ParsingTemplateParams,
                                              DeclEnd, AS_none);
 }
+
+SourceRange Parser::ParsedTemplateInfo::getSourceRange() const {
+  if (TemplateParams)
+    return getTemplateParamsRange(TemplateParams->data(),
+                                  TemplateParams->size());
+
+  SourceRange R(TemplateLoc);
+  if (ExternLoc.isValid())
+    R.setBegin(ExternLoc);
+  return R;
+}
diff --git a/lib/Parse/ParseTentative.cpp b/lib/Parse/ParseTentative.cpp
index c22d99fa9d38..a603c37a53e5 100644
--- a/lib/Parse/ParseTentative.cpp
+++ b/lib/Parse/ParseTentative.cpp
@@ -111,10 +111,7 @@ bool Parser::isCXXSimpleDeclaration() {
   // We need tentative parsing...
 
   TentativeParsingAction PA(*this);
-
   TPR = TryParseSimpleDeclaration();
-  SourceLocation TentativeParseLoc = Tok.getLocation();
-
   PA.Revert();
 
   // In case of an error, let the declaration parsing code handle it.
@@ -139,9 +136,13 @@ Parser::TPResult Parser::TryParseSimpleDeclaration() {
 
   if (Tok.is(tok::kw_typeof))
     TryParseTypeofSpecifier();
-  else
+  else {
     ConsumeToken();
-
+    
+    if (getLang().ObjC1 && Tok.is(tok::less))
+      TryParseProtocolQualifiers();
+  }
+  
   assert(Tok.is(tok::l_paren) && "Expected '('");
 
   TPResult TPR = TryParseInitDeclaratorList();
@@ -242,8 +243,12 @@ bool Parser::isCXXConditionDeclaration() {
   // type-specifier-seq
   if (Tok.is(tok::kw_typeof))
     TryParseTypeofSpecifier();
-  else
+  else {
     ConsumeToken();
+    
+    if (getLang().ObjC1 && Tok.is(tok::less))
+      TryParseProtocolQualifiers();
+  }
   assert(Tok.is(tok::l_paren) && "Expected '('");
 
   // declarator
@@ -313,8 +318,13 @@ bool Parser::isCXXTypeId(TentativeCXXTypeIdContext Context, bool &isAmbiguous) {
   // type-specifier-seq
   if (Tok.is(tok::kw_typeof))
     TryParseTypeofSpecifier();
-  else
+  else {
     ConsumeToken();
+    
+    if (getLang().ObjC1 && Tok.is(tok::less))
+      TryParseProtocolQualifiers();
+  }
+  
   assert(Tok.is(tok::l_paren) && "Expected '('");
 
   // declarator
@@ -448,6 +458,7 @@ bool Parser::isCXX0XAttributeSpecifier (bool CheckClosing,
 ///         abstract-declarator:
 ///           ptr-operator abstract-declarator[opt]
 ///           direct-abstract-declarator
+///           ...
 ///
 ///         direct-abstract-declarator:
 ///           direct-abstract-declarator[opt]
@@ -470,7 +481,7 @@ bool Parser::isCXX0XAttributeSpecifier (bool CheckClosing,
 ///           'volatile'
 ///
 ///         declarator-id:
-///           id-expression
+///           '...'[opt] id-expression
 ///
 ///         id-expression:
 ///           unqualified-id
@@ -495,6 +506,7 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
         return TPResult::Error();
 
     if (Tok.is(tok::star) || Tok.is(tok::amp) || Tok.is(tok::caret) ||
+        Tok.is(tok::ampamp) ||
         (Tok.is(tok::annot_cxxscope) && NextToken().is(tok::star))) {
       // ptr-operator
       ConsumeToken();
@@ -509,7 +521,9 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
 
   // direct-declarator:
   // direct-abstract-declarator:
-
+  if (Tok.is(tok::ellipsis))
+    ConsumeToken();
+  
   if ((Tok.is(tok::identifier) ||
        (Tok.is(tok::annot_cxxscope) && NextToken().is(tok::identifier))) &&
       mayHaveIdentifier) {
@@ -532,7 +546,12 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
       // '(' declarator ')'
       // '(' attributes declarator ')'
       // '(' abstract-declarator ')'
-      if (Tok.is(tok::kw___attribute))
+      if (Tok.is(tok::kw___attribute) ||
+          Tok.is(tok::kw___declspec) ||
+          Tok.is(tok::kw___cdecl) ||
+          Tok.is(tok::kw___stdcall) ||
+          Tok.is(tok::kw___fastcall) ||
+          Tok.is(tok::kw___thiscall))
         return TPResult::True(); // attributes indicate declaration
       TPResult TPR = TryParseDeclarator(mayBeAbstract, mayHaveIdentifier);
       if (TPR != TPResult::Ambiguous())
@@ -548,6 +567,10 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
   while (1) {
     TPResult TPR(TPResult::Ambiguous());
 
+    // abstract-declarator: ...
+    if (Tok.is(tok::ellipsis))
+      ConsumeToken();
+
     if (Tok.is(tok::l_paren)) {
       // Check whether we have a function declarator or a possible ctor-style
       // initializer that follows the declarator. Note that ctor-style
@@ -575,6 +598,118 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
   return TPResult::Ambiguous();
 }
 
+Parser::TPResult 
+Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
+  switch (Kind) {
+  // Obviously starts an expression.
+  case tok::numeric_constant:
+  case tok::char_constant:
+  case tok::string_literal:
+  case tok::wide_string_literal:
+  case tok::l_square:
+  case tok::l_paren:
+  case tok::amp:
+  case tok::ampamp:
+  case tok::star:
+  case tok::plus:
+  case tok::plusplus:
+  case tok::minus:
+  case tok::minusminus:
+  case tok::tilde:
+  case tok::exclaim:
+  case tok::kw_sizeof:
+  case tok::kw___func__:
+  case tok::kw_const_cast:
+  case tok::kw_delete:
+  case tok::kw_dynamic_cast:
+  case tok::kw_false:
+  case tok::kw_new:
+  case tok::kw_operator:
+  case tok::kw_reinterpret_cast:
+  case tok::kw_static_cast:
+  case tok::kw_this:
+  case tok::kw_throw:
+  case tok::kw_true:
+  case tok::kw_typeid:
+  case tok::kw_alignof:
+  case tok::kw_noexcept:
+  case tok::kw_nullptr:
+  case tok::kw___null:
+  case tok::kw___alignof:
+  case tok::kw___builtin_choose_expr:
+  case tok::kw___builtin_offsetof:
+  case tok::kw___builtin_types_compatible_p:
+  case tok::kw___builtin_va_arg:
+  case tok::kw___imag:
+  case tok::kw___real:
+  case tok::kw___FUNCTION__:
+  case tok::kw___PRETTY_FUNCTION__:
+  case tok::kw___has_nothrow_assign:
+  case tok::kw___has_nothrow_copy:
+  case tok::kw___has_nothrow_constructor:
+  case tok::kw___has_trivial_assign:
+  case tok::kw___has_trivial_copy:
+  case tok::kw___has_trivial_constructor:
+  case tok::kw___has_trivial_destructor:
+  case tok::kw___has_virtual_destructor:
+  case tok::kw___is_abstract:
+  case tok::kw___is_base_of:
+  case tok::kw___is_class:
+  case tok::kw___is_convertible_to:
+  case tok::kw___is_empty:
+  case tok::kw___is_enum:
+  case tok::kw___is_pod:
+  case tok::kw___is_polymorphic:
+  case tok::kw___is_union:
+  case tok::kw___is_literal:
+  case tok::kw___uuidof:
+    return TPResult::True();
+      
+  // Obviously starts a type-specifier-seq:
+  case tok::kw_char:
+  case tok::kw_const:
+  case tok::kw_double:
+  case tok::kw_enum:
+  case tok::kw_float:
+  case tok::kw_int:
+  case tok::kw_long:
+  case tok::kw_restrict:
+  case tok::kw_short:
+  case tok::kw_signed:
+  case tok::kw_struct:
+  case tok::kw_union:
+  case tok::kw_unsigned:
+  case tok::kw_void:
+  case tok::kw_volatile:
+  case tok::kw__Bool:
+  case tok::kw__Complex:
+  case tok::kw_class:
+  case tok::kw_typename:
+  case tok::kw_wchar_t:
+  case tok::kw_char16_t:
+  case tok::kw_char32_t:
+  case tok::kw_decltype:
+  case tok::kw_thread_local:
+  case tok::kw__Decimal32:
+  case tok::kw__Decimal64:
+  case tok::kw__Decimal128:
+  case tok::kw___thread:
+  case tok::kw_typeof:
+  case tok::kw___cdecl:
+  case tok::kw___stdcall:
+  case tok::kw___fastcall:
+  case tok::kw___thiscall:
+  case tok::kw___vector:
+  case tok::kw___pixel:
+    return TPResult::False();
+
+  default:
+    break;
+  }
+  
+  return TPResult::Ambiguous();
+}
+
 /// isCXXDeclarationSpecifier - Returns TPResult::True() if it is a declaration
 /// specifier, TPResult::False() if it is not, TPResult::Ambiguous() if it could
 /// be either a decl-specifier or a function-style cast, and TPResult::Error()
@@ -803,6 +938,28 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
 
     // simple-type-specifier:
 
+  case tok::annot_typename:
+  case_typename:
+    // In Objective-C, we might have a protocol-qualified type.
+    if (getLang().ObjC1 && NextToken().is(tok::less)) {
+      // Tentatively parse the 
+      TentativeParsingAction PA(*this);
+      ConsumeToken(); // The type token
+      
+      TPResult TPR = TryParseProtocolQualifiers();
+      bool isFollowedByParen = Tok.is(tok::l_paren);
+      
+      PA.Revert();
+      
+      if (TPR == TPResult::Error())
+        return TPResult::Error();
+      
+      if (isFollowedByParen)
+        return TPResult::Ambiguous();
+      
+      return TPResult::True();
+    }
+      
   case tok::kw_char:
   case tok::kw_wchar_t:
   case tok::kw_char16_t:
@@ -816,11 +973,12 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw_float:
   case tok::kw_double:
   case tok::kw_void:
-  case tok::annot_typename:
-  case_typename:
     if (NextToken().is(tok::l_paren))
       return TPResult::Ambiguous();
 
+    if (isStartOfObjCClassMessageMissingOpenBracket())
+      return TPResult::False();
+      
     return TPResult::True();
 
   // GNU typeof support.
@@ -870,6 +1028,30 @@ Parser::TPResult Parser::TryParseTypeofSpecifier() {
   return TPResult::Ambiguous();
 }
 
+/// [ObjC] protocol-qualifiers:
+////         '<' identifier-list '>'
+Parser::TPResult Parser::TryParseProtocolQualifiers() {
+  assert(Tok.is(tok::less) && "Expected '<' for qualifier list");
+  ConsumeToken();
+  do {
+    if (Tok.isNot(tok::identifier))
+      return TPResult::Error();
+    ConsumeToken();
+    
+    if (Tok.is(tok::comma)) {
+      ConsumeToken();
+      continue;
+    }
+    
+    if (Tok.is(tok::greater)) {
+      ConsumeToken();
+      return TPResult::Ambiguous();
+    }
+  } while (false);
+  
+  return TPResult::Error();
+}
+
 Parser::TPResult Parser::TryParseDeclarationSpecifier() {
   TPResult TPR = isCXXDeclarationSpecifier();
   if (TPR != TPResult::Ambiguous())
@@ -877,8 +1059,12 @@ Parser::TPResult Parser::TryParseDeclarationSpecifier() {
 
   if (Tok.is(tok::kw_typeof))
     TryParseTypeofSpecifier();
-  else
+  else {
     ConsumeToken();
+    
+    if (getLang().ObjC1 && Tok.is(tok::less))
+      TryParseProtocolQualifiers();
+  }
 
   assert(Tok.is(tok::l_paren) && "Expected '('!");
   return TPResult::Ambiguous();
@@ -940,10 +1126,11 @@ bool Parser::isCXXFunctionDeclarator(bool warnIfAmbiguous) {
 ///   parameter-declaration-list ',' parameter-declaration
 ///
 /// parameter-declaration:
-///   decl-specifier-seq declarator
-///   decl-specifier-seq declarator '=' assignment-expression
-///   decl-specifier-seq abstract-declarator[opt]
-///   decl-specifier-seq abstract-declarator[opt] '=' assignment-expression
+///   decl-specifier-seq declarator attributes[opt]
+///   decl-specifier-seq declarator attributes[opt] '=' assignment-expression
+///   decl-specifier-seq abstract-declarator[opt] attributes[opt]
+///   decl-specifier-seq abstract-declarator[opt] attributes[opt]
+///     '=' assignment-expression
 ///
 Parser::TPResult Parser::TryParseParameterDeclarationClause() {
 
@@ -964,6 +1151,9 @@ Parser::TPResult Parser::TryParseParameterDeclarationClause() {
       return TPResult::True(); // '...' is a sign of a function declarator.
     }
 
+    ParsedAttributes attrs;
+    MaybeParseMicrosoftAttributes(attrs);
+
     // decl-specifier-seq
     TPResult TPR = TryParseDeclarationSpecifier();
     if (TPR != TPResult::Ambiguous())
@@ -975,11 +1165,15 @@ Parser::TPResult Parser::TryParseParameterDeclarationClause() {
     if (TPR != TPResult::Ambiguous())
       return TPR;
 
+    // [GNU] attributes[opt]
+    if (Tok.is(tok::kw___attribute))
+      return TPResult::True();
+
     if (Tok.is(tok::equal)) {
       // '=' assignment-expression
       // Parse through assignment-expression.
-      tok::TokenKind StopToks[3] ={ tok::comma, tok::ellipsis, tok::r_paren };
-      if (!SkipUntil(StopToks, 3, true/*StopAtSemi*/, true/*DontConsume*/))
+      tok::TokenKind StopToks[2] ={ tok::comma, tok::r_paren };
+      if (!SkipUntil(StopToks, 2, true/*StopAtSemi*/, true/*DontConsume*/))
         return TPResult::Error();
     }
 
@@ -1029,6 +1223,10 @@ Parser::TPResult Parser::TryParseFunctionDeclarator() {
          Tok.is(tok::kw_restrict)   )
     ConsumeToken();
 
+  // ref-qualifier[opt]
+  if (Tok.is(tok::amp) || Tok.is(tok::ampamp))
+    ConsumeToken();
+  
   // exception-specification
   if (Tok.is(tok::kw_throw)) {
     ConsumeToken();
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index 44bd0fbc0c03..a50763a0e38e 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -23,8 +23,8 @@ using namespace clang;
 
 Parser::Parser(Preprocessor &pp, Sema &actions)
   : CrashInfo(*this), PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
-    GreaterThanIsOperator(true), ColonIsSacred(false),
-    TemplateParameterDepth(0) {
+    GreaterThanIsOperator(true), ColonIsSacred(false), 
+    InMessageExpression(false), TemplateParameterDepth(0) {
   Tok.setKind(tok::eof);
   Actions.CurScope = 0;
   NumCachedScopes = 0;
@@ -50,6 +50,17 @@ Parser::Parser(Preprocessor &pp, Sema &actions)
 
   WeakHandler.reset(new PragmaWeakHandler(actions));
   PP.AddPragmaHandler(WeakHandler.get());
+
+  FPContractHandler.reset(new PragmaFPContractHandler(actions, *this));
+  PP.AddPragmaHandler("STDC", FPContractHandler.get());
+
+  if (getLang().OpenCL) {
+    OpenCLExtensionHandler.reset(
+                  new PragmaOpenCLExtensionHandler(actions, *this));
+    PP.AddPragmaHandler("OPENCL", OpenCLExtensionHandler.get());
+
+    PP.AddPragmaHandler("OPENCL", FPContractHandler.get());
+  }
       
   PP.setCodeCompletionHandler(*this);
 }
@@ -78,7 +89,7 @@ void PrettyStackTraceParserEntry::print(llvm::raw_ostream &OS) const {
 
 
 DiagnosticBuilder Parser::Diag(SourceLocation Loc, unsigned DiagID) {
-  return Diags.Report(FullSourceLoc(Loc, PP.getSourceManager()), DiagID);
+  return Diags.Report(Loc, DiagID);
 }
 
 DiagnosticBuilder Parser::Diag(const Token &Tok, unsigned DiagID) {
@@ -126,6 +137,8 @@ SourceLocation Parser::MatchRHSPunctuation(tok::TokenKind RHSTok,
   case tok::r_brace : LHSName = "{"; DID = diag::err_expected_rbrace; break;
   case tok::r_square: LHSName = "["; DID = diag::err_expected_rsquare; break;
   case tok::greater:  LHSName = "<"; DID = diag::err_expected_greater; break;
+  case tok::greatergreatergreater:
+                      LHSName = "<<<"; DID = diag::err_expected_ggg; break;
   }
   Diag(Tok, DID);
   Diag(LHSLoc, diag::note_matching) << LHSName;
@@ -133,6 +146,13 @@ SourceLocation Parser::MatchRHSPunctuation(tok::TokenKind RHSTok,
   return R;
 }
 
+static bool IsCommonTypo(tok::TokenKind ExpectedTok, const Token &Tok) {
+  switch (ExpectedTok) {
+  case tok::semi: return Tok.is(tok::colon); // : for ;
+  default: return false;
+  }
+}
+
 /// ExpectAndConsume - The parser expects that 'ExpectedTok' is next in the
 /// input.  If so, it is consumed and false is returned.
 ///
@@ -146,6 +166,19 @@ bool Parser::ExpectAndConsume(tok::TokenKind ExpectedTok, unsigned DiagID,
     return false;
   }
 
+  // Detect common single-character typos and resume.
+  if (IsCommonTypo(ExpectedTok, Tok)) {
+    SourceLocation Loc = Tok.getLocation();
+    Diag(Loc, DiagID)
+      << Msg
+      << FixItHint::CreateReplacement(SourceRange(Loc),
+                                      getTokenSimpleSpelling(ExpectedTok));
+    ConsumeAnyToken();
+
+    // Pretend there wasn't a problem.
+    return false;
+  }
+
   const char *Spelling = 0;
   SourceLocation EndLoc = PP.getLocForEndOfToken(PrevTokLocation);
   if (EndLoc.isValid() &&
@@ -162,6 +195,25 @@ bool Parser::ExpectAndConsume(tok::TokenKind ExpectedTok, unsigned DiagID,
   return true;
 }
 
+bool Parser::ExpectAndConsumeSemi(unsigned DiagID) {
+  if (Tok.is(tok::semi) || Tok.is(tok::code_completion)) {
+    ConsumeAnyToken();
+    return false;
+  }
+  
+  if ((Tok.is(tok::r_paren) || Tok.is(tok::r_square)) && 
+      NextToken().is(tok::semi)) {
+    Diag(Tok, diag::err_extraneous_token_before_semi)
+      << PP.getSpelling(Tok)
+      << FixItHint::CreateRemoval(Tok.getLocation());
+    ConsumeAnyToken(); // The ')' or ']'.
+    ConsumeToken(); // The ';'.
+    return false;
+  }
+  
+  return ExpectAndConsume(tok::semi, DiagID);
+}
+
 //===----------------------------------------------------------------------===//
 // Error recovery.
 //===----------------------------------------------------------------------===//
@@ -175,7 +227,8 @@ bool Parser::ExpectAndConsume(tok::TokenKind ExpectedTok, unsigned DiagID,
 /// If SkipUntil finds the specified token, it returns true, otherwise it
 /// returns false.
 bool Parser::SkipUntil(const tok::TokenKind *Toks, unsigned NumToks,
-                       bool StopAtSemi, bool DontConsume) {
+                       bool StopAtSemi, bool DontConsume,
+                       bool StopAtCodeCompletion) {
   // We always want this function to skip at least one token if the first token
   // isn't T and if not at EOF.
   bool isFirstTokenSkipped = true;
@@ -198,23 +251,24 @@ bool Parser::SkipUntil(const tok::TokenKind *Toks, unsigned NumToks,
       return false;
         
     case tok::code_completion:
-      ConsumeToken();
+      if (!StopAtCodeCompletion)
+        ConsumeToken();
       return false;
         
     case tok::l_paren:
       // Recursively skip properly-nested parens.
       ConsumeParen();
-      SkipUntil(tok::r_paren, false);
+      SkipUntil(tok::r_paren, false, false, StopAtCodeCompletion);
       break;
     case tok::l_square:
       // Recursively skip properly-nested square brackets.
       ConsumeBracket();
-      SkipUntil(tok::r_square, false);
+      SkipUntil(tok::r_square, false, false, StopAtCodeCompletion);
       break;
     case tok::l_brace:
       // Recursively skip properly-nested braces.
       ConsumeBrace();
-      SkipUntil(tok::r_brace, false);
+      SkipUntil(tok::r_brace, false, false, StopAtCodeCompletion);
       break;
 
     // Okay, we found a ']' or '}' or ')', which we think should be balanced.
@@ -266,9 +320,8 @@ void Parser::EnterScope(unsigned ScopeFlags) {
     N->Init(getCurScope(), ScopeFlags);
     Actions.CurScope = N;
   } else {
-    Actions.CurScope = new Scope(getCurScope(), ScopeFlags);
+    Actions.CurScope = new Scope(getCurScope(), ScopeFlags, Diags);
   }
-  getCurScope()->setNumErrorsAtStart(Diags.getNumErrors());
 }
 
 /// ExitScope - Pop a scope off the scope stack.
@@ -318,6 +371,15 @@ Parser::~Parser() {
   UnusedHandler.reset();
   PP.RemovePragmaHandler(WeakHandler.get());
   WeakHandler.reset();
+
+  if (getLang().OpenCL) {
+    PP.RemovePragmaHandler("OPENCL", OpenCLExtensionHandler.get());
+    OpenCLExtensionHandler.reset();
+    PP.RemovePragmaHandler("OPENCL", FPContractHandler.get());
+  }
+
+  PP.RemovePragmaHandler("STDC", FPContractHandler.get());
+  FPContractHandler.reset();
   PP.clearCodeCompletionHandler();
 }
 
@@ -347,6 +409,9 @@ void Parser::Initialize() {
     ObjCTypeQuals[objc_byref] = &PP.getIdentifierTable().get("byref");
   }
 
+  Ident_final = 0;
+  Ident_override = 0;
+
   Ident_super = &PP.getIdentifierTable().get("super");
 
   if (getLang().AltiVec) {
@@ -358,16 +423,21 @@ void Parser::Initialize() {
 /// ParseTopLevelDecl - Parse one top-level declaration, return whatever the
 /// action tells us to.  This returns true if the EOF was encountered.
 bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result) {
+
+  while (Tok.is(tok::annot_pragma_unused))
+    HandlePragmaUnused();
+
   Result = DeclGroupPtrTy();
   if (Tok.is(tok::eof)) {
     Actions.ActOnEndOfTranslationUnit();
     return true;
   }
 
-  CXX0XAttributeList Attr;
-  if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
-    Attr = ParseCXX0XAttributes();
-  Result = ParseExternalDeclaration(Attr);
+  ParsedAttributesWithRange attrs;
+  MaybeParseCXX0XAttributes(attrs);
+  MaybeParseMicrosoftAttributes(attrs);
+  
+  Result = ParseExternalDeclaration(attrs);
   return false;
 }
 
@@ -408,8 +478,9 @@ void Parser::ParseTranslationUnit() {
 ///           ';'
 ///
 /// [C++0x/GNU] 'extern' 'template' declaration
-Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
-                                                        ParsingDeclSpec *DS) {
+Parser::DeclGroupPtrTy
+Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
+                                 ParsingDeclSpec *DS) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   
   Decl *SingleDecl = 0;
@@ -433,12 +504,10 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
     // __extension__ silences extension warnings in the subexpression.
     ExtensionRAIIObject O(Diags);  // Use RAII to do this.
     ConsumeToken();
-    return ParseExternalDeclaration(Attr);
+    return ParseExternalDeclaration(attrs);
   }
   case tok::kw_asm: {
-    if (Attr.HasAttr)
-      Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
-        << Attr.Range;
+    ProhibitAttributes(attrs);
 
     ExprResult Result(ParseSimpleAsm());
 
@@ -470,7 +539,7 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
                                    ObjCImpDecl? Sema::PCC_ObjCImplementation
                                               : Sema::PCC_Namespace);
     ConsumeCodeCompletionToken();
-    return ParseExternalDeclaration(Attr);
+    return ParseExternalDeclaration(attrs);
   case tok::kw_using:
   case tok::kw_namespace:
   case tok::kw_typedef:
@@ -480,14 +549,42 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
     // A function definition cannot start with a these keywords.
     {
       SourceLocation DeclEnd;
-      return ParseDeclaration(Declarator::FileContext, DeclEnd, Attr);
+      StmtVector Stmts(Actions);
+      return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);
     }
 
+  case tok::kw_static:
+    // Parse (then ignore) 'static' prior to a template instantiation. This is
+    // a GCC extension that we intentionally do not support.
+    if (getLang().CPlusPlus && NextToken().is(tok::kw_template)) {
+      Diag(ConsumeToken(), diag::warn_static_inline_explicit_inst_ignored)
+        << 0;
+      SourceLocation DeclEnd;
+      StmtVector Stmts(Actions);
+      return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);  
+    }
+    goto dont_know;
+      
   case tok::kw_inline:
-    if (getLang().CPlusPlus && NextToken().is(tok::kw_namespace)) {
+    if (getLang().CPlusPlus) {
+      tok::TokenKind NextKind = NextToken().getKind();
+      
       // Inline namespaces. Allowed as an extension even in C++03.
-      SourceLocation DeclEnd;
-      return ParseDeclaration(Declarator::FileContext, DeclEnd, Attr);
+      if (NextKind == tok::kw_namespace) {
+        SourceLocation DeclEnd;
+        StmtVector Stmts(Actions);
+        return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);
+      }
+      
+      // Parse (then ignore) 'inline' prior to a template instantiation. This is
+      // a GCC extension that we intentionally do not support.
+      if (NextKind == tok::kw_template) {
+        Diag(ConsumeToken(), diag::warn_static_inline_explicit_inst_ignored)
+          << 1;
+        SourceLocation DeclEnd;
+        StmtVector Stmts(Actions);
+        return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);  
+      }
     }
     goto dont_know;
 
@@ -506,10 +603,12 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
   default:
   dont_know:
     // We can't tell whether this is a function-definition or declaration yet.
-    if (DS)
-      return ParseDeclarationOrFunctionDefinition(*DS, Attr.AttrList);
-    else
-      return ParseDeclarationOrFunctionDefinition(Attr.AttrList);
+    if (DS) {
+      DS->takeAttributesFrom(attrs);
+      return ParseDeclarationOrFunctionDefinition(*DS);
+    } else {
+      return ParseDeclarationOrFunctionDefinition(attrs);
+    }
   }
 
   // This routine returns a DeclGroup, if the thing we parsed only contains a
@@ -532,14 +631,13 @@ bool Parser::isDeclarationAfterDeclarator() const {
 /// \brief Determine whether the current token, if it occurs after a
 /// declarator, indicates the start of a function definition.
 bool Parser::isStartOfFunctionDefinition(const ParsingDeclarator &Declarator) {
-  assert(Declarator.getTypeObject(0).Kind == DeclaratorChunk::Function &&
-         "Isn't a function declarator");
+  assert(Declarator.isFunctionDeclarator() && "Isn't a function declarator");
   if (Tok.is(tok::l_brace))   // int X() {}
     return true;
   
   // Handle K&R C argument lists: int X(f) int f; {}
   if (!getLang().CPlusPlus &&
-      Declarator.getTypeObject(0).Fun.isKNRPrototype()) 
+      Declarator.getFunctionTypeInfo().isKNRPrototype()) 
     return isDeclarationSpecifier();
   
   return Tok.is(tok::colon) ||         // X() : Base() {} (used for ctors)
@@ -564,12 +662,8 @@ bool Parser::isStartOfFunctionDefinition(const ParsingDeclarator &Declarator) {
 ///
 Parser::DeclGroupPtrTy
 Parser::ParseDeclarationOrFunctionDefinition(ParsingDeclSpec &DS,
-                                             AttributeList *Attr,
                                              AccessSpecifier AS) {
   // Parse the common declaration-specifiers piece.
-  if (Attr)
-    DS.AddAttributes(Attr);
-
   ParseDeclarationSpecifiers(DS, ParsedTemplateInfo(), AS, DSC_top_level);
 
   // C99 6.7.2.3p6: Handle "struct-or-union identifier;", "enum { X };"
@@ -622,10 +716,11 @@ Parser::ParseDeclarationOrFunctionDefinition(ParsingDeclSpec &DS,
 }
 
 Parser::DeclGroupPtrTy
-Parser::ParseDeclarationOrFunctionDefinition(AttributeList *Attr,
+Parser::ParseDeclarationOrFunctionDefinition(ParsedAttributes &attrs,
                                              AccessSpecifier AS) {
   ParsingDeclSpec DS(*this);
-  return ParseDeclarationOrFunctionDefinition(DS, Attr, AS);
+  DS.takeAttributesFrom(attrs);
+  return ParseDeclarationOrFunctionDefinition(DS, AS);
 }
 
 /// ParseFunctionDefinition - We parsed and verified that the specified
@@ -643,11 +738,8 @@ Parser::ParseDeclarationOrFunctionDefinition(AttributeList *Attr,
 ///         decl-specifier-seq[opt] declarator function-try-block
 ///
 Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
-                                     const ParsedTemplateInfo &TemplateInfo) {
-  const DeclaratorChunk &FnTypeInfo = D.getTypeObject(0);
-  assert(FnTypeInfo.Kind == DeclaratorChunk::Function &&
-         "This isn't a function declarator!");
-  const DeclaratorChunk::FunctionTypeInfo &FTI = FnTypeInfo.Fun;
+                                      const ParsedTemplateInfo &TemplateInfo) {
+  const DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
   // If this is C90 and the declspecs were completely missing, fudge in an
   // implicit int.  We do this here because this is the only place where
@@ -669,8 +761,9 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 
   // We should have either an opening brace or, in a C++ constructor,
   // we may have a colon.
-  if (Tok.isNot(tok::l_brace) && Tok.isNot(tok::colon) &&
-      Tok.isNot(tok::kw_try)) {
+  if (Tok.isNot(tok::l_brace) && 
+      (!getLang().CPlusPlus ||
+       (Tok.isNot(tok::colon) && Tok.isNot(tok::kw_try)))) {
     Diag(Tok, diag::err_expected_fn_body);
 
     // Skip over garbage, until we get to '{'.  Don't eat the '{'.
@@ -724,7 +817,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 /// types for a function with a K&R-style identifier list for arguments.
 void Parser::ParseKNRParamDeclarations(Declarator &D) {
   // We know that the top-level of this declarator is a function.
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
   // Enter function-declaration scope, limiting any declarators to the
   // function prototype scope, including parameter declarators.
@@ -770,11 +863,7 @@ void Parser::ParseKNRParamDeclarations(Declarator &D) {
     // Handle the full declarator list.
     while (1) {
       // If attributes are present, parse them.
-      if (Tok.is(tok::kw___attribute)) {
-        SourceLocation Loc;
-        AttributeList *AttrList = ParseGNUAttributes(&Loc);
-        ParmDeclarator.AddAttributes(AttrList, Loc);
-      }
+      MaybeParseGNUAttributes(ParmDeclarator);
 
       // Ask the actions module to compute the type for this declarator.
       Decl *Param =
@@ -994,7 +1083,8 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
     // Determine whether the identifier is a type name.
     if (ParsedType Ty = Actions.getTypeName(*Tok.getIdentifierInfo(),
                                             Tok.getLocation(), getCurScope(),
-                                            &SS)) {
+                                            &SS, false, 
+                                            NextToken().is(tok::period))) {
       // This is a typename. Replace the current token in-place with an
       // annotation type token.
       Tok.setKind(tok::annot_typename);
@@ -1115,6 +1205,20 @@ bool Parser::TryAnnotateCXXScopeToken(bool EnteringContext) {
   return false;
 }
 
+bool Parser::isTokenEqualOrMistypedEqualEqual(unsigned DiagID) {
+  if (Tok.is(tok::equalequal)) {
+    // We have '==' in a context that we would expect a '='.
+    // The user probably made a typo, intending to type '='. Emit diagnostic,
+    // fixit hint to turn '==' -> '=' and continue as if the user typed '='.
+    Diag(Tok, DiagID)
+      << FixItHint::CreateReplacement(SourceRange(Tok.getLocation()),
+                                      getTokenSimpleSpelling(tok::equal));
+    return true;
+  }
+
+  return Tok.is(tok::equal);
+}
+
 void Parser::CodeCompletionRecovery() {
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->getFlags() & Scope::FnScope) {
diff --git a/lib/Parse/RAIIObjectsForParser.h b/lib/Parse/RAIIObjectsForParser.h
index addc79508399..583f18428d68 100644
--- a/lib/Parse/RAIIObjectsForParser.h
+++ b/lib/Parse/RAIIObjectsForParser.h
@@ -80,6 +80,22 @@ namespace clang {
     }
   };
   
+  class InMessageExpressionRAIIObject {
+    bool &InMessageExpression;
+    bool OldValue;
+    
+  public:
+    InMessageExpressionRAIIObject(Parser &P, bool Value)
+      : InMessageExpression(P.InMessageExpression), 
+        OldValue(P.InMessageExpression) {
+      InMessageExpression = Value;
+    }
+    
+    ~InMessageExpressionRAIIObject() {
+      InMessageExpression = OldValue;
+    }
+  };
+  
   /// \brief RAII object that makes sure paren/bracket/brace count is correct
   /// after declaration/statement parsing, even when there's a parsing error.
   class ParenBraceBracketBalancer {
diff --git a/lib/Rewrite/CMakeLists.txt b/lib/Rewrite/CMakeLists.txt
index ffeb3e66ebcb..ee4cba21e0f9 100644
--- a/lib/Rewrite/CMakeLists.txt
+++ b/lib/Rewrite/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST clangParse clangFrontend)
 
 add_clang_library(clangRewrite
   DeltaTree.cpp
@@ -14,7 +14,7 @@ add_clang_library(clangRewrite
   TokenRewriter.cpp
   )
 
-add_dependencies(clangBasic
+add_dependencies(clangRewrite
   ClangAttrClasses
   ClangAttrList
   ClangDeclNodes
diff --git a/lib/Rewrite/FixItRewriter.cpp b/lib/Rewrite/FixItRewriter.cpp
index 582096978d7b..8dcc5dcba789 100644
--- a/lib/Rewrite/FixItRewriter.cpp
+++ b/lib/Rewrite/FixItRewriter.cpp
@@ -19,7 +19,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 #include "llvm/ADT/OwningPtr.h"
 #include <cstdio>
 
@@ -80,6 +80,9 @@ bool FixItRewriter::IncludeInDiagnosticCounts() const {
 
 void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
                                      const DiagnosticInfo &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticClient::HandleDiagnostic(DiagLevel, Info);
+
   Client->HandleDiagnostic(DiagLevel, Info);
 
   // Skip over any diagnostics that are ignored or notes.
@@ -141,7 +144,7 @@ void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
 }
 
 /// \brief Emit a diagnostic via the adapted diagnostic client.
-void FixItRewriter::Diag(FullSourceLoc Loc, unsigned DiagID) {
+void FixItRewriter::Diag(SourceLocation Loc, unsigned DiagID) {
   // When producing this diagnostic, we temporarily bypass ourselves,
   // clear out any current diagnostic, and let the downstream client
   // format the diagnostic.
diff --git a/lib/Rewrite/FrontendActions.cpp b/lib/Rewrite/FrontendActions.cpp
index 977e0cfba6d0..33e79edaf93b 100644
--- a/lib/Rewrite/FrontendActions.cpp
+++ b/lib/Rewrite/FrontendActions.cpp
@@ -20,7 +20,7 @@
 #include "clang/Rewrite/Rewriters.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -42,6 +42,7 @@ ASTConsumer *FixItAction::CreateASTConsumer(CompilerInstance &CI,
   return new ASTConsumer();
 }
 
+namespace {
 class FixItRewriteInPlace : public FixItOptions {
 public:
   std::string RewriteFilename(const std::string &Filename) { return Filename; }
@@ -57,13 +58,13 @@ public:
   }
 
   std::string RewriteFilename(const std::string &Filename) {
-    llvm::sys::Path Path(Filename);
-    std::string Suffix = Path.getSuffix();
-    Path.eraseSuffix();
-    Path.appendSuffix(NewSuffix + "." + Suffix);
-    return Path.c_str();
+    llvm::SmallString<128> Path(Filename);
+    llvm::sys::path::replace_extension(Path,
+      NewSuffix + llvm::sys::path::extension(Path));
+    return Path.str();
   }
 };
+} // end anonymous namespace
 
 bool FixItAction::BeginSourceFileAction(CompilerInstance &CI,
                                         llvm::StringRef Filename) {
diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp
index b461df462e2e..df08cd7cbf3d 100644
--- a/lib/Rewrite/HTMLRewrite.cpp
+++ b/lib/Rewrite/HTMLRewrite.cpp
@@ -20,6 +20,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
@@ -378,14 +379,16 @@ void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
     unsigned TokLen = Tok.getLength();
     switch (Tok.getKind()) {
     default: break;
-    case tok::identifier: {
-      // Fill in Result.IdentifierInfo, looking up the identifier in the
-      // identifier table.
-      const IdentifierInfo *II =
-        PP.LookUpIdentifierInfo(Tok, BufferStart+TokOffs);
+    case tok::identifier:
+      llvm_unreachable("tok::identifier in raw lexing mode!");
+      break;
+    case tok::raw_identifier: {
+      // Fill in Result.IdentifierInfo and update the token kind,
+      // looking up the identifier in the identifier table.
+      PP.LookUpIdentifierInfo(Tok);
 
       // If this is a pp-identifier, for a keyword, highlight it as such.
-      if (II->getTokenID() != tok::identifier)
+      if (Tok.isNot(tok::identifier))
         HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
                        "<span class='keyword'>", "</span>");
       break;
@@ -473,11 +476,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // If this raw token is an identifier, the raw lexer won't have looked up
     // the corresponding identifier info for it.  Do this now so that it will be
     // macro expanded when we re-preprocess it.
-    if (Tok.is(tok::identifier)) {
-      // Change the kind of this identifier to the appropriate token kind, e.g.
-      // turning "for" into a keyword.
-      Tok.setKind(PP.LookUpIdentifierInfo(Tok)->getTokenID());
-    }
+    if (Tok.is(tok::raw_identifier))
+      PP.LookUpIdentifierInfo(Tok);
 
     TokenStream.push_back(Tok);
 
@@ -486,7 +486,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
 
   // Temporarily change the diagnostics object so that we ignore any generated
   // diagnostics from this pass.
-  Diagnostic TmpDiags(new IgnoringDiagClient);
+  Diagnostic TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
+                      new IgnoringDiagClient);
 
   // FIXME: This is a huge hack; we reuse the input preprocessor because we want
   // its state, but we aren't actually changing it (we hope). This should really
diff --git a/lib/Rewrite/RewriteMacros.cpp b/lib/Rewrite/RewriteMacros.cpp
index 910fa6ba4c2d..0453098a56d2 100644
--- a/lib/Rewrite/RewriteMacros.cpp
+++ b/lib/Rewrite/RewriteMacros.cpp
@@ -17,7 +17,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 #include "llvm/ADT/OwningPtr.h"
 #include <cstdio>
 
@@ -78,7 +78,7 @@ static void LexRawTokensFromMainFile(Preprocessor &PP,
     // If we have an identifier with no identifier info for our raw token, look
     // up the indentifier info.  This is important for equality comparison of
     // identifier tokens.
-    if (RawTok.is(tok::identifier) && !RawTok.getIdentifierInfo())
+    if (RawTok.is(tok::raw_identifier))
       PP.LookUpIdentifierInfo(RawTok);
 
     RawTokens.push_back(RawTok);
diff --git a/lib/Rewrite/RewriteObjC.cpp b/lib/Rewrite/RewriteObjC.cpp
index 578a063614a1..875a0c7a84c5 100644
--- a/lib/Rewrite/RewriteObjC.cpp
+++ b/lib/Rewrite/RewriteObjC.cpp
@@ -33,14 +33,14 @@ using llvm::utostr;
 namespace {
   class RewriteObjC : public ASTConsumer {
     enum {
-      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)), 
+      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
                                         block, ... */
       BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
       BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the 
                                         __block variable */
-      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy 
+      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
                                         helpers */
-      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose 
+      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose
                                         support routines */
       BLOCK_BYREF_CURRENT_MAX = 256
     };
@@ -139,10 +139,10 @@ namespace {
     llvm::DenseMap<BlockExpr *, std::string> RewrittenBlockExprs;
 
     // This maps a property to it's assignment statement.
-    llvm::DenseMap<ObjCPropertyRefExpr *, BinaryOperator *> PropSetters;
+    llvm::DenseMap<Expr *, BinaryOperator *> PropSetters;
     // This maps a property to it's synthesied message expression.
     // This allows us to rewrite chained getters (e.g. o.a.b.c).
-    llvm::DenseMap<ObjCPropertyRefExpr *, Stmt *> PropGetters;
+    llvm::DenseMap<Expr *, Stmt *> PropGetters;
 
     // This maps an original source AST to it's rewritten form. This allows
     // us to avoid rewriting the same node twice (which is very uncommon).
@@ -155,7 +155,7 @@ namespace {
 
     bool DisableReplaceStmt;
 
-    static const int OBJC_ABI_VERSION =7 ;
+    static const int OBJC_ABI_VERSION = 7;
   public:
     virtual void Initialize(ASTContext &context);
 
@@ -195,7 +195,7 @@ namespace {
     }
 
     void ReplaceStmtWithRange(Stmt *Old, Stmt *New, SourceRange SrcRange) {
-      // Measaure the old text.
+      // Measure the old text.
       int Size = Rewrite.getRangeSize(SrcRange);
       if (Size == -1) {
         Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
@@ -247,11 +247,12 @@ namespace {
                                  ObjCCategoryImplDecl *CID);
     void RewriteInterfaceDecl(ObjCInterfaceDecl *Dcl);
     void RewriteImplementationDecl(Decl *Dcl);
-    void RewriteObjCMethodDecl(ObjCMethodDecl *MDecl, std::string &ResultStr);
+    void RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                               ObjCMethodDecl *MDecl, std::string &ResultStr);
     void RewriteTypeIntoString(QualType T, std::string &ResultStr,
                                const FunctionType *&FPRetType);
     void RewriteByRefString(std::string &ResultStr, const std::string &Name,
-                            ValueDecl *VD);
+                            ValueDecl *VD, bool def=false);
     void RewriteCategoryDecl(ObjCCategoryDecl *Dcl);
     void RewriteProtocolDecl(ObjCProtocolDecl *Dcl);
     void RewriteForwardProtocolDecl(ObjCForwardProtocolDecl *Dcl);
@@ -281,8 +282,8 @@ namespace {
     Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV, SourceLocation OrigStart,
                                  bool &replaced);
     Stmt *RewriteObjCNestedIvarRefExpr(Stmt *S, bool &replaced);
-    Stmt *RewritePropertyGetter(ObjCPropertyRefExpr *PropRefExpr);
-    Stmt *RewritePropertySetter(BinaryOperator *BinOp, Expr *newStmt,
+    Stmt *RewritePropertyOrImplicitGetter(Expr *PropOrGetterRefExpr);
+    Stmt *RewritePropertyOrImplicitSetter(BinaryOperator *BinOp, Expr *newStmt,
                                 SourceRange SrcRange);
     Stmt *RewriteAtSelector(ObjCSelectorExpr *Exp);
     Stmt *RewriteMessageExpr(ObjCMessageExpr *Exp);
@@ -344,8 +345,7 @@ namespace {
                                          std::string &Result);
     void SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
                                       std::string &Result);
-    void SynthesizeIvarOffsetComputation(ObjCContainerDecl *IDecl,
-                                         ObjCIvarDecl *ivar,
+    void SynthesizeIvarOffsetComputation(ObjCIvarDecl *ivar,
                                          std::string &Result);
     void RewriteImplementations();
     void SynthesizeMetaDataIntoBuffer(std::string &Result);
@@ -404,6 +404,16 @@ namespace {
       return false;
     }
     
+    void convertToUnqualifiedObjCType(QualType &T) {
+      if (T->isObjCQualifiedIdType())
+        T = Context->getObjCIdType();
+      else if (T->isObjCQualifiedClassType())
+        T = Context->getObjCClassType();
+      else if (T->isObjCObjectPointerType() &&
+               T->getPointeeType()->isObjCQualifiedInterfaceType())
+        T = Context->getObjCIdType();
+    }
+    
     // FIXME: This predicate seems like it would be useful to add to ASTContext.
     bool isObjCType(QualType T) {
       if (!LangOpts.ObjC1 && !LangOpts.ObjC2)
@@ -423,6 +433,7 @@ namespace {
       return false;
     }
     bool PointerTypeTakesAnyBlockArguments(QualType QT);
+    bool PointerTypeTakesAnyObjCQualifiedType(QualType QT);
     void GetExtentOfArgList(const char *Name, const char *&LParen,
                             const char *&RParen);
     void RewriteCastExpr(CStyleCastExpr *CE);
@@ -439,20 +450,30 @@ namespace {
           To += From[i];
       }
     }
+
+    QualType getSimpleFunctionType(QualType result,
+                                   const QualType *args,
+                                   unsigned numArgs,
+                                   bool variadic = false) {
+      FunctionProtoType::ExtProtoInfo fpi;
+      fpi.Variadic = variadic;
+      return Context->getFunctionType(result, args, numArgs, fpi);
+    }
   };
 
   // Helper function: create a CStyleCastExpr with trivial type source info.
   CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
                                            CastKind Kind, Expr *E) {
     TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
-    return CStyleCastExpr::Create(*Ctx, Ty, Kind, E, 0, TInfo,
+    return CStyleCastExpr::Create(*Ctx, Ty, VK_RValue, Kind, E, 0, TInfo,
                                   SourceLocation(), SourceLocation());
   }
 }
 
 void RewriteObjC::RewriteBlocksInFunctionProtoType(QualType funcType,
                                                    NamedDecl *D) {
-  if (FunctionProtoType *fproto = dyn_cast<FunctionProtoType>(funcType)) {
+  if (const FunctionProtoType *fproto
+      = dyn_cast<FunctionProtoType>(funcType.IgnoreParens())) {
     for (FunctionProtoType::arg_type_iterator I = fproto->arg_type_begin(),
          E = fproto->arg_type_end(); I && (I != E); ++I)
       if (isTopLevelBlockPointerType(*I)) {
@@ -733,8 +754,8 @@ void RewriteObjC::RewriteInclude() {
   }
 }
 
-static std::string getIvarAccessString(ObjCInterfaceDecl *ClassDecl,
-                                       ObjCIvarDecl *OID) {
+static std::string getIvarAccessString(ObjCIvarDecl *OID) {
+  const ObjCInterfaceDecl *ClassDecl = OID->getContainingInterface();
   std::string S;
   S = "((struct ";
   S += ClassDecl->getIdentifier()->getName();
@@ -762,64 +783,67 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
 
   // Generate the 'getter' function.
   ObjCPropertyDecl *PD = PID->getPropertyDecl();
-  ObjCInterfaceDecl *ClassDecl = PD->getGetterMethodDecl()->getClassInterface();
   ObjCIvarDecl *OID = PID->getPropertyIvarDecl();
 
   if (!OID)
     return;
   unsigned Attributes = PD->getPropertyAttributes();
-  bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
-                         (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
-                                        ObjCPropertyDecl::OBJC_PR_copy));
-  std::string Getr;
-  if (GenGetProperty && !objcGetPropertyDefined) {
-    objcGetPropertyDefined = true;
-    // FIXME. Is this attribute correct in all cases?
-    Getr = "\nextern \"C\" __declspec(dllimport) "
-           "id objc_getProperty(id, SEL, long, bool);\n";
-  }
-  RewriteObjCMethodDecl(PD->getGetterMethodDecl(), Getr);
-  Getr += "{ ";
-  // Synthesize an explicit cast to gain access to the ivar.
-  // See objc-act.c:objc_synthesize_new_getter() for details.
-  if (GenGetProperty) {
-    // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
-    Getr += "typedef ";
-    const FunctionType *FPRetType = 0;
-    RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
-                          FPRetType);
-    Getr += " _TYPE";
-    if (FPRetType) {
-      Getr += ")"; // close the precedence "scope" for "*".
+  if (!PD->getGetterMethodDecl()->isDefined()) {
+    bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
+                          (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
+                                         ObjCPropertyDecl::OBJC_PR_copy));
+    std::string Getr;
+    if (GenGetProperty && !objcGetPropertyDefined) {
+      objcGetPropertyDefined = true;
+      // FIXME. Is this attribute correct in all cases?
+      Getr = "\nextern \"C\" __declspec(dllimport) "
+            "id objc_getProperty(id, SEL, long, bool);\n";
+    }
+    RewriteObjCMethodDecl(OID->getContainingInterface(),  
+                          PD->getGetterMethodDecl(), Getr);
+    Getr += "{ ";
+    // Synthesize an explicit cast to gain access to the ivar.
+    // See objc-act.c:objc_synthesize_new_getter() for details.
+    if (GenGetProperty) {
+      // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
+      Getr += "typedef ";
+      const FunctionType *FPRetType = 0;
+      RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
+                            FPRetType);
+      Getr += " _TYPE";
+      if (FPRetType) {
+        Getr += ")"; // close the precedence "scope" for "*".
       
-      // Now, emit the argument types (if any).
-      if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
-        Getr += "(";
-        for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
-          if (i) Getr += ", ";
-          std::string ParamStr = FT->getArgType(i).getAsString(
-            Context->PrintingPolicy);
-          Getr += ParamStr;
-        }
-        if (FT->isVariadic()) {
-          if (FT->getNumArgs()) Getr += ", ";
-          Getr += "...";
-        }
-        Getr += ")";
-      } else
-        Getr += "()";
+        // Now, emit the argument types (if any).
+        if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
+          Getr += "(";
+          for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
+            if (i) Getr += ", ";
+            std::string ParamStr = FT->getArgType(i).getAsString(
+                                                          Context->PrintingPolicy);
+            Getr += ParamStr;
+          }
+          if (FT->isVariadic()) {
+            if (FT->getNumArgs()) Getr += ", ";
+            Getr += "...";
+          }
+          Getr += ")";
+        } else
+          Getr += "()";
+      }
+      Getr += ";\n";
+      Getr += "return (_TYPE)";
+      Getr += "objc_getProperty(self, _cmd, ";
+      SynthesizeIvarOffsetComputation(OID, Getr);
+      Getr += ", 1)";
     }
-    Getr += ";\n";
-    Getr += "return (_TYPE)";
-    Getr += "objc_getProperty(self, _cmd, ";
-    SynthesizeIvarOffsetComputation(ClassDecl, OID, Getr);
-    Getr += ", 1)";
+    else
+      Getr += "return " + getIvarAccessString(OID);
+    Getr += "; }";
+    InsertText(onePastSemiLoc, Getr);
   }
-  else
-    Getr += "return " + getIvarAccessString(ClassDecl, OID);
-  Getr += "; }";
-  InsertText(onePastSemiLoc, Getr);
-  if (PD->isReadOnly())
+  
+  if (PD->isReadOnly() || PD->getSetterMethodDecl()->isDefined())
     return;
 
   // Generate the 'setter' function.
@@ -833,13 +857,14 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
     "void objc_setProperty (id, SEL, long, id, bool, bool);\n";
   }
   
-  RewriteObjCMethodDecl(PD->getSetterMethodDecl(), Setr);
+  RewriteObjCMethodDecl(OID->getContainingInterface(), 
+                        PD->getSetterMethodDecl(), Setr);
   Setr += "{ ";
   // Synthesize an explicit cast to initialize the ivar.
   // See objc-act.c:objc_synthesize_new_setter() for details.
   if (GenSetProperty) {
     Setr += "objc_setProperty (self, _cmd, ";
-    SynthesizeIvarOffsetComputation(ClassDecl, OID, Setr);
+    SynthesizeIvarOffsetComputation(OID, Setr);
     Setr += ", (id)";
     Setr += PD->getName();
     Setr += ", ";
@@ -853,7 +878,7 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
       Setr += "0)";
   }
   else {
-    Setr += getIvarAccessString(ClassDecl, OID) + " = ";
+    Setr += getIvarAccessString(OID) + " = ";
     Setr += PD->getName();
   }
   Setr += "; }";
@@ -962,6 +987,10 @@ void RewriteObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
        I != E; ++I)
     RewriteMethodDeclaration(*I);
 
+  for (ObjCInterfaceDecl::prop_iterator I = PDecl->prop_begin(),
+       E = PDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  
   // Lastly, comment out the @end.
   SourceLocation LocEnd = PDecl->getAtEndRange().getBegin();
   ReplaceText(LocEnd, strlen("@end"), "/* @end */");
@@ -1014,7 +1043,8 @@ void RewriteObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
     ResultStr += T.getAsString(Context->PrintingPolicy);
 }
 
-void RewriteObjC::RewriteObjCMethodDecl(ObjCMethodDecl *OMD,
+void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                                        ObjCMethodDecl *OMD,
                                         std::string &ResultStr) {
   //fprintf(stderr,"In RewriteObjCMethodDecl\n");
   const FunctionType *FPRetType = 0;
@@ -1030,7 +1060,7 @@ void RewriteObjC::RewriteObjCMethodDecl(ObjCMethodDecl *OMD,
   else
     NameStr += "_C_";
 
-  NameStr += OMD->getClassInterface()->getNameAsString();
+  NameStr += IDecl->getNameAsString();
   NameStr += "_";
 
   if (ObjCCategoryImplDecl *CID =
@@ -1056,14 +1086,14 @@ void RewriteObjC::RewriteObjCMethodDecl(ObjCMethodDecl *OMD,
 
   // invisible arguments
   if (OMD->isInstanceMethod()) {
-    QualType selfTy = Context->getObjCInterfaceType(OMD->getClassInterface());
+    QualType selfTy = Context->getObjCInterfaceType(IDecl);
     selfTy = Context->getPointerType(selfTy);
     if (!LangOpts.Microsoft) {
-      if (ObjCSynthesizedStructs.count(OMD->getClassInterface()))
+      if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
         ResultStr += "struct ";
     }
     // When rewriting for Microsoft, explicitly omit the structure name.
-    ResultStr += OMD->getClassInterface()->getNameAsString();
+    ResultStr += IDecl->getNameAsString();
     ResultStr += " *";
   }
   else
@@ -1131,7 +1161,7 @@ void RewriteObjC::RewriteImplementationDecl(Decl *OID) {
        I != E; ++I) {
     std::string ResultStr;
     ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD, ResultStr);
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
     SourceLocation LocStart = OMD->getLocStart();
     SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
 
@@ -1146,7 +1176,7 @@ void RewriteObjC::RewriteImplementationDecl(Decl *OID) {
        I != E; ++I) {
     std::string ResultStr;
     ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD, ResultStr);
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
     SourceLocation LocStart = OMD->getLocStart();
     SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
 
@@ -1199,42 +1229,74 @@ void RewriteObjC::RewriteInterfaceDecl(ObjCInterfaceDecl *ClassDecl) {
               "/* @end */");
 }
 
-Stmt *RewriteObjC::RewritePropertySetter(BinaryOperator *BinOp, Expr *newStmt,
+Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(BinaryOperator *BinOp, Expr *newStmt,
                                          SourceRange SrcRange) {
-  // Synthesize a ObjCMessageExpr from a ObjCPropertyRefExpr.
+  ObjCMethodDecl *OMD = 0;
+  QualType Ty;
+  Selector Sel;
+  Stmt *Receiver = 0;
+  bool Super = false;
+  QualType SuperTy;
+  SourceLocation SuperLocation;
+  SourceLocation SelectorLoc;
+  // Synthesize a ObjCMessageExpr from a ObjCPropertyRefExpr or ObjCImplicitSetterGetterRefExpr.
   // This allows us to reuse all the fun and games in SynthMessageExpr().
-  ObjCPropertyRefExpr *PropRefExpr = dyn_cast<ObjCPropertyRefExpr>(BinOp->getLHS());
-  ObjCMessageExpr *MsgExpr;
-  ObjCPropertyDecl *PDecl = PropRefExpr->getProperty();
+  if (ObjCPropertyRefExpr *PropRefExpr =
+        dyn_cast<ObjCPropertyRefExpr>(BinOp->getLHS())) {
+    SelectorLoc = PropRefExpr->getLocation();
+    if (PropRefExpr->isExplicitProperty()) {
+      ObjCPropertyDecl *PDecl = PropRefExpr->getExplicitProperty();
+      OMD = PDecl->getSetterMethodDecl();
+      Ty = PDecl->getType();
+      Sel = PDecl->getSetterName();
+    } else {
+      OMD = PropRefExpr->getImplicitPropertySetter();
+      Sel = OMD->getSelector();
+      Ty = PropRefExpr->getType();
+    }
+    Super = PropRefExpr->isSuperReceiver();
+    if (!Super) {
+      Receiver = PropRefExpr->getBase();
+    } else {
+      SuperTy = PropRefExpr->getSuperReceiverType();
+      SuperLocation = PropRefExpr->getReceiverLocation();
+    }
+  }
+  
+  assert(OMD && "RewritePropertyOrImplicitSetter - null OMD");
   llvm::SmallVector<Expr *, 1> ExprVec;
   ExprVec.push_back(newStmt);
 
-  Stmt *Receiver = PropRefExpr->getBase();
-  ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(Receiver);
-  if (PRE && PropGetters[PRE]) {
-    // This allows us to handle chain/nested property getters.
-    Receiver = PropGetters[PRE];
-  }
-  if (isa<ObjCSuperExpr>(Receiver))
+  ObjCMessageExpr *MsgExpr;
+  if (Super)
     MsgExpr = ObjCMessageExpr::Create(*Context, 
-                                      PDecl->getType().getNonReferenceType(),
+                                      Ty.getNonReferenceType(),
+                                      Expr::getValueKindForType(Ty),
                                       /*FIXME?*/SourceLocation(),
-                                      Receiver->getLocStart(),
+                                      SuperLocation,
                                       /*IsInstanceSuper=*/true,
-                                      cast<Expr>(Receiver)->getType(),
-                                      PDecl->getSetterName(),
-                                      PDecl->getSetterMethodDecl(),
+                                      SuperTy,
+                                      Sel, SelectorLoc, OMD,
                                       &ExprVec[0], 1,
                                       /*FIXME:*/SourceLocation());
-  else
+  else {
+    // FIXME. Refactor this into common code with that in 
+    // RewritePropertyOrImplicitGetter
+    assert(Receiver && "RewritePropertyOrImplicitSetter - null Receiver");
+    if (Expr *Exp = dyn_cast<Expr>(Receiver))
+      if (PropGetters[Exp])
+        // This allows us to handle chain/nested property/implicit getters.
+        Receiver = PropGetters[Exp];
+  
     MsgExpr = ObjCMessageExpr::Create(*Context, 
-                                      PDecl->getType().getNonReferenceType(),
+                                      Ty.getNonReferenceType(),
+                                      Expr::getValueKindForType(Ty),
                                       /*FIXME: */SourceLocation(),
                                       cast<Expr>(Receiver),
-                                      PDecl->getSetterName(),
-                                      PDecl->getSetterMethodDecl(),
+                                      Sel, SelectorLoc, OMD,
                                       &ExprVec[0], 1,
                                       /*FIXME:*/SourceLocation());
+  }
   Stmt *ReplacingStmt = SynthMessageExpr(MsgExpr);
 
   // Now do the actual rewrite.
@@ -1246,57 +1308,93 @@ Stmt *RewriteObjC::RewritePropertySetter(BinaryOperator *BinOp, Expr *newStmt,
   return ReplacingStmt;
 }
 
-Stmt *RewriteObjC::RewritePropertyGetter(ObjCPropertyRefExpr *PropRefExpr) {
-  // Synthesize a ObjCMessageExpr from a ObjCPropertyRefExpr.
+Stmt *RewriteObjC::RewritePropertyOrImplicitGetter(Expr *PropOrGetterRefExpr) {
+  // Synthesize a ObjCMessageExpr from a ObjCPropertyRefExpr or ImplicitGetter.
   // This allows us to reuse all the fun and games in SynthMessageExpr().
-  ObjCMessageExpr *MsgExpr;
-  ObjCPropertyDecl *PDecl = PropRefExpr->getProperty();
-
-  Stmt *Receiver = PropRefExpr->getBase();
-
-  ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(Receiver);
-  if (PRE && PropGetters[PRE]) {
-    // This allows us to handle chain/nested property getters.
-    Receiver = PropGetters[PRE];
+  Stmt *Receiver = 0;
+  ObjCMethodDecl *OMD = 0;
+  QualType Ty;
+  Selector Sel;
+  bool Super = false;
+  QualType SuperTy;
+  SourceLocation SuperLocation;
+  SourceLocation SelectorLoc;
+  if (ObjCPropertyRefExpr *PropRefExpr = 
+        dyn_cast<ObjCPropertyRefExpr>(PropOrGetterRefExpr)) {
+    SelectorLoc = PropRefExpr->getLocation();
+    if (PropRefExpr->isExplicitProperty()) {
+      ObjCPropertyDecl *PDecl = PropRefExpr->getExplicitProperty();
+      OMD = PDecl->getGetterMethodDecl();
+      Ty = PDecl->getType();
+      Sel = PDecl->getGetterName();
+    } else {
+      OMD = PropRefExpr->getImplicitPropertyGetter();
+      Sel = OMD->getSelector();
+      Ty = PropRefExpr->getType();
+    }
+    Super = PropRefExpr->isSuperReceiver();
+    if (!Super)
+      Receiver = PropRefExpr->getBase();
+    else {
+      SuperTy = PropRefExpr->getSuperReceiverType();
+      SuperLocation = PropRefExpr->getReceiverLocation();
+    }
   }
-
-  if (isa<ObjCSuperExpr>(Receiver))
+  
+  assert (OMD && "RewritePropertyOrImplicitGetter - OMD is null");
+  
+  ObjCMessageExpr *MsgExpr;
+  if (Super)
     MsgExpr = ObjCMessageExpr::Create(*Context, 
-                                      PDecl->getType().getNonReferenceType(),
-                                      /*FIXME:*/SourceLocation(),
-                                      Receiver->getLocStart(),
+                                      Ty.getNonReferenceType(),
+                                      Expr::getValueKindForType(Ty),
+                                      /*FIXME?*/SourceLocation(),
+                                      SuperLocation,
                                       /*IsInstanceSuper=*/true,
-                                      cast<Expr>(Receiver)->getType(),
-                                      PDecl->getGetterName(), 
-                                      PDecl->getGetterMethodDecl(),
+                                      SuperTy,
+                                      Sel, SelectorLoc, OMD,
                                       0, 0, 
                                       /*FIXME:*/SourceLocation());
-  else
+  else {
+    assert (Receiver && "RewritePropertyOrImplicitGetter - Receiver is null");
+    if (Expr *Exp = dyn_cast<Expr>(Receiver))
+      if (PropGetters[Exp])
+        // This allows us to handle chain/nested property/implicit getters.
+        Receiver = PropGetters[Exp];
     MsgExpr = ObjCMessageExpr::Create(*Context, 
-                                      PDecl->getType().getNonReferenceType(),
+                                      Ty.getNonReferenceType(),
+                                      Expr::getValueKindForType(Ty),
                                       /*FIXME:*/SourceLocation(),
                                       cast<Expr>(Receiver),
-                                      PDecl->getGetterName(), 
-                                      PDecl->getGetterMethodDecl(),
+                                      Sel, SelectorLoc, OMD,
                                       0, 0, 
                                       /*FIXME:*/SourceLocation());
+  }
 
   Stmt *ReplacingStmt = SynthMessageExpr(MsgExpr);
 
   if (!PropParentMap)
     PropParentMap = new ParentMap(CurrentBody);
-
-  Stmt *Parent = PropParentMap->getParent(PropRefExpr);
-  if (Parent && isa<ObjCPropertyRefExpr>(Parent)) {
+  bool NestedPropertyRef = false;
+  Stmt *Parent = PropParentMap->getParent(PropOrGetterRefExpr);
+  ImplicitCastExpr*ICE=0;
+  if (Parent)
+    if ((ICE = dyn_cast<ImplicitCastExpr>(Parent))) {
+      assert((ICE->getCastKind() == CK_GetObjCProperty)
+             && "RewritePropertyOrImplicitGetter");
+      Parent = PropParentMap->getParent(Parent);
+      NestedPropertyRef = (Parent && isa<ObjCPropertyRefExpr>(Parent));
+    }
+  if (NestedPropertyRef) {
     // We stash away the ReplacingStmt since actually doing the
     // replacement/rewrite won't work for nested getters (e.g. obj.p.i)
-    PropGetters[PropRefExpr] = ReplacingStmt;
+    PropGetters[ICE] = ReplacingStmt;
     // NOTE: We don't want to call MsgExpr->Destroy(), as it holds references
     // to things that stay around.
     Context->Deallocate(MsgExpr);
-    return PropRefExpr; // return the original...
+    return PropOrGetterRefExpr; // return the original...
   } else {
-    ReplaceStmt(PropRefExpr, ReplacingStmt);
+    ReplaceStmt(PropOrGetterRefExpr, ReplacingStmt);
     // delete PropRefExpr; elsewhere...
     // NOTE: We don't want to call MsgExpr->Destroy(), as it holds references
     // to things that stay around.
@@ -1312,7 +1410,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
   const Expr *BaseExpr = IV->getBase();
   if (CurMethodDef) {
     if (BaseExpr->getType()->isObjCObjectPointerType()) {
-      ObjCInterfaceType *iFaceDecl =
+      const ObjCInterfaceType *iFaceDecl =
         dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
       assert(iFaceDecl && "RewriteObjCIvarRefExpr - iFaceDecl is null");
       // lookup which class implements the instance variable.
@@ -1330,19 +1428,20 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CK_Unknown,
+                                                    CK_BitCast,
                                                     IV->getBase());
       // Don't forget the parens to enforce the proper binding.
       ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
-                                               IV->getBase()->getLocEnd(),
-                                               castExpr);
+                                              IV->getBase()->getLocEnd(),
+                                              castExpr);
       replaced = true;
       if (IV->isFreeIvar() &&
           CurMethodDef->getClassInterface() == iFaceDecl->getDecl()) {
         MemberExpr *ME = new (Context) MemberExpr(PE, true, D,
-                                                   IV->getLocation(),
-                                                   D->getType());
-        // delete IV; leak for now, see RewritePropertySetter() usage for more info.
+                                                  IV->getLocation(),
+                                                  D->getType(),
+                                                  VK_LValue, OK_Ordinary);
+        // delete IV; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
         return ME;
       }
       // Get the new text
@@ -1358,7 +1457,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
     // Explicit ivar refs need to have a cast inserted.
     // FIXME: consider sharing some of this code with the code above.
     if (BaseExpr->getType()->isObjCObjectPointerType()) {
-      ObjCInterfaceType *iFaceDecl =
+      const ObjCInterfaceType *iFaceDecl =
         dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
       // lookup which class implements the instance variable.
       ObjCInterfaceDecl *clsDeclared = 0;
@@ -1375,7 +1474,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CK_Unknown,
+                                                    CK_BitCast,
                                                     IV->getBase());
       // Don't forget the parens to enforce the proper binding.
       ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
@@ -1392,8 +1491,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
 }
 
 Stmt *RewriteObjC::RewriteObjCNestedIvarRefExpr(Stmt *S, bool &replaced) {
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI) {
     if (*CI) {
       Stmt *newStmt = RewriteObjCNestedIvarRefExpr(*CI, replaced);
       if (newStmt)
@@ -1711,7 +1809,7 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   std::string syncBuf;
   syncBuf += " objc_sync_exit(";
   Expr *syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK_Unknown,
+                                            CK_BitCast,
                                             S->getSynchExpr());
   std::string syncExprBufS;
   llvm::raw_string_ostream syncExprBuf(syncExprBufS);
@@ -1738,8 +1836,7 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
 void RewriteObjC::WarnAboutReturnGotoStmts(Stmt *S)
 {
   // Perform a bottom up traversal of all children.
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI)
       WarnAboutReturnGotoStmts(*CI);
 
@@ -1753,8 +1850,7 @@ void RewriteObjC::WarnAboutReturnGotoStmts(Stmt *S)
 void RewriteObjC::HasReturnStmts(Stmt *S, bool &hasReturns) 
 {  
   // Perform a bottom up traversal of all children.
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-        CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
    if (*CI)
      HasReturnStmts(*CI, hasReturns);
 
@@ -1765,8 +1861,7 @@ void RewriteObjC::HasReturnStmts(Stmt *S, bool &hasReturns)
 
 void RewriteObjC::RewriteTryReturnStmts(Stmt *S) {
  // Perform a bottom up traversal of all children.
- for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-      CI != E; ++CI)
+ for (Stmt::child_range CI = S->children(); CI; ++CI)
    if (*CI) {
      RewriteTryReturnStmts(*CI);
    }
@@ -1789,8 +1884,7 @@ void RewriteObjC::RewriteTryReturnStmts(Stmt *S) {
 
 void RewriteObjC::RewriteSyncReturnStmts(Stmt *S, std::string syncExitBuf) {
   // Perform a bottom up traversal of all children.
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI) {
       RewriteSyncReturnStmts(*CI, syncExitBuf);
     }
@@ -1853,7 +1947,6 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
     buf += "}";
     ReplaceText(lastCurlyLoc, 1, buf);
   }
-  bool sawIdTypedCatch = false;
   Stmt *lastCatchBody = 0;
   for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
     ObjCAtCatchStmt *Catch = S->getCatchStmt(I);
@@ -1886,7 +1979,6 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
       if (t == Context->getObjCIdType()) {
         buf += "1) { ";
         ReplaceText(startLoc, lParenLoc-startBuf+1, buf);
-        sawIdTypedCatch = true;
       } else if (const ObjCObjectPointerType *Ptr =
                    t->getAs<ObjCObjectPointerType>()) {
         // Should be a pointer to a class.
@@ -2020,7 +2112,7 @@ Stmt *RewriteObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
   ReplaceStmt(Exp, Replacement);
 
   // Replace this subexpr in the parent.
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return Replacement;
 }
 
@@ -2038,7 +2130,7 @@ Stmt *RewriteObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
   CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
                                                  &SelExprs[0], SelExprs.size());
   ReplaceStmt(Exp, SelExp);
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return SelExp;
 }
 
@@ -2049,19 +2141,21 @@ CallExpr *RewriteObjC::SynthesizeCallToFunctionDecl(
   QualType msgSendType = FD->getType();
 
   // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, msgSendType, SourceLocation());
+  DeclRefExpr *DRE =
+    new (Context) DeclRefExpr(FD, msgSendType, VK_LValue, SourceLocation());
 
   // Now, we cast the reference to a pointer to the objc_msgSend type.
   QualType pToFunc = Context->getPointerType(msgSendType);
   ImplicitCastExpr *ICE = 
-    ImplicitCastExpr::Create(*Context, pToFunc, CK_Unknown,
+    ImplicitCastExpr::Create(*Context, pToFunc, CK_FunctionToPointerDecay,
                              DRE, 0, VK_RValue);
 
   const FunctionType *FT = msgSendType->getAs<FunctionType>();
 
   CallExpr *Exp =  
     new (Context) CallExpr(*Context, ICE, args, nargs, 
-                           FT->getCallResultType(*Context), EndLoc);
+                           FT->getCallResultType(*Context),
+                           VK_RValue, EndLoc);
   return Exp;
 }
 
@@ -2105,6 +2199,10 @@ bool RewriteObjC::needToScanForQualifiers(QualType T) {
     T = T->getPointeeType();
     return T->isObjCQualifiedInterfaceType();
   }
+  if (T->isArrayType()) {
+    QualType ElemTy = Context->getBaseElementType(T);
+    return needToScanForQualifiers(ElemTy);
+  }
   return false;
 }
 
@@ -2257,11 +2355,8 @@ void RewriteObjC::SynthSelGetUidFunctionDecl() {
   IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
   llvm::SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getFuncType = Context->getFunctionType(Context->getObjCSelType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  false /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType getFuncType =
+    getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
   SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                            SourceLocation(),
                                            SelGetUidIdent, getFuncType, 0,
@@ -2356,11 +2451,8 @@ void RewriteObjC::SynthSuperContructorFunctionDecl() {
   assert(!argT.isNull() && "Can't find 'id' type");
   ArgTys.push_back(argT);
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->getObjCIdType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  false, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size());
   SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
@@ -2378,11 +2470,9 @@ void RewriteObjC::SynthMsgSendFunctionDecl() {
   argT = Context->getObjCSelType();
   assert(!argT.isNull() && "Can't find 'SEL' type");
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->getObjCIdType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  true /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
   MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
@@ -2403,11 +2493,9 @@ void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
   argT = Context->getObjCSelType();
   assert(!argT.isNull() && "Can't find 'SEL' type");
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->getObjCIdType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  true /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
   MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
@@ -2425,11 +2513,9 @@ void RewriteObjC::SynthMsgSendStretFunctionDecl() {
   argT = Context->getObjCSelType();
   assert(!argT.isNull() && "Can't find 'SEL' type");
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->getObjCIdType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  true /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
   MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
@@ -2452,11 +2538,9 @@ void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
   argT = Context->getObjCSelType();
   assert(!argT.isNull() && "Can't find 'SEL' type");
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->getObjCIdType(),
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  true /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
   MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                                        SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
@@ -2474,11 +2558,9 @@ void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
   argT = Context->getObjCSelType();
   assert(!argT.isNull() && "Can't find 'SEL' type");
   ArgTys.push_back(argT);
-  QualType msgSendType = Context->getFunctionType(Context->DoubleTy,
-                                                  &ArgTys[0], ArgTys.size(),
-                                                  true /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                  FunctionType::ExtInfo());
+  QualType msgSendType = getSimpleFunctionType(Context->DoubleTy,
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
   MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
@@ -2491,11 +2573,8 @@ void RewriteObjC::SynthGetClassFunctionDecl() {
   IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
   llvm::SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = Context->getFunctionType(Context->getObjCIdType(),
-                                                   &ArgTys[0], ArgTys.size(),
-                                                   false /*isVariadic*/, 0,
-                                                  false, false, 0, 0,
-                                                   FunctionType::ExtInfo());
+  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
+                                                &ArgTys[0], ArgTys.size());
   GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                           SourceLocation(),
                                           getClassIdent, getClassType, 0,
@@ -2509,11 +2588,8 @@ void RewriteObjC::SynthGetSuperClassFunctionDecl() {
     &Context->Idents.get("class_getSuperclass");
   llvm::SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getObjCClassType());
-  QualType getClassType = Context->getFunctionType(Context->getObjCClassType(),
-                                                   &ArgTys[0], ArgTys.size(),
-                                                   false /*isVariadic*/, 0,
-                                                   false, false, 0, 0,
-                                                   FunctionType::ExtInfo());
+  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
+                                                &ArgTys[0], ArgTys.size());
   GetSuperClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                                    SourceLocation(),
                                                    getSuperClassIdent,
@@ -2528,11 +2604,8 @@ void RewriteObjC::SynthGetMetaClassFunctionDecl() {
   IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
   llvm::SmallVector<QualType, 16> ArgTys;
   ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = Context->getFunctionType(Context->getObjCIdType(),
-                                                   &ArgTys[0], ArgTys.size(),
-                                                   false /*isVariadic*/, 0,
-                                                   false, false, 0, 0,
-                                                   FunctionType::ExtInfo());
+  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
+                                                &ArgTys[0], ArgTys.size());
   GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               getClassIdent, getClassType, 0,
@@ -2572,15 +2645,17 @@ Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
   VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
                                     &Context->Idents.get(S), strType, 0,
                                     SC_Static, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, strType, SourceLocation());
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, strType, VK_LValue,
+                                               SourceLocation());
   Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
                                  Context->getPointerType(DRE->getType()),
-                                 SourceLocation());
+                                           VK_RValue, OK_Ordinary,
+                                           SourceLocation());
   // cast to NSConstantString *
   CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
-                                            CK_Unknown, Unop);
+                                            CK_BitCast, Unop);
   ReplaceStmt(Exp, cast);
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return cast;
 }
 
@@ -2692,10 +2767,11 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // set the receiver to self, the first argument to all methods.
     InitExprs.push_back(
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_Unknown,
+                               CK_BitCast,
                    new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                   Context->getObjCIdType(),
-                                   SourceLocation()))
+                                             Context->getObjCIdType(),
+                                             VK_RValue,
+                                             SourceLocation()))
                         ); // set the 'receiver'.
 
     // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
@@ -2713,7 +2789,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // (Class)objc_getClass("CurrentClass")
     CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
                                              Context->getObjCClassType(),
-                                             CK_Unknown, Cls);
+                                             CK_BitCast, Cls);
     ClsExprs.clear();
     ClsExprs.push_back(ArgExpr);
     Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
@@ -2725,7 +2801,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     InitExprs.push_back( // set 'super class', using class_getSuperclass().
                         NoTypeInfoCStyleCastExpr(Context,
                                                  Context->getObjCIdType(),
-                                                 CK_Unknown, Cls));
+                                                 CK_BitCast, Cls));
     // struct objc_super
     QualType superType = getSuperStructType();
     Expr *SuperRep;
@@ -2734,10 +2810,12 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SynthSuperContructorFunctionDecl();
       // Simulate a contructor call...
       DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                         superType, SourceLocation());
+                                                   superType, VK_LValue,
+                                                   SourceLocation());
       SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
                                         InitExprs.size(),
-                                        superType, SourceLocation());
+                                        superType, VK_LValue,
+                                        SourceLocation());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -2746,10 +2824,11 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       //
       SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
-                               SourceLocation());
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                           Context->getPointerType(superType),
-                                          CK_Unknown, SuperRep);
+                                          CK_BitCast, SuperRep);
     } else {
       // (struct objc_super) { <exprs from above> }
       InitListExpr *ILE =
@@ -2759,11 +2838,13 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       TypeSourceInfo *superTInfo
         = Context->getTrivialTypeSourceInfo(superType);
       SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, ILE, false);
+                                                   superType, VK_LValue,
+                                                   ILE, false);
       // struct objc_super *
       SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
-                               SourceLocation());
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
     }
     MsgExprs.push_back(SuperRep);
     break;
@@ -2798,10 +2879,10 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
     InitExprs.push_back(
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_Unknown,
+                               CK_BitCast,
                    new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                   Context->getObjCIdType(),
-                                   SourceLocation()))
+                                             Context->getObjCIdType(),
+                                             VK_RValue, SourceLocation()))
                         ); // set the 'receiver'.
     
     // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
@@ -2818,7 +2899,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // (Class)objc_getClass("CurrentClass")
     CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
                                                  Context->getObjCClassType(),
-                                                 CK_Unknown, Cls);
+                                                 CK_BitCast, Cls);
     ClsExprs.clear();
     ClsExprs.push_back(ArgExpr);
     Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
@@ -2830,7 +2911,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     InitExprs.push_back(
       // set 'super class', using class_getSuperclass().
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_Unknown, Cls));
+                               CK_BitCast, Cls));
     // struct objc_super
     QualType superType = getSuperStructType();
     Expr *SuperRep;
@@ -2839,10 +2920,11 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SynthSuperContructorFunctionDecl();
       // Simulate a contructor call...
       DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                         superType, SourceLocation());
+                                                   superType, VK_LValue,
+                                                   SourceLocation());
       SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
                                         InitExprs.size(),
-                                        superType, SourceLocation());
+                                        superType, VK_LValue, SourceLocation());
       // The code for super is a little tricky to prevent collision with
       // the structure definition in the header. The rewriter has it's own
       // internal definition (__rw_objc_super) that is uses. This is why
@@ -2851,10 +2933,11 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       //
       SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
+                               VK_RValue, OK_Ordinary,
                                SourceLocation());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                Context->getPointerType(superType),
-                               CK_Unknown, SuperRep);
+                               CK_BitCast, SuperRep);
     } else {
       // (struct objc_super) { <exprs from above> }
       InitListExpr *ILE =
@@ -2864,7 +2947,8 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       TypeSourceInfo *superTInfo
         = Context->getTrivialTypeSourceInfo(superType);
       SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, ILE, false);
+                                                   superType, VK_RValue, ILE,
+                                                   false);
     }
     MsgExprs.push_back(SuperRep);
     break;
@@ -2877,7 +2961,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
       recExpr = CE->getSubExpr();
     recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                       CK_Unknown, recExpr);
+                                       CK_BitCast, recExpr);
     MsgExprs.push_back(recExpr);
     break;
   }
@@ -2907,7 +2991,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                 : ICE->getType();
       // Make sure we convert "type (^)(...)" to "type (*)(...)".
       (void)convertBlockPointerToFunctionPointer(type);
-      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK_Unknown,
+      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK_BitCast,
                                           userExpr);
     }
     // Make id<P...> cast into an 'id' cast.
@@ -2916,13 +3000,13 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
         while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
           userExpr = CE->getSubExpr();
         userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK_Unknown, userExpr);
+                                            CK_BitCast, userExpr);
       }
     }
     MsgExprs.push_back(userExpr);
     // We've transferred the ownership to MsgExprs. For now, we *don't* null
     // out the argument in the original expression (since we aren't deleting
-    // the ObjCMessageExpr). See RewritePropertySetter() usage for more info.
+    // the ObjCMessageExpr). See RewritePropertyOrImplicitSetter() usage for more info.
     //Exp->setArg(i, 0);
   }
   // Generate the funky cast.
@@ -2958,7 +3042,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
   // Create a reference to the objc_msgSend() declaration.
   DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, msgSendType,
-                                     SourceLocation());
+                                               VK_LValue, SourceLocation());
 
   // Need to cast objc_msgSend to "void *" (to workaround a GCC bandaid).
   // If we don't do this cast, we get the following bizarre warning/note:
@@ -2966,17 +3050,15 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   // xx.m:13: note: if this code is reached, the program will abort
   cast = NoTypeInfoCStyleCastExpr(Context,
                                   Context->getPointerType(Context->VoidTy),
-                                  CK_Unknown, DRE);
+                                  CK_BitCast, DRE);
 
   // Now do the "normal" pointer to function cast.
-  QualType castType = Context->getFunctionType(returnType,
-    &ArgTypes[0], ArgTypes.size(),
-    // If we don't have a method decl, force a variadic cast.
-    Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true, 0,
-                                               false, false, 0, 0,
-                                               FunctionType::ExtInfo());
+  QualType castType =
+    getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+      // If we don't have a method decl, force a variadic cast.
+      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true);
   castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_Unknown,
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
                                   cast);
 
   // Don't forget the parens to enforce the proper binding.
@@ -2985,7 +3067,8 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   const FunctionType *FT = msgSendType->getAs<FunctionType>();
   CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
                                         MsgExprs.size(),
-                                        FT->getResultType(), EndLoc);
+                                        FT->getResultType(), VK_RValue,
+                                        EndLoc);
   Stmt *ReplacingStmt = CE;
   if (MsgSendStretFlavor) {
     // We have the method which returns a struct/union. Must also generate
@@ -2995,19 +3078,16 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
     // Create a reference to the objc_msgSend_stret() declaration.
     DeclRefExpr *STDRE = new (Context) DeclRefExpr(MsgSendStretFlavor, msgSendType,
-                                         SourceLocation());
+                                                   VK_LValue, SourceLocation());
     // Need to cast objc_msgSend_stret to "void *" (see above comment).
     cast = NoTypeInfoCStyleCastExpr(Context,
                                     Context->getPointerType(Context->VoidTy),
-                                    CK_Unknown, STDRE);
+                                    CK_BitCast, STDRE);
     // Now do the "normal" pointer to function cast.
-    castType = Context->getFunctionType(returnType,
-      &ArgTypes[0], ArgTypes.size(),
-      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : false, 0,
-                                        false, false, 0, 0,
-                                        FunctionType::ExtInfo());
+    castType = getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : false);
     castType = Context->getPointerType(castType);
-    cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_Unknown,
+    cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
                                     cast);
 
     // Don't forget the parens to enforce the proper binding.
@@ -3016,7 +3096,8 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     FT = msgSendType->getAs<FunctionType>();
     CallExpr *STCE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
                                             MsgExprs.size(),
-                                            FT->getResultType(), SourceLocation());
+                                            FT->getResultType(), VK_RValue,
+                                            SourceLocation());
 
     // Build sizeof(returnType)
     SizeOfAlignOfExpr *sizeofExpr = new (Context) SizeOfAlignOfExpr(true,
@@ -3033,20 +3114,19 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                                    llvm::APInt(IntSize, 8),
                                                    Context->IntTy,
                                                    SourceLocation());
-    BinaryOperator *lessThanExpr = new (Context) BinaryOperator(sizeofExpr, limit,
-                                                      BO_LE,
-                                                      Context->IntTy,
-                                                      SourceLocation());
+    BinaryOperator *lessThanExpr = 
+      new (Context) BinaryOperator(sizeofExpr, limit, BO_LE, Context->IntTy,
+                                   VK_RValue, OK_Ordinary, SourceLocation());
     // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
     ConditionalOperator *CondExpr =
       new (Context) ConditionalOperator(lessThanExpr,
                                         SourceLocation(), CE,
-                                        SourceLocation(), STCE, (Expr*)0,
-                                        returnType);
+                                        SourceLocation(), STCE,
+                                        returnType, VK_RValue, OK_Ordinary);
     ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
                                             CondExpr);
   }
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return ReplacingStmt;
 }
 
@@ -3057,7 +3137,7 @@ Stmt *RewriteObjC::RewriteMessageExpr(ObjCMessageExpr *Exp) {
   // Now do the actual rewrite.
   ReplaceStmt(Exp, ReplacingStmt);
 
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return ReplacingStmt;
 }
 
@@ -3084,16 +3164,17 @@ Stmt *RewriteObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
   VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
                                 ID, getProtocolType(), 0,
                                 SC_Extern, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, getProtocolType(), SourceLocation());
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, getProtocolType(), VK_LValue,
+                                               SourceLocation());
   Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
                              Context->getPointerType(DRE->getType()),
-                             SourceLocation());
+                             VK_RValue, OK_Ordinary, SourceLocation());
   CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
-                                                CK_Unknown,
+                                                CK_BitCast,
                                                 DerefExpr);
   ReplaceStmt(Exp, castExpr);
   ProtocolExprDecls.insert(Exp->getProtocol());
-  // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
   return castExpr;
 
 }
@@ -3637,8 +3718,7 @@ void RewriteObjC::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
 
 /// SynthesizeIvarOffsetComputation - This rutine synthesizes computation of
 /// ivar offset.
-void RewriteObjC::SynthesizeIvarOffsetComputation(ObjCContainerDecl *IDecl,
-                                                  ObjCIvarDecl *ivar,
+void RewriteObjC::SynthesizeIvarOffsetComputation(ObjCIvarDecl *ivar,
                                                   std::string &Result) {
   if (ivar->isBitField()) {
     // FIXME: The hack below doesn't work for bitfields. For now, we simply
@@ -3646,7 +3726,7 @@ void RewriteObjC::SynthesizeIvarOffsetComputation(ObjCContainerDecl *IDecl,
     Result += "0";
   } else {
     Result += "__OFFSETOFIVAR__(struct ";
-    Result += IDecl->getNameAsString();
+    Result += ivar->getContainingInterface()->getNameAsString();
     if (LangOpts.Microsoft)
       Result += "_IMPL";
     Result += ", ";
@@ -3729,7 +3809,7 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
     QuoteDoublequotes(TmpString, StrEncoding);
     Result += StrEncoding;
     Result += "\", ";
-    SynthesizeIvarOffsetComputation(IDecl, *IVI, Result);
+    SynthesizeIvarOffsetComputation(*IVI, Result);
     Result += "}\n";
     for (++IVI; IVI != IVE; ++IVI) {
       Result += "\t  ,{\"";
@@ -3740,7 +3820,7 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
       QuoteDoublequotes(TmpString, StrEncoding);
       Result += StrEncoding;
       Result += "\", ";
-      SynthesizeIvarOffsetComputation(IDecl, (*IVI), Result);
+      SynthesizeIvarOffsetComputation((*IVI), Result);
       Result += "}\n";
     }
 
@@ -3764,11 +3844,13 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
     if (!PD)
       continue;
     if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
-      InstanceMethods.push_back(Getter);
+      if (!Getter->isDefined())
+        InstanceMethods.push_back(Getter);
     if (PD->isReadOnly())
       continue;
     if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
-      InstanceMethods.push_back(Setter);
+      if (!Setter->isDefined())
+        InstanceMethods.push_back(Setter);
   }
   RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
                              true, "", IDecl->getName(), Result);
@@ -4031,10 +4113,12 @@ void RewriteObjC::SynthesizeMetaDataIntoBuffer(std::string &Result) {
 
 void RewriteObjC::RewriteByRefString(std::string &ResultStr, 
                                      const std::string &Name,
-                                     ValueDecl *VD) {
+                                     ValueDecl *VD, bool def) {
   assert(BlockByRefDeclNo.count(VD) && 
          "RewriteByRefString: ByRef decl missing");
-  ResultStr += "struct __Block_byref_" + Name + 
+  if (def)
+    ResultStr += "struct ";
+  ResultStr += "__Block_byref_" + Name + 
     "_" + utostr(BlockByRefDeclNo[VD]) ;
 }
 
@@ -4373,6 +4457,12 @@ void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
         BlockByRefDeclsPtrSet.insert(VD);
         BlockByRefDecls.push_back(VD);
       }
+      // imported objects in the inner blocks not used in the outer
+      // blocks must be copied/disposed in the outer block as well.
+      if (Exp->isByRef() ||
+          VD->getType()->isObjCObjectPointerType() || 
+          VD->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(VD);
     }
 
     std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
@@ -4450,8 +4540,7 @@ void RewriteObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
 }
 
 void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI) {
       if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI))
         GetBlockDeclRefExprs(CBE->getBody());
@@ -4467,8 +4556,9 @@ void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
   else if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S))
     if (HasLocalVariableExternalStorage(DRE->getDecl())) {
         BlockDeclRefExpr *BDRE = 
-          new (Context)BlockDeclRefExpr(DRE->getDecl(), DRE->getType(), 
-                                        DRE->getLocation(), false);
+          new (Context)BlockDeclRefExpr(cast<VarDecl>(DRE->getDecl()),
+                                        DRE->getType(), 
+                                        VK_LValue, DRE->getLocation(), false);
         BlockDeclRefs.push_back(BDRE);
     }
   
@@ -4478,8 +4568,7 @@ void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
 void RewriteObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
                 llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs,
                 llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI) {
       if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI)) {
         InnerContexts.insert(cast<DeclContext>(CBE->getBlockDecl()));
@@ -4534,9 +4623,7 @@ QualType RewriteObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
   // FIXME. Does this work if block takes no argument but has a return type
   // which is of block type?
   if (HasBlockType)
-    FuncType = Context->getFunctionType(Res,
-                        &ArgTypes[0], ArgTypes.size(), false/*no variadic*/, 0,
-                        false, false, 0, 0, FunctionType::ExtInfo());
+    FuncType = getSimpleFunctionType(Res, &ArgTypes[0], ArgTypes.size());
   else FuncType = QualType(FT, 0);
   return FuncType;
 }
@@ -4569,8 +4656,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
       new (Context) ConditionalOperator(CONDExp,
                                       SourceLocation(), cast<Expr>(LHSStmt),
                                       SourceLocation(), cast<Expr>(RHSStmt),
-                                      (Expr*)0,
-                                      Exp->getType());
+                                      Exp->getType(), VK_RValue, OK_Ordinary);
     return CondExpr;
   } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
     CPT = IRE->getType()->getAs<BlockPointerType>();
@@ -4598,20 +4684,19 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
          E = FTP->arg_type_end(); I && (I != E); ++I) {
       QualType t = *I;
       // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(t);
+      if (!convertBlockPointerToFunctionPointer(t))
+        convertToUnqualifiedObjCType(t);
       ArgTypes.push_back(t);
     }
   }
   // Now do the pointer to function cast.
-  QualType PtrToFuncCastType = Context->getFunctionType(Exp->getType(),
-    &ArgTypes[0], ArgTypes.size(), false/*no variadic*/, 0,
-                                                        false, false, 0, 0, 
-                                                       FunctionType::ExtInfo());
+  QualType PtrToFuncCastType
+    = getSimpleFunctionType(Exp->getType(), &ArgTypes[0], ArgTypes.size());
 
   PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
 
   CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
-                                               CK_Unknown,
+                                               CK_BitCast,
                                                const_cast<Expr*>(BlockExp));
   // Don't forget the parens to enforce the proper binding.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
@@ -4622,10 +4707,12 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
                      &Context->Idents.get("FuncPtr"), Context->VoidPtrTy, 0,
                                     /*BitWidth=*/0, /*Mutable=*/true);
   MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
-                                            FD->getType());
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
 
+  
   CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
-                                                CK_Unknown, ME);
+                                                CK_BitCast, ME);
   PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
 
   llvm::SmallVector<Expr*, 8> BlkExprs;
@@ -4638,7 +4725,8 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
   }
   CallExpr *CE = new (Context) CallExpr(*Context, PE, &BlkExprs[0],
                                         BlkExprs.size(),
-                                        Exp->getType(), SourceLocation());
+                                        Exp->getType(), VK_RValue,
+                                        SourceLocation());
   return CE;
 }
 
@@ -4673,7 +4761,8 @@ Stmt *RewriteObjC::RewriteBlockDeclRefExpr(Expr *DeclRefExp) {
                                     /*BitWidth=*/0, /*Mutable=*/true);
   MemberExpr *ME = new (Context) MemberExpr(DeclRefExp, isArrow,
                                             FD, SourceLocation(),
-                                            FD->getType());
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
 
   llvm::StringRef Name = VD->getName();
   FD = FieldDecl::Create(*Context, 0, SourceLocation(),
@@ -4681,7 +4770,7 @@ Stmt *RewriteObjC::RewriteBlockDeclRefExpr(Expr *DeclRefExp) {
                          Context->VoidPtrTy, 0,
                          /*BitWidth=*/0, /*Mutable=*/true);
   ME = new (Context) MemberExpr(ME, true, FD, SourceLocation(),
-                                DeclRefExp->getType());
+                                DeclRefExp->getType(), VK_LValue, OK_Ordinary);
   
   
   
@@ -4700,8 +4789,9 @@ Stmt *RewriteObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
   if (VarDecl *Var = dyn_cast<VarDecl>(VD))
     if (!ImportedLocalExternalDecls.count(Var))
       return DRE;
-  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref,
-                                    DRE->getType(), DRE->getLocation());
+  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref, DRE->getType(),
+                                          VK_LValue, OK_Ordinary,
+                                          DRE->getLocation());
   // Need parens to enforce precedence.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
                                           Exp);
@@ -4802,6 +4892,30 @@ bool RewriteObjC::PointerTypeTakesAnyBlockArguments(QualType QT) {
   return false;
 }
 
+bool RewriteObjC::PointerTypeTakesAnyObjCQualifiedType(QualType QT) {
+  const FunctionProtoType *FTP;
+  const PointerType *PT = QT->getAs<PointerType>();
+  if (PT) {
+    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
+  } else {
+    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
+    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
+    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
+  }
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I != E; ++I) {
+      if ((*I)->isObjCQualifiedIdType())
+        return true;
+      if ((*I)->isObjCObjectPointerType() &&
+          (*I)->getPointeeType()->isObjCQualifiedInterfaceType())
+        return true;
+    }
+        
+  }
+  return false;
+}
+
 void RewriteObjC::GetExtentOfArgList(const char *Name, const char *&LParen,
                                      const char *&RParen) {
   const char *argPtr = strchr(Name, '(');
@@ -4845,28 +4959,57 @@ void RewriteObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
   // scan backward (from the decl location) for the end of the previous decl.
   while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
     startBuf--;
-
+  SourceLocation Start = DeclLoc.getFileLocWithOffset(startBuf-endBuf);
+  std::string buf;
+  unsigned OrigLength=0;
   // *startBuf != '^' if we are dealing with a pointer to function that
   // may take block argument types (which will be handled below).
   if (*startBuf == '^') {
     // Replace the '^' with '*', computing a negative offset.
-    DeclLoc = DeclLoc.getFileLocWithOffset(startBuf-endBuf);
-    ReplaceText(DeclLoc, 1, "*");
+    buf = '*';
+    startBuf++;
+    OrigLength++;
   }
-  if (PointerTypeTakesAnyBlockArguments(DeclT)) {
+  while (*startBuf != ')') {
+    buf += *startBuf;
+    startBuf++;
+    OrigLength++;
+  }
+  buf += ')';
+  OrigLength++;
+  
+  if (PointerTypeTakesAnyBlockArguments(DeclT) ||
+      PointerTypeTakesAnyObjCQualifiedType(DeclT)) {
     // Replace the '^' with '*' for arguments.
+    // Replace id<P> with id/*<>*/
     DeclLoc = ND->getLocation();
     startBuf = SM->getCharacterData(DeclLoc);
     const char *argListBegin, *argListEnd;
     GetExtentOfArgList(startBuf, argListBegin, argListEnd);
     while (argListBegin < argListEnd) {
-      if (*argListBegin == '^') {
-        SourceLocation CaretLoc = DeclLoc.getFileLocWithOffset(argListBegin-startBuf);
-        ReplaceText(CaretLoc, 1, "*");
+      if (*argListBegin == '^')
+        buf += '*';
+      else if (*argListBegin ==  '<') {
+        buf += "/*"; 
+        buf += *argListBegin++;
+        OrigLength++;;
+        while (*argListBegin != '>') {
+          buf += *argListBegin++;
+          OrigLength++;
+        }
+        buf += *argListBegin;
+        buf += "*/";
       }
+      else
+        buf += *argListBegin;
       argListBegin++;
+      OrigLength++;
     }
+    buf += ')';
+    OrigLength++;
   }
+  ReplaceText(Start, OrigLength, buf);
+  
   return;
 }
 
@@ -4964,7 +5107,7 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
   const char *endBuf = SM->getCharacterData(X);
   std::string Name(ND->getNameAsString());
   std::string ByrefType;
-  RewriteByRefString(ByrefType, Name, ND);
+  RewriteByRefString(ByrefType, Name, ND, true);
   ByrefType += " {\n";
   ByrefType += "  void *__isa;\n";
   RewriteByRefString(ByrefType, Name, ND);
@@ -5125,6 +5268,7 @@ FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(llvm::StringRef name) {
 
 Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
           const llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs) {
+  const BlockDecl *block = Exp->getBlockDecl();
   Blocks.push_back(Exp);
 
   CollectBlockDeclRefInfo(Exp);
@@ -5183,16 +5327,17 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
 
   // Simulate a contructor call...
   FD = SynthBlockInitFunctionDecl(Tag);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, FType, SourceLocation());
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, FType, VK_RValue,
+                                               SourceLocation());
 
   llvm::SmallVector<Expr*, 4> InitExprs;
 
   // Initialize the block function.
   FD = SynthBlockInitFunctionDecl(Func);
-  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, FD->getType(),
+  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, FD->getType(), VK_LValue,
                                                SourceLocation());
   CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                                CK_Unknown, Arg);
+                                                CK_BitCast, Arg);
   InitExprs.push_back(castExpr);
 
   // Initialize the block descriptor.
@@ -5202,12 +5347,15 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
                                     &Context->Idents.get(DescData.c_str()), 
                                     Context->VoidPtrTy, 0,
                                     SC_Static, SC_None);
-  UnaryOperator *DescRefExpr = new (Context) UnaryOperator(
-                                  new (Context) DeclRefExpr(NewVD, 
-                                    Context->VoidPtrTy, SourceLocation()), 
-                                  UO_AddrOf,
-                                  Context->getPointerType(Context->VoidPtrTy), 
-                                  SourceLocation());
+  UnaryOperator *DescRefExpr =
+    new (Context) UnaryOperator(new (Context) DeclRefExpr(NewVD,
+                                                          Context->VoidPtrTy,
+                                                          VK_LValue,
+                                                          SourceLocation()), 
+                                UO_AddrOf,
+                                Context->getPointerType(Context->VoidPtrTy), 
+                                VK_RValue, OK_Ordinary,
+                                SourceLocation());
   InitExprs.push_back(DescRefExpr); 
   
   // Add initializers for any closure decl refs.
@@ -5219,26 +5367,29 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       if (isObjCType((*I)->getType())) {
         // FIXME: Conform to ABI ([[obj retain] autorelease]).
         FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
+        Exp = new (Context) DeclRefExpr(FD, FD->getType(), VK_LValue,
+                                        SourceLocation());
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT,
-                                            SourceLocation());
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
         }
       } else if (isTopLevelBlockPointerType((*I)->getType())) {
         FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Arg = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
+        Arg = new (Context) DeclRefExpr(FD, FD->getType(), VK_LValue,
+                                        SourceLocation());
         Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                       CK_Unknown, Arg);
+                                       CK_BitCast, Arg);
       } else {
         FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
+        Exp = new (Context) DeclRefExpr(FD, FD->getType(), VK_LValue,
+                                        SourceLocation());
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, 
-                                            SourceLocation());
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
         }
         
       }
@@ -5250,7 +5401,7 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       ValueDecl *ND = (*I);
       std::string Name(ND->getNameAsString());
       std::string RecName;
-      RewriteByRefString(RecName, Name, ND);
+      RewriteByRefString(RecName, Name, ND, true);
       IdentifierInfo *II = &Context->Idents.get(RecName.c_str() 
                                                 + sizeof("struct"));
       RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
@@ -5259,11 +5410,27 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       
       FD = SynthBlockInitFunctionDecl((*I)->getName());
-      Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
-      Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
-                              Context->getPointerType(Exp->getType()),
-                              SourceLocation());
-      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_Unknown, Exp);
+      Exp = new (Context) DeclRefExpr(FD, FD->getType(), VK_LValue,
+                                      SourceLocation());
+      bool isNestedCapturedVar = false;
+      if (block)
+        for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
+             ce = block->capture_end(); ci != ce; ++ci) {
+          const VarDecl *variable = ci->getVariable();
+          if (variable == ND && ci->isNested()) {
+            assert (ci->isByRef() && 
+                    "SynthBlockInitExpr - captured block variable is not byref");
+            isNestedCapturedVar = true;
+            break;
+          }
+        }
+      // captured nested byref variable has its address passed. Do not take
+      // its address again.
+      if (!isNestedCapturedVar)
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
+                                     Context->getPointerType(Exp->getType()),
+                                     VK_RValue, OK_Ordinary, SourceLocation());
+      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
       InitExprs.push_back(Exp);
     }
   }
@@ -5277,11 +5444,11 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
     InitExprs.push_back(FlagExp);
   }
   NewRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0], InitExprs.size(),
-                                  FType, SourceLocation());
+                                  FType, VK_LValue, SourceLocation());
   NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
                              Context->getPointerType(NewRep->getType()),
-                             SourceLocation());
-  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_Unknown,
+                             VK_RValue, OK_Ordinary, SourceLocation());
+  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
                                     NewRep);
   BlockDeclRefs.clear();
   BlockByRefDecls.clear();
@@ -5304,15 +5471,14 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
 // we get this right.
 void RewriteObjC::CollectPropertySetters(Stmt *S) {
   // Perform a bottom up traversal of all children.
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI)
       CollectPropertySetters(*CI);
 
   if (BinaryOperator *BinOp = dyn_cast<BinaryOperator>(S)) {
     if (BinOp->isAssignmentOp()) {
-      if (ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(BinOp->getLHS()))
-        PropSetters[PRE] = BinOp;
+      if (isa<ObjCPropertyRefExpr>(BinOp->getLHS()))
+        PropSetters[BinOp->getLHS()] = BinOp;
     }
   }
 }
@@ -5329,8 +5495,7 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
   SourceRange OrigStmtRange = S->getSourceRange();
 
   // Perform a bottom up rewrite of all children.
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end();
-       CI != E; ++CI)
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
     if (*CI) {
       Stmt *newStmt;
       Stmt *S = (*CI);
@@ -5349,6 +5514,17 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
         newStmt = RewriteFunctionBodyOrGlobalInitializer(S);
       if (newStmt)
         *CI = newStmt;
+      // If dealing with an assignment with LHS being a property reference
+      // expression, the entire assignment tree is rewritten into a property
+      // setter messaging. This involvs the RHS too. Do not attempt to rewrite
+      // RHS again.
+      if (Expr *Exp = dyn_cast<Expr>(S))
+        if (isa<ObjCPropertyRefExpr>(Exp)) {
+          if (PropSetters[Exp]) {
+            ++CI;
+            continue;
+          }
+        }
     }
 
   if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
@@ -5359,7 +5535,12 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
     GetInnerBlockDeclRefExprs(BE->getBody(),
                               InnerBlockDeclRefs, InnerContexts);
     // Rewrite the block body in place.
+    Stmt *SaveCurrentBody = CurrentBody;
+    CurrentBody = BE->getBody();
+    PropParentMap = 0;
     RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
+    CurrentBody = SaveCurrentBody;
+    PropParentMap = 0;
     ImportedLocalExternalDecls.clear();
     // Now we snarf the rewritten text and stash it away for later use.
     std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
@@ -5375,8 +5556,11 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
   if (ObjCEncodeExpr *AtEncode = dyn_cast<ObjCEncodeExpr>(S))
     return RewriteAtEncode(AtEncode);
 
-  if (ObjCPropertyRefExpr *PropRefExpr = dyn_cast<ObjCPropertyRefExpr>(S)) {
-    BinaryOperator *BinOp = PropSetters[PropRefExpr];
+  if (isa<ObjCPropertyRefExpr>(S)) {
+    Expr *PropOrImplicitRefExpr = dyn_cast<Expr>(S);
+    assert(PropOrImplicitRefExpr && "Property or implicit setter/getter is null");
+    
+    BinaryOperator *BinOp = PropSetters[PropOrImplicitRefExpr];
     if (BinOp) {
       // Because the rewriter doesn't allow us to rewrite rewritten code,
       // we need to rewrite the right hand side prior to rewriting the setter.
@@ -5388,6 +5572,12 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
       //              ^(NSURL *errorURL, NSError *error) { return (BOOL)1; };
       SourceRange SrcRange = BinOp->getSourceRange();
       Stmt *newStmt = RewriteFunctionBodyOrGlobalInitializer(BinOp->getRHS());
+      // Need to rewrite the ivar access expression if need be.
+      if (isa<ObjCIvarRefExpr>(newStmt)) {
+        bool replaced = false;
+        newStmt = RewriteObjCNestedIvarRefExpr(newStmt, replaced);
+      }
+      
       DisableReplaceStmt = false;
       //
       // Unlike the main iterator, we explicily avoid changing 'BinOp'. If
@@ -5414,18 +5604,19 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
       //       (CStyleCastExpr 0x231d220 'void *'
       //         (DeclRefExpr 0x231d200 'id (id, SEL, ...)' FunctionDecl='objc_msgSend' 0x231cdc0))))
       //
-      // Note that 'newStmt' is passed to RewritePropertySetter so that it
+      // Note that 'newStmt' is passed to RewritePropertyOrImplicitSetter so that it
       // can be used as the setter argument. ReplaceStmt() will still 'see'
       // the original RHS (since we haven't altered BinOp).
       //
       // This implies the Rewrite* routines can no longer delete the original
       // node. As a result, we now leak the original AST nodes.
       //
-      return RewritePropertySetter(BinOp, dyn_cast<Expr>(newStmt), SrcRange);
+      return RewritePropertyOrImplicitSetter(BinOp, dyn_cast<Expr>(newStmt), SrcRange);
     } else {
-      return RewritePropertyGetter(PropRefExpr);
+      return RewritePropertyOrImplicitGetter(PropOrImplicitRefExpr);
     }
   }
+  
   if (ObjCSelectorExpr *AtSelector = dyn_cast<ObjCSelectorExpr>(S))
     return RewriteAtSelector(AtSelector);
 
diff --git a/lib/Rewrite/RewriteRope.cpp b/lib/Rewrite/RewriteRope.cpp
index e29092184789..cfedd4b41faf 100644
--- a/lib/Rewrite/RewriteRope.cpp
+++ b/lib/Rewrite/RewriteRope.cpp
@@ -119,7 +119,7 @@ namespace {
     /// guaranteed that there is a split at Offset.
     void erase(unsigned Offset, unsigned NumBytes);
 
-    static inline bool classof(const RopePieceBTreeNode *) { return true; }
+    //static inline bool classof(const RopePieceBTreeNode *) { return true; }
 
   };
 } // end anonymous namespace
@@ -223,7 +223,7 @@ namespace {
     /// guaranteed that there is a split at Offset.
     void erase(unsigned Offset, unsigned NumBytes);
 
-    static inline bool classof(const RopePieceBTreeLeaf *) { return true; }
+    //static inline bool classof(const RopePieceBTreeLeaf *) { return true; }
     static inline bool classof(const RopePieceBTreeNode *N) {
       return N->isLeaf();
     }
@@ -455,7 +455,7 @@ namespace {
     /// guaranteed that there is a split at Offset.
     void erase(unsigned Offset, unsigned NumBytes);
 
-    static inline bool classof(const RopePieceBTreeInterior *) { return true; }
+    //static inline bool classof(const RopePieceBTreeInterior *) { return true; }
     static inline bool classof(const RopePieceBTreeNode *N) {
       return !N->isLeaf();
     }
diff --git a/lib/Rewrite/TokenRewriter.cpp b/lib/Rewrite/TokenRewriter.cpp
index 789d53f4afaf..03ce63ea1174 100644
--- a/lib/Rewrite/TokenRewriter.cpp
+++ b/lib/Rewrite/TokenRewriter.cpp
@@ -34,10 +34,10 @@ TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
   RawLex.LexFromRawLexer(RawTok);
   while (RawTok.isNot(tok::eof)) {
 #if 0
-    if (Tok.is(tok::identifier)) {
+    if (Tok.is(tok::raw_identifier)) {
       // Look up the identifier info for the token.  This should use
       // IdentifierTable directly instead of PP.
-      Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
+      PP.LookUpIdentifierInfo(Tok);
     }
 #endif
 
@@ -73,7 +73,7 @@ TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
   bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
                                                         Where)).second;
   assert(InsertSuccess && "Token location already in rewriter!");
-  InsertSuccess = InsertSuccess;
+  (void)InsertSuccess;
   return Where;
 }
 
diff --git a/lib/Sema/AnalysisBasedWarnings.cpp b/lib/Sema/AnalysisBasedWarnings.cpp
index cfebed6a05b2..63f561d6ab19 100644
--- a/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/lib/Sema/AnalysisBasedWarnings.cpp
@@ -16,6 +16,7 @@
 #include "clang/Sema/AnalysisBasedWarnings.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExprObjC.h"
@@ -25,6 +26,7 @@
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/CFG.h"
 #include "clang/Analysis/Analyses/ReachableCode.h"
+#include "clang/Analysis/Analyses/UninitializedValuesV2.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/Support/Casting.h"
 
@@ -108,24 +110,41 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
   bool HasFakeEdge = false;
   bool HasPlainEdge = false;
   bool HasAbnormalEdge = false;
-  for (CFGBlock::pred_iterator I=cfg->getExit().pred_begin(),
-       E = cfg->getExit().pred_end();
-       I != E;
-       ++I) {
-    CFGBlock& B = **I;
+
+  // Ignore default cases that aren't likely to be reachable because all
+  // enums in a switch(X) have explicit case statements.
+  CFGBlock::FilterOptions FO;
+  FO.IgnoreDefaultsWithCoveredEnums = 1;
+
+  for (CFGBlock::filtered_pred_iterator
+	 I = cfg->getExit().filtered_pred_start_end(FO); I.hasMore(); ++I) {
+    const CFGBlock& B = **I;
     if (!live[B.getBlockID()])
       continue;
-    if (B.size() == 0) {
+
+    // Destructors can appear after the 'return' in the CFG.  This is
+    // normal.  We need to look pass the destructors for the return
+    // statement (if it exists).
+    CFGBlock::const_reverse_iterator ri = B.rbegin(), re = B.rend();
+    for ( ; ri != re ; ++ri) {
+      CFGElement CE = *ri;
+      if (isa<CFGStmt>(CE))
+        break;
+    }
+    
+    // No more CFGElements in the block?
+    if (ri == re) {
       if (B.getTerminator() && isa<CXXTryStmt>(B.getTerminator())) {
         HasAbnormalEdge = true;
         continue;
       }
-
       // A labeled empty statement, or the entry block...
       HasPlainEdge = true;
       continue;
     }
-    Stmt *S = B[B.size()-1];
+
+    CFGStmt CS = cast<CFGStmt>(*ri);
+    Stmt *S = CS.getStmt();
     if (isa<ReturnStmt>(S)) {
       HasLiveReturn = true;
       continue;
@@ -169,19 +188,6 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
         }
       }
     }
-    // FIXME: Remove this hack once temporaries and their destructors are
-    // modeled correctly by the CFG.
-    if (CXXExprWithTemporaries *E = dyn_cast<CXXExprWithTemporaries>(S)) {
-      for (unsigned I = 0, N = E->getNumTemporaries(); I != N; ++I) {
-        const FunctionDecl *FD = E->getTemporary(I)->getDestructor();
-        if (FD->hasAttr<NoReturnAttr>() ||
-            FD->getType()->getAs<FunctionType>()->getNoReturnAttr()) {
-          NoReturnEdge = true;
-          HasFakeEdge = true;
-          break;
-        }
-      }
-    }
     // FIXME: Add noreturn message sends.
     if (NoReturnEdge == false)
       HasPlainEdge = true;
@@ -208,9 +214,11 @@ struct CheckFallThroughDiagnostics {
   unsigned diag_AlwaysFallThrough_ReturnsNonVoid;
   unsigned diag_NeverFallThroughOrReturn;
   bool funMode;
+  SourceLocation FuncLoc;
 
   static CheckFallThroughDiagnostics MakeForFunction(const Decl *Func) {
     CheckFallThroughDiagnostics D;
+    D.FuncLoc = Func->getLocation();
     D.diag_MaybeFallThrough_HasNoReturn =
       diag::warn_falloff_noreturn_function;
     D.diag_MaybeFallThrough_ReturnsNonVoid =
@@ -255,18 +263,22 @@ struct CheckFallThroughDiagnostics {
   bool checkDiagnostics(Diagnostic &D, bool ReturnsVoid,
                         bool HasNoReturn) const {
     if (funMode) {
-      return (D.getDiagnosticLevel(diag::warn_maybe_falloff_nonvoid_function)
-              == Diagnostic::Ignored || ReturnsVoid)
-        && (D.getDiagnosticLevel(diag::warn_noreturn_function_has_return_expr)
-              == Diagnostic::Ignored || !HasNoReturn)
-        && (D.getDiagnosticLevel(diag::warn_suggest_noreturn_block)
-              == Diagnostic::Ignored || !ReturnsVoid);
+      return (ReturnsVoid ||
+              D.getDiagnosticLevel(diag::warn_maybe_falloff_nonvoid_function,
+                                   FuncLoc) == Diagnostic::Ignored)
+        && (!HasNoReturn ||
+            D.getDiagnosticLevel(diag::warn_noreturn_function_has_return_expr,
+                                 FuncLoc) == Diagnostic::Ignored)
+        && (!ReturnsVoid ||
+            D.getDiagnosticLevel(diag::warn_suggest_noreturn_block, FuncLoc)
+              == Diagnostic::Ignored);
     }
 
     // For blocks.
     return  ReturnsVoid && !HasNoReturn
-            && (D.getDiagnosticLevel(diag::warn_suggest_noreturn_block)
-                == Diagnostic::Ignored || !ReturnsVoid);
+            && (!ReturnsVoid ||
+                D.getDiagnosticLevel(diag::warn_suggest_noreturn_block, FuncLoc)
+                  == Diagnostic::Ignored);
   }
 };
 
@@ -343,6 +355,112 @@ static void CheckFallThroughForBody(Sema &S, const Decl *D, const Stmt *Body,
 }
 
 //===----------------------------------------------------------------------===//
+// -Wuninitialized
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct SLocSort {
+  bool operator()(const Expr *a, const Expr *b) {
+    SourceLocation aLoc = a->getLocStart();
+    SourceLocation bLoc = b->getLocStart();
+    return aLoc.getRawEncoding() < bLoc.getRawEncoding();
+  }
+};
+
+class UninitValsDiagReporter : public UninitVariablesHandler {
+  Sema &S;
+  typedef llvm::SmallVector<const Expr *, 2> UsesVec;
+  typedef llvm::DenseMap<const VarDecl *, UsesVec*> UsesMap;
+  UsesMap *uses;
+  
+public:
+  UninitValsDiagReporter(Sema &S) : S(S), uses(0) {}
+  ~UninitValsDiagReporter() { 
+    flushDiagnostics();
+  }
+  
+  void handleUseOfUninitVariable(const Expr *ex, const VarDecl *vd) {
+    if (!uses)
+      uses = new UsesMap();
+    
+    UsesVec *&vec = (*uses)[vd];
+    if (!vec)
+      vec = new UsesVec();
+    
+    vec->push_back(ex);
+  }
+  
+  void flushDiagnostics() {
+    if (!uses)
+      return;
+    
+    for (UsesMap::iterator i = uses->begin(), e = uses->end(); i != e; ++i) {
+      const VarDecl *vd = i->first;
+      UsesVec *vec = i->second;
+
+      bool fixitIssued = false;
+            
+      // Sort the uses by their SourceLocations.  While not strictly
+      // guaranteed to produce them in line/column order, this will provide
+      // a stable ordering.
+      std::sort(vec->begin(), vec->end(), SLocSort());
+      
+      for (UsesVec::iterator vi = vec->begin(), ve = vec->end(); vi != ve; ++vi)
+      {
+        if (const DeclRefExpr *dr = dyn_cast<DeclRefExpr>(*vi)) {
+          S.Diag(dr->getLocStart(), diag::warn_uninit_var)
+            << vd->getDeclName() << dr->getSourceRange();          
+        }
+        else {
+          const BlockExpr *be = cast<BlockExpr>(*vi);
+          S.Diag(be->getLocStart(), diag::warn_uninit_var_captured_by_block)
+            << vd->getDeclName();
+        }
+        
+        // Report where the variable was declared.
+        S.Diag(vd->getLocStart(), diag::note_uninit_var_def)
+          << vd->getDeclName();
+
+        // Only report the fixit once.
+        if (fixitIssued)
+          continue;
+      
+        fixitIssued = true;
+
+        // Suggest possible initialization (if any).
+        const char *initialization = 0;
+        QualType vdTy = vd->getType().getCanonicalType();
+
+        if (vdTy->getAs<ObjCObjectPointerType>()) {
+          // Check if 'nil' is defined.
+          if (S.PP.getMacroInfo(&S.getASTContext().Idents.get("nil")))
+            initialization = " = nil";
+          else
+            initialization = " = 0";
+        }
+        else if (vdTy->isRealFloatingType())
+          initialization = " = 0.0";
+        else if (vdTy->isBooleanType() && S.Context.getLangOptions().CPlusPlus)
+          initialization = " = false";
+        else if (vdTy->isEnumeralType())
+          continue;
+        else if (vdTy->isScalarType())
+          initialization = " = 0";
+      
+        if (initialization) {
+          SourceLocation loc = S.PP.getLocForEndOfToken(vd->getLocEnd());
+          S.Diag(loc, diag::note_var_fixit_add_initialization)
+            << FixItHint::CreateInsertion(loc, initialization);
+        }
+      }
+      delete vec;
+    }
+    delete uses;
+  }
+};
+}
+
+//===----------------------------------------------------------------------===//
 // AnalysisBasedWarnings - Worker object used by Sema to execute analysis-based
 //  warnings on a function, method, or block.
 //===----------------------------------------------------------------------===//
@@ -355,7 +473,8 @@ clang::sema::AnalysisBasedWarnings::Policy::Policy() {
 clang::sema::AnalysisBasedWarnings::AnalysisBasedWarnings(Sema &s) : S(s) {
   Diagnostic &D = S.getDiagnostics();
   DefaultPolicy.enableCheckUnreachable = (unsigned)
-    (D.getDiagnosticLevel(diag::warn_unreachable) != Diagnostic::Ignored);
+    (D.getDiagnosticLevel(diag::warn_unreachable, SourceLocation()) !=
+        Diagnostic::Ignored);
 }
 
 void clang::sema::
@@ -390,7 +509,8 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
 
   // Don't generate EH edges for CallExprs as we'd like to avoid the n^2
   // explosion for destrutors that can result and the compile time hit.
-  AnalysisContext AC(D, 0, false);
+  AnalysisContext AC(D, 0, /*useUnoptimizedCFG=*/false, /*addehedges=*/false,
+                     /*addImplicitDtors=*/true, /*addInitializers=*/true);
 
   // Warning: check missing 'return'
   if (P.enableCheckFallThrough) {
@@ -403,6 +523,32 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   // Warning: check for unreachable code
   if (P.enableCheckUnreachable)
     CheckUnreachable(S, AC);
+  
+  if (Diags.getDiagnosticLevel(diag::warn_uninit_var, D->getLocStart())
+      != Diagnostic::Ignored) {
+    ASTContext &ctx = D->getASTContext();
+    llvm::OwningPtr<CFG> tmpCFG;
+    bool useAlternateCFG = false;
+    if (ctx.getLangOptions().CPlusPlus) {
+      // Temporary workaround: implicit dtors in the CFG can confuse
+      // the path-sensitivity in the uninitialized values analysis.
+      // For now create (if necessary) a separate CFG without implicit dtors.
+      // FIXME: We should not need to do this, as it results in multiple
+      // CFGs getting constructed.
+      CFG::BuildOptions B;
+      B.AddEHEdges = false;
+      B.AddImplicitDtors = false;
+      B.AddInitializers = true;
+      tmpCFG.reset(CFG::buildCFG(D, AC.getBody(), &ctx, B));
+      useAlternateCFG = true;
+    }
+    CFG *cfg = useAlternateCFG ? tmpCFG.get() : AC.getCFG();
+    if (cfg) {
+      UninitValsDiagReporter reporter(S);
+      runUninitializedVariablesAnalysis(*cast<DeclContext>(D), *cfg, AC,
+                                        reporter);
+    }
+  }
 }
 
 void clang::sema::
diff --git a/lib/Sema/AttributeList.cpp b/lib/Sema/AttributeList.cpp
index 8ccb2ca586eb..c0a305365afc 100644
--- a/lib/Sema/AttributeList.cpp
+++ b/lib/Sema/AttributeList.cpp
@@ -16,37 +16,26 @@
 #include "llvm/ADT/StringSwitch.h"
 using namespace clang;
 
-AttributeList::AttributeList(IdentifierInfo *aName, SourceLocation aLoc,
+AttributeList::AttributeList(llvm::BumpPtrAllocator &Alloc,
+                             IdentifierInfo *aName, SourceLocation aLoc,
                              IdentifierInfo *sName, SourceLocation sLoc,
                              IdentifierInfo *pName, SourceLocation pLoc,
                              Expr **ExprList, unsigned numArgs,
-                             AttributeList *n, bool declspec, bool cxx0x)
-  : AttrName(aName), AttrLoc(aLoc), ScopeName(sName), ScopeLoc(sLoc),
-    ParmName(pName), ParmLoc(pLoc), NumArgs(numArgs), Next(n),
+                             bool declspec, bool cxx0x)
+  : AttrName(aName), AttrLoc(aLoc), ScopeName(sName),
+    ScopeLoc(sLoc),
+    ParmName(pName), ParmLoc(pLoc), NumArgs(numArgs), Next(0),
     DeclspecAttribute(declspec), CXX0XAttribute(cxx0x), Invalid(false) {
 
   if (numArgs == 0)
     Args = 0;
   else {
-    Args = new Expr*[numArgs];
+    // Allocate the Args array using the BumpPtrAllocator.
+    Args = Alloc.Allocate<Expr*>(numArgs);
     memcpy(Args, ExprList, numArgs*sizeof(Args[0]));
   }
 }
 
-AttributeList::~AttributeList() {
-  if (Args) {
-    // FIXME: before we delete the vector, we need to make sure the Expr's
-    // have been deleted. Since ActionBase::ExprTy is "void", we are dependent
-    // on the actions module for actually freeing the memory. The specific
-    // hooks are ActOnDeclarator, ActOnTypeName, ActOnParamDeclaratorType,
-    // ParseField, ParseTag. Once these routines have freed the expression,
-    // they should zero out the Args slot (to indicate the memory has been
-    // freed). If any element of the vector is non-null, we should assert.
-    delete [] Args;
-  }
-  delete Next;
-}
-
 AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
   llvm::StringRef AttrName = Name->getName();
 
@@ -62,17 +51,17 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("used", AT_used)
     .Case("alias", AT_alias)
     .Case("align", AT_aligned)
-    .Case("final", AT_final)
     .Case("cdecl", AT_cdecl)
     .Case("const", AT_const)
+    .Case("__const", AT_const) // some GCC headers do contain this spelling
     .Case("blocks", AT_blocks)
     .Case("format", AT_format)
-    .Case("hiding", AT_hiding)
     .Case("malloc", AT_malloc)
     .Case("packed", AT_packed)
     .Case("unused", AT_unused)
     .Case("aligned", AT_aligned)
     .Case("cleanup", AT_cleanup)
+    .Case("naked", AT_naked)
     .Case("nodebug", AT_nodebug)
     .Case("nonnull", AT_nonnull)
     .Case("nothrow", AT_nothrow)
@@ -87,12 +76,11 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("iboutletcollection", AT_IBOutletCollection)
     .Case("noreturn", AT_noreturn)
     .Case("noinline", AT_noinline)
-    .Case("override", AT_override)
     .Case("sentinel", AT_sentinel)
     .Case("NSObject", AT_nsobject)
     .Case("dllimport", AT_dllimport)
     .Case("dllexport", AT_dllexport)
-    .Case("may_alias", IgnoredAttribute) // FIXME: TBAA
+    .Case("may_alias", AT_may_alias)
     .Case("base_check", AT_base_check)
     .Case("deprecated", AT_deprecated)
     .Case("visibility", AT_visibility)
@@ -111,12 +99,18 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("vec_type_hint", IgnoredAttribute)
     .Case("objc_exception", AT_objc_exception)
     .Case("ext_vector_type", AT_ext_vector_type)
+    .Case("neon_vector_type", AT_neon_vector_type)
+    .Case("neon_polyvector_type", AT_neon_polyvector_type)
     .Case("transparent_union", AT_transparent_union)
     .Case("analyzer_noreturn", AT_analyzer_noreturn)
     .Case("warn_unused_result", AT_warn_unused_result)
     .Case("carries_dependency", AT_carries_dependency)
+    .Case("ns_consumed", AT_ns_consumed)
+    .Case("ns_consumes_self", AT_ns_consumes_self)
+    .Case("ns_returns_autoreleased", AT_ns_returns_autoreleased)
     .Case("ns_returns_not_retained", AT_ns_returns_not_retained)
     .Case("ns_returns_retained", AT_ns_returns_retained)
+    .Case("cf_consumed", AT_cf_consumed)
     .Case("cf_returns_not_retained", AT_cf_returns_not_retained)
     .Case("cf_returns_retained", AT_cf_returns_retained)
     .Case("ownership_returns", AT_ownership_returns)
@@ -126,11 +120,22 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("init_priority", AT_init_priority)
     .Case("no_instrument_function", AT_no_instrument_function)
     .Case("thiscall", AT_thiscall)
+    .Case("bounded", IgnoredAttribute)       // OpenBSD
     .Case("pascal", AT_pascal)
     .Case("__cdecl", AT_cdecl)
     .Case("__stdcall", AT_stdcall)
     .Case("__fastcall", AT_fastcall)
     .Case("__thiscall", AT_thiscall)
     .Case("__pascal", AT_pascal)
+    .Case("constant", AT_constant)
+    .Case("device", AT_device)
+    .Case("global", AT_global)
+    .Case("host", AT_host)
+    .Case("shared", AT_shared)
+    .Case("launch_bounds", AT_launch_bounds)
+    .Case("common", AT_common)
+    .Case("nocommon", AT_nocommon)
+    .Case("opencl_kernel_function", AT_opencl_kernel_function)
+    .Case("uuid", AT_uuid)
     .Default(UnknownAttribute);
 }
diff --git a/lib/Sema/CMakeLists.txt b/lib/Sema/CMakeLists.txt
index e65bb227f159..0d66e259d484 100644
--- a/lib/Sema/CMakeLists.txt
+++ b/lib/Sema/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_USED_LIBS clangBasic clangAST clangLex clangAnalysis)
 
 add_clang_library(clangSema
   AnalysisBasedWarnings.cpp
@@ -31,6 +31,7 @@ add_clang_library(clangSema
   SemaTemplateDeduction.cpp
   SemaTemplateInstantiate.cpp
   SemaTemplateInstantiateDecl.cpp
+  SemaTemplateVariadic.cpp
   SemaType.cpp
   TargetAttributesSema.cpp
   )
diff --git a/lib/Sema/CodeCompleteConsumer.cpp b/lib/Sema/CodeCompleteConsumer.cpp
index 58a1627b478a..b7037ce83e7f 100644
--- a/lib/Sema/CodeCompleteConsumer.cpp
+++ b/lib/Sema/CodeCompleteConsumer.cpp
@@ -19,6 +19,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang-c/Index.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cstring>
@@ -28,9 +29,51 @@ using namespace clang;
 using llvm::StringRef;
 
 //===----------------------------------------------------------------------===//
+// Code completion context implementation
+//===----------------------------------------------------------------------===//
+
+bool CodeCompletionContext::wantConstructorResults() const {
+  switch (Kind) {
+  case CCC_Recovery:
+  case CCC_Statement:
+  case CCC_Expression:
+  case CCC_ObjCMessageReceiver:
+  case CCC_ParenthesizedExpression:
+    return true;
+    
+  case CCC_TopLevel:
+  case CCC_ObjCInterface:
+  case CCC_ObjCImplementation:
+  case CCC_ObjCIvarList:
+  case CCC_ClassStructUnion:
+  case CCC_MemberAccess:
+  case CCC_EnumTag:
+  case CCC_UnionTag:
+  case CCC_ClassOrStructTag:
+  case CCC_ObjCProtocolName:
+  case CCC_Namespace:
+  case CCC_Type:
+  case CCC_Name:
+  case CCC_PotentiallyQualifiedName:
+  case CCC_MacroName:
+  case CCC_MacroNameUse:
+  case CCC_PreprocessorExpression:
+  case CCC_PreprocessorDirective:
+  case CCC_NaturalLanguage:
+  case CCC_SelectorName:
+  case CCC_TypeQualifiers:
+  case CCC_Other:
+  case CCC_OtherWithMacros:
+    return false;
+  }
+  
+  return false;
+}
+
+//===----------------------------------------------------------------------===//
 // Code completion string implementation
 //===----------------------------------------------------------------------===//
-CodeCompletionString::Chunk::Chunk(ChunkKind Kind, llvm::StringRef Text) 
+CodeCompletionString::Chunk::Chunk(ChunkKind Kind, const char *Text) 
   : Kind(Kind), Text("")
 {
   switch (Kind) {
@@ -39,13 +82,9 @@ CodeCompletionString::Chunk::Chunk(ChunkKind Kind, llvm::StringRef Text)
   case CK_Placeholder:
   case CK_Informative:
   case CK_ResultType:
-  case CK_CurrentParameter: {
-    char *New = new char [Text.size() + 1];
-    std::memcpy(New, Text.data(), Text.size());
-    New[Text.size()] = '\0';
-    this->Text = New;
+  case CK_CurrentParameter:
+    this->Text = Text;
     break;
-  }
 
   case CK_Optional:
     llvm_unreachable("Optional strings cannot be created from text");
@@ -110,112 +149,48 @@ CodeCompletionString::Chunk::Chunk(ChunkKind Kind, llvm::StringRef Text)
 }
 
 CodeCompletionString::Chunk
-CodeCompletionString::Chunk::CreateText(StringRef Text) {
+CodeCompletionString::Chunk::CreateText(const char *Text) {
   return Chunk(CK_Text, Text);
 }
 
 CodeCompletionString::Chunk 
-CodeCompletionString::Chunk::CreateOptional(
-                                 std::auto_ptr<CodeCompletionString> Optional) {
+CodeCompletionString::Chunk::CreateOptional(CodeCompletionString *Optional) {
   Chunk Result;
   Result.Kind = CK_Optional;
-  Result.Optional = Optional.release();
+  Result.Optional = Optional;
   return Result;
 }
 
 CodeCompletionString::Chunk 
-CodeCompletionString::Chunk::CreatePlaceholder(StringRef Placeholder) {
+CodeCompletionString::Chunk::CreatePlaceholder(const char *Placeholder) {
   return Chunk(CK_Placeholder, Placeholder);
 }
 
 CodeCompletionString::Chunk 
-CodeCompletionString::Chunk::CreateInformative(StringRef Informative) {
+CodeCompletionString::Chunk::CreateInformative(const char *Informative) {
   return Chunk(CK_Informative, Informative);
 }
 
 CodeCompletionString::Chunk 
-CodeCompletionString::Chunk::CreateResultType(StringRef ResultType) {
+CodeCompletionString::Chunk::CreateResultType(const char *ResultType) {
   return Chunk(CK_ResultType, ResultType);
 }
 
 CodeCompletionString::Chunk 
 CodeCompletionString::Chunk::CreateCurrentParameter(
-                                                StringRef CurrentParameter) {
+                                                const char *CurrentParameter) {
   return Chunk(CK_CurrentParameter, CurrentParameter);
 }
 
-CodeCompletionString::Chunk CodeCompletionString::Chunk::Clone() const {
-  switch (Kind) {
-  case CK_TypedText:
-  case CK_Text:
-  case CK_Placeholder:
-  case CK_Informative:
-  case CK_ResultType:
-  case CK_CurrentParameter:
-  case CK_LeftParen:
-  case CK_RightParen:
-  case CK_LeftBracket:
-  case CK_RightBracket:
-  case CK_LeftBrace:
-  case CK_RightBrace:
-  case CK_LeftAngle:
-  case CK_RightAngle:
-  case CK_Comma:
-  case CK_Colon:
-  case CK_SemiColon:
-  case CK_Equal:
-  case CK_HorizontalSpace:
-  case CK_VerticalSpace:
-    return Chunk(Kind, Text);
-      
-  case CK_Optional: {
-    std::auto_ptr<CodeCompletionString> Opt(Optional->Clone());
-    return CreateOptional(Opt);
-  }
-  }
-
-  // Silence GCC warning.
-  return Chunk();
-}
-
-void
-CodeCompletionString::Chunk::Destroy() {
-  switch (Kind) {
-  case CK_Optional: 
-    delete Optional; 
-    break;
-      
-  case CK_TypedText:
-  case CK_Text: 
-  case CK_Placeholder:
-  case CK_Informative:
-  case CK_ResultType:
-  case CK_CurrentParameter:
-    delete [] Text;
-    break;
-
-  case CK_LeftParen:
-  case CK_RightParen:
-  case CK_LeftBracket:
-  case CK_RightBracket:
-  case CK_LeftBrace:
-  case CK_RightBrace:
-  case CK_LeftAngle:
-  case CK_RightAngle:
-  case CK_Comma:
-  case CK_Colon:
-  case CK_SemiColon:
-  case CK_Equal:
-  case CK_HorizontalSpace:
-  case CK_VerticalSpace:
-    break;
-  }
-}
-
-void CodeCompletionString::clear() {
-  std::for_each(Chunks.begin(), Chunks.end(), 
-                std::mem_fun_ref(&Chunk::Destroy));
-  Chunks.clear();
+CodeCompletionString::CodeCompletionString(const Chunk *Chunks, 
+                                           unsigned NumChunks,
+                                           unsigned Priority, 
+                                           CXAvailabilityKind Availability) 
+  : NumChunks(NumChunks), Priority(Priority), Availability(Availability) 
+{ 
+  Chunk *StoredChunks = reinterpret_cast<Chunk *>(this + 1);
+  for (unsigned I = 0; I != NumChunks; ++I)
+    StoredChunks[I] = Chunks[I];
 }
 
 std::string CodeCompletionString::getAsString() const {
@@ -247,140 +222,30 @@ const char *CodeCompletionString::getTypedText() const {
   return 0;
 }
 
-CodeCompletionString *
-CodeCompletionString::Clone(CodeCompletionString *Result) const {
-  if (!Result)
-    Result = new CodeCompletionString;
-  for (iterator C = begin(), CEnd = end(); C != CEnd; ++C)
-    Result->AddChunk(C->Clone());
-  return Result;
+const char *CodeCompletionAllocator::CopyString(llvm::StringRef String) {
+  char *Mem = (char *)Allocate(String.size() + 1, 1);
+  std::copy(String.begin(), String.end(), Mem);
+  Mem[String.size()] = 0;
+  return Mem;
 }
 
-static void WriteUnsigned(llvm::raw_ostream &OS, unsigned Value) {
-  OS.write((const char *)&Value, sizeof(unsigned));
+const char *CodeCompletionAllocator::CopyString(llvm::Twine String) {
+  // FIXME: It would be more efficient to teach Twine to tell us its size and
+  // then add a routine there to fill in an allocated char* with the contents
+  // of the string.
+  llvm::SmallString<128> Data;
+  return CopyString(String.toStringRef(Data));
 }
 
-static bool ReadUnsigned(const char *&Memory, const char *MemoryEnd,
-                         unsigned &Value) {
-  if (Memory + sizeof(unsigned) > MemoryEnd)
-    return true;
-
-  memmove(&Value, Memory, sizeof(unsigned));
-  Memory += sizeof(unsigned);
-  return false;
-}
-
-void CodeCompletionString::Serialize(llvm::raw_ostream &OS) const {
-  // Write the number of chunks.
-  WriteUnsigned(OS, size());
-
-  for (iterator C = begin(), CEnd = end(); C != CEnd; ++C) {
-    WriteUnsigned(OS, C->Kind);
-
-    switch (C->Kind) {
-    case CK_TypedText:
-    case CK_Text:
-    case CK_Placeholder:
-    case CK_Informative:
-    case CK_ResultType:
-    case CK_CurrentParameter: {
-      const char *Text = C->Text;
-      unsigned StrLen = strlen(Text);
-      WriteUnsigned(OS, StrLen);
-      OS.write(Text, StrLen);
-      break;
-    }
-
-    case CK_Optional:
-      C->Optional->Serialize(OS);
-      break;
-
-    case CK_LeftParen:
-    case CK_RightParen:
-    case CK_LeftBracket:
-    case CK_RightBracket:
-    case CK_LeftBrace:
-    case CK_RightBrace:
-    case CK_LeftAngle:
-    case CK_RightAngle:
-    case CK_Comma:
-    case CK_Colon:
-    case CK_SemiColon:
-    case CK_Equal:
-    case CK_HorizontalSpace:
-    case CK_VerticalSpace:
-      break;
-    }
-  }
-}
-
-bool CodeCompletionString::Deserialize(const char *&Str, const char *StrEnd) {
-  if (Str == StrEnd || *Str == 0)
-    return false;
-
-  unsigned NumBlocks;
-  if (ReadUnsigned(Str, StrEnd, NumBlocks))
-    return false;
-
-  for (unsigned I = 0; I != NumBlocks; ++I) {
-    if (Str + 1 >= StrEnd)
-      break;
-
-    // Parse the next kind.
-    unsigned KindValue;
-    if (ReadUnsigned(Str, StrEnd, KindValue))
-      return false;
-
-    switch (ChunkKind Kind = (ChunkKind)KindValue) {
-    case CK_TypedText:
-    case CK_Text:
-    case CK_Placeholder:
-    case CK_Informative:
-    case CK_ResultType:
-    case CK_CurrentParameter: {
-      unsigned StrLen;
-      if (ReadUnsigned(Str, StrEnd, StrLen) || (Str + StrLen > StrEnd))
-        return false;
-
-      AddChunk(Chunk(Kind, StringRef(Str, StrLen)));
-      Str += StrLen;
-      break;
-    }
-
-    case CK_Optional: {
-      std::auto_ptr<CodeCompletionString> Optional(new CodeCompletionString());
-      if (Optional->Deserialize(Str, StrEnd))
-        AddOptionalChunk(Optional);
-      break;
-    }
-
-    case CK_LeftParen:
-    case CK_RightParen:
-    case CK_LeftBracket:
-    case CK_RightBracket:
-    case CK_LeftBrace:
-    case CK_RightBrace:
-    case CK_LeftAngle:
-    case CK_RightAngle:
-    case CK_Comma:
-    case CK_Colon:
-    case CK_SemiColon:
-    case CK_Equal:
-    case CK_HorizontalSpace:
-    case CK_VerticalSpace:
-      AddChunk(Chunk(Kind));
-      break;      
-    }
-  };
-  
-  return true;
-}
-
-void CodeCompletionResult::Destroy() {
-  if (Kind == RK_Pattern) {
-    delete Pattern;
-    Pattern = 0;
-  }
+CodeCompletionString *CodeCompletionBuilder::TakeString() {
+  void *Mem = Allocator.Allocate(
+                  sizeof(CodeCompletionString) + sizeof(Chunk) * Chunks.size(), 
+                                 llvm::alignOf<CodeCompletionString>());
+  CodeCompletionString *Result 
+    = new (Mem) CodeCompletionString(Chunks.data(), Chunks.size(),
+                               Priority, Availability);
+  Chunks.clear();
+  return Result;
 }
 
 unsigned CodeCompletionResult::getPriorityFromDecl(NamedDecl *ND) {
@@ -389,8 +254,15 @@ unsigned CodeCompletionResult::getPriorityFromDecl(NamedDecl *ND) {
   
   // Context-based decisions.
   DeclContext *DC = ND->getDeclContext()->getRedeclContext();
-  if (DC->isFunctionOrMethod() || isa<BlockDecl>(DC))
+  if (DC->isFunctionOrMethod() || isa<BlockDecl>(DC)) {
+    // _cmd is relatively rare
+    if (ImplicitParamDecl *ImplicitParam = dyn_cast<ImplicitParamDecl>(ND))
+      if (ImplicitParam->getIdentifier() &&
+          ImplicitParam->getIdentifier()->isStr("_cmd"))
+        return CCP_ObjC_cmd;
+    
     return CCP_LocalDeclaration;
+  }
   if (DC->isRecord() || isa<ObjCContainerDecl>(DC))
     return CCP_MemberDeclaration;
   
@@ -399,6 +271,7 @@ unsigned CodeCompletionResult::getPriorityFromDecl(NamedDecl *ND) {
     return CCP_Constant;
   if (isa<TypeDecl>(ND) || isa<ObjCInterfaceDecl>(ND))
     return CCP_Type;
+  
   return CCP_Declaration;
 }
 
@@ -454,9 +327,8 @@ PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
       if (Results[I].Hidden)
         OS << " (Hidden)";
       if (CodeCompletionString *CCS 
-            = Results[I].CreateCodeCompletionString(SemaRef)) {
+            = Results[I].CreateCodeCompletionString(SemaRef, Allocator)) {
         OS << " : " << CCS->getAsString();
-        delete CCS;
       }
         
       OS << '\n';
@@ -469,9 +341,8 @@ PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
     case CodeCompletionResult::RK_Macro: {
       OS << Results[I].Macro->getName();
       if (CodeCompletionString *CCS 
-            = Results[I].CreateCodeCompletionString(SemaRef)) {
+            = Results[I].CreateCodeCompletionString(SemaRef, Allocator)) {
         OS << " : " << CCS->getAsString();
-        delete CCS;
       }
       OS << '\n';
       break;
@@ -493,9 +364,9 @@ PrintingCodeCompleteConsumer::ProcessOverloadCandidates(Sema &SemaRef,
                                                      unsigned NumCandidates) {
   for (unsigned I = 0; I != NumCandidates; ++I) {
     if (CodeCompletionString *CCS
-          = Candidates[I].CreateSignatureString(CurrentArg, SemaRef)) {
+          = Candidates[I].CreateSignatureString(CurrentArg, SemaRef,
+                                                Allocator)) {
       OS << "OVERLOAD: " << CCS->getAsString() << "\n";
-      delete CCS;
     }
   }
 }
@@ -587,37 +458,3 @@ bool clang::operator<(const CodeCompletionResult &X,
   
   return false;
 }
-
-void 
-CIndexCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
-                                                 CodeCompletionContext Context,
-                                                 CodeCompletionResult *Results,
-                                                       unsigned NumResults) {
-  // Print the results.
-  for (unsigned I = 0; I != NumResults; ++I) {
-    WriteUnsigned(OS, Results[I].CursorKind);
-    WriteUnsigned(OS, Results[I].Priority);
-    WriteUnsigned(OS, Results[I].Availability);
-    CodeCompletionString *CCS = Results[I].CreateCodeCompletionString(SemaRef);
-    assert(CCS && "No code-completion string?");
-    CCS->Serialize(OS);
-    delete CCS;
-  }
-}
-
-void 
-CIndexCodeCompleteConsumer::ProcessOverloadCandidates(Sema &SemaRef,
-                                                      unsigned CurrentArg,
-                                                OverloadCandidate *Candidates,
-                                                       unsigned NumCandidates) {
-  for (unsigned I = 0; I != NumCandidates; ++I) {
-    WriteUnsigned(OS, CXCursor_NotImplemented);
-    WriteUnsigned(OS, /*Priority=*/I);
-    WriteUnsigned(OS, /*Availability=*/CXAvailability_Available);
-    CodeCompletionString *CCS
-      = Candidates[I].CreateSignatureString(CurrentArg, SemaRef);
-    assert(CCS && "No code-completion string?");
-    CCS->Serialize(OS);
-    delete CCS;
-  }
-}
diff --git a/lib/Sema/DeclSpec.cpp b/lib/Sema/DeclSpec.cpp
index b46e8af9db86..bc289ec42c99 100644
--- a/lib/Sema/DeclSpec.cpp
+++ b/lib/Sema/DeclSpec.cpp
@@ -23,8 +23,8 @@ using namespace clang;
 
 
 static DiagnosticBuilder Diag(Diagnostic &D, SourceLocation Loc,
-                              SourceManager &SrcMgr, unsigned DiagID) {
-  return D.Report(FullSourceLoc(Loc, SrcMgr), DiagID);
+                              unsigned DiagID) {
+  return D.Report(Loc, DiagID);
 }
 
 
@@ -46,11 +46,14 @@ void UnqualifiedId::setConstructorTemplateId(TemplateIdAnnotation *TemplateId) {
 
 /// DeclaratorChunk::getFunction - Return a DeclaratorChunk for a function.
 /// "TheDeclarator" is the declarator that this will be added to.
-DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
+DeclaratorChunk DeclaratorChunk::getFunction(const ParsedAttributes &attrs,
+                                             bool hasProto, bool isVariadic,
                                              SourceLocation EllipsisLoc,
                                              ParamInfo *ArgInfo,
                                              unsigned NumArgs,
                                              unsigned TypeQuals,
+                                             bool RefQualifierIsLvalueRef,
+                                             SourceLocation RefQualifierLoc,
                                              bool hasExceptionSpec,
                                              SourceLocation ThrowLoc,
                                              bool hasAnyExceptionSpec,
@@ -59,11 +62,13 @@ DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
                                              unsigned NumExceptions,
                                              SourceLocation LPLoc,
                                              SourceLocation RPLoc,
-                                             Declarator &TheDeclarator) {
+                                             Declarator &TheDeclarator,
+                                             ParsedType TrailingReturnType) {
   DeclaratorChunk I;
   I.Kind                 = Function;
   I.Loc                  = LPLoc;
   I.EndLoc               = RPLoc;
+  I.Fun.AttrList         = attrs.getList();
   I.Fun.hasPrototype     = hasProto;
   I.Fun.isVariadic       = isVariadic;
   I.Fun.EllipsisLoc      = EllipsisLoc.getRawEncoding();
@@ -71,11 +76,14 @@ DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
   I.Fun.TypeQuals        = TypeQuals;
   I.Fun.NumArgs          = NumArgs;
   I.Fun.ArgInfo          = 0;
+  I.Fun.RefQualifierIsLValueRef = RefQualifierIsLvalueRef;
+  I.Fun.RefQualifierLoc = RefQualifierLoc.getRawEncoding();
   I.Fun.hasExceptionSpec = hasExceptionSpec;
   I.Fun.ThrowLoc         = ThrowLoc.getRawEncoding();
   I.Fun.hasAnyExceptionSpec = hasAnyExceptionSpec;
   I.Fun.NumExceptions    = NumExceptions;
   I.Fun.Exceptions       = 0;
+  I.Fun.TrailingReturnType   = TrailingReturnType.getAsOpaquePtr();
 
   // new[] an argument array if needed.
   if (NumArgs) {
@@ -218,7 +226,25 @@ const char *DeclSpec::getSpecifierName(TQ T) {
 
 bool DeclSpec::SetStorageClassSpec(SCS S, SourceLocation Loc,
                                    const char *&PrevSpec,
-                                   unsigned &DiagID) {
+                                   unsigned &DiagID,
+                                   const LangOptions &Lang) {
+  // OpenCL prohibits extern, auto, register, and static
+  // It seems sensible to prohibit private_extern too
+  if (Lang.OpenCL) {
+    switch (S) {
+    case SCS_extern:
+    case SCS_private_extern:
+    case SCS_auto:
+    case SCS_register:
+    case SCS_static:
+      DiagID   = diag::err_not_opencl_storage_class_specifier;
+      PrevSpec = getSpecifierName(S);
+      return true;
+    default:
+      break;
+    }
+  }
+
   if (StorageClassSpec != SCS_unspecified) {
     // Changing storage class is allowed only if the previous one
     // was the 'extern' that is part of a linkage specification and
@@ -481,7 +507,7 @@ void DeclSpec::SaveWrittenBuiltinSpecs() {
   writtenBS.Type = getTypeSpecType();
   // Search the list of attributes for the presence of a mode attribute.
   writtenBS.ModeAttr = false;
-  AttributeList* attrs = getAttributes();
+  AttributeList* attrs = getAttributes().getList();
   while (attrs) {
     if (attrs->getKind() == AttributeList::AT_mode) {
       writtenBS.ModeAttr = true;
@@ -510,28 +536,27 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
   SaveStorageSpecifierAsWritten();
 
   // Check the type specifier components first.
-  SourceManager &SrcMgr = PP.getSourceManager();
 
   // Validate and finalize AltiVec vector declspec.
   if (TypeAltiVecVector) {
     if (TypeAltiVecBool) {
       // Sign specifiers are not allowed with vector bool. (PIM 2.1)
       if (TypeSpecSign != TSS_unspecified) {
-        Diag(D, TSSLoc, SrcMgr, diag::err_invalid_vector_bool_decl_spec)
+        Diag(D, TSSLoc, diag::err_invalid_vector_bool_decl_spec)
           << getSpecifierName((TSS)TypeSpecSign);
       }
 
       // Only char/int are valid with vector bool. (PIM 2.1)
       if (((TypeSpecType != TST_unspecified) && (TypeSpecType != TST_char) &&
            (TypeSpecType != TST_int)) || TypeAltiVecPixel) {
-        Diag(D, TSTLoc, SrcMgr, diag::err_invalid_vector_bool_decl_spec)
+        Diag(D, TSTLoc, diag::err_invalid_vector_bool_decl_spec)
           << (TypeAltiVecPixel ? "__pixel" :
                                  getSpecifierName((TST)TypeSpecType));
       }
 
       // Only 'short' is valid with vector bool. (PIM 2.1)
       if ((TypeSpecWidth != TSW_unspecified) && (TypeSpecWidth != TSW_short))
-        Diag(D, TSWLoc, SrcMgr, diag::err_invalid_vector_bool_decl_spec)
+        Diag(D, TSWLoc, diag::err_invalid_vector_bool_decl_spec)
           << getSpecifierName((TSW)TypeSpecWidth);
 
       // Elements of vector bool are interpreted as unsigned. (PIM 2.1)
@@ -555,7 +580,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
       TypeSpecType = TST_int; // unsigned -> unsigned int, signed -> signed int.
     else if (TypeSpecType != TST_int  &&
              TypeSpecType != TST_char && TypeSpecType != TST_wchar) {
-      Diag(D, TSSLoc, SrcMgr, diag::err_invalid_sign_spec)
+      Diag(D, TSSLoc, diag::err_invalid_sign_spec)
         << getSpecifierName((TST)TypeSpecType);
       // signed double -> double.
       TypeSpecSign = TSS_unspecified;
@@ -570,7 +595,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     if (TypeSpecType == TST_unspecified)
       TypeSpecType = TST_int; // short -> short int, long long -> long long int.
     else if (TypeSpecType != TST_int) {
-      Diag(D, TSWLoc, SrcMgr,
+      Diag(D, TSWLoc,
            TypeSpecWidth == TSW_short ? diag::err_invalid_short_spec
                                       : diag::err_invalid_longlong_spec)
         <<  getSpecifierName((TST)TypeSpecType);
@@ -582,7 +607,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     if (TypeSpecType == TST_unspecified)
       TypeSpecType = TST_int;  // long -> long int.
     else if (TypeSpecType != TST_int && TypeSpecType != TST_double) {
-      Diag(D, TSWLoc, SrcMgr, diag::err_invalid_long_spec)
+      Diag(D, TSWLoc, diag::err_invalid_long_spec)
         << getSpecifierName((TST)TypeSpecType);
       TypeSpecType = TST_int;
       TypeSpecOwned = false;
@@ -594,16 +619,16 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
   // disallow their use.  Need information about the backend.
   if (TypeSpecComplex != TSC_unspecified) {
     if (TypeSpecType == TST_unspecified) {
-      Diag(D, TSCLoc, SrcMgr, diag::ext_plain_complex)
+      Diag(D, TSCLoc, diag::ext_plain_complex)
         << FixItHint::CreateInsertion(
                               PP.getLocForEndOfToken(getTypeSpecComplexLoc()),
                                                  " double");
       TypeSpecType = TST_double;   // _Complex -> _Complex double.
     } else if (TypeSpecType == TST_int || TypeSpecType == TST_char) {
       // Note that this intentionally doesn't include _Complex _Bool.
-      Diag(D, TSTLoc, SrcMgr, diag::ext_integer_complex);
+      Diag(D, TSTLoc, diag::ext_integer_complex);
     } else if (TypeSpecType != TST_float && TypeSpecType != TST_double) {
-      Diag(D, TSCLoc, SrcMgr, diag::err_invalid_complex_spec)
+      Diag(D, TSCLoc, diag::err_invalid_complex_spec)
         << getSpecifierName((TST)TypeSpecType);
       TypeSpecComplex = TSC_unspecified;
     }
@@ -619,7 +644,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     SourceLocation SCLoc = getStorageClassSpecLoc();
     SourceLocation SCEndLoc = SCLoc.getFileLocWithOffset(strlen(SpecName));
 
-    Diag(D, SCLoc, SrcMgr, diag::err_friend_storage_spec)
+    Diag(D, SCLoc, diag::err_friend_storage_spec)
       << SpecName
       << FixItHint::CreateRemoval(SourceRange(SCLoc, SCEndLoc));
 
@@ -665,3 +690,58 @@ void UnqualifiedId::setOperatorFunctionId(SourceLocation OperatorLoc,
       EndLocation = SymbolLocations[I];
   }
 }
+
+bool VirtSpecifiers::SetSpecifier(Specifier VS, SourceLocation Loc,
+                                  const char *&PrevSpec) {
+  if (Specifiers & VS) {
+    PrevSpec = getSpecifierName(VS);
+    return true;
+  }
+
+  Specifiers |= VS;
+
+  switch (VS) {
+  default: assert(0 && "Unknown specifier!");
+  case VS_Override: VS_overrideLoc = Loc; break;
+  case VS_Final:    VS_finalLoc = Loc; break;
+  case VS_New:      VS_newLoc = Loc; break;
+  }
+
+  return false;
+}
+
+const char *VirtSpecifiers::getSpecifierName(Specifier VS) {
+  switch (VS) {
+  default: assert(0 && "Unknown specifier");
+  case VS_Override: return "override";
+  case VS_Final: return "final";
+  case VS_New: return "new";
+  }
+}
+
+bool ClassVirtSpecifiers::SetSpecifier(Specifier CVS, SourceLocation Loc,
+                                       const char *&PrevSpec) {
+  if (Specifiers & CVS) {
+    PrevSpec = getSpecifierName(CVS);
+    return true;
+  }
+
+  Specifiers |= CVS;
+
+  switch (CVS) {
+  default: assert(0 && "Unknown specifier!");
+  case CVS_Final: CVS_finalLoc = Loc; break;
+  case CVS_Explicit: CVS_explicitLoc = Loc; break;
+  }
+
+  return false;
+}
+
+const char *ClassVirtSpecifiers::getSpecifierName(Specifier CVS) {
+  switch (CVS) {
+  default: assert(0 && "Unknown specifier");
+  case CVS_Final: return "final";
+  case CVS_Explicit: return "explicit";
+  }
+}
+
diff --git a/lib/Sema/JumpDiagnostics.cpp b/lib/Sema/JumpDiagnostics.cpp
index b23f615af7b7..b73f0e9f1452 100644
--- a/lib/Sema/JumpDiagnostics.cpp
+++ b/lib/Sema/JumpDiagnostics.cpp
@@ -62,7 +62,7 @@ class JumpScopeChecker {
   llvm::SmallVector<Stmt*, 16> Jumps;
 
   llvm::SmallVector<IndirectGotoStmt*, 4> IndirectJumps;
-  llvm::SmallVector<LabelStmt*, 4> IndirectJumpTargets;
+  llvm::SmallVector<LabelDecl*, 4> IndirectJumpTargets;
 public:
   JumpScopeChecker(Stmt *Body, Sema &S);
 private:
@@ -71,7 +71,7 @@ private:
   void VerifyJumps();
   void VerifyIndirectJumps();
   void DiagnoseIndirectJump(IndirectGotoStmt *IG, unsigned IGScope,
-                            LabelStmt *Target, unsigned TargetScope);
+                            LabelDecl *Target, unsigned TargetScope);
   void CheckJump(Stmt *From, Stmt *To,
                  SourceLocation DiagLoc, unsigned JumpDiag);
 
@@ -186,6 +186,17 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned ParentScope) {
     break;
 
   case Stmt::IndirectGotoStmtClass:
+    // "goto *&&lbl;" is a special case which we treat as equivalent
+    // to a normal goto.  In addition, we don't calculate scope in the
+    // operand (to avoid recording the address-of-label use), which
+    // works only because of the restricted set of expressions which
+    // we detect as constant targets.
+    if (cast<IndirectGotoStmt>(S)->getConstantTarget()) {
+      LabelAndGotoScopes[S] = ParentScope;
+      Jumps.push_back(S);
+      return;
+    }
+
     LabelAndGotoScopes[S] = ParentScope;
     IndirectJumps.push_back(cast<IndirectGotoStmt>(S));
     break;
@@ -210,8 +221,7 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned ParentScope) {
     break;
   }
 
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end(); CI != E;
-       ++CI) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI) {
     if (SkipFirstSubStmt) {
       SkipFirstSubStmt = false;
       continue;
@@ -225,12 +235,12 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned ParentScope) {
     // order to avoid blowing out the stack.
     while (true) {
       Stmt *Next;
-      if (isa<CaseStmt>(SubStmt))
-        Next = cast<CaseStmt>(SubStmt)->getSubStmt();
-      else if (isa<DefaultStmt>(SubStmt))
-        Next = cast<DefaultStmt>(SubStmt)->getSubStmt();
-      else if (isa<LabelStmt>(SubStmt))
-        Next = cast<LabelStmt>(SubStmt)->getSubStmt();
+      if (CaseStmt *CS = dyn_cast<CaseStmt>(SubStmt))
+        Next = CS->getSubStmt();
+      else if (DefaultStmt *DS = dyn_cast<DefaultStmt>(SubStmt))
+        Next = DS->getSubStmt();
+      else if (LabelStmt *LS = dyn_cast<LabelStmt>(SubStmt))
+        Next = LS->getSubStmt();
       else
         break;
 
@@ -336,7 +346,15 @@ void JumpScopeChecker::VerifyJumps() {
 
     // With a goto,
     if (GotoStmt *GS = dyn_cast<GotoStmt>(Jump)) {
-      CheckJump(GS, GS->getLabel(), GS->getGotoLoc(),
+      CheckJump(GS, GS->getLabel()->getStmt(), GS->getGotoLoc(),
+                diag::err_goto_into_protected_scope);
+      continue;
+    }
+
+    // We only get indirect gotos here when they have a constant target.
+    if (IndirectGotoStmt *IGS = dyn_cast<IndirectGotoStmt>(Jump)) {
+      LabelDecl *Target = IGS->getConstantTarget();
+      CheckJump(IGS, Target->getStmt(), IGS->getGotoLoc(),
                 diag::err_goto_into_protected_scope);
       continue;
     }
@@ -406,15 +424,15 @@ void JumpScopeChecker::VerifyIndirectJumps() {
   // Collect a single representative of every scope containing a
   // label whose address was taken somewhere in the function.
   // For most code bases, there will be only one such scope.
-  llvm::DenseMap<unsigned, LabelStmt*> TargetScopes;
-  for (llvm::SmallVectorImpl<LabelStmt*>::iterator
+  llvm::DenseMap<unsigned, LabelDecl*> TargetScopes;
+  for (llvm::SmallVectorImpl<LabelDecl*>::iterator
          I = IndirectJumpTargets.begin(), E = IndirectJumpTargets.end();
        I != E; ++I) {
-    LabelStmt *TheLabel = *I;
-    assert(LabelAndGotoScopes.count(TheLabel) &&
+    LabelDecl *TheLabel = *I;
+    assert(LabelAndGotoScopes.count(TheLabel->getStmt()) &&
            "Referenced label didn't get added to scopes?");
-    unsigned LabelScope = LabelAndGotoScopes[TheLabel];
-    LabelStmt *&Target = TargetScopes[LabelScope];
+    unsigned LabelScope = LabelAndGotoScopes[TheLabel->getStmt()];
+    LabelDecl *&Target = TargetScopes[LabelScope];
     if (!Target) Target = TheLabel;
   }
 
@@ -427,10 +445,10 @@ void JumpScopeChecker::VerifyIndirectJumps() {
   // entered, then verify that every jump scope can be trivially
   // exitted to reach a scope in S.
   llvm::BitVector Reachable(Scopes.size(), false);
-  for (llvm::DenseMap<unsigned,LabelStmt*>::iterator
+  for (llvm::DenseMap<unsigned,LabelDecl*>::iterator
          TI = TargetScopes.begin(), TE = TargetScopes.end(); TI != TE; ++TI) {
     unsigned TargetScope = TI->first;
-    LabelStmt *TargetLabel = TI->second;
+    LabelDecl *TargetLabel = TI->second;
 
     Reachable.reset();
 
@@ -493,12 +511,12 @@ void JumpScopeChecker::VerifyIndirectJumps() {
 /// Diagnose an indirect jump which is known to cross scopes.
 void JumpScopeChecker::DiagnoseIndirectJump(IndirectGotoStmt *Jump,
                                             unsigned JumpScope,
-                                            LabelStmt *Target,
+                                            LabelDecl *Target,
                                             unsigned TargetScope) {
   assert(JumpScope != TargetScope);
 
-  S.Diag(Jump->getGotoLoc(), diag::warn_indirect_goto_in_protected_scope);
-  S.Diag(Target->getIdentLoc(), diag::note_indirect_goto_target);
+  S.Diag(Jump->getGotoLoc(), diag::err_indirect_goto_in_protected_scope);
+  S.Diag(Target->getStmt()->getIdentLoc(), diag::note_indirect_goto_target);
 
   unsigned Common = GetDeepestCommonScope(JumpScope, TargetScope);
 
diff --git a/lib/Sema/Sema.cpp b/lib/Sema/Sema.cpp
index 17817d41691b..23a3c24804e6 100644
--- a/lib/Sema/Sema.cpp
+++ b/lib/Sema/Sema.cpp
@@ -19,7 +19,9 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/APFloat.h"
 #include "clang/Sema/CXXFieldCollector.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
@@ -29,6 +31,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/StmtCXX.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
@@ -37,15 +40,14 @@ using namespace sema;
 
 FunctionScopeInfo::~FunctionScopeInfo() { }
 
-void FunctionScopeInfo::Clear(unsigned NumErrors) {
+void FunctionScopeInfo::Clear() {
   HasBranchProtectedScope = false;
   HasBranchIntoScope = false;
   HasIndirectGoto = false;
   
-  LabelMap.clear();
   SwitchStack.clear();
   Returns.clear();
-  NumErrorsAtStartOfFunction = NumErrors;
+  ErrorTrap.reset();
 }
 
 BlockScopeInfo::~BlockScopeInfo() { }
@@ -129,15 +131,18 @@ void Sema::ActOnTranslationUnitScope(Scope *S) {
 Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
            bool CompleteTranslationUnit,
            CodeCompleteConsumer *CodeCompleter)
-  : TheTargetAttributesSema(0),
+  : TheTargetAttributesSema(0), FPFeatures(pp.getLangOptions()),
     LangOpts(pp.getLangOptions()), PP(pp), Context(ctxt), Consumer(consumer),
     Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()),
     ExternalSource(0), CodeCompleter(CodeCompleter), CurContext(0), 
-    PackContext(0), VisContext(0), ParsingDeclDepth(0),
-    IdResolver(pp.getLangOptions()), GlobalNewDeleteDeclared(false), 
+    PackContext(0), VisContext(0),
+    IdResolver(pp.getLangOptions()), CXXTypeInfoDecl(0), MSVCGuidDecl(0),
+    GlobalNewDeleteDeclared(false), 
     CompleteTranslationUnit(CompleteTranslationUnit),
-    NumSFINAEErrors(0), SuppressAccessChecking(false),
-    NonInstantiationEntries(0), CurrentInstantiationScope(0), TyposCorrected(0),
+    NumSFINAEErrors(0), SuppressAccessChecking(false), 
+    AccessCheckingSFINAE(false), InNonInstantiationSFINAEContext(false),
+    NonInstantiationEntries(0), ArgumentPackSubstitutionIndex(-1),
+    CurrentInstantiationScope(0), TyposCorrected(0),
     AnalysisWarnings(*this)
 {
   TUScope = 0;
@@ -151,7 +156,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
   ExprEvalContexts.push_back(
                   ExpressionEvaluationContextRecord(PotentiallyEvaluated, 0));  
 
-  FunctionScopes.push_back(new FunctionScopeInfo(Diags.getNumErrors()));
+  FunctionScopes.push_back(new FunctionScopeInfo(Diags));
 }
 
 void Sema::Initialize() {
@@ -201,15 +206,6 @@ void Sema::ImpCastExprToType(Expr *&Expr, QualType Ty,
   if (ExprTy == TypeTy)
     return;
 
-  if (Expr->getType()->isPointerType() && Ty->isPointerType()) {
-    QualType ExprBaseType = cast<PointerType>(ExprTy)->getPointeeType();
-    QualType BaseType = cast<PointerType>(TypeTy)->getPointeeType();
-    if (ExprBaseType.getAddressSpace() != BaseType.getAddressSpace()) {
-      Diag(Expr->getExprLoc(), diag::err_implicit_pointer_address_space_cast)
-        << Expr->getSourceRange();
-    }
-  }
-
   // If this is a derived-to-base cast to a through a virtual base, we
   // need a vtable.
   if (Kind == CK_DerivedToBase && 
@@ -275,32 +271,103 @@ static bool ShouldRemoveFromUnused(Sema *SemaRef, const DeclaratorDecl *D) {
   return false;
 }
 
+namespace {
+  struct UndefinedInternal {
+    NamedDecl *decl;
+    FullSourceLoc useLoc;
+
+    UndefinedInternal(NamedDecl *decl, FullSourceLoc useLoc)
+      : decl(decl), useLoc(useLoc) {}
+  };
+
+  bool operator<(const UndefinedInternal &l, const UndefinedInternal &r) {
+    return l.useLoc.isBeforeInTranslationUnitThan(r.useLoc);
+  }
+}
+
+/// checkUndefinedInternals - Check for undefined objects with internal linkage.
+static void checkUndefinedInternals(Sema &S) {
+  if (S.UndefinedInternals.empty()) return;
+
+  // Collect all the still-undefined entities with internal linkage.
+  llvm::SmallVector<UndefinedInternal, 16> undefined;
+  for (llvm::DenseMap<NamedDecl*,SourceLocation>::iterator
+         i = S.UndefinedInternals.begin(), e = S.UndefinedInternals.end();
+       i != e; ++i) {
+    NamedDecl *decl = i->first;
+
+    // Ignore attributes that have become invalid.
+    if (decl->isInvalidDecl()) continue;
+
+    // __attribute__((weakref)) is basically a definition.
+    if (decl->hasAttr<WeakRefAttr>()) continue;
+
+    if (FunctionDecl *fn = dyn_cast<FunctionDecl>(decl)) {
+      if (fn->isPure() || fn->hasBody())
+        continue;
+    } else {
+      if (cast<VarDecl>(decl)->hasDefinition() != VarDecl::DeclarationOnly)
+        continue;
+    }
+
+    // We build a FullSourceLoc so that we can sort with array_pod_sort.
+    FullSourceLoc loc(i->second, S.Context.getSourceManager());
+    undefined.push_back(UndefinedInternal(decl, loc));
+  }
+
+  if (undefined.empty()) return;
+
+  // Sort (in order of use site) so that we're not (as) dependent on
+  // the iteration order through an llvm::DenseMap.
+  llvm::array_pod_sort(undefined.begin(), undefined.end());
+
+  for (llvm::SmallVectorImpl<UndefinedInternal>::iterator
+         i = undefined.begin(), e = undefined.end(); i != e; ++i) {
+    NamedDecl *decl = i->decl;
+    S.Diag(decl->getLocation(), diag::warn_undefined_internal)
+      << isa<VarDecl>(decl) << decl;
+    S.Diag(i->useLoc, diag::note_used_here);
+  }
+}
+
 /// ActOnEndOfTranslationUnit - This is called at the very end of the
 /// translation unit when EOF is reached and all but the top-level scope is
 /// popped.
 void Sema::ActOnEndOfTranslationUnit() {
   // At PCH writing, implicit instantiations and VTable handling info are
   // stored and performed when the PCH is included.
-  if (CompleteTranslationUnit)
-    while (1) {
-      // C++: Perform implicit template instantiations.
-      //
-      // FIXME: When we perform these implicit instantiations, we do not
-      // carefully keep track of the point of instantiation (C++ [temp.point]).
-      // This means that name lookup that occurs within the template
-      // instantiation will always happen at the end of the translation unit,
-      // so it will find some names that should not be found. Although this is
-      // common behavior for C++ compilers, it is technically wrong. In the
-      // future, we either need to be able to filter the results of name lookup
-      // or we need to perform template instantiations earlier.
-      PerformPendingInstantiations();
-
-      /// If DefinedUsedVTables ends up marking any virtual member
-      /// functions it might lead to more pending template
-      /// instantiations, which is why we need to loop here.
-      if (!DefineUsedVTables())
-        break;
+  if (CompleteTranslationUnit) {
+    // If any dynamic classes have their key function defined within
+    // this translation unit, then those vtables are considered "used" and must
+    // be emitted.
+    for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I) {
+      assert(!DynamicClasses[I]->isDependentType() &&
+             "Should not see dependent types here!");
+      if (const CXXMethodDecl *KeyFunction
+          = Context.getKeyFunction(DynamicClasses[I])) {
+        const FunctionDecl *Definition = 0;
+        if (KeyFunction->hasBody(Definition))
+          MarkVTableUsed(Definition->getLocation(), DynamicClasses[I], true);
+      }
     }
+
+    // If DefinedUsedVTables ends up marking any virtual member functions it
+    // might lead to more pending template instantiations, which we then need
+    // to instantiate.
+    DefineUsedVTables();
+
+    // C++: Perform implicit template instantiations.
+    //
+    // FIXME: When we perform these implicit instantiations, we do not
+    // carefully keep track of the point of instantiation (C++ [temp.point]).
+    // This means that name lookup that occurs within the template
+    // instantiation will always happen at the end of the translation unit,
+    // so it will find some names that should not be found. Although this is
+    // common behavior for C++ compilers, it is technically wrong. In the
+    // future, we either need to be able to filter the results of name lookup
+    // or we need to perform template instantiations earlier.
+    PerformPendingInstantiations();
+  }
   
   // Remove file scoped decls that turned out to be used.
   UnusedFileScopedDecls.erase(std::remove_if(UnusedFileScopedDecls.begin(),
@@ -371,26 +438,32 @@ void Sema::ActOnEndOfTranslationUnit() {
       Consumer.CompleteTentativeDefinition(VD);
 
   }
-  
-  // Output warning for unused file scoped decls.
-  for (llvm::SmallVectorImpl<const DeclaratorDecl*>::iterator
-         I = UnusedFileScopedDecls.begin(),
-         E = UnusedFileScopedDecls.end(); I != E; ++I) {
-    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
-      const FunctionDecl *DiagD;
-      if (!FD->hasBody(DiagD))
-        DiagD = FD;
-      Diag(DiagD->getLocation(),
-           isa<CXXMethodDecl>(DiagD) ? diag::warn_unused_member_function
-                                     : diag::warn_unused_function)
-            << DiagD->getDeclName();
-    } else {
-      const VarDecl *DiagD = cast<VarDecl>(*I)->getDefinition();
-      if (!DiagD)
-        DiagD = cast<VarDecl>(*I);
-      Diag(DiagD->getLocation(), diag::warn_unused_variable)
-            << DiagD->getDeclName();
+
+  // If there were errors, disable 'unused' warnings since they will mostly be
+  // noise.
+  if (!Diags.hasErrorOccurred()) {
+    // Output warning for unused file scoped decls.
+    for (llvm::SmallVectorImpl<const DeclaratorDecl*>::iterator
+           I = UnusedFileScopedDecls.begin(),
+           E = UnusedFileScopedDecls.end(); I != E; ++I) {
+      if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
+        const FunctionDecl *DiagD;
+        if (!FD->hasBody(DiagD))
+          DiagD = FD;
+        Diag(DiagD->getLocation(),
+             isa<CXXMethodDecl>(DiagD) ? diag::warn_unused_member_function
+                                       : diag::warn_unused_function)
+              << DiagD->getDeclName();
+      } else {
+        const VarDecl *DiagD = cast<VarDecl>(*I)->getDefinition();
+        if (!DiagD)
+          DiagD = cast<VarDecl>(*I);
+        Diag(DiagD->getLocation(), diag::warn_unused_variable)
+              << DiagD->getDeclName();
+      }
     }
+
+    checkUndefinedInternals(*this);
   }
 
   TUScope = 0;
@@ -431,6 +504,50 @@ NamedDecl *Sema::getCurFunctionOrMethodDecl() {
 }
 
 Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() {
+  if (!isActive())
+    return;
+  
+  if (llvm::Optional<TemplateDeductionInfo*> Info = SemaRef.isSFINAEContext()) {
+    switch (DiagnosticIDs::getDiagnosticSFINAEResponse(getDiagID())) {
+    case DiagnosticIDs::SFINAE_Report:
+      // Fall through; we'll report the diagnostic below.
+      break;
+      
+    case DiagnosticIDs::SFINAE_AccessControl:
+      // Unless access checking is specifically called out as a SFINAE
+      // error, report this diagnostic.
+      if (!SemaRef.AccessCheckingSFINAE)
+        break;
+        
+    case DiagnosticIDs::SFINAE_SubstitutionFailure:
+      // Count this failure so that we know that template argument deduction
+      // has failed.
+      ++SemaRef.NumSFINAEErrors;
+      SemaRef.Diags.setLastDiagnosticIgnored();
+      SemaRef.Diags.Clear();
+      Clear();
+      return;
+      
+    case DiagnosticIDs::SFINAE_Suppress:
+      // Make a copy of this suppressed diagnostic and store it with the
+      // template-deduction information;
+      FlushCounts();
+      DiagnosticInfo DiagInfo(&SemaRef.Diags);
+        
+      if (*Info)
+        (*Info)->addSuppressedDiagnostic(DiagInfo.getLocation(),
+                        PartialDiagnostic(DiagInfo,
+                                          SemaRef.Context.getDiagAllocator()));
+        
+      // Suppress this diagnostic.        
+      SemaRef.Diags.setLastDiagnosticIgnored();
+      SemaRef.Diags.Clear();
+      Clear();
+      return;
+    }
+  }
+  
+  // Emit the diagnostic.
   if (!this->Emit())
     return;
 
@@ -438,7 +555,7 @@ Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() {
   // that is different from the last template instantiation where
   // we emitted an error, print a template instantiation
   // backtrace.
-  if (!SemaRef.Diags.isBuiltinNote(DiagID) &&
+  if (!DiagnosticIDs::isBuiltinNote(DiagID) &&
       !SemaRef.ActiveTemplateInstantiations.empty() &&
       SemaRef.ActiveTemplateInstantiations.back()
         != SemaRef.LastTemplateInstantiationErrorContext) {
@@ -449,26 +566,7 @@ Sema::SemaDiagnosticBuilder::~SemaDiagnosticBuilder() {
 }
 
 Sema::SemaDiagnosticBuilder Sema::Diag(SourceLocation Loc, unsigned DiagID) {
-  if (isSFINAEContext()) {
-    switch (Diagnostic::getDiagnosticSFINAEResponse(DiagID)) {
-    case Diagnostic::SFINAE_Report:
-      // Fall through; we'll report the diagnostic below.
-      break;
-
-    case Diagnostic::SFINAE_SubstitutionFailure:
-      // Count this failure so that we know that template argument deduction
-      // has failed.
-      ++NumSFINAEErrors;
-      // Fall through
-        
-    case Diagnostic::SFINAE_Suppress:
-      // Suppress this diagnostic.
-      Diags.setLastDiagnosticIgnored();
-      return SemaDiagnosticBuilder(*this);
-    }
-  }
-  
-  DiagnosticBuilder DB = Diags.Report(FullSourceLoc(Loc, SourceMgr), DiagID);
+  DiagnosticBuilder DB = Diags.Report(Loc, DiagID);
   return SemaDiagnosticBuilder(DB, *this, DiagID);
 }
 
@@ -514,17 +612,16 @@ void Sema::PushFunctionScope() {
   if (FunctionScopes.size() == 1) {
     // Use the "top" function scope rather than having to allocate
     // memory for a new scope.
-    FunctionScopes.back()->Clear(getDiagnostics().getNumErrors());
+    FunctionScopes.back()->Clear();
     FunctionScopes.push_back(FunctionScopes.back());
     return;
   }
   
-  FunctionScopes.push_back(
-                      new FunctionScopeInfo(getDiagnostics().getNumErrors()));
+  FunctionScopes.push_back(new FunctionScopeInfo(getDiagnostics()));
 }
 
 void Sema::PushBlockScope(Scope *BlockScope, BlockDecl *Block) {
-  FunctionScopes.push_back(new BlockScopeInfo(getDiagnostics().getNumErrors(),
+  FunctionScopes.push_back(new BlockScopeInfo(getDiagnostics(),
                                               BlockScope, Block));
 }
 
@@ -538,8 +635,7 @@ void Sema::PopFunctionOrBlockScope() {
 /// \brief Determine whether any errors occurred within this function/method/
 /// block.
 bool Sema::hasAnyErrorsInThisFunction() const {
-  return getCurFunction()->NumErrorsAtStartOfFunction
-             != getDiagnostics().getNumErrors();
+  return getCurFunction()->ErrorTrap.hasErrorOccurred();
 }
 
 BlockScopeInfo *Sema::getCurBlock() {
@@ -552,6 +648,11 @@ BlockScopeInfo *Sema::getCurBlock() {
 // Pin this vtable to this file.
 ExternalSemaSource::~ExternalSemaSource() {}
 
+std::pair<ObjCMethodList, ObjCMethodList>
+ExternalSemaSource::ReadMethodPool(Selector Sel) {
+  return std::pair<ObjCMethodList, ObjCMethodList>();
+}
+
 void PrettyDeclStackTraceEntry::print(llvm::raw_ostream &OS) const {
   SourceLocation Loc = this->Loc;
   if (!Loc.isValid() && TheDecl) Loc = TheDecl->getLocation();
diff --git a/lib/Sema/SemaAccess.cpp b/lib/Sema/SemaAccess.cpp
index e629f0fd35bf..4f9bf1c5edd7 100644
--- a/lib/Sema/SemaAccess.cpp
+++ b/lib/Sema/SemaAccess.cpp
@@ -134,7 +134,7 @@ struct EffectiveContext {
   bool Dependent;
 };
 
-/// Like sema:;AccessedEntity, but kindly lets us scribble all over
+/// Like sema::AccessedEntity, but kindly lets us scribble all over
 /// it.
 struct AccessTarget : public AccessedEntity {
   AccessTarget(const AccessedEntity &Entity)
@@ -516,6 +516,11 @@ static AccessResult MatchesFriend(Sema &S,
 static AccessResult MatchesFriend(Sema &S,
                                   const EffectiveContext &EC,
                                   FriendDecl *FriendD) {
+  // Whitelist accesses if there's an invalid or unsupported friend
+  // declaration.
+  if (FriendD->isInvalidDecl() || FriendD->isUnsupportedFriend())
+    return AR_accessible;
+
   if (TypeSourceInfo *T = FriendD->getFriendType())
     return MatchesFriend(S, EC, T->getType()->getCanonicalTypeUnqualified());
 
@@ -1003,9 +1008,51 @@ static void DiagnoseAccessPath(Sema &S,
           TryDiagnoseProtectedAccess(S, EC, Entity))
         return;
 
+      // Find an original declaration.
+      while (D->isOutOfLine()) {
+        NamedDecl *PrevDecl = 0;
+        if (isa<VarDecl>(D))
+          PrevDecl = cast<VarDecl>(D)->getPreviousDeclaration();
+        else if (isa<FunctionDecl>(D))
+          PrevDecl = cast<FunctionDecl>(D)->getPreviousDeclaration();
+        else if (isa<TypedefDecl>(D))
+          PrevDecl = cast<TypedefDecl>(D)->getPreviousDeclaration();
+        else if (isa<TagDecl>(D)) {
+          if (isa<RecordDecl>(D) && cast<RecordDecl>(D)->isInjectedClassName())
+            break;
+          PrevDecl = cast<TagDecl>(D)->getPreviousDeclaration();
+        }
+        if (!PrevDecl) break;
+        D = PrevDecl;
+      }
+
+      CXXRecordDecl *DeclaringClass = FindDeclaringClass(D);
+      Decl *ImmediateChild;
+      if (D->getDeclContext() == DeclaringClass)
+        ImmediateChild = D;
+      else {
+        DeclContext *DC = D->getDeclContext();
+        while (DC->getParent() != DeclaringClass)
+          DC = DC->getParent();
+        ImmediateChild = cast<Decl>(DC);
+      }
+      
+      // Check whether there's an AccessSpecDecl preceding this in the
+      // chain of the DeclContext.
+      bool Implicit = true;
+      for (CXXRecordDecl::decl_iterator
+             I = DeclaringClass->decls_begin(), E = DeclaringClass->decls_end();
+           I != E; ++I) {
+        if (*I == ImmediateChild) break;
+        if (isa<AccessSpecDecl>(*I)) {
+          Implicit = false;
+          break;
+        }
+      }
+
       S.Diag(D->getLocation(), diag::note_access_natural)
         << (unsigned) (Access == AS_protected)
-        << /*FIXME: not implicitly*/ 0;
+        << Implicit;
       return;
     }
 
@@ -1213,13 +1260,19 @@ static Sema::AccessResult CheckAccess(Sema &S, SourceLocation Loc,
   if (S.SuppressAccessChecking)
     return Sema::AR_accessible;
 
-  // If we're currently parsing a top-level declaration, delay
-  // diagnostics.  This is the only case where parsing a declaration
-  // can actually change our effective context for the purposes of
-  // access control.
-  if (S.CurContext->isFileContext() && S.ParsingDeclDepth) {
-    S.DelayedDiagnostics.push_back(
-        DelayedDiagnostic::makeAccess(Loc, Entity));
+  // If we're currently parsing a declaration, we may need to delay
+  // access control checking, because our effective context might be
+  // different based on what the declaration comes out as.
+  //
+  // For example, we might be parsing a declaration with a scope
+  // specifier, like this:
+  //   A::private_type A::foo() { ... }
+  //
+  // Or we might be parsing something that will turn out to be a friend:
+  //   void foo(A::private_type);
+  //   void B::foo(A::private_type);
+  if (S.DelayedDiagnostics.shouldDelayDiagnostics()) {
+    S.DelayedDiagnostics.add(DelayedDiagnostic::makeAccess(Loc, Entity));
     return Sema::AR_delayed;
   }
 
@@ -1233,16 +1286,20 @@ static Sema::AccessResult CheckAccess(Sema &S, SourceLocation Loc,
   return Sema::AR_accessible;
 }
 
-void Sema::HandleDelayedAccessCheck(DelayedDiagnostic &DD, Decl *Ctx) {
-  // Pretend we did this from the context of the newly-parsed
-  // declaration. If that declaration itself forms a declaration context,
-  // include it in the effective context so that parameters and return types of
-  // befriended functions have that function's access priveledges.
-  DeclContext *DC = Ctx->getDeclContext();
-  if (isa<FunctionDecl>(Ctx))
-    DC = cast<DeclContext>(Ctx);
-  else if (FunctionTemplateDecl *FnTpl = dyn_cast<FunctionTemplateDecl>(Ctx))
-    DC = cast<DeclContext>(FnTpl->getTemplatedDecl());
+void Sema::HandleDelayedAccessCheck(DelayedDiagnostic &DD, Decl *decl) {
+  // Access control for names used in the declarations of functions
+  // and function templates should normally be evaluated in the context
+  // of the declaration, just in case it's a friend of something.
+  // However, this does not apply to local extern declarations.
+
+  DeclContext *DC = decl->getDeclContext();
+  if (FunctionDecl *fn = dyn_cast<FunctionDecl>(decl)) {
+    if (!DC->isFunctionOrMethod()) DC = fn;
+  } else if (FunctionTemplateDecl *fnt = dyn_cast<FunctionTemplateDecl>(decl)) {
+    // Never a local declaration.
+    DC = fnt->getTemplatedDecl();
+  }
+
   EffectiveContext EC(DC);
 
   AccessTarget Target(DD.getAccessData());
diff --git a/lib/Sema/SemaAttr.cpp b/lib/Sema/SemaAttr.cpp
index 0921156b932f..794b0b1f1cfa 100644
--- a/lib/Sema/SemaAttr.cpp
+++ b/lib/Sema/SemaAttr.cpp
@@ -263,37 +263,35 @@ void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
   }
 }
 
-void Sema::ActOnPragmaUnused(const Token *Identifiers, unsigned NumIdentifiers,
-                             Scope *curScope,
-                             SourceLocation PragmaLoc,
-                             SourceLocation LParenLoc,
-                             SourceLocation RParenLoc) {
-
-  for (unsigned i = 0; i < NumIdentifiers; ++i) {
-    const Token &Tok = Identifiers[i];
-    IdentifierInfo *Name = Tok.getIdentifierInfo();
-    LookupResult Lookup(*this, Name, Tok.getLocation(), LookupOrdinaryName);
-    LookupParsedName(Lookup, curScope, NULL, true);
-
-    if (Lookup.empty()) {
-      Diag(PragmaLoc, diag::warn_pragma_unused_undeclared_var)
-        << Name << SourceRange(Tok.getLocation());
-      continue;
-    }
+void Sema::ActOnPragmaUnused(const Token &IdTok, Scope *curScope,
+                             SourceLocation PragmaLoc) {
 
-    VarDecl *VD = Lookup.getAsSingle<VarDecl>();
-    if (!VD || !VD->hasLocalStorage()) {
-      Diag(PragmaLoc, diag::warn_pragma_unused_expected_localvar)
-        << Name << SourceRange(Tok.getLocation());
-      continue;
-    }
+  IdentifierInfo *Name = IdTok.getIdentifierInfo();
+  LookupResult Lookup(*this, Name, IdTok.getLocation(), LookupOrdinaryName);
+  LookupParsedName(Lookup, curScope, NULL, true);
+
+  if (Lookup.empty()) {
+    Diag(PragmaLoc, diag::warn_pragma_unused_undeclared_var)
+      << Name << SourceRange(IdTok.getLocation());
+    return;
+  }
 
-    VD->addAttr(::new (Context) UnusedAttr(Tok.getLocation(), Context));
+  VarDecl *VD = Lookup.getAsSingle<VarDecl>();
+  if (!VD) {
+    Diag(PragmaLoc, diag::warn_pragma_unused_expected_var_arg)
+      << Name << SourceRange(IdTok.getLocation());
+    return;
   }
+
+  // Warn if this was used before being marked unused.
+  if (VD->isUsed())
+    Diag(PragmaLoc, diag::warn_used_but_marked_unused) << Name;
+
+  VD->addAttr(::new (Context) UnusedAttr(IdTok.getLocation(), Context));
 }
 
-typedef std::vector<std::pair<VisibilityAttr::VisibilityType,
-                              SourceLocation> > VisStack;
+typedef std::vector<std::pair<unsigned, SourceLocation> > VisStack;
+enum { NoVisibility = (unsigned) -1 };
 
 void Sema::AddPushedVisibilityAttribute(Decl *D) {
   if (!VisContext)
@@ -303,7 +301,11 @@ void Sema::AddPushedVisibilityAttribute(Decl *D) {
     return;
 
   VisStack *Stack = static_cast<VisStack*>(VisContext);
-  VisibilityAttr::VisibilityType type = Stack->back().first;
+  unsigned rawType = Stack->back().first;
+  if (rawType == NoVisibility) return;
+
+  VisibilityAttr::VisibilityType type
+    = (VisibilityAttr::VisibilityType) rawType;
   SourceLocation loc = Stack->back().second;
 
   D->addAttr(::new (Context) VisibilityAttr(loc, Context, type));
@@ -315,8 +317,7 @@ void Sema::FreeVisContext() {
   VisContext = 0;
 }
 
-static void PushPragmaVisibility(Sema &S, VisibilityAttr::VisibilityType type,
-                                 SourceLocation loc) {
+static void PushPragmaVisibility(Sema &S, unsigned type, SourceLocation loc) {
   // Put visibility on stack.
   if (!S.VisContext)
     S.VisContext = new VisStack;
@@ -349,8 +350,26 @@ void Sema::ActOnPragmaVisibility(bool IsPush, const IdentifierInfo* VisType,
   }
 }
 
-void Sema::PushVisibilityAttr(const VisibilityAttr *Attr) {
-  PushPragmaVisibility(*this, Attr->getVisibility(), Attr->getLocation());
+void Sema::ActOnPragmaFPContract(tok::OnOffSwitch OOS) {
+  switch (OOS) {
+  case tok::OOS_ON:
+    FPFeatures.fp_contract = 1;
+    break;
+  case tok::OOS_OFF:
+    FPFeatures.fp_contract = 0; 
+    break;
+  case tok::OOS_DEFAULT:
+    FPFeatures.fp_contract = getLangOptions().DefaultFPContract;
+    break;
+  }
+}
+
+void Sema::PushNamespaceVisibilityAttr(const VisibilityAttr *Attr) {
+  // Visibility calculations will consider the namespace's visibility.
+  // Here we just want to note that we're in a visibility context
+  // which overrides any enclosing #pragma context, but doesn't itself
+  // contribute visibility.
+  PushPragmaVisibility(*this, NoVisibility, SourceLocation());
 }
 
 void Sema::PopPragmaVisibility() {
diff --git a/lib/Sema/SemaCXXCast.cpp b/lib/Sema/SemaCXXCast.cpp
index 21b1a73aa3bc..506d2612ffbc 100644
--- a/lib/Sema/SemaCXXCast.cpp
+++ b/lib/Sema/SemaCXXCast.cpp
@@ -21,6 +21,8 @@
 #include <set>
 using namespace clang;
 
+
+
 enum TryCastResult {
   TC_NotApplicable, ///< The cast method is not applicable.
   TC_Success,       ///< The cast method is appropriate and successful.
@@ -37,18 +39,25 @@ enum CastType {
   CT_Functional   ///< Type(expr)
 };
 
+
+
+
 static void CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
+                           ExprValueKind &VK,
                            const SourceRange &OpRange,
                            const SourceRange &DestRange);
 static void CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
+                                 ExprValueKind &VK,
                                  const SourceRange &OpRange,
                                  const SourceRange &DestRange,
                                  CastKind &Kind);
 static void CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
+                            ExprValueKind &VK,
                             const SourceRange &OpRange,
                             CastKind &Kind,
                             CXXCastPath &BasePath);
 static void CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
+                             ExprValueKind &VK,
                              const SourceRange &OpRange,
                              const SourceRange &DestRange,
                              CastKind &Kind,
@@ -68,7 +77,10 @@ static bool CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType);
 // %1: Source Type
 // %2: Destination Type
 static TryCastResult TryLValueToRValueCast(Sema &Self, Expr *SrcExpr,
-                                           QualType DestType, unsigned &msg);
+                                           QualType DestType, bool CStyle,
+                                           CastKind &Kind,
+                                           CXXCastPath &BasePath,
+                                           unsigned &msg);
 static TryCastResult TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr,
                                                QualType DestType, bool CStyle,
                                                const SourceRange &OpRange,
@@ -109,12 +121,24 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
                                    CXXCastPath &BasePath);
 static TryCastResult TryConstCast(Sema &Self, Expr *SrcExpr, QualType DestType,
                                   bool CStyle, unsigned &msg);
-static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
+static TryCastResult TryReinterpretCast(Sema &Self, Expr *&SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
                                         unsigned &msg,
                                         CastKind &Kind);
 
+
+static ExprResult 
+ResolveAndFixSingleFunctionTemplateSpecialization(
+                    Sema &Self, Expr *SrcExpr, 
+                    bool DoFunctionPointerConverion = false, 
+                    bool Complain = false, 
+                    const SourceRange& OpRangeForComplaining = SourceRange(), 
+                    QualType DestTypeForComplaining = QualType(), 
+                    unsigned DiagIDForComplaining = 0);
+
+
+
 /// ActOnCXXNamedCast - Parse {dynamic,static,reinterpret,const}_cast's.
 ExprResult
 Sema::ActOnCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
@@ -146,51 +170,147 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
   // FIXME: should we check this in a more fine-grained manner?
   bool TypeDependent = DestType->isDependentType() || Ex->isTypeDependent();
 
+  if (Ex->isBoundMemberFunction(Context))
+    Diag(Ex->getLocStart(), diag::err_invalid_use_of_bound_member_func)
+      << Ex->getSourceRange();
+
+  ExprValueKind VK = VK_RValue;
+  if (TypeDependent)
+    VK = Expr::getValueKindForType(DestType);
+
   switch (Kind) {
-  default: assert(0 && "Unknown C++ cast!");
+  default: llvm_unreachable("Unknown C++ cast!");
 
   case tok::kw_const_cast:
     if (!TypeDependent)
-      CheckConstCast(*this, Ex, DestType, OpRange, DestRange);
+      CheckConstCast(*this, Ex, DestType, VK, OpRange, DestRange);
     return Owned(CXXConstCastExpr::Create(Context,
                                         DestType.getNonLValueExprType(Context),
-                                          Ex, DestTInfo, OpLoc));
+                                          VK, Ex, DestTInfo, OpLoc,
+                                          Parens.getEnd()));
 
   case tok::kw_dynamic_cast: {
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Dependent;
     CXXCastPath BasePath;
     if (!TypeDependent)
-      CheckDynamicCast(*this, Ex, DestType, OpRange, DestRange, Kind, BasePath);
+      CheckDynamicCast(*this, Ex, DestType, VK, OpRange, DestRange,
+                       Kind, BasePath);
     return Owned(CXXDynamicCastExpr::Create(Context,
                                           DestType.getNonLValueExprType(Context),
-                                            Kind, Ex, &BasePath, DestTInfo,
-                                            OpLoc));
+                                            VK, Kind, Ex, &BasePath, DestTInfo,
+                                            OpLoc, Parens.getEnd()));
   }
   case tok::kw_reinterpret_cast: {
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Dependent;
     if (!TypeDependent)
-      CheckReinterpretCast(*this, Ex, DestType, OpRange, DestRange, Kind);
+      CheckReinterpretCast(*this, Ex, DestType, VK, OpRange, DestRange, Kind);
     return Owned(CXXReinterpretCastExpr::Create(Context,
                                   DestType.getNonLValueExprType(Context),
-                                  Kind, Ex, 0,
-                                  DestTInfo, OpLoc));
+                                  VK, Kind, Ex, 0,
+                                  DestTInfo, OpLoc, Parens.getEnd()));
   }
   case tok::kw_static_cast: {
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Dependent;
     CXXCastPath BasePath;
     if (!TypeDependent)
-      CheckStaticCast(*this, Ex, DestType, OpRange, Kind, BasePath);
+      CheckStaticCast(*this, Ex, DestType, VK, OpRange, Kind, BasePath);
     
     return Owned(CXXStaticCastExpr::Create(Context,
                                          DestType.getNonLValueExprType(Context),
-                                           Kind, Ex, &BasePath,
-                                           DestTInfo, OpLoc));
+                                           VK, Kind, Ex, &BasePath,
+                                           DestTInfo, OpLoc, Parens.getEnd()));
   }
   }
 
   return ExprError();
 }
 
+/// Try to diagnose a failed overloaded cast.  Returns true if
+/// diagnostics were emitted.
+static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
+                                      SourceRange range, Expr *src,
+                                      QualType destType) {
+  switch (CT) {
+  // These cast kinds don't consider user-defined conversions.
+  case CT_Const:
+  case CT_Reinterpret:
+  case CT_Dynamic:
+    return false;
+
+  // These do.
+  case CT_Static:
+  case CT_CStyle:
+  case CT_Functional:
+    break;
+  }
+
+  QualType srcType = src->getType();
+  if (!destType->isRecordType() && !srcType->isRecordType())
+    return false;
+
+  InitializedEntity entity = InitializedEntity::InitializeTemporary(destType);
+  InitializationKind initKind
+    = InitializationKind::CreateCast(/*type range?*/ range,
+                                     (CT == CT_CStyle || CT == CT_Functional));
+  InitializationSequence sequence(S, entity, initKind, &src, 1);
+
+  assert(sequence.getKind() == InitializationSequence::FailedSequence &&
+         "initialization succeeded on second try?");
+  switch (sequence.getFailureKind()) {
+  default: return false;
+
+  case InitializationSequence::FK_ConstructorOverloadFailed:
+  case InitializationSequence::FK_UserConversionOverloadFailed:
+    break;
+  }
+
+  OverloadCandidateSet &candidates = sequence.getFailedCandidateSet();
+
+  unsigned msg = 0;
+  OverloadCandidateDisplayKind howManyCandidates = OCD_AllCandidates;
+
+  switch (sequence.getFailedOverloadResult()) {
+  case OR_Success: llvm_unreachable("successful failed overload");
+    return false;
+  case OR_No_Viable_Function:
+    if (candidates.empty())
+      msg = diag::err_ovl_no_conversion_in_cast;
+    else
+      msg = diag::err_ovl_no_viable_conversion_in_cast;
+    howManyCandidates = OCD_AllCandidates;
+    break;
+
+  case OR_Ambiguous:
+    msg = diag::err_ovl_ambiguous_conversion_in_cast;
+    howManyCandidates = OCD_ViableCandidates;
+    break;
+
+  case OR_Deleted:
+    msg = diag::err_ovl_deleted_conversion_in_cast;
+    howManyCandidates = OCD_ViableCandidates;
+    break;
+  }
+
+  S.Diag(range.getBegin(), msg)
+    << CT << srcType << destType
+    << range << src->getSourceRange();
+
+  candidates.NoteCandidates(S, howManyCandidates, &src, 1);
+
+  return true;
+}
+
+/// Diagnose a failed cast.
+static void diagnoseBadCast(Sema &S, unsigned msg, CastType castType,
+                            SourceRange opRange, Expr *src, QualType destType) {
+  if (msg == diag::err_bad_cxx_cast_generic &&
+      tryDiagnoseOverloadedCast(S, castType, opRange, src, destType))
+    return;
+
+  S.Diag(opRange.getBegin(), msg) << castType
+    << src->getType() << destType << opRange << src->getSourceRange();
+}
+
 /// UnwrapDissimilarPointerTypes - Like Sema::UnwrapSimilarPointerTypes,
 /// this removes one level of indirection from both types, provided that they're
 /// the same kind of pointer (plain or to-member). Unlike the Sema function,
@@ -297,7 +417,7 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType) {
 
   // Test if they're compatible.
   return SrcConstruct != DestConstruct &&
-    !Self.IsQualificationConversion(SrcConstruct, DestConstruct);
+    !Self.IsQualificationConversion(SrcConstruct, DestConstruct, false);
 }
 
 /// CheckDynamicCast - Check that a dynamic_cast\<DestType\>(SrcExpr) is valid.
@@ -305,7 +425,7 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType) {
 /// checked downcasts in class hierarchies.
 static void
 CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
-                 const SourceRange &OpRange,
+                 ExprValueKind &VK, const SourceRange &OpRange,
                  const SourceRange &DestRange, CastKind &Kind,
                  CXXCastPath &BasePath) {
   QualType OrigDestType = DestType, OrigSrcType = SrcExpr->getType();
@@ -316,11 +436,14 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 
   QualType DestPointee;
   const PointerType *DestPointer = DestType->getAs<PointerType>();
-  const ReferenceType *DestReference = DestType->getAs<ReferenceType>();
+  const ReferenceType *DestReference = 0;
   if (DestPointer) {
     DestPointee = DestPointer->getPointeeType();
-  } else if (DestReference) {
+  } else if ((DestReference = DestType->getAs<ReferenceType>())) {
     DestPointee = DestReference->getPointeeType();
+    VK = isa<LValueReferenceType>(DestReference) ? VK_LValue 
+       : isa<RValueReferenceType>(DestReference) ? VK_XValue
+       : VK_RValue;
   } else {
     Self.Diag(OpRange.getBegin(), diag::err_bad_dynamic_cast_not_ref_or_ptr)
       << OrigDestType << DestRange;
@@ -343,10 +466,8 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 
   // C++0x 5.2.7p2: If T is a pointer type, v shall be an rvalue of a pointer to
   //   complete class type, [...]. If T is an lvalue reference type, v shall be
-  //   an lvalue of a complete class type, [...]. If T is an rvalue reference
-  //   type, v shall be an expression having a complete effective class type,
-  //   [...]
-
+  //   an lvalue of a complete class type, [...]. If T is an rvalue reference 
+  //   type, v shall be an expression having a complete class type, [...]
   QualType SrcType = Self.Context.getCanonicalType(OrigSrcType);
   QualType SrcPointee;
   if (DestPointer) {
@@ -358,7 +479,7 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
       return;
     }
   } else if (DestReference->isLValueReferenceType()) {
-    if (SrcExpr->isLvalue(Self.Context) != Expr::LV_Valid) {
+    if (!SrcExpr->isLValue()) {
       Self.Diag(OpRange.getBegin(), diag::err_bad_cxx_cast_rvalue)
         << CT_Dynamic << OrigSrcType << OrigDestType << OpRange;
     }
@@ -437,9 +558,10 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 /// const char *str = "literal";
 /// legacy_function(const_cast\<char*\>(str));
 void
-CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
+CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType, ExprValueKind &VK,
                const SourceRange &OpRange, const SourceRange &DestRange) {
-  if (!DestType->isLValueReferenceType())
+  VK = Expr::getValueKindForType(DestType);
+  if (VK == VK_RValue)
     Self.DefaultFunctionArrayLvalueConversion(SrcExpr);
 
   unsigned msg = diag::err_bad_cxx_cast_generic;
@@ -456,17 +578,28 @@ CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 /// char *bytes = reinterpret_cast\<char*\>(int_ptr);
 void
 CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
-                     const SourceRange &OpRange, const SourceRange &DestRange,
-                     CastKind &Kind) {
-  if (!DestType->isLValueReferenceType())
+                     ExprValueKind &VK, const SourceRange &OpRange,
+                     const SourceRange &DestRange, CastKind &Kind) {
+  VK = Expr::getValueKindForType(DestType);
+  if (VK == VK_RValue)
     Self.DefaultFunctionArrayLvalueConversion(SrcExpr);
 
   unsigned msg = diag::err_bad_cxx_cast_generic;
   if (TryReinterpretCast(Self, SrcExpr, DestType, /*CStyle*/false, OpRange,
                          msg, Kind)
       != TC_Success && msg != 0)
-    Self.Diag(OpRange.getBegin(), msg) << CT_Reinterpret
-      << SrcExpr->getType() << DestType << OpRange;
+  {
+    if (SrcExpr->getType() == Self.Context.OverloadTy) {
+      //FIXME: &f<int>; is overloaded and resolvable 
+      Self.Diag(OpRange.getBegin(), diag::err_bad_reinterpret_cast_overload) 
+        << OverloadExpr::find(SrcExpr).Expression->getName()
+        << DestType << OpRange;
+      Self.NoteAllOverloadCandidates(SrcExpr);
+
+    } else {
+      diagnoseBadCast(Self, msg, CT_Reinterpret, OpRange, SrcExpr, DestType);
+    }
+  }    
 }
 
 
@@ -475,25 +608,47 @@ CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 /// implicit conversions explicit and getting rid of data loss warnings.
 void
 CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
-                const SourceRange &OpRange, CastKind &Kind,
-                CXXCastPath &BasePath) {
+                ExprValueKind &VK, const SourceRange &OpRange,
+                CastKind &Kind, CXXCastPath &BasePath) {
   // This test is outside everything else because it's the only case where
   // a non-lvalue-reference target type does not lead to decay.
   // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
   if (DestType->isVoidType()) {
-    Kind = CK_ToVoid;
+    Self.IgnoredValueConversions(SrcExpr);
+    if (SrcExpr->getType() == Self.Context.OverloadTy) {
+      ExprResult SingleFunctionExpression = 
+         ResolveAndFixSingleFunctionTemplateSpecialization(Self, SrcExpr, 
+                false, // Decay Function to ptr 
+                true, // Complain
+                OpRange, DestType, diag::err_bad_static_cast_overload);
+        if (SingleFunctionExpression.isUsable())
+        {
+          SrcExpr = SingleFunctionExpression.release();
+          Kind = CK_ToVoid;
+        }
+    }
+    else
+      Kind = CK_ToVoid;
     return;
   }
 
-  if (!DestType->isLValueReferenceType() && !DestType->isRecordType())
+  VK = Expr::getValueKindForType(DestType);
+  if (VK == VK_RValue && !DestType->isRecordType())
     Self.DefaultFunctionArrayLvalueConversion(SrcExpr);
 
   unsigned msg = diag::err_bad_cxx_cast_generic;
   if (TryStaticCast(Self, SrcExpr, DestType, /*CStyle*/false, OpRange, msg,
-                    Kind, BasePath) != TC_Success && msg != 0)
-    Self.Diag(OpRange.getBegin(), msg) << CT_Static
-      << SrcExpr->getType() << DestType << OpRange;
-  else if (Kind == CK_Unknown || Kind == CK_BitCast)
+                    Kind, BasePath) != TC_Success && msg != 0) {
+    if (SrcExpr->getType() == Self.Context.OverloadTy) {
+      OverloadExpr* oe = OverloadExpr::find(SrcExpr).Expression;
+      Self.Diag(OpRange.getBegin(), diag::err_bad_static_cast_overload)
+        << oe->getName() << DestType << OpRange << oe->getQualifierRange();
+      Self.NoteAllOverloadCandidates(SrcExpr);
+    } else {
+      diagnoseBadCast(Self, msg, CT_Static, OpRange, SrcExpr, DestType);
+    }
+  }
+  else if (Kind == CK_BitCast)
     Self.CheckCastAlign(SrcExpr, DestType, OpRange);
 }
 
@@ -530,13 +685,13 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
   if (tcr != TC_NotApplicable)
     return tcr;
 
-  // N2844 5.2.9p3: An lvalue of type "cv1 T1" can be cast to type "rvalue
-  //   reference to cv2 T2" if "cv2 T2" is reference-compatible with "cv1 T1".
-  tcr = TryLValueToRValueCast(Self, SrcExpr, DestType, msg);
-  if (tcr != TC_NotApplicable) {
-    Kind = CK_NoOp;
+  // C++0x [expr.static.cast]p3: 
+  //   A glvalue of type "cv1 T1" can be cast to type "rvalue reference to cv2
+  //   T2" if "cv2 T2" is reference-compatible with "cv1 T1".
+  tcr = TryLValueToRValueCast(Self, SrcExpr, DestType, CStyle, Kind, BasePath, 
+                              msg);
+  if (tcr != TC_NotApplicable)
     return tcr;
-  }
 
   // C++ 5.2.9p2: An expression e can be explicitly converted to a type T
   //   [...] if the declaration "T t(e);" is well-formed, [...].
@@ -553,10 +708,26 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
   // In the CStyle case, the earlier attempt to const_cast should have taken
   // care of reverse qualification conversions.
 
-  QualType OrigSrcType = SrcExpr->getType();
-
   QualType SrcType = Self.Context.getCanonicalType(SrcExpr->getType());
 
+  // C++0x 5.2.9p9: A value of a scoped enumeration type can be explicitly
+  // converted to an integral type. [...] A value of a scoped enumeration type
+  // can also be explicitly converted to a floating-point type [...].
+  if (const EnumType *Enum = SrcType->getAs<EnumType>()) {
+    if (Enum->getDecl()->isScoped()) {
+      if (DestType->isBooleanType()) {
+        Kind = CK_IntegralToBoolean;
+        return TC_Success;
+      } else if (DestType->isIntegralType(Self.Context)) {
+        Kind = CK_IntegralCast;
+        return TC_Success;
+      } else if (DestType->isRealFloatingType()) {
+        Kind = CK_IntegralToFloating;
+        return TC_Success;
+      }
+    }
+  }
+  
   // Reverse integral promotion/conversion. All such conversions are themselves
   // again integral promotions or conversions and are thus already handled by
   // p2 (TryDirectInitialization above).
@@ -623,8 +794,10 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
   }
   // Allow arbitray objective-c pointer conversion with static casts.
   if (SrcType->isObjCObjectPointerType() &&
-      DestType->isObjCObjectPointerType())
+      DestType->isObjCObjectPointerType()) {
+    Kind = CK_BitCast;
     return TC_Success;
+  }
   
   // We tried everything. Everything! Nothing works! :-(
   return TC_NotApplicable;
@@ -633,14 +806,16 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
 /// Tests whether a conversion according to N2844 is valid.
 TryCastResult
 TryLValueToRValueCast(Sema &Self, Expr *SrcExpr, QualType DestType,
+                      bool CStyle, CastKind &Kind, CXXCastPath &BasePath, 
                       unsigned &msg) {
-  // N2844 5.2.9p3: An lvalue of type "cv1 T1" can be cast to type "rvalue
-  //   reference to cv2 T2" if "cv2 T2" is reference-compatible with "cv1 T1".
+  // C++0x [expr.static.cast]p3:
+  //   A glvalue of type "cv1 T1" can be cast to type "rvalue reference to 
+  //   cv2 T2" if "cv2 T2" is reference-compatible with "cv1 T1".
   const RValueReferenceType *R = DestType->getAs<RValueReferenceType>();
   if (!R)
     return TC_NotApplicable;
 
-  if (SrcExpr->isLvalue(Self.Context) != Expr::LV_Valid)
+  if (!SrcExpr->isGLValue())
     return TC_NotApplicable;
 
   // Because we try the reference downcast before this function, from now on
@@ -648,16 +823,32 @@ TryLValueToRValueCast(Sema &Self, Expr *SrcExpr, QualType DestType,
   // FIXME: Should allow casting away constness if CStyle.
   bool DerivedToBase;
   bool ObjCConversion;
+  QualType FromType = SrcExpr->getType();
+  QualType ToType = R->getPointeeType();
+  if (CStyle) {
+    FromType = FromType.getUnqualifiedType();
+    ToType = ToType.getUnqualifiedType();
+  }
+  
   if (Self.CompareReferenceRelationship(SrcExpr->getLocStart(),
-                                        SrcExpr->getType(), R->getPointeeType(),
+                                        ToType, FromType,
                                         DerivedToBase, ObjCConversion) <
         Sema::Ref_Compatible_With_Added_Qualification) {
     msg = diag::err_bad_lvalue_to_rvalue_cast;
     return TC_Failed;
   }
 
-  // FIXME: We should probably have an AST node for lvalue-to-rvalue 
-  // conversions.
+  if (DerivedToBase) {
+    Kind = CK_DerivedToBase;
+    CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
+                       /*DetectVirtual=*/true);
+    if (!Self.IsDerivedFrom(SrcExpr->getType(), R->getPointeeType(), Paths))
+      return TC_NotApplicable;
+  
+    Self.BuildBasePathArray(Paths, BasePath);
+  } else
+    Kind = CK_NoOp;
+  
   return TC_Success;
 }
 
@@ -681,7 +872,7 @@ TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
     return TC_NotApplicable;
   }
   bool RValueRef = DestReference->isRValueReferenceType();
-  if (!RValueRef && SrcExpr->isLvalue(Self.Context) != Expr::LV_Valid) {
+  if (!RValueRef && !SrcExpr->isLValue()) {
     // We know the left side is an lvalue reference, so we can suggest a reason.
     msg = diag::err_bad_cxx_cast_rvalue;
     return TC_NotApplicable;
@@ -818,12 +1009,20 @@ TryStaticDowncast(Sema &Self, CanQualType SrcType, CanQualType DestType,
     return TC_Failed;
   }
 
-  if (!CStyle && Self.CheckBaseClassAccess(OpRange.getBegin(),
-                                           SrcType, DestType,
-                                           Paths.front(),
+  if (!CStyle) {
+    switch (Self.CheckBaseClassAccess(OpRange.getBegin(),
+                                      SrcType, DestType,
+                                      Paths.front(),
                                 diag::err_downcast_from_inaccessible_base)) {
-    msg = 0;
-    return TC_Failed;
+    case Sema::AR_accessible:
+    case Sema::AR_delayed:     // be optimistic
+    case Sema::AR_dependent:   // be optimistic
+      break;
+
+    case Sema::AR_inaccessible:
+      msg = 0;
+      return TC_Failed;
+    }
   }
 
   Self.BuildBasePathArray(Paths, BasePath);
@@ -887,7 +1086,7 @@ TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType,
     Paths.setRecordingPaths(true);
     bool StillOkay = Self.IsDerivedFrom(SrcClass, DestClass, Paths);
     assert(StillOkay);
-    StillOkay = StillOkay;
+    (void)StillOkay;
     std::string PathDisplayStr = Self.getAmbiguousPathsDisplayString(Paths);
     Self.Diag(OpRange.getBegin(), diag::err_ambiguous_memptr_conv)
       << 1 << SrcClass << DestClass << PathDisplayStr << OpRange;
@@ -902,12 +1101,22 @@ TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType,
     return TC_Failed;
   }
 
-  if (!CStyle && Self.CheckBaseClassAccess(OpRange.getBegin(),
-                                           DestClass, SrcClass,
-                                           Paths.front(),
-                                     diag::err_upcast_to_inaccessible_base)) {
-    msg = 0;
-    return TC_Failed;
+  if (!CStyle) {
+    switch (Self.CheckBaseClassAccess(OpRange.getBegin(),
+                                      DestClass, SrcClass,
+                                      Paths.front(),
+                                      diag::err_upcast_to_inaccessible_base)) {
+    case Sema::AR_accessible:
+    case Sema::AR_delayed:
+    case Sema::AR_dependent:
+      // Optimistically assume that the delayed and dependent cases
+      // will work out.
+      break;
+
+    case Sema::AR_inaccessible:
+      msg = 0;
+      return TC_Failed;
+    }
   }
 
   if (WasOverloadedFunction) {
@@ -951,17 +1160,19 @@ TryStaticImplicitCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
     }
   }
   
-  // At this point of CheckStaticCast, if the destination is a reference,
-  // this has to work. There is no other way that works.
-  // On the other hand, if we're checking a C-style cast, we've still got
-  // the reinterpret_cast way.
   InitializedEntity Entity = InitializedEntity::InitializeTemporary(DestType);
   InitializationKind InitKind
-    = InitializationKind::CreateCast(/*FIXME:*/OpRange, 
-                                                               CStyle);    
+    = InitializationKind::CreateCast(/*FIXME:*/OpRange, CStyle);    
   InitializationSequence InitSeq(Self, Entity, InitKind, &SrcExpr, 1);
+
+  // At this point of CheckStaticCast, if the destination is a reference,
+  // or the expression is an overload expression this has to work. 
+  // There is no other way that works.
+  // On the other hand, if we're checking a C-style cast, we've still got
+  // the reinterpret_cast way.
+  
   if (InitSeq.getKind() == InitializationSequence::FailedSequence && 
-      (CStyle || !DestType->isReferenceType()))
+    (CStyle || !DestType->isReferenceType()))
     return TC_NotApplicable;
     
   ExprResult Result
@@ -986,9 +1197,8 @@ static TryCastResult TryConstCast(Sema &Self, Expr *SrcExpr, QualType DestType,
                                   bool CStyle, unsigned &msg) {
   DestType = Self.Context.getCanonicalType(DestType);
   QualType SrcType = SrcExpr->getType();
-  if (const LValueReferenceType *DestTypeTmp =
-        DestType->getAs<LValueReferenceType>()) {
-    if (SrcExpr->isLvalue(Self.Context) != Expr::LV_Valid) {
+  if (const ReferenceType *DestTypeTmp =DestType->getAs<ReferenceType>()) {
+    if (DestTypeTmp->isLValueReferenceType() && !SrcExpr->isLValue()) {
       // Cannot const_cast non-lvalue to lvalue reference type. But if this
       // is C-style, static_cast might find a way, so we simply suggest a
       // message and tell the parent to keep searching.
@@ -1049,7 +1259,43 @@ static TryCastResult TryConstCast(Sema &Self, Expr *SrcExpr, QualType DestType,
   return TC_Success;
 }
 
-static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
+// A helper function to resolve and fix an overloaded expression that
+// can be resolved because it identifies a single function
+// template specialization
+// Last three arguments should only be supplied if Complain = true
+static ExprResult ResolveAndFixSingleFunctionTemplateSpecialization(
+                    Sema &Self, Expr *SrcExpr, 
+                    bool DoFunctionPointerConverion, 
+                    bool Complain, 
+                    const SourceRange& OpRangeForComplaining, 
+                    QualType DestTypeForComplaining, 
+                    unsigned DiagIDForComplaining) {
+  assert(SrcExpr->getType() == Self.Context.OverloadTy);
+  DeclAccessPair Found;
+  Expr* SingleFunctionExpression = 0;
+  if (FunctionDecl* Fn = Self.ResolveSingleFunctionTemplateSpecialization(
+                                    SrcExpr, false, // false -> Complain 
+                                    &Found)) {
+    if (!Self.DiagnoseUseOfDecl(Fn, SrcExpr->getSourceRange().getBegin())) {
+      // mark the expression as resolved to Fn
+      SingleFunctionExpression = Self.FixOverloadedFunctionReference(SrcExpr, 
+                                                                    Found, Fn);
+      
+      if (DoFunctionPointerConverion)
+        Self.DefaultFunctionArrayLvalueConversion(SingleFunctionExpression);
+    }      
+  }
+  if (!SingleFunctionExpression && Complain) {
+    OverloadExpr* oe = OverloadExpr::find(SrcExpr).Expression;
+    Self.Diag(OpRangeForComplaining.getBegin(), DiagIDForComplaining)
+      << oe->getName() << DestTypeForComplaining << OpRangeForComplaining 
+              << oe->getQualifierRange();
+    Self.NoteAllOverloadCandidates(SrcExpr);
+  }
+  return SingleFunctionExpression;
+}
+
+static TryCastResult TryReinterpretCast(Sema &Self, Expr *&SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
                                         unsigned &msg,
@@ -1058,11 +1304,28 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
   
   DestType = Self.Context.getCanonicalType(DestType);
   QualType SrcType = SrcExpr->getType();
+
+  // Is the source an overloaded name? (i.e. &foo)
+  // If so, reinterpret_cast can not help us here (13.4, p1, bullet 5) ...
+  if (SrcType == Self.Context.OverloadTy) {
+    // ... unless foo<int> resolves to an lvalue unambiguously
+    ExprResult SingleFunctionExpr = 
+        ResolveAndFixSingleFunctionTemplateSpecialization(Self, SrcExpr, 
+          Expr::getValueKindForType(DestType) == VK_RValue // Convert Fun to Ptr 
+        );
+    if (SingleFunctionExpr.isUsable()) {
+      SrcExpr = SingleFunctionExpr.release();
+      SrcType = SrcExpr->getType();
+    }      
+    else  
+      return TC_NotApplicable;
+  }
+
   if (const ReferenceType *DestTypeTmp = DestType->getAs<ReferenceType>()) {
     bool LValue = DestTypeTmp->isLValueReferenceType();
-    if (LValue && SrcExpr->isLvalue(Self.Context) != Expr::LV_Valid) {
-      // Cannot cast non-lvalue to reference type. See the similar comment in
-      // const_cast.
+    if (LValue && !SrcExpr->isLValue()) {
+      // Cannot cast non-lvalue to lvalue reference type. See the similar 
+      // comment in const_cast.
       msg = diag::err_bad_cxx_cast_rvalue;
       return TC_NotApplicable;
     }
@@ -1073,6 +1336,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     // This code does this transformation for the checked types.
     DestType = Self.Context.getPointerType(DestTypeTmp->getPointeeType());
     SrcType = Self.Context.getPointerType(SrcType);
+    
     IsLValueCast = true;
   }
 
@@ -1193,6 +1457,8 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     assert(destIsPtr && "One type must be a pointer");
     // C++ 5.2.10p5: A value of integral or enumeration type can be explicitly
     //   converted to a pointer.
+    // C++ 5.2.10p9: [Note: ...a null pointer constant of integral type is not
+    //   necessarily converted to a null pointer value.]
     Kind = CK_IntegralToPointer;
     return TC_Success;
   }
@@ -1257,26 +1523,54 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
   // So we finish by allowing everything that remains - it's got to be two
   // object pointers.
   return TC_Success;
-}
+}                                     
 
 bool 
-Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
-                         CastKind &Kind, 
+Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, ExprValueKind &VK,
+                         Expr *&CastExpr, CastKind &Kind, 
                          CXXCastPath &BasePath,
                          bool FunctionalStyle) {
+  if (CastExpr->isBoundMemberFunction(Context))
+    return Diag(CastExpr->getLocStart(),
+                diag::err_invalid_use_of_bound_member_func)
+        << CastExpr->getSourceRange();
+
   // This test is outside everything else because it's the only case where
   // a non-lvalue-reference target type does not lead to decay.
   // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
   if (CastTy->isVoidType()) {
-    Kind = CK_ToVoid;
-    return false;
+    IgnoredValueConversions(CastExpr);    
+    bool ret = false; // false is 'able to convert'
+    if (CastExpr->getType() == Context.OverloadTy) {
+      ExprResult SingleFunctionExpr = 
+        ResolveAndFixSingleFunctionTemplateSpecialization(*this, 
+              CastExpr, 
+              /* Decay Function to ptr */ false, 
+              /* Complain */ true, 
+              R, CastTy, diag::err_bad_cstyle_cast_overload);
+      if (SingleFunctionExpr.isUsable()) {
+        CastExpr = SingleFunctionExpr.release();
+        Kind = CK_ToVoid;
+      }
+      else
+        ret = true;  
+    }
+    else
+      Kind = CK_ToVoid;
+    return ret;
   }
 
+  // Make sure we determine the value kind before we bail out for
+  // dependent types.
+  VK = Expr::getValueKindForType(CastTy);
+
   // If the type is dependent, we won't do any other semantic analysis now.
-  if (CastTy->isDependentType() || CastExpr->isTypeDependent())
+  if (CastTy->isDependentType() || CastExpr->isTypeDependent()) {
+    Kind = CK_Dependent;
     return false;
+  }
 
-  if (!CastTy->isLValueReferenceType() && !CastTy->isRecordType())
+  if (VK == VK_RValue && !CastTy->isRecordType())
     DefaultFunctionArrayLvalueConversion(CastExpr);
 
   // C++ [expr.cast]p5: The conversions performed by
@@ -1307,10 +1601,24 @@ Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
     }
   }
 
-  if (tcr != TC_Success && msg != 0)
-    Diag(R.getBegin(), msg) << (FunctionalStyle ? CT_Functional : CT_CStyle)
-      << CastExpr->getType() << CastTy << R;
-  else if (Kind == CK_Unknown || Kind == CK_BitCast)
+  if (tcr != TC_Success && msg != 0) {
+    if (CastExpr->getType() == Context.OverloadTy) {
+      DeclAccessPair Found;
+      FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(CastExpr, 
+                                CastTy,
+                                /* Complain */ true,
+                                Found);
+      
+      assert(!Fn 
+                && "cast failed but able to resolve overload expression!!");
+      (void)Fn;
+
+    } else {
+      diagnoseBadCast(*this, msg, (FunctionalStyle ? CT_Functional : CT_CStyle),
+                      R, CastExpr, CastTy);
+    }
+  }
+  else if (Kind == CK_BitCast)
     CheckCastAlign(CastExpr, CastTy, R);
 
   return tcr != TC_Success;
diff --git a/lib/Sema/SemaCXXScopeSpec.cpp b/lib/Sema/SemaCXXScopeSpec.cpp
index 631308eebb04..aa0efcdc70e1 100644
--- a/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/lib/Sema/SemaCXXScopeSpec.cpp
@@ -24,14 +24,26 @@
 using namespace clang;
 
 /// \brief Find the current instantiation that associated with the given type.
-static CXXRecordDecl *getCurrentInstantiationOf(QualType T) {
+static CXXRecordDecl *getCurrentInstantiationOf(QualType T, 
+                                                DeclContext *CurContext) {
   if (T.isNull())
     return 0;
 
   const Type *Ty = T->getCanonicalTypeInternal().getTypePtr();
-  if (isa<RecordType>(Ty))
-    return cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
-  else if (isa<InjectedClassNameType>(Ty))
+  if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
+    CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
+    if (!T->isDependentType())
+      return Record;
+
+    // This may be a member of a class template or class template partial
+    // specialization. If it's part of the current semantic context, then it's
+    // an injected-class-name;
+    for (; !CurContext->isFileContext(); CurContext = CurContext->getParent())
+      if (CurContext->Equals(Record))
+        return Record;
+    
+    return 0;
+  } else if (isa<InjectedClassNameType>(Ty))
     return cast<InjectedClassNameType>(Ty)->getDecl();
   else
     return 0;
@@ -45,10 +57,11 @@ static CXXRecordDecl *getCurrentInstantiationOf(QualType T) {
 /// or NULL if the declaration context cannot be computed (e.g., because it is
 /// dependent and not the current instantiation).
 DeclContext *Sema::computeDeclContext(QualType T) {
-  if (const TagType *Tag = T->getAs<TagType>())
-    return Tag->getDecl();
+  if (!T->isDependentType())
+    if (const TagType *Tag = T->getAs<TagType>())
+      return Tag->getDecl();
 
-  return ::getCurrentInstantiationOf(T);
+  return ::getCurrentInstantiationOf(T, CurContext);
 }
 
 /// \brief Compute the DeclContext that is associated with the given
@@ -174,7 +187,7 @@ CXXRecordDecl *Sema::getCurrentInstantiationOf(NestedNameSpecifier *NNS) {
     return 0;
 
   QualType T = QualType(NNS->getAsType(), 0);
-  return ::getCurrentInstantiationOf(T);
+  return ::getCurrentInstantiationOf(T, CurContext);
 }
 
 /// \brief Require that the context specified by SS be complete.
@@ -518,7 +531,6 @@ Sema::CXXScopeTy *Sema::BuildCXXNestedNameSpecifier(Scope *S,
     // a declaration context.
     if (NamespaceAliasDecl *Alias = dyn_cast<NamespaceAliasDecl>(SD))
       return NestedNameSpecifier::Create(Context, Prefix,
-
                                          Alias->getNamespace());
 
     QualType T = Context.getTypeDeclType(cast<TypeDecl>(SD));
diff --git a/lib/Sema/SemaChecking.cpp b/lib/Sema/SemaChecking.cpp
index a0b4b988dbeb..556665483e64 100644
--- a/lib/Sema/SemaChecking.cpp
+++ b/lib/Sema/SemaChecking.cpp
@@ -25,82 +25,23 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtObjC.h"
-#include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/ConvertUTF.h"
 #include <limits>
 using namespace clang;
 using namespace sema;
 
-/// getLocationOfStringLiteralByte - Return a source location that points to the
-/// specified byte of the specified string literal.
-///
-/// Strings are amazingly complex.  They can be formed from multiple tokens and
-/// can have escape sequences in them in addition to the usual trigraph and
-/// escaped newline business.  This routine handles this complexity.
-///
 SourceLocation Sema::getLocationOfStringLiteralByte(const StringLiteral *SL,
                                                     unsigned ByteNo) const {
-  assert(!SL->isWide() && "This doesn't work for wide strings yet");
-
-  // Loop over all of the tokens in this string until we find the one that
-  // contains the byte we're looking for.
-  unsigned TokNo = 0;
-  while (1) {
-    assert(TokNo < SL->getNumConcatenated() && "Invalid byte number!");
-    SourceLocation StrTokLoc = SL->getStrTokenLoc(TokNo);
-
-    // Get the spelling of the string so that we can get the data that makes up
-    // the string literal, not the identifier for the macro it is potentially
-    // expanded through.
-    SourceLocation StrTokSpellingLoc = SourceMgr.getSpellingLoc(StrTokLoc);
-
-    // Re-lex the token to get its length and original spelling.
-    std::pair<FileID, unsigned> LocInfo =
-      SourceMgr.getDecomposedLoc(StrTokSpellingLoc);
-    bool Invalid = false;
-    llvm::StringRef Buffer = SourceMgr.getBufferData(LocInfo.first, &Invalid);
-    if (Invalid)
-      return StrTokSpellingLoc;
-      
-    const char *StrData = Buffer.data()+LocInfo.second;
-
-    // Create a langops struct and enable trigraphs.  This is sufficient for
-    // relexing tokens.
-    LangOptions LangOpts;
-    LangOpts.Trigraphs = true;
-
-    // Create a lexer starting at the beginning of this token.
-    Lexer TheLexer(StrTokSpellingLoc, LangOpts, Buffer.begin(), StrData,
-                   Buffer.end());
-    Token TheTok;
-    TheLexer.LexFromRawLexer(TheTok);
-
-    // Use the StringLiteralParser to compute the length of the string in bytes.
-    StringLiteralParser SLP(&TheTok, 1, PP, /*Complain=*/false);
-    unsigned TokNumBytes = SLP.GetStringLength();
-
-    // If the byte is in this token, return the location of the byte.
-    if (ByteNo < TokNumBytes ||
-        (ByteNo == TokNumBytes && TokNo == SL->getNumConcatenated())) {
-      unsigned Offset =
-        StringLiteralParser::getOffsetOfStringByte(TheTok, ByteNo, PP,
-                                                   /*Complain=*/false);
-
-      // Now that we know the offset of the token in the spelling, use the
-      // preprocessor to get the offset in the original source.
-      return PP.AdvanceToTokenCharacter(StrTokLoc, Offset);
-    }
-
-    // Move to the next string token.
-    ++TokNo;
-    ByteNo -= TokNumBytes;
-  }
+  return SL->getLocationOfByte(ByteNo, PP.getSourceManager(),
+                               PP.getLangOptions(), PP.getTargetInfo());
 }
+  
 
 /// CheckablePrintfAttr - does a function call have a "printf" attribute
 /// and arguments that merit checking?
@@ -129,6 +70,24 @@ ExprResult
 Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   ExprResult TheCallResult(Owned(TheCall));
 
+  // Find out if any arguments are required to be integer constant expressions.
+  unsigned ICEArguments = 0;
+  ASTContext::GetBuiltinTypeError Error;
+  Context.GetBuiltinType(BuiltinID, Error, &ICEArguments);
+  if (Error != ASTContext::GE_None)
+    ICEArguments = 0;  // Don't diagnose previously diagnosed errors.
+  
+  // If any arguments are required to be ICE's, check and diagnose.
+  for (unsigned ArgNo = 0; ICEArguments != 0; ++ArgNo) {
+    // Skip arguments not required to be ICE's.
+    if ((ICEArguments & (1 << ArgNo)) == 0) continue;
+    
+    llvm::APSInt Result;
+    if (SemaBuiltinConstantArg(TheCall, ArgNo, Result))
+      return true;
+    ICEArguments &= ~(1 << ArgNo);
+  }
+  
   switch (BuiltinID) {
   case Builtin::BI__builtin___CFStringMakeConstantString:
     assert(TheCall->getNumArgs() == 1 &&
@@ -162,19 +121,6 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
     if (SemaBuiltinFPClassification(TheCall, 1))
       return ExprError();
     break;
-  case Builtin::BI__builtin_return_address:
-  case Builtin::BI__builtin_frame_address: {
-    llvm::APSInt Result;
-    if (SemaBuiltinConstantArg(TheCall, 0, Result))
-      return ExprError();
-    break;
-  }
-  case Builtin::BI__builtin_eh_return_data_regno: {
-    llvm::APSInt Result;
-    if (SemaBuiltinConstantArg(TheCall, 0, Result))
-      return ExprError();
-    break;
-  }
   case Builtin::BI__builtin_shufflevector:
     return SemaBuiltinShuffleVector(TheCall);
     // TheCall will be freed by the smart pointer here, but that's fine, since
@@ -191,6 +137,16 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
     if (SemaBuiltinLongjmp(TheCall))
       return ExprError();
     break;
+  case Builtin::BI__builtin_constant_p:
+    if (TheCall->getNumArgs() == 0)
+      return Diag(TheCall->getLocEnd(), diag::err_typecheck_call_too_few_args)
+        << 0 /*function call*/ << 1 << 0 << TheCall->getSourceRange();
+    if (TheCall->getNumArgs() > 1)
+      return Diag(TheCall->getArg(1)->getLocStart(),
+                  diag::err_typecheck_call_too_many_args)
+        << 0 /*function call*/ << 1 << TheCall->getNumArgs()
+        << TheCall->getArg(1)->getSourceRange();
+    break;
   case Builtin::BI__sync_fetch_and_add:
   case Builtin::BI__sync_fetch_and_sub:
   case Builtin::BI__sync_fetch_and_or:
@@ -217,11 +173,6 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
         if (CheckARMBuiltinFunctionCall(BuiltinID, TheCall))
           return ExprError();
         break;
-      case llvm::Triple::x86:
-      case llvm::Triple::x86_64:
-        if (CheckX86BuiltinFunctionCall(BuiltinID, TheCall))
-          return ExprError();
-        break;
       default:
         break;
     }
@@ -230,19 +181,6 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   return move(TheCallResult);
 }
 
-bool Sema::CheckX86BuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
-  switch (BuiltinID) {
-  case X86::BI__builtin_ia32_palignr128:
-  case X86::BI__builtin_ia32_palignr: {
-    llvm::APSInt Result;
-    if (SemaBuiltinConstantArg(TheCall, 2, Result))
-      return true;
-    break;
-  }
-  }
-  return false;
-}
-
 // Get the valid immediate range for the specified NEON type code.
 static unsigned RFT(unsigned t, bool shift = false) {
   bool quad = t & 0x10;
@@ -260,11 +198,9 @@ static unsigned RFT(unsigned t, bool shift = false) {
       assert(!shift && "cannot shift float types!");
       return (2 << (int)quad) - 1;
     case 5: // poly8
-      assert(!shift && "cannot shift polynomial types!");
-      return (8 << (int)quad) - 1;
+      return shift ? 7 : (8 << (int)quad) - 1;
     case 6: // poly16
-      assert(!shift && "cannot shift polynomial types!");
-      return (4 << (int)quad) - 1;
+      return shift ? 15 : (4 << (int)quad) - 1;
     case 7: // float16
       assert(!shift && "cannot shift float types!");
       return (4 << (int)quad) - 1;
@@ -339,8 +275,12 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
   // more efficient. For example, just map function ids to custom
   // handlers.
 
-  // Printf checking.
-  if (const FormatAttr *Format = FDecl->getAttr<FormatAttr>()) {
+  // Printf and scanf checking.
+  for (specific_attr_iterator<FormatAttr>
+         i = FDecl->specific_attr_begin<FormatAttr>(),
+         e = FDecl->specific_attr_end<FormatAttr>(); i != e ; ++i) {
+
+    const FormatAttr *Format = *i;
     const bool b = Format->getType() == "scanf";
     if (b || CheckablePrintfAttr(Format, TheCall)) {
       bool HasVAListArg = Format->getFirstArg() == 0;
@@ -351,12 +291,11 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
     }
   }
 
-  specific_attr_iterator<NonNullAttr>
-    i = FDecl->specific_attr_begin<NonNullAttr>(),
-    e = FDecl->specific_attr_end<NonNullAttr>();
-
-  for (; i != e; ++i)
+  for (specific_attr_iterator<NonNullAttr>
+         i = FDecl->specific_attr_begin<NonNullAttr>(),
+         e = FDecl->specific_attr_end<NonNullAttr>(); i != e; ++i) {
     CheckNonNullArguments(*i, TheCall);
+  }
 
   return false;
 }
@@ -422,7 +361,7 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
 
   QualType ValType =
     FirstArg->getType()->getAs<PointerType>()->getPointeeType();
-  if (!ValType->isIntegerType() && !ValType->isPointerType() &&
+  if (!ValType->isIntegerType() && !ValType->isAnyPointerType() &&
       !ValType->isBlockPointerType()) {
     Diag(DRE->getLocStart(), diag::err_atomic_builtin_must_be_pointer_intptr)
       << FirstArg->getType() << FirstArg->getSourceRange();
@@ -545,9 +484,10 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
 
     // GCC does an implicit conversion to the pointer or integer ValType.  This
     // can fail in some cases (1i -> int**), check for this error case now.
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Invalid;
+    ExprValueKind VK = VK_RValue;
     CXXCastPath BasePath;
-    if (CheckCastTypes(Arg->getSourceRange(), ValType, Arg, Kind, BasePath))
+    if (CheckCastTypes(Arg->getSourceRange(), ValType, Arg, Kind, VK, BasePath))
       return ExprError();
 
     // Okay, we have something that *can* be converted to the right type.  Check
@@ -556,7 +496,7 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
     // pass in 42.  The 42 gets converted to char.  This is even more strange
     // for things like 45.123 -> char, etc.
     // FIXME: Do this check.
-    ImpCastExprToType(Arg, ValType, Kind, VK_RValue, &BasePath);
+    ImpCastExprToType(Arg, ValType, Kind, VK, &BasePath);
     TheCall->setArg(i+1, Arg);
   }
 
@@ -581,9 +521,6 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
 
 /// CheckObjCString - Checks that the argument to the builtin
 /// CFString constructor is correct
-/// FIXME: GCC currently emits the following warning:
-/// "warning: input conversion stopped due to an input byte that does not
-///           belong to the input codeset UTF-8"
 /// Note: It might also make sense to do the UTF-16 conversion here (would
 /// simplify the backend).
 bool Sema::CheckObjCString(Expr *Arg) {
@@ -602,7 +539,21 @@ bool Sema::CheckObjCString(Expr *Arg) {
          diag::warn_cfstring_literal_contains_nul_character)
       << Arg->getSourceRange();
   }
-
+  if (Literal->containsNonAsciiOrNull()) {
+    llvm::StringRef String = Literal->getString();
+    unsigned NumBytes = String.size();
+    llvm::SmallVector<UTF16, 128> ToBuf(NumBytes);
+    const UTF8 *FromPtr = (UTF8 *)String.data();
+    UTF16 *ToPtr = &ToBuf[0];
+    
+    ConversionResult Result = ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
+                                                 &ToPtr, ToPtr + NumBytes,
+                                                 strictConversion);
+    // Check for conversion failure.
+    if (Result != conversionOK)
+      Diag(Arg->getLocStart(),
+           diag::warn_cfstring_truncated) << Arg->getSourceRange();
+  }
   return false;
 }
 
@@ -798,7 +749,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
     } else if (numElements != numResElements) {
       QualType eltType = LHSType->getAs<VectorType>()->getElementType();
       resType = Context.getVectorType(eltType, numResElements,
-                                      VectorType::NotAltiVec);
+                                      VectorType::GenericVector);
     }
   }
 
@@ -929,31 +880,44 @@ bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
                                   bool HasVAListArg,
                                   unsigned format_idx, unsigned firstDataArg,
                                   bool isPrintf) {
-
+ tryAgain:
   if (E->isTypeDependent() || E->isValueDependent())
     return false;
 
   switch (E->getStmtClass()) {
+  case Stmt::BinaryConditionalOperatorClass:
   case Stmt::ConditionalOperatorClass: {
-    const ConditionalOperator *C = cast<ConditionalOperator>(E);
+    const AbstractConditionalOperator *C = cast<AbstractConditionalOperator>(E);
     return SemaCheckStringLiteral(C->getTrueExpr(), TheCall, HasVAListArg,
                                   format_idx, firstDataArg, isPrintf)
-        && SemaCheckStringLiteral(C->getRHS(), TheCall, HasVAListArg,
+        && SemaCheckStringLiteral(C->getFalseExpr(), TheCall, HasVAListArg,
                                   format_idx, firstDataArg, isPrintf);
   }
 
+  case Stmt::IntegerLiteralClass:
+    // Technically -Wformat-nonliteral does not warn about this case.
+    // The behavior of printf and friends in this case is implementation
+    // dependent.  Ideally if the format string cannot be null then
+    // it should have a 'nonnull' attribute in the function prototype.
+    return true;
+
   case Stmt::ImplicitCastExprClass: {
-    const ImplicitCastExpr *Expr = cast<ImplicitCastExpr>(E);
-    return SemaCheckStringLiteral(Expr->getSubExpr(), TheCall, HasVAListArg,
-                                  format_idx, firstDataArg, isPrintf);
+    E = cast<ImplicitCastExpr>(E)->getSubExpr();
+    goto tryAgain;
   }
 
   case Stmt::ParenExprClass: {
-    const ParenExpr *Expr = cast<ParenExpr>(E);
-    return SemaCheckStringLiteral(Expr->getSubExpr(), TheCall, HasVAListArg,
-                                  format_idx, firstDataArg, isPrintf);
+    E = cast<ParenExpr>(E)->getSubExpr();
+    goto tryAgain;
   }
 
+  case Stmt::OpaqueValueExprClass:
+    if (const Expr *src = cast<OpaqueValueExpr>(E)->getSourceExpr()) {
+      E = src;
+      goto tryAgain;
+    }
+    return false;
+
   case Stmt::DeclRefExprClass: {
     const DeclRefExpr *DR = cast<DeclRefExpr>(E);
 
@@ -1072,12 +1036,16 @@ Sema::CheckPrintfScanfArguments(const CallExpr *TheCall, bool HasVAListArg,
   // of member functions is counted. However, it doesn't appear in our own
   // lists, so decrement format_idx in that case.
   if (isa<CXXMemberCallExpr>(TheCall)) {
-    // Catch a format attribute mistakenly referring to the object argument.
-    if (format_idx == 0)
-      return;
-    --format_idx;
-    if(firstDataArg != 0)
-      --firstDataArg;
+    const CXXMethodDecl *method_decl =
+      dyn_cast<CXXMethodDecl>(TheCall->getCalleeDecl());
+    if (method_decl && method_decl->isInstance()) {
+      // Catch a format attribute mistakenly referring to the object argument.
+      if (format_idx == 0)
+        return;
+      --format_idx;
+      if(firstDataArg != 0)
+        --firstDataArg;
+    }
   }
 
   // CHECK: printf/scanf-like function is called with no format string.
@@ -1531,6 +1499,8 @@ CheckPrintfHandler::HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier
   }
 
   // Check each flag does not conflict with any other component.
+  if (!FS.hasValidThousandsGroupingPrefix())
+    HandleFlag(FS, FS.hasThousandsGrouping(), startSpecifier, specifierLen);
   if (!FS.hasValidLeadingZeros())
     HandleFlag(FS, FS.hasLeadingZeros(), startSpecifier, specifierLen);
   if (!FS.hasValidPlusPrefix())
@@ -1585,9 +1555,12 @@ CheckPrintfHandler::HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier
     // or 'short' to an 'int'.  This is done because printf is a varargs
     // function.
     if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Ex))
-      if (ICE->getType() == S.Context.IntTy)
-        if (ATR.matchesType(S.Context, ICE->getSubExpr()->getType()))
+      if (ICE->getType() == S.Context.IntTy) {
+        // All further checking is done on the subexpression.
+        Ex = ICE->getSubExpr();
+        if (ATR.matchesType(S.Context, Ex->getType()))
           return true;
+      }
 
     // We may be able to offer a FixItHint if it is a supported type.
     PrintfSpecifier fixedFS = FS;
@@ -1794,8 +1767,8 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
 
 //===--- CHECK: Return Address of Stack Variable --------------------------===//
 
-static DeclRefExpr* EvalVal(Expr *E);
-static DeclRefExpr* EvalAddr(Expr* E);
+static Expr *EvalVal(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars);
+static Expr *EvalAddr(Expr* E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars);
 
 /// CheckReturnStackAddr - Check if a return statement returns the address
 ///   of a stack variable.
@@ -1803,45 +1776,79 @@ void
 Sema::CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
                            SourceLocation ReturnLoc) {
 
-  // Perform checking for returned stack addresses.
-  if (lhsType->isPointerType() || lhsType->isBlockPointerType()) {
-    if (DeclRefExpr *DR = EvalAddr(RetValExp))
-      Diag(DR->getLocStart(), diag::warn_ret_stack_addr)
-       << DR->getDecl()->getDeclName() << RetValExp->getSourceRange();
-
-    // Skip over implicit cast expressions when checking for block expressions.
-    RetValExp = RetValExp->IgnoreParenCasts();
+  Expr *stackE = 0;
+  llvm::SmallVector<DeclRefExpr *, 8> refVars;
 
-    if (BlockExpr *C = dyn_cast<BlockExpr>(RetValExp))
-      if (C->hasBlockDeclRefExprs())
-        Diag(C->getLocStart(), diag::err_ret_local_block)
-          << C->getSourceRange();
+  // Perform checking for returned stack addresses, local blocks,
+  // label addresses or references to temporaries.
+  if (lhsType->isPointerType() || lhsType->isBlockPointerType()) {
+    stackE = EvalAddr(RetValExp, refVars);
+  } else if (lhsType->isReferenceType()) {
+    stackE = EvalVal(RetValExp, refVars);
+  }
 
-    if (AddrLabelExpr *ALE = dyn_cast<AddrLabelExpr>(RetValExp))
-      Diag(ALE->getLocStart(), diag::warn_ret_addr_label)
-        << ALE->getSourceRange();
+  if (stackE == 0)
+    return; // Nothing suspicious was found.
 
-  } else if (lhsType->isReferenceType()) {
-    // Perform checking for stack values returned by reference.
-    // Check for a reference to the stack
-    if (DeclRefExpr *DR = EvalVal(RetValExp))
-      Diag(DR->getLocStart(), diag::warn_ret_stack_ref)
-        << DR->getDecl()->getDeclName() << RetValExp->getSourceRange();
+  SourceLocation diagLoc;
+  SourceRange diagRange;
+  if (refVars.empty()) {
+    diagLoc = stackE->getLocStart();
+    diagRange = stackE->getSourceRange();
+  } else {
+    // We followed through a reference variable. 'stackE' contains the
+    // problematic expression but we will warn at the return statement pointing
+    // at the reference variable. We will later display the "trail" of
+    // reference variables using notes.
+    diagLoc = refVars[0]->getLocStart();
+    diagRange = refVars[0]->getSourceRange();
+  }
+
+  if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(stackE)) { //address of local var.
+    Diag(diagLoc, lhsType->isReferenceType() ? diag::warn_ret_stack_ref
+                                             : diag::warn_ret_stack_addr)
+     << DR->getDecl()->getDeclName() << diagRange;
+  } else if (isa<BlockExpr>(stackE)) { // local block.
+    Diag(diagLoc, diag::err_ret_local_block) << diagRange;
+  } else if (isa<AddrLabelExpr>(stackE)) { // address of label.
+    Diag(diagLoc, diag::warn_ret_addr_label) << diagRange;
+  } else { // local temporary.
+    Diag(diagLoc, lhsType->isReferenceType() ? diag::warn_ret_local_temp_ref
+                                             : diag::warn_ret_local_temp_addr)
+     << diagRange;
+  }
+
+  // Display the "trail" of reference variables that we followed until we
+  // found the problematic expression using notes.
+  for (unsigned i = 0, e = refVars.size(); i != e; ++i) {
+    VarDecl *VD = cast<VarDecl>(refVars[i]->getDecl());
+    // If this var binds to another reference var, show the range of the next
+    // var, otherwise the var binds to the problematic expression, in which case
+    // show the range of the expression.
+    SourceRange range = (i < e-1) ? refVars[i+1]->getSourceRange()
+                                  : stackE->getSourceRange();
+    Diag(VD->getLocation(), diag::note_ref_var_local_bind)
+      << VD->getDeclName() << range;
   }
 }
 
 /// EvalAddr - EvalAddr and EvalVal are mutually recursive functions that
 ///  check if the expression in a return statement evaluates to an address
-///  to a location on the stack.  The recursion is used to traverse the
+///  to a location on the stack, a local block, an address of a label, or a
+///  reference to local temporary. The recursion is used to traverse the
 ///  AST of the return expression, with recursion backtracking when we
-///  encounter a subexpression that (1) clearly does not lead to the address
-///  of a stack variable or (2) is something we cannot determine leads to
-///  the address of a stack variable based on such local checking.
+///  encounter a subexpression that (1) clearly does not lead to one of the
+///  above problematic expressions (2) is something we cannot determine leads to
+///  a problematic expression based on such local checking.
+///
+///  Both EvalAddr and EvalVal follow through reference variables to evaluate
+///  the expression that they point to. Such variables are added to the
+///  'refVars' vector so that we know what the reference variable "trail" was.
 ///
 ///  EvalAddr processes expressions that are pointers that are used as
 ///  references (and not L-values).  EvalVal handles all other values.
-///  At the base case of the recursion is a check for a DeclRefExpr* in
-///  the refers to a stack variable.
+///  At the base case of the recursion is a check for the above problematic
+///  expressions.
 ///
 ///  This implementation handles:
 ///
@@ -1851,7 +1858,10 @@ Sema::CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
 ///   * arbitrary interplay between "&" and "*" operators
 ///   * pointer arithmetic from an address of a stack variable
 ///   * taking the address of an array element where the array is on the stack
-static DeclRefExpr* EvalAddr(Expr *E) {
+static Expr *EvalAddr(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars) {
+  if (E->isTypeDependent())
+      return NULL;
+
   // We should only be called for evaluating pointer expressions.
   assert((E->getType()->isAnyPointerType() ||
           E->getType()->isBlockPointerType() ||
@@ -1864,7 +1874,23 @@ static DeclRefExpr* EvalAddr(Expr *E) {
   switch (E->getStmtClass()) {
   case Stmt::ParenExprClass:
     // Ignore parentheses.
-    return EvalAddr(cast<ParenExpr>(E)->getSubExpr());
+    return EvalAddr(cast<ParenExpr>(E)->getSubExpr(), refVars);
+
+  case Stmt::DeclRefExprClass: {
+    DeclRefExpr *DR = cast<DeclRefExpr>(E);
+
+    if (VarDecl *V = dyn_cast<VarDecl>(DR->getDecl()))
+      // If this is a reference variable, follow through to the expression that
+      // it points to.
+      if (V->hasLocalStorage() &&
+          V->getType()->isReferenceType() && V->hasInit()) {
+        // Add the reference variable to the "trail".
+        refVars.push_back(DR);
+        return EvalAddr(V->getInit(), refVars);
+      }
+
+    return NULL;
+  }
 
   case Stmt::UnaryOperatorClass: {
     // The only unary operator that make sense to handle here
@@ -1872,7 +1898,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     UnaryOperator *U = cast<UnaryOperator>(E);
 
     if (U->getOpcode() == UO_AddrOf)
-      return EvalVal(U->getSubExpr());
+      return EvalVal(U->getSubExpr(), refVars);
     else
       return NULL;
   }
@@ -1893,7 +1919,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     if (!Base->getType()->isPointerType()) Base = B->getRHS();
 
     assert (Base->getType()->isPointerType());
-    return EvalAddr(Base);
+    return EvalAddr(Base, refVars);
   }
 
   // For conditional operators we need to see if either the LHS or RHS are
@@ -1902,12 +1928,27 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     ConditionalOperator *C = cast<ConditionalOperator>(E);
 
     // Handle the GNU extension for missing LHS.
-    if (Expr *lhsExpr = C->getLHS())
-      if (DeclRefExpr* LHS = EvalAddr(lhsExpr))
-        return LHS;
+    if (Expr *lhsExpr = C->getLHS()) {
+    // In C++, we can have a throw-expression, which has 'void' type.
+      if (!lhsExpr->getType()->isVoidType())
+        if (Expr* LHS = EvalAddr(lhsExpr, refVars))
+          return LHS;
+    }
 
-     return EvalAddr(C->getRHS());
+    // In C++, we can have a throw-expression, which has 'void' type.
+    if (C->getRHS()->getType()->isVoidType())
+      return NULL;
+
+    return EvalAddr(C->getRHS(), refVars);
   }
+  
+  case Stmt::BlockExprClass:
+    if (cast<BlockExpr>(E)->getBlockDecl()->hasCaptures())
+      return E; // local block.
+    return NULL;
+
+  case Stmt::AddrLabelExprClass:
+    return E; // address of label.
 
   // For casts, we need to handle conversions from arrays to
   // pointer values, and pointer-to-pointer conversions.
@@ -1920,9 +1961,9 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     if (SubExpr->getType()->isPointerType() ||
         SubExpr->getType()->isBlockPointerType() ||
         SubExpr->getType()->isObjCQualifiedIdType())
-      return EvalAddr(SubExpr);
+      return EvalAddr(SubExpr, refVars);
     else if (T->isArrayType())
-      return EvalVal(SubExpr);
+      return EvalVal(SubExpr, refVars);
     else
       return 0;
   }
@@ -1941,7 +1982,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
   case Stmt::CXXReinterpretCastExprClass: {
       Expr *S = cast<CXXNamedCastExpr>(E)->getSubExpr();
       if (S->getType()->isPointerType() || S->getType()->isBlockPointerType())
-        return EvalAddr(S);
+        return EvalAddr(S, refVars);
       else
         return NULL;
   }
@@ -1955,7 +1996,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
 
 ///  EvalVal - This function is complements EvalAddr in the mutual recursion.
 ///   See the comments for EvalAddr for more details.
-static DeclRefExpr* EvalVal(Expr *E) {
+static Expr *EvalVal(Expr *E, llvm::SmallVectorImpl<DeclRefExpr *> &refVars) {
 do {
   // We should only be called for evaluating non-pointer expressions, or
   // expressions with a pointer type that are not used as references but instead
@@ -1975,13 +2016,24 @@ do {
   }
 
   case Stmt::DeclRefExprClass: {
-    // DeclRefExpr: the base case.  When we hit a DeclRefExpr we are looking
-    //  at code that refers to a variable's name.  We check if it has local
-    //  storage within the function, and if so, return the expression.
+    // When we hit a DeclRefExpr we are looking at code that refers to a
+    // variable's name. If it's not a reference variable we check if it has
+    // local storage within the function, and if so, return the expression.
     DeclRefExpr *DR = cast<DeclRefExpr>(E);
 
     if (VarDecl *V = dyn_cast<VarDecl>(DR->getDecl()))
-      if (V->hasLocalStorage() && !V->getType()->isReferenceType()) return DR;
+      if (V->hasLocalStorage()) {
+        if (!V->getType()->isReferenceType())
+          return DR;
+
+        // Reference variable, follow through to the expression that
+        // it points to.
+        if (V->hasInit()) {
+          // Add the reference variable to the "trail".
+          refVars.push_back(DR);
+          return EvalVal(V->getInit(), refVars);
+        }
+      }
 
     return NULL;
   }
@@ -1999,7 +2051,7 @@ do {
     UnaryOperator *U = cast<UnaryOperator>(E);
 
     if (U->getOpcode() == UO_Deref)
-      return EvalAddr(U->getSubExpr());
+      return EvalAddr(U->getSubExpr(), refVars);
 
     return NULL;
   }
@@ -2008,20 +2060,20 @@ do {
     // Array subscripts are potential references to data on the stack.  We
     // retrieve the DeclRefExpr* for the array variable if it indeed
     // has local storage.
-    return EvalAddr(cast<ArraySubscriptExpr>(E)->getBase());
+    return EvalAddr(cast<ArraySubscriptExpr>(E)->getBase(), refVars);
   }
 
   case Stmt::ConditionalOperatorClass: {
     // For conditional operators we need to see if either the LHS or RHS are
-    // non-NULL DeclRefExpr's.  If one is non-NULL, we return it.
+    // non-NULL Expr's.  If one is non-NULL, we return it.
     ConditionalOperator *C = cast<ConditionalOperator>(E);
 
     // Handle the GNU extension for missing LHS.
     if (Expr *lhsExpr = C->getLHS())
-      if (DeclRefExpr *LHS = EvalVal(lhsExpr))
+      if (Expr *LHS = EvalVal(lhsExpr, refVars))
         return LHS;
 
-    return EvalVal(C->getRHS());
+    return EvalVal(C->getRHS(), refVars);
   }
 
   // Accesses to members are potential references to data on the stack.
@@ -2037,11 +2089,16 @@ do {
     if (M->getMemberDecl()->getType()->isReferenceType())
       return NULL;
 
-    return EvalVal(M->getBase());
+    return EvalVal(M->getBase(), refVars);
   }
 
-  // Everything else: we simply don't reason about them.
   default:
+    // Check that we don't return or take the address of a reference to a
+    // temporary. This is only useful in C++.
+    if (!E->isTypeDependent() && E->isRValue())
+      return E;
+
+    // Everything else: we simply don't reason about them.
     return NULL;
   }
 } while (true);
@@ -2055,8 +2112,8 @@ do {
 void Sema::CheckFloatComparison(SourceLocation loc, Expr* lex, Expr *rex) {
   bool EmitWarning = true;
 
-  Expr* LeftExprSansParen = lex->IgnoreParens();
-  Expr* RightExprSansParen = rex->IgnoreParens();
+  Expr* LeftExprSansParen = lex->IgnoreParenImpCasts();
+  Expr* RightExprSansParen = rex->IgnoreParenImpCasts();
 
   // Special case: check for x == x (which is OK).
   // Do not emit warnings for such cases.
@@ -2117,19 +2174,19 @@ struct IntRange {
     : Width(Width), NonNegative(NonNegative)
   {}
 
-  // Returns the range of the bool type.
+  /// Returns the range of the bool type.
   static IntRange forBoolType() {
     return IntRange(1, true);
   }
 
-  // Returns the range of an integral type.
-  static IntRange forType(ASTContext &C, QualType T) {
-    return forCanonicalType(C, T->getCanonicalTypeInternal().getTypePtr());
+  /// Returns the range of an opaque value of the given integral type.
+  static IntRange forValueOfType(ASTContext &C, QualType T) {
+    return forValueOfCanonicalType(C,
+                          T->getCanonicalTypeInternal().getTypePtr());
   }
 
-  // Returns the range of an integeral type based on its canonical
-  // representation.
-  static IntRange forCanonicalType(ASTContext &C, const Type *T) {
+  /// Returns the range of an opaque value of a canonical integral type.
+  static IntRange forValueOfCanonicalType(ASTContext &C, const Type *T) {
     assert(T->isCanonicalUnqualified());
 
     if (const VectorType *VT = dyn_cast<VectorType>(T))
@@ -2137,8 +2194,12 @@ struct IntRange {
     if (const ComplexType *CT = dyn_cast<ComplexType>(T))
       T = CT->getElementType().getTypePtr();
 
+    // For enum types, use the known bit width of the enumerators.
     if (const EnumType *ET = dyn_cast<EnumType>(T)) {
       EnumDecl *Enum = ET->getDecl();
+      if (!Enum->isDefinition())
+        return IntRange(C.getIntWidth(QualType(T, 0)), false);
+
       unsigned NumPositive = Enum->getNumPositiveBits();
       unsigned NumNegative = Enum->getNumNegativeBits();
 
@@ -2151,13 +2212,34 @@ struct IntRange {
     return IntRange(C.getIntWidth(QualType(T, 0)), BT->isUnsignedInteger());
   }
 
-  // Returns the supremum of two ranges: i.e. their conservative merge.
+  /// Returns the "target" range of a canonical integral type, i.e.
+  /// the range of values expressible in the type.
+  ///
+  /// This matches forValueOfCanonicalType except that enums have the
+  /// full range of their type, not the range of their enumerators.
+  static IntRange forTargetOfCanonicalType(ASTContext &C, const Type *T) {
+    assert(T->isCanonicalUnqualified());
+
+    if (const VectorType *VT = dyn_cast<VectorType>(T))
+      T = VT->getElementType().getTypePtr();
+    if (const ComplexType *CT = dyn_cast<ComplexType>(T))
+      T = CT->getElementType().getTypePtr();
+    if (const EnumType *ET = dyn_cast<EnumType>(T))
+      T = ET->getDecl()->getIntegerType().getTypePtr();
+
+    const BuiltinType *BT = cast<BuiltinType>(T);
+    assert(BT->isInteger());
+
+    return IntRange(C.getIntWidth(QualType(T, 0)), BT->isUnsignedInteger());
+  }
+
+  /// Returns the supremum of two ranges: i.e. their conservative merge.
   static IntRange join(IntRange L, IntRange R) {
     return IntRange(std::max(L.Width, R.Width),
                     L.NonNegative && R.NonNegative);
   }
 
-  // Returns the infinum of two ranges: i.e. their aggressive merge.
+  /// Returns the infinum of two ranges: i.e. their aggressive merge.
   static IntRange meet(IntRange L, IntRange R) {
     return IntRange(std::min(L.Width, R.Width),
                     L.NonNegative || R.NonNegative);
@@ -2169,7 +2251,7 @@ IntRange GetValueRange(ASTContext &C, llvm::APSInt &value, unsigned MaxWidth) {
     return IntRange(value.getMinSignedBits(), false);
 
   if (value.getBitWidth() > MaxWidth)
-    value.trunc(MaxWidth);
+    value = value.trunc(MaxWidth);
 
   // isNonNegative() just checks the sign bit without considering
   // signedness.
@@ -2224,11 +2306,9 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     if (CE->getCastKind() == CK_NoOp)
       return GetExprRange(C, CE->getSubExpr(), MaxWidth);
 
-    IntRange OutputTypeRange = IntRange::forType(C, CE->getType());
+    IntRange OutputTypeRange = IntRange::forValueOfType(C, CE->getType());
 
     bool isIntegerCast = (CE->getCastKind() == CK_IntegralCast);
-    if (!isIntegerCast && CE->getCastKind() == CK_Unknown)
-      isIntegerCast = CE->getSubExpr()->getType()->isIntegerType();
 
     // Assume that non-integer casts can span the full range of the type.
     if (!isIntegerCast)
@@ -2283,12 +2363,12 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     case BO_RemAssign:
     case BO_AddAssign:
     case BO_SubAssign:
-      return IntRange::forType(C, E->getType());
+      return IntRange::forValueOfType(C, E->getType());
 
     // Operations with opaque sources are black-listed.
     case BO_PtrMemD:
     case BO_PtrMemI:
-      return IntRange::forType(C, E->getType());
+      return IntRange::forValueOfType(C, E->getType());
 
     // Bitwise-and uses the *infinum* of the two source ranges.
     case BO_And:
@@ -2303,14 +2383,14 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
       if (IntegerLiteral *I
             = dyn_cast<IntegerLiteral>(BO->getLHS()->IgnoreParenCasts())) {
         if (I->getValue() == 1) {
-          IntRange R = IntRange::forType(C, E->getType());
+          IntRange R = IntRange::forValueOfType(C, E->getType());
           return IntRange(R.Width, /*NonNegative*/ true);
         }
       }
       // fallthrough
 
     case BO_ShlAssign:
-      return IntRange::forType(C, E->getType());
+      return IntRange::forValueOfType(C, E->getType());
 
     // Right shift by a constant can narrow its left argument.
     case BO_Shr:
@@ -2339,7 +2419,7 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     // Black-list pointer subtractions.
     case BO_Sub:
       if (BO->getLHS()->getType()->isPointerType())
-        return IntRange::forType(C, E->getType());
+        return IntRange::forValueOfType(C, E->getType());
       // fallthrough
 
     default:
@@ -2362,7 +2442,7 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     // Operations with opaque sources are black-listed.
     case UO_Deref:
     case UO_AddrOf: // should be impossible
-      return IntRange::forType(C, E->getType());
+      return IntRange::forValueOfType(C, E->getType());
 
     default:
       return GetExprRange(C, UO->getSubExpr(), MaxWidth);
@@ -2370,7 +2450,7 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
   }
   
   if (dyn_cast<OffsetOfExpr>(E)) {
-    IntRange::forType(C, E->getType());
+    IntRange::forValueOfType(C, E->getType());
   }
 
   FieldDecl *BitField = E->getBitField();
@@ -2381,7 +2461,7 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     return IntRange(BitWidth, BitField->getType()->isUnsignedIntegerType());
   }
 
-  return IntRange::forType(C, E->getType());
+  return IntRange::forValueOfType(C, E->getType());
 }
 
 IntRange GetExprRange(ASTContext &C, Expr *E) {
@@ -2426,30 +2506,55 @@ bool IsSameFloatAfterCast(const APValue &value,
           IsSameFloatAfterCast(value.getComplexFloatImag(), Src, Tgt));
 }
 
-void AnalyzeImplicitConversions(Sema &S, Expr *E);
+void AnalyzeImplicitConversions(Sema &S, Expr *E, SourceLocation CC);
+
+static bool IsZero(Sema &S, Expr *E) {
+  // Suppress cases where we are comparing against an enum constant.
+  if (const DeclRefExpr *DR =
+      dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()))
+    if (isa<EnumConstantDecl>(DR->getDecl()))
+      return false;
+
+  // Suppress cases where the '0' value is expanded from a macro.
+  if (E->getLocStart().isMacroID())
+    return false;
 
-bool IsZero(Sema &S, Expr *E) {
   llvm::APSInt Value;
   return E->isIntegerConstantExpr(Value, S.Context) && Value == 0;
 }
 
+static bool HasEnumType(Expr *E) {
+  // Strip off implicit integral promotions.
+  while (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
+    if (ICE->getCastKind() != CK_IntegralCast &&
+        ICE->getCastKind() != CK_NoOp)
+      break;
+    E = ICE->getSubExpr();
+  }
+
+  return E->getType()->isEnumeralType();
+}
+
 void CheckTrivialUnsignedComparison(Sema &S, BinaryOperator *E) {
   BinaryOperatorKind op = E->getOpcode();
+  if (E->isValueDependent())
+    return;
+
   if (op == BO_LT && IsZero(S, E->getRHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_lunsigned_always_true_comparison)
-      << "< 0" << "false"
+      << "< 0" << "false" << HasEnumType(E->getLHS())
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
   } else if (op == BO_GE && IsZero(S, E->getRHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_lunsigned_always_true_comparison)
-      << ">= 0" << "true"
+      << ">= 0" << "true" << HasEnumType(E->getLHS())
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
   } else if (op == BO_GT && IsZero(S, E->getLHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_runsigned_always_true_comparison)
-      << "0 >" << "false" 
+      << "0 >" << "false" << HasEnumType(E->getRHS())
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
   } else if (op == BO_LE && IsZero(S, E->getLHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_runsigned_always_true_comparison)
-      << "0 <=" << "true" 
+      << "0 <=" << "true" << HasEnumType(E->getRHS())
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
   }
 }
@@ -2457,8 +2562,8 @@ void CheckTrivialUnsignedComparison(Sema &S, BinaryOperator *E) {
 /// Analyze the operands of the given comparison.  Implements the
 /// fallback case from AnalyzeComparison.
 void AnalyzeImpConvsInComparison(Sema &S, BinaryOperator *E) {
-  AnalyzeImplicitConversions(S, E->getLHS());
-  AnalyzeImplicitConversions(S, E->getRHS());
+  AnalyzeImplicitConversions(S, E->getLHS(), E->getOperatorLoc());
+  AnalyzeImplicitConversions(S, E->getRHS(), E->getOperatorLoc());
 }
 
 /// \brief Implements -Wsign-compare.
@@ -2476,7 +2581,11 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   // We don't do anything special if this isn't an unsigned integral
   // comparison:  we're only interested in integral comparisons, and
   // signed comparisons only happen in cases we don't care to warn about.
-  if (!T->hasUnsignedIntegerRepresentation())
+  //
+  // We also don't care about value-dependent expressions or expressions
+  // whose result is a constant.
+  if (!T->hasUnsignedIntegerRepresentation()
+      || E->isValueDependent() || E->isIntegerConstantExpr(S.Context))
     return AnalyzeImpConvsInComparison(S, E);
 
   Expr *lex = E->getLHS()->IgnoreParenImpCasts();
@@ -2503,8 +2612,8 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
 
   // Go ahead and analyze implicit conversions in the operands.  Note
   // that we skip the implicit conversions on both sides.
-  AnalyzeImplicitConversions(S, lex);
-  AnalyzeImplicitConversions(S, rex);
+  AnalyzeImplicitConversions(S, lex, E->getOperatorLoc());
+  AnalyzeImplicitConversions(S, rex, E->getOperatorLoc());
 
   // If the signed range is non-negative, -Wsign-compare won't fire,
   // but we should still check for comparisons which are always true
@@ -2533,13 +2642,103 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
     << lex->getSourceRange() << rex->getSourceRange();
 }
 
+/// Analyzes an attempt to assign the given value to a bitfield.
+///
+/// Returns true if there was something fishy about the attempt.
+bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
+                               SourceLocation InitLoc) {
+  assert(Bitfield->isBitField());
+  if (Bitfield->isInvalidDecl())
+    return false;
+
+  // White-list bool bitfields.
+  if (Bitfield->getType()->isBooleanType())
+    return false;
+
+  // Ignore value- or type-dependent expressions.
+  if (Bitfield->getBitWidth()->isValueDependent() ||
+      Bitfield->getBitWidth()->isTypeDependent() ||
+      Init->isValueDependent() ||
+      Init->isTypeDependent())
+    return false;
+
+  Expr *OriginalInit = Init->IgnoreParenImpCasts();
+
+  llvm::APSInt Width(32);
+  Expr::EvalResult InitValue;
+  if (!Bitfield->getBitWidth()->isIntegerConstantExpr(Width, S.Context) ||
+      !OriginalInit->Evaluate(InitValue, S.Context) ||
+      !InitValue.Val.isInt())
+    return false;
+
+  const llvm::APSInt &Value = InitValue.Val.getInt();
+  unsigned OriginalWidth = Value.getBitWidth();
+  unsigned FieldWidth = Width.getZExtValue();
+
+  if (OriginalWidth <= FieldWidth)
+    return false;
+
+  llvm::APSInt TruncatedValue = Value.trunc(FieldWidth);
+
+  // It's fairly common to write values into signed bitfields
+  // that, if sign-extended, would end up becoming a different
+  // value.  We don't want to warn about that.
+  if (Value.isSigned() && Value.isNegative())
+    TruncatedValue = TruncatedValue.sext(OriginalWidth);
+  else
+    TruncatedValue = TruncatedValue.zext(OriginalWidth);
+
+  if (Value == TruncatedValue)
+    return false;
+
+  std::string PrettyValue = Value.toString(10);
+  std::string PrettyTrunc = TruncatedValue.toString(10);
+
+  S.Diag(InitLoc, diag::warn_impcast_bitfield_precision_constant)
+    << PrettyValue << PrettyTrunc << OriginalInit->getType()
+    << Init->getSourceRange();
+
+  return true;
+}
+
+/// Analyze the given simple or compound assignment for warning-worthy
+/// operations.
+void AnalyzeAssignment(Sema &S, BinaryOperator *E) {
+  // Just recurse on the LHS.
+  AnalyzeImplicitConversions(S, E->getLHS(), E->getOperatorLoc());
+
+  // We want to recurse on the RHS as normal unless we're assigning to
+  // a bitfield.
+  if (FieldDecl *Bitfield = E->getLHS()->getBitField()) {
+    if (AnalyzeBitFieldAssignment(S, Bitfield, E->getRHS(),
+                                  E->getOperatorLoc())) {
+      // Recurse, ignoring any implicit conversions on the RHS.
+      return AnalyzeImplicitConversions(S, E->getRHS()->IgnoreParenImpCasts(),
+                                        E->getOperatorLoc());
+    }
+  }
+
+  AnalyzeImplicitConversions(S, E->getRHS(), E->getOperatorLoc());
+}
+
 /// Diagnose an implicit cast;  purely a helper for CheckImplicitConversion.
-void DiagnoseImpCast(Sema &S, Expr *E, QualType T, unsigned diag) {
-  S.Diag(E->getExprLoc(), diag) << E->getType() << T << E->getSourceRange();
+void DiagnoseImpCast(Sema &S, Expr *E, QualType T, SourceLocation CContext,
+                     unsigned diag) {
+  S.Diag(E->getExprLoc(), diag)
+    << E->getType() << T << E->getSourceRange() << SourceRange(CContext);
+}
+
+std::string PrettyPrintInRange(const llvm::APSInt &Value, IntRange Range) {
+  if (!Range.Width) return "0";
+
+  llvm::APSInt ValueInRange = Value;
+  ValueInRange.setIsSigned(!Range.NonNegative);
+  ValueInRange = ValueInRange.trunc(Range.Width);
+  return ValueInRange.toString(10);
 }
 
 void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
-                             bool *ICContext = 0) {
+                             SourceLocation CC, bool *ICContext = 0) {
   if (E->isTypeDependent() || E->isValueDependent()) return;
 
   const Type *Source = S.Context.getCanonicalType(E->getType()).getTypePtr();
@@ -2547,6 +2746,13 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   if (Source == Target) return;
   if (Target->isDependentType()) return;
 
+  // If the conversion context location is invalid or instantiated
+  // from a system macro, don't complain.
+  if (CC.isInvalid() ||
+      (CC.isMacroID() && S.Context.getSourceManager().isInSystemHeader(
+                           S.Context.getSourceManager().getSpellingLoc(CC))))
+    return;
+
   // Never diagnose implicit casts to bool.
   if (Target->isSpecificBuiltinType(BuiltinType::Bool))
     return;
@@ -2554,7 +2760,7 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   // Strip vector types.
   if (isa<VectorType>(Source)) {
     if (!isa<VectorType>(Target))
-      return DiagnoseImpCast(S, E, T, diag::warn_impcast_vector_scalar);
+      return DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_vector_scalar);
 
     Source = cast<VectorType>(Source)->getElementType().getTypePtr();
     Target = cast<VectorType>(Target)->getElementType().getTypePtr();
@@ -2563,7 +2769,7 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   // Strip complex types.
   if (isa<ComplexType>(Source)) {
     if (!isa<ComplexType>(Target))
-      return DiagnoseImpCast(S, E, T, diag::warn_impcast_complex_scalar);
+      return DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_complex_scalar);
 
     Source = cast<ComplexType>(Source)->getElementType().getTypePtr();
     Target = cast<ComplexType>(Target)->getElementType().getTypePtr();
@@ -2591,15 +2797,21 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
             return;
         }
 
-        DiagnoseImpCast(S, E, T, diag::warn_impcast_float_precision);
+        DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_float_precision);
       }
       return;
     }
 
     // If the target is integral, always warn.
-    if ((TargetBT && TargetBT->isInteger()))
-      // TODO: don't warn for integer values?
-      DiagnoseImpCast(S, E, T, diag::warn_impcast_float_integer);
+    if ((TargetBT && TargetBT->isInteger())) {
+      Expr *InnerE = E->IgnoreParenImpCasts();
+      if (FloatingLiteral *LiteralExpr = dyn_cast<FloatingLiteral>(InnerE)) {
+        DiagnoseImpCast(S, LiteralExpr, T, CC,
+                        diag::warn_impcast_literal_float_to_integer);
+      } else {
+        DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_float_integer);
+      }
+    }
 
     return;
   }
@@ -2608,14 +2820,27 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
     return;
 
   IntRange SourceRange = GetExprRange(S.Context, E);
-  IntRange TargetRange = IntRange::forCanonicalType(S.Context, Target);
+  IntRange TargetRange = IntRange::forTargetOfCanonicalType(S.Context, Target);
 
   if (SourceRange.Width > TargetRange.Width) {
+    // If the source is a constant, use a default-on diagnostic.
+    // TODO: this should happen for bitfield stores, too.
+    llvm::APSInt Value(32);
+    if (E->isIntegerConstantExpr(Value, S.Context)) {
+      std::string PrettySourceValue = Value.toString(10);
+      std::string PrettyTargetValue = PrettyPrintInRange(Value, TargetRange);
+
+      S.Diag(E->getExprLoc(), diag::warn_impcast_integer_precision_constant)
+        << PrettySourceValue << PrettyTargetValue
+        << E->getType() << T << E->getSourceRange() << clang::SourceRange(CC);
+      return;
+    }
+
     // People want to build with -Wshorten-64-to-32 and not -Wconversion
     // and by god we'll let them.
     if (SourceRange.Width == 64 && TargetRange.Width == 32)
-      return DiagnoseImpCast(S, E, T, diag::warn_impcast_integer_64_32);
-    return DiagnoseImpCast(S, E, T, diag::warn_impcast_integer_precision);
+      return DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_integer_64_32);
+    return DiagnoseImpCast(S, E, T, CC, diag::warn_impcast_integer_precision);
   }
 
   if ((TargetRange.NonNegative && !SourceRange.NonNegative) ||
@@ -2633,7 +2858,7 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
       *ICContext = true;
     }
 
-    return DiagnoseImpCast(S, E, T, DiagID);
+    return DiagnoseImpCast(S, E, T, CC, DiagID);
   }
 
   return;
@@ -2642,35 +2867,38 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
 void CheckConditionalOperator(Sema &S, ConditionalOperator *E, QualType T);
 
 void CheckConditionalOperand(Sema &S, Expr *E, QualType T,
-                             bool &ICContext) {
+                             SourceLocation CC, bool &ICContext) {
   E = E->IgnoreParenImpCasts();
 
   if (isa<ConditionalOperator>(E))
     return CheckConditionalOperator(S, cast<ConditionalOperator>(E), T);
 
-  AnalyzeImplicitConversions(S, E);
+  AnalyzeImplicitConversions(S, E, CC);
   if (E->getType() != T)
-    return CheckImplicitConversion(S, E, T, &ICContext);
+    return CheckImplicitConversion(S, E, T, CC, &ICContext);
   return;
 }
 
 void CheckConditionalOperator(Sema &S, ConditionalOperator *E, QualType T) {
-  AnalyzeImplicitConversions(S, E->getCond());
+  SourceLocation CC = E->getQuestionLoc();
+
+  AnalyzeImplicitConversions(S, E->getCond(), CC);
 
   bool Suspicious = false;
-  CheckConditionalOperand(S, E->getTrueExpr(), T, Suspicious);
-  CheckConditionalOperand(S, E->getFalseExpr(), T, Suspicious);
+  CheckConditionalOperand(S, E->getTrueExpr(), T, CC, Suspicious);
+  CheckConditionalOperand(S, E->getFalseExpr(), T, CC, Suspicious);
 
   // If -Wconversion would have warned about either of the candidates
   // for a signedness conversion to the context type...
   if (!Suspicious) return;
 
   // ...but it's currently ignored...
-  if (S.Diags.getDiagnosticLevel(diag::warn_impcast_integer_sign_conditional))
+  if (S.Diags.getDiagnosticLevel(diag::warn_impcast_integer_sign_conditional,
+                                 CC))
     return;
 
   // ...and -Wsign-compare isn't...
-  if (!S.Diags.getDiagnosticLevel(diag::warn_mixed_sign_conditional))
+  if (!S.Diags.getDiagnosticLevel(diag::warn_mixed_sign_conditional, CC))
     return;
 
   // ...then check whether it would have warned about either of the
@@ -2678,10 +2906,10 @@ void CheckConditionalOperator(Sema &S, ConditionalOperator *E, QualType T) {
   if (E->getType() != T) {
     Suspicious = false;
     CheckImplicitConversion(S, E->getTrueExpr()->IgnoreParenImpCasts(),
-                            E->getType(), &Suspicious);
+                            E->getType(), CC, &Suspicious);
     if (!Suspicious)
       CheckImplicitConversion(S, E->getFalseExpr()->IgnoreParenImpCasts(),
-                              E->getType(), &Suspicious);
+                              E->getType(), CC, &Suspicious);
     if (!Suspicious)
       return;
   }
@@ -2697,7 +2925,7 @@ void CheckConditionalOperator(Sema &S, ConditionalOperator *E, QualType T) {
 /// AnalyzeImplicitConversions - Find and report any interesting
 /// implicit conversions in the given expression.  There are a couple
 /// of competing diagnostics here, -Wconversion and -Wsign-compare.
-void AnalyzeImplicitConversions(Sema &S, Expr *OrigE) {
+void AnalyzeImplicitConversions(Sema &S, Expr *OrigE, SourceLocation CC) {
   QualType T = OrigE->getType();
   Expr *E = OrigE->IgnoreParenImpCasts();
 
@@ -2713,19 +2941,25 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE) {
   // The non-canonical typecheck is just an optimization;
   // CheckImplicitConversion will filter out dead implicit conversions.
   if (E->getType() != T)
-    CheckImplicitConversion(S, E, T);
+    CheckImplicitConversion(S, E, T, CC);
 
   // Now continue drilling into this expression.
 
   // Skip past explicit casts.
   if (isa<ExplicitCastExpr>(E)) {
     E = cast<ExplicitCastExpr>(E)->getSubExpr()->IgnoreParenImpCasts();
-    return AnalyzeImplicitConversions(S, E);
+    return AnalyzeImplicitConversions(S, E, CC);
   }
 
-  // Do a somewhat different check with comparison operators.
-  if (isa<BinaryOperator>(E) && cast<BinaryOperator>(E)->isComparisonOp())
-    return AnalyzeComparison(S, cast<BinaryOperator>(E));
+  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
+    // Do a somewhat different check with comparison operators.
+    if (BO->isComparisonOp())
+      return AnalyzeComparison(S, BO);
+
+    // And with assignments and compound assignments.
+    if (BO->isAssignmentOp())
+      return AnalyzeAssignment(S, BO);
+  }
 
   // These break the otherwise-useful invariant below.  Fortunately,
   // we don't really need to recurse into them, because any internal
@@ -2737,9 +2971,9 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE) {
   if (isa<SizeOfAlignOfExpr>(E)) return;
 
   // Now just recurse over the expression's children.
-  for (Stmt::child_iterator I = E->child_begin(), IE = E->child_end();
-         I != IE; ++I)
-    AnalyzeImplicitConversions(S, cast<Expr>(*I));
+  CC = E->getExprLoc();
+  for (Stmt::child_range I = E->children(); I; ++I)
+    AnalyzeImplicitConversions(S, cast<Expr>(*I), CC);
 }
 
 } // end anonymous namespace
@@ -2747,7 +2981,11 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE) {
 /// Diagnoses "dangerous" implicit conversions within the given
 /// expression (which is a full expression).  Implements -Wconversion
 /// and -Wsign-compare.
-void Sema::CheckImplicitConversions(Expr *E) {
+///
+/// \param CC the "context" location of the implicit conversion, i.e.
+///   the most location of the syntactic entity requiring the implicit
+///   conversion
+void Sema::CheckImplicitConversions(Expr *E, SourceLocation CC) {
   // Don't diagnose in unevaluated contexts.
   if (ExprEvalContexts.back().Context == Sema::Unevaluated)
     return;
@@ -2756,7 +2994,14 @@ void Sema::CheckImplicitConversions(Expr *E) {
   if (E->isTypeDependent() || E->isValueDependent())
     return;
 
-  AnalyzeImplicitConversions(*this, E);
+  // This is not the right CC for (e.g.) a variable initialization.
+  AnalyzeImplicitConversions(*this, E, CC);
+}
+
+void Sema::CheckBitFieldInitialization(SourceLocation InitLoc,
+                                       FieldDecl *BitField,
+                                       Expr *Init) {
+  (void) AnalyzeBitFieldAssignment(*this, BitField, Init, InitLoc);
 }
 
 /// CheckParmsForFunctionDef - Check that the parameters of the given
@@ -2764,11 +3009,12 @@ void Sema::CheckImplicitConversions(Expr *E) {
 /// takes care of any checks that cannot be performed on the
 /// declaration itself, e.g., that the types of each of the function
 /// parameters are complete.
-bool Sema::CheckParmsForFunctionDef(FunctionDecl *FD) {
+bool Sema::CheckParmsForFunctionDef(ParmVarDecl **P, ParmVarDecl **PEnd,
+                                    bool CheckParameterNames) {
   bool HasInvalidParm = false;
-  for (unsigned p = 0, NumParams = FD->getNumParams(); p < NumParams; ++p) {
-    ParmVarDecl *Param = FD->getParamDecl(p);
-
+  for (; P != PEnd; ++P) {
+    ParmVarDecl *Param = *P;
+    
     // C99 6.7.5.3p4: the parameters in a parameter type list in a
     // function declarator that is part of a function definition of
     // that function shall not have incomplete type.
@@ -2783,7 +3029,8 @@ bool Sema::CheckParmsForFunctionDef(FunctionDecl *FD) {
 
     // C99 6.9.1p5: If the declarator includes a parameter type list, the
     // declaration of each parameter shall include an identifier.
-    if (Param->getIdentifier() == 0 &&
+    if (CheckParameterNames &&
+        Param->getIdentifier() == 0 &&
         !Param->isImplicit() &&
         !getLangOptions().CPlusPlus)
       Diag(Param->getLocation(), diag::err_parameter_name_omitted);
@@ -2811,7 +3058,8 @@ bool Sema::CheckParmsForFunctionDef(FunctionDecl *FD) {
 void Sema::CheckCastAlign(Expr *Op, QualType T, SourceRange TRange) {
   // This is actually a lot of work to potentially be doing on every
   // cast; don't do it if we're ignoring -Wcast_align (as is the default).
-  if (getDiagnostics().getDiagnosticLevel(diag::warn_cast_align)
+  if (getDiagnostics().getDiagnosticLevel(diag::warn_cast_align,
+                                          TRange.getBegin())
         == Diagnostic::Ignored)
     return;
 
@@ -2850,3 +3098,47 @@ void Sema::CheckCastAlign(Expr *Op, QualType T, SourceRange TRange) {
     << TRange << Op->getSourceRange();
 }
 
+void Sema::CheckArrayAccess(const clang::ArraySubscriptExpr *E) {
+  const Expr *BaseExpr = E->getBase()->IgnoreParenImpCasts();
+  const ConstantArrayType *ArrayTy =
+    Context.getAsConstantArrayType(BaseExpr->getType());
+  if (!ArrayTy)
+    return;
+
+  const Expr *IndexExpr = E->getIdx();
+  if (IndexExpr->isValueDependent())
+    return;
+  llvm::APSInt index;
+  if (!IndexExpr->isIntegerConstantExpr(index, Context))
+    return;
+
+  if (!index.isNegative()) {
+    llvm::APInt size = ArrayTy->getSize();
+    if (!size.isStrictlyPositive())
+      return;
+    if (size.getBitWidth() > index.getBitWidth())
+      index = index.sext(size.getBitWidth());
+    else if (size.getBitWidth() < index.getBitWidth())
+      size = size.sext(index.getBitWidth());
+
+    if (index.slt(size))
+      return;
+
+    Diag(E->getBase()->getLocStart(), diag::warn_array_index_exceeds_bounds)
+      << index.toString(10, true) << size.toString(10, true)
+      << IndexExpr->getSourceRange();
+  } else {
+    Diag(E->getBase()->getLocStart(), diag::warn_array_index_precedes_bounds)
+      << index.toString(10, true) << IndexExpr->getSourceRange();
+  }
+
+  const NamedDecl *ND = NULL;
+  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BaseExpr))
+    ND = dyn_cast<NamedDecl>(DRE->getDecl());
+  if (const MemberExpr *ME = dyn_cast<MemberExpr>(BaseExpr))
+    ND = dyn_cast<NamedDecl>(ME->getMemberDecl());
+  if (ND)
+    Diag(ND->getLocStart(), diag::note_array_index_out_of_bounds)
+      << ND->getDeclName();
+}
+
diff --git a/lib/Sema/SemaCodeComplete.cpp b/lib/Sema/SemaCodeComplete.cpp
index f00d1cd20f1e..bab665a38da7 100644
--- a/lib/Sema/SemaCodeComplete.cpp
+++ b/lib/Sema/SemaCodeComplete.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -116,6 +117,9 @@ namespace {
     /// \brief The semantic analysis object for which results are being 
     /// produced.
     Sema &SemaRef;
+
+    /// \brief The allocator used to allocate new code-completion strings.
+    CodeCompletionAllocator &Allocator;
     
     /// \brief If non-NULL, a filter function used to remove any code-completion
     /// results that are not desirable.
@@ -146,12 +150,44 @@ namespace {
     /// \brief The selector that we prefer.
     Selector PreferredSelector;
     
-    void AdjustResultPriorityForPreferredType(Result &R);
+    /// \brief The completion context in which we are gathering results.
+    CodeCompletionContext CompletionContext;
+    
+    /// \brief If we are in an instance method definition, the @implementation
+    /// object.
+    ObjCImplementationDecl *ObjCImplementation;
+    
+    void AdjustResultPriorityForDecl(Result &R);
 
+    void MaybeAddConstructorResults(Result R);
+    
   public:
-    explicit ResultBuilder(Sema &SemaRef, LookupFilter Filter = 0)
-      : SemaRef(SemaRef), Filter(Filter), AllowNestedNameSpecifiers(false),
-        HasObjectTypeQualifiers(false) { }
+    explicit ResultBuilder(Sema &SemaRef, CodeCompletionAllocator &Allocator,
+                           const CodeCompletionContext &CompletionContext,
+                           LookupFilter Filter = 0)
+      : SemaRef(SemaRef), Allocator(Allocator), Filter(Filter), 
+        AllowNestedNameSpecifiers(false), HasObjectTypeQualifiers(false), 
+        CompletionContext(CompletionContext),
+        ObjCImplementation(0) 
+    { 
+      // If this is an Objective-C instance method definition, dig out the 
+      // corresponding implementation.
+      switch (CompletionContext.getKind()) {
+      case CodeCompletionContext::CCC_Expression:
+      case CodeCompletionContext::CCC_ObjCMessageReceiver:
+      case CodeCompletionContext::CCC_ParenthesizedExpression:
+      case CodeCompletionContext::CCC_Statement:
+      case CodeCompletionContext::CCC_Recovery:
+        if (ObjCMethodDecl *Method = SemaRef.getCurMethodDecl())
+          if (Method->isInstanceMethod())
+            if (ObjCInterfaceDecl *Interface = Method->getClassInterface())
+              ObjCImplementation = Interface->getImplementation();
+        break;
+          
+      default:
+        break;
+      }
+    }
     
     /// \brief Whether we should include code patterns in the completion
     /// results.
@@ -165,10 +201,6 @@ namespace {
       this->Filter = Filter;
     }
     
-    typedef std::vector<Result>::iterator iterator;
-    iterator begin() { return Results.begin(); }
-    iterator end() { return Results.end(); }
-    
     Result *data() { return Results.empty()? 0 : &Results.front(); }
     unsigned size() const { return Results.size(); }
     bool empty() const { return Results.empty(); }
@@ -198,12 +230,25 @@ namespace {
     void setPreferredSelector(Selector Sel) {
       PreferredSelector = Sel;
     }
+        
+    /// \brief Retrieve the code-completion context for which results are
+    /// being collected.
+    const CodeCompletionContext &getCompletionContext() const { 
+      return CompletionContext; 
+    }
     
     /// \brief Specify whether nested-name-specifiers are allowed.
     void allowNestedNameSpecifiers(bool Allow = true) {
       AllowNestedNameSpecifiers = Allow;
     }
 
+    /// \brief Return the semantic analysis object for which we are collecting
+    /// code completion results.
+    Sema &getSema() const { return SemaRef; }
+    
+    /// \brief Retrieve the allocator used to allocate code completion strings.
+    CodeCompletionAllocator &getAllocator() const { return Allocator; }
+    
     /// \brief Determine whether the given declaration is at all interesting
     /// as a code-completion result.
     ///
@@ -278,6 +323,7 @@ namespace {
     bool IsObjCIvar(NamedDecl *ND) const;
     bool IsObjCMessageReceiver(NamedDecl *ND) const;
     bool IsObjCCollection(NamedDecl *ND) const;
+    bool IsImpossibleToSatisfy(NamedDecl *ND) const;
     //@}    
   };  
 }
@@ -324,11 +370,11 @@ public:
     return *this;
   }
 
-  iterator operator++(int) {
+  /*iterator operator++(int) {
     iterator tmp(*this);
     ++(*this);
     return tmp;
-  }
+  }*/
 
   reference operator*() const {
     if (NamedDecl *ND = DeclOrIterator.dyn_cast<NamedDecl *>())
@@ -464,15 +510,20 @@ bool ResultBuilder::isInterestingDecl(NamedDecl *ND,
         return false;
     }
   }
- 
-  // C++ constructors are never found by name lookup.
-  if (isa<CXXConstructorDecl>(ND))
+   
+  // Skip out-of-line declarations and definitions.
+  // NOTE: Unless it's an Objective-C property, method, or ivar, where
+  // the contexts can be messy.
+  if (!ND->getDeclContext()->Equals(ND->getLexicalDeclContext()) &&
+      !(isa<ObjCPropertyDecl>(ND) || isa<ObjCIvarDecl>(ND) ||
+        isa<ObjCMethodDecl>(ND)))
     return false;
-  
+
   if (Filter == &ResultBuilder::IsNestedNameSpecifier ||
       ((isa<NamespaceDecl>(ND) || isa<NamespaceAliasDecl>(ND)) &&
        Filter != &ResultBuilder::IsNamespace &&
-       Filter != &ResultBuilder::IsNamespaceOrAlias))
+       Filter != &ResultBuilder::IsNamespaceOrAlias &&
+       Filter != 0))
     AsNestedNameSpecifier = true;
 
   // Filter out any unwanted results.
@@ -535,7 +586,6 @@ SimplifiedTypeClass clang::getSimplifiedTypeClass(CanQualType T) {
         
       case BuiltinType::Overload:
       case BuiltinType::Dependent:
-      case BuiltinType::UndeducedAuto:
         return STC_Other;
         
       case BuiltinType::ObjCId:
@@ -621,23 +671,66 @@ QualType clang::getDeclUsageType(ASTContext &C, NamedDecl *ND) {
   return T.getNonReferenceType();
 }
 
-void ResultBuilder::AdjustResultPriorityForPreferredType(Result &R) {
-  QualType T = getDeclUsageType(SemaRef.Context, R.Declaration);
-  if (T.isNull())
-    return;
+void ResultBuilder::AdjustResultPriorityForDecl(Result &R) {
+  // If this is an Objective-C method declaration whose selector matches our
+  // preferred selector, give it a priority boost.
+  if (!PreferredSelector.isNull())
+    if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(R.Declaration))
+      if (PreferredSelector == Method->getSelector())
+        R.Priority += CCD_SelectorMatch;
   
-  CanQualType TC = SemaRef.Context.getCanonicalType(T);
-  // Check for exactly-matching types (modulo qualifiers).
-  if (SemaRef.Context.hasSameUnqualifiedType(PreferredType, TC)) {
-    if (PreferredType->isVoidType())
-      R.Priority += CCD_VoidMatch;
-    else
-      R.Priority /= CCF_ExactTypeMatch;
-  } // Check for nearly-matching types, based on classification of each.
-  else if ((getSimplifiedTypeClass(PreferredType)
+  // If we have a preferred type, adjust the priority for results with exactly-
+  // matching or nearly-matching types.
+  if (!PreferredType.isNull()) {
+    QualType T = getDeclUsageType(SemaRef.Context, R.Declaration);
+    if (!T.isNull()) {
+      CanQualType TC = SemaRef.Context.getCanonicalType(T);
+      // Check for exactly-matching types (modulo qualifiers).
+      if (SemaRef.Context.hasSameUnqualifiedType(PreferredType, TC))
+        R.Priority /= CCF_ExactTypeMatch;
+      // Check for nearly-matching types, based on classification of each.
+      else if ((getSimplifiedTypeClass(PreferredType)
                                                == getSimplifiedTypeClass(TC)) &&
-           !(PreferredType->isEnumeralType() && TC->isEnumeralType()))
-    R.Priority /= CCF_SimilarTypeMatch;  
+               !(PreferredType->isEnumeralType() && TC->isEnumeralType()))
+        R.Priority /= CCF_SimilarTypeMatch;  
+    }
+  }  
+}
+
+void ResultBuilder::MaybeAddConstructorResults(Result R) {
+  if (!SemaRef.getLangOptions().CPlusPlus || !R.Declaration ||
+      !CompletionContext.wantConstructorResults())
+    return;
+  
+  ASTContext &Context = SemaRef.Context;
+  NamedDecl *D = R.Declaration;
+  CXXRecordDecl *Record = 0;
+  if (ClassTemplateDecl *ClassTemplate = dyn_cast<ClassTemplateDecl>(D))
+    Record = ClassTemplate->getTemplatedDecl();
+  else if ((Record = dyn_cast<CXXRecordDecl>(D))) {
+    // Skip specializations and partial specializations.
+    if (isa<ClassTemplateSpecializationDecl>(Record))
+      return;
+  } else {
+    // There are no constructors here.
+    return;
+  }
+  
+  Record = Record->getDefinition();
+  if (!Record)
+    return;
+
+  
+  QualType RecordTy = Context.getTypeDeclType(Record);
+  DeclarationName ConstructorName
+    = Context.DeclarationNames.getCXXConstructorName(
+                                           Context.getCanonicalType(RecordTy));
+  for (DeclContext::lookup_result Ctors = Record->lookup(ConstructorName);
+       Ctors.first != Ctors.second; ++Ctors.first) {
+    R.Declaration = *Ctors.first;
+    R.CursorKind = getCursorKindForDecl(R.Declaration);
+    Results.push_back(R);
+  }
 }
 
 void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
@@ -662,6 +755,10 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
   if (!isInterestingDecl(R.Declaration, AsNestedNameSpecifier))
     return;
       
+  // C++ constructors are never found by name lookup.
+  if (isa<CXXConstructorDecl>(R.Declaration))
+    return;
+
   ShadowMap &SMap = ShadowMaps.back();
   ShadowMapEntry::iterator I, IEnd;
   ShadowMap::iterator NamePos = SMap.find(R.Declaration->getDeclName());
@@ -719,20 +816,13 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
   if (!AllDeclsFound.insert(CanonDecl))
     return;
 
-  // If this is an Objective-C method declaration whose selector matches our
-  // preferred selector, give it a priority boost.
-  if (!PreferredSelector.isNull())
-    if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(R.Declaration))
-      if (PreferredSelector == Method->getSelector())
-        R.Priority += CCD_SelectorMatch;
-
   // If the filter is for nested-name-specifiers, then this result starts a
   // nested-name-specifier.
   if (AsNestedNameSpecifier) {
     R.StartsNestedNameSpecifier = true;
     R.Priority = CCP_NestedNameSpecifier;
-  } else if (!PreferredType.isNull())
-      AdjustResultPriorityForPreferredType(R);
+  } else 
+      AdjustResultPriorityForDecl(R);
       
   // If this result is supposed to have an informative qualifier, add one.
   if (R.QualifierIsInformative && !R.Qualifier &&
@@ -751,6 +841,9 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
   // map.
   SMap[R.Declaration->getDeclName()].Add(R.Declaration, Results.size());
   Results.push_back(R);
+  
+  if (!AsNestedNameSpecifier)
+    MaybeAddConstructorResults(R);
 }
 
 void ResultBuilder::AddResult(Result R, DeclContext *CurContext, 
@@ -771,6 +864,10 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
   if (!isInterestingDecl(R.Declaration, AsNestedNameSpecifier))
     return;
   
+  // C++ constructors are never found by name lookup.
+  if (isa<CXXConstructorDecl>(R.Declaration))
+    return;
+
   if (Hiding && CheckHiddenResult(R, CurContext, Hiding))
     return;
       
@@ -806,15 +903,7 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
   if (InBaseClass)
     R.Priority += CCD_InBaseClass;
   
-  // If this is an Objective-C method declaration whose selector matches our
-  // preferred selector, give it a priority boost.
-  if (!PreferredSelector.isNull())
-    if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(R.Declaration))
-      if (PreferredSelector == Method->getSelector())
-        R.Priority += CCD_SelectorMatch;
-
-  if (!PreferredType.isNull())
-    AdjustResultPriorityForPreferredType(R);
+  AdjustResultPriorityForDecl(R);
   
   if (HasObjectTypeQualifiers)
     if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(R.Declaration))
@@ -832,6 +921,9 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
   
   // Insert this result into the set of results.
   Results.push_back(R);
+  
+  if (!AsNestedNameSpecifier)
+    MaybeAddConstructorResults(R);
 }
 
 void ResultBuilder::AddResult(Result R) {
@@ -864,9 +956,14 @@ bool ResultBuilder::IsOrdinaryName(NamedDecl *ND) const {
   unsigned IDNS = Decl::IDNS_Ordinary;
   if (SemaRef.getLangOptions().CPlusPlus)
     IDNS |= Decl::IDNS_Tag | Decl::IDNS_Namespace | Decl::IDNS_Member;
-  else if (SemaRef.getLangOptions().ObjC1 && isa<ObjCIvarDecl>(ND))
-    return true;
-
+  else if (SemaRef.getLangOptions().ObjC1) {
+    if (isa<ObjCIvarDecl>(ND))
+      return true;
+    if (isa<ObjCPropertyDecl>(ND) &&
+        SemaRef.canSynthesizeProvisionalIvar(cast<ObjCPropertyDecl>(ND)))
+      return true;
+  }
+  
   return ND->getIdentifierNamespace() & IDNS;
 }
 
@@ -880,9 +977,14 @@ bool ResultBuilder::IsOrdinaryNonTypeName(NamedDecl *ND) const {
   unsigned IDNS = Decl::IDNS_Ordinary;
   if (SemaRef.getLangOptions().CPlusPlus)
     IDNS |= Decl::IDNS_Tag | Decl::IDNS_Namespace | Decl::IDNS_Member;
-  else if (SemaRef.getLangOptions().ObjC1 && isa<ObjCIvarDecl>(ND))
-    return true;
-  
+  else if (SemaRef.getLangOptions().ObjC1) {
+    if (isa<ObjCIvarDecl>(ND))
+      return true;
+    if (isa<ObjCPropertyDecl>(ND) &&
+        SemaRef.canSynthesizeProvisionalIvar(cast<ObjCPropertyDecl>(ND)))
+      return true;
+  }
+ 
   return ND->getIdentifierNamespace() & IDNS;
 }
 
@@ -1038,6 +1140,10 @@ bool ResultBuilder::IsObjCCollection(NamedDecl *ND) const {
          (SemaRef.getLangOptions().CPlusPlus && T->isRecordType());
 }
 
+bool ResultBuilder::IsImpossibleToSatisfy(NamedDecl *ND) const {
+  return false;
+}
+
 /// \rief Determines whether the given declaration is an Objective-C
 /// instance variable.
 bool ResultBuilder::IsObjCIvar(NamedDecl *ND) const {
@@ -1088,32 +1194,32 @@ static void AddTypeSpecifierResults(const LangOptions &LangOpts,
     Results.AddResult(Result("restrict", CCP_Type));
   }
   
+  CodeCompletionBuilder Builder(Results.getAllocator());
   if (LangOpts.CPlusPlus) {
     // C++-specific
-    Results.AddResult(Result("bool", CCP_Type));
+    Results.AddResult(Result("bool", CCP_Type + 
+                             (LangOpts.ObjC1? CCD_bool_in_ObjC : 0)));
     Results.AddResult(Result("class", CCP_Type));
     Results.AddResult(Result("wchar_t", CCP_Type));
     
     // typename qualified-id
-    CodeCompletionString *Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("typename");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("qualifier");
-    Pattern->AddTextChunk("::");
-    Pattern->AddPlaceholderChunk("name");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk("typename");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("qualifier");
+    Builder.AddTextChunk("::");
+    Builder.AddPlaceholderChunk("name");
+    Results.AddResult(Result(Builder.TakeString()));
     
     if (LangOpts.CPlusPlus0x) {
       Results.AddResult(Result("auto", CCP_Type));
       Results.AddResult(Result("char16_t", CCP_Type));
       Results.AddResult(Result("char32_t", CCP_Type));
       
-      CodeCompletionString *Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("decltype");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("decltype");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));
     }
   }
   
@@ -1124,18 +1230,16 @@ static void AddTypeSpecifierResults(const LangOptions &LangOpts,
     //    Results.AddResult(Result("_Decimal64"));
     //    Results.AddResult(Result("_Decimal128"));
     
-    CodeCompletionString *Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("typeof");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("expression");
-    Results.AddResult(Result(Pattern));
-
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("typeof");
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("type");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk("typeof");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("expression");
+    Results.AddResult(Result(Builder.TakeString()));
+
+    Builder.AddTypedTextChunk("typeof");
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("type");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(Result(Builder.TakeString()));
   }
 }
 
@@ -1180,6 +1284,8 @@ static void AddFunctionSpecifiers(Sema::ParserCompletionContext CCC,
   case Sema::PCC_Condition:
   case Sema::PCC_RecoveryInFunction:
   case Sema::PCC_Type:
+  case Sema::PCC_ParenthesizedExpression:
+  case Sema::PCC_LocalDeclarationSpecifiers:
     break;
   }
 }
@@ -1198,20 +1304,17 @@ static void AddObjCInterfaceResults(const LangOptions &LangOpts,
 static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt);
 
 static void AddTypedefResult(ResultBuilder &Results) {
-  CodeCompletionString *Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("typedef");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("type");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("name");
-  Results.AddResult(CodeCompletionResult(Pattern));        
+  CodeCompletionBuilder Builder(Results.getAllocator());
+  Builder.AddTypedTextChunk("typedef");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("type");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("name");
+  Results.AddResult(CodeCompletionResult(Builder.TakeString()));        
 }
 
 static bool WantTypesInContext(Sema::ParserCompletionContext CCC,
                                const LangOptions &LangOpts) {
-  if (LangOpts.CPlusPlus)
-    return true;
-  
   switch (CCC) {
   case Sema::PCC_Namespace:
   case Sema::PCC_Class:
@@ -1221,16 +1324,20 @@ static bool WantTypesInContext(Sema::ParserCompletionContext CCC,
   case Sema::PCC_Statement:
   case Sema::PCC_RecoveryInFunction:
   case Sema::PCC_Type:
+  case Sema::PCC_ParenthesizedExpression:
+  case Sema::PCC_LocalDeclarationSpecifiers:
     return true;
     
-  case Sema::PCC_ObjCInterface:
-  case Sema::PCC_ObjCImplementation:
   case Sema::PCC_Expression:
   case Sema::PCC_Condition:
+    return LangOpts.CPlusPlus;
+      
+  case Sema::PCC_ObjCInterface:
+  case Sema::PCC_ObjCImplementation:
     return false;
     
   case Sema::PCC_ForInit:
-    return LangOpts.ObjC1 || LangOpts.C99;
+    return LangOpts.CPlusPlus || LangOpts.ObjC1 || LangOpts.C99;
   }
   
   return false;
@@ -1241,58 +1348,53 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
                                    Scope *S,
                                    Sema &SemaRef,
                                    ResultBuilder &Results) {
+  CodeCompletionBuilder Builder(Results.getAllocator());
+  
   typedef CodeCompletionResult Result;
   switch (CCC) {
   case Sema::PCC_Namespace:
     if (SemaRef.getLangOptions().CPlusPlus) {
-      CodeCompletionString *Pattern = 0;
-      
       if (Results.includeCodePatterns()) {
         // namespace <identifier> { declarations }
-        CodeCompletionString *Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("namespace");
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddPlaceholderChunk("identifier");
-        Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-        Pattern->AddPlaceholderChunk("declarations");
-        Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-        Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("namespace");
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddPlaceholderChunk("identifier");
+        Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+        Builder.AddPlaceholderChunk("declarations");
+        Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+        Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+        Results.AddResult(Result(Builder.TakeString()));
       }
       
       // namespace identifier = identifier ;
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("namespace");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("name");
-      Pattern->AddChunk(CodeCompletionString::CK_Equal);
-      Pattern->AddPlaceholderChunk("namespace");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("namespace");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("name");
+      Builder.AddChunk(CodeCompletionString::CK_Equal);
+      Builder.AddPlaceholderChunk("namespace");
+      Results.AddResult(Result(Builder.TakeString()));
 
       // Using directives
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("using");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddTextChunk("namespace");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("identifier");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("using");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTextChunk("namespace");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("identifier");
+      Results.AddResult(Result(Builder.TakeString()));
 
       // asm(string-literal)      
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("asm");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("string-literal");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("asm");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("string-literal");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));
 
       if (Results.includeCodePatterns()) {
         // Explicit template instantiation
-        Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("template");
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddPlaceholderChunk("declaration");
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("template");
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddPlaceholderChunk("declaration");
+        Results.AddResult(Result(Builder.TakeString()));
       }
     }
       
@@ -1305,47 +1407,42 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
   case Sema::PCC_Class:
     if (SemaRef.getLangOptions().CPlusPlus) {
       // Using declaration
-      CodeCompletionString *Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("using");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("qualifier");
-      Pattern->AddTextChunk("::");
-      Pattern->AddPlaceholderChunk("name");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("using");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("qualifier");
+      Builder.AddTextChunk("::");
+      Builder.AddPlaceholderChunk("name");
+      Results.AddResult(Result(Builder.TakeString()));
       
       // using typename qualifier::name (only in a dependent context)
       if (SemaRef.CurContext->isDependentContext()) {
-        Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("using");
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddTextChunk("typename");
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddPlaceholderChunk("qualifier");
-        Pattern->AddTextChunk("::");
-        Pattern->AddPlaceholderChunk("name");
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("using");
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddTextChunk("typename");
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddPlaceholderChunk("qualifier");
+        Builder.AddTextChunk("::");
+        Builder.AddPlaceholderChunk("name");
+        Results.AddResult(Result(Builder.TakeString()));
       }
 
       if (CCC == Sema::PCC_Class) {
         AddTypedefResult(Results);
 
         // public:
-        Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("public");
-        Pattern->AddChunk(CodeCompletionString::CK_Colon);
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("public");
+        Builder.AddChunk(CodeCompletionString::CK_Colon);
+        Results.AddResult(Result(Builder.TakeString()));
 
         // protected:
-        Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("protected");
-        Pattern->AddChunk(CodeCompletionString::CK_Colon);
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("protected");
+        Builder.AddChunk(CodeCompletionString::CK_Colon);
+        Results.AddResult(Result(Builder.TakeString()));
 
         // private:
-        Pattern = new CodeCompletionString;
-        Pattern->AddTypedTextChunk("private");
-        Pattern->AddChunk(CodeCompletionString::CK_Colon);
-        Results.AddResult(Result(Pattern));
+        Builder.AddTypedTextChunk("private");
+        Builder.AddChunk(CodeCompletionString::CK_Colon);
+        Results.AddResult(Result(Builder.TakeString()));
       }
     }
     // Fall through
@@ -1354,12 +1451,11 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
   case Sema::PCC_MemberTemplate:
     if (SemaRef.getLangOptions().CPlusPlus && Results.includeCodePatterns()) {
       // template < parameters >
-      CodeCompletionString *Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("template");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftAngle);
-      Pattern->AddPlaceholderChunk("parameters");
-      Pattern->AddChunk(CodeCompletionString::CK_RightAngle);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("template");
+      Builder.AddChunk(CodeCompletionString::CK_LeftAngle);
+      Builder.AddPlaceholderChunk("parameters");
+      Builder.AddChunk(CodeCompletionString::CK_RightAngle);
+      Results.AddResult(Result(Builder.TakeString()));
     }
 
     AddStorageSpecifiers(CCC, SemaRef.getLangOptions(), Results);
@@ -1386,136 +1482,126 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
   case Sema::PCC_Statement: {
     AddTypedefResult(Results);
 
-    CodeCompletionString *Pattern = 0;
     if (SemaRef.getLangOptions().CPlusPlus && Results.includeCodePatterns()) {
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("try");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Pattern->AddTextChunk("catch");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("declaration");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("try");
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Builder.AddTextChunk("catch");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("declaration");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Results.AddResult(Result(Builder.TakeString()));
     }
     if (SemaRef.getLangOptions().ObjC1)
       AddObjCStatementResults(Results, true);
     
     if (Results.includeCodePatterns()) {
       // if (condition) { statements }
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("if");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTypedTextChunk("if");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
       if (SemaRef.getLangOptions().CPlusPlus)
-        Pattern->AddPlaceholderChunk("condition");
+        Builder.AddPlaceholderChunk("condition");
       else
-        Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Results.AddResult(Result(Pattern));
+        Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Results.AddResult(Result(Builder.TakeString()));
 
       // switch (condition) { }
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("switch");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTypedTextChunk("switch");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
       if (SemaRef.getLangOptions().CPlusPlus)
-        Pattern->AddPlaceholderChunk("condition");
+        Builder.AddPlaceholderChunk("condition");
       else
-        Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Results.AddResult(Result(Pattern));
+        Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Results.AddResult(Result(Builder.TakeString()));
     }
     
     // Switch-specific statements.
     if (!SemaRef.getCurFunction()->SwitchStack.empty()) {
       // case expression:
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("case");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_Colon);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("case");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_Colon);
+      Results.AddResult(Result(Builder.TakeString()));
 
       // default:
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("default");
-      Pattern->AddChunk(CodeCompletionString::CK_Colon);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("default");
+      Builder.AddChunk(CodeCompletionString::CK_Colon);
+      Results.AddResult(Result(Builder.TakeString()));
     }
 
     if (Results.includeCodePatterns()) {
       /// while (condition) { statements }
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("while");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTypedTextChunk("while");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
       if (SemaRef.getLangOptions().CPlusPlus)
-        Pattern->AddPlaceholderChunk("condition");
+        Builder.AddPlaceholderChunk("condition");
       else
-        Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Results.AddResult(Result(Pattern));
+        Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Results.AddResult(Result(Builder.TakeString()));
 
       // do { statements } while ( expression );
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("do");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Pattern->AddTextChunk("while");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("do");
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Builder.AddTextChunk("while");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));
 
       // for ( for-init-statement ; condition ; expression ) { statements }
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("for");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTypedTextChunk("for");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
       if (SemaRef.getLangOptions().CPlusPlus || SemaRef.getLangOptions().C99)
-        Pattern->AddPlaceholderChunk("init-statement");
+        Builder.AddPlaceholderChunk("init-statement");
       else
-        Pattern->AddPlaceholderChunk("init-expression");
-      Pattern->AddChunk(CodeCompletionString::CK_SemiColon);
-      Pattern->AddPlaceholderChunk("condition");
-      Pattern->AddChunk(CodeCompletionString::CK_SemiColon);
-      Pattern->AddPlaceholderChunk("inc-expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddPlaceholderChunk("statements");
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-      Results.AddResult(Result(Pattern));
+        Builder.AddPlaceholderChunk("init-expression");
+      Builder.AddChunk(CodeCompletionString::CK_SemiColon);
+      Builder.AddPlaceholderChunk("condition");
+      Builder.AddChunk(CodeCompletionString::CK_SemiColon);
+      Builder.AddPlaceholderChunk("inc-expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddPlaceholderChunk("statements");
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+      Results.AddResult(Result(Builder.TakeString()));
     }
     
     if (S->getContinueParent()) {
       // continue ;
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("continue");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("continue");
+      Results.AddResult(Result(Builder.TakeString()));
     }
 
     if (S->getBreakParent()) {
       // break ;
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("break");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("break");
+      Results.AddResult(Result(Builder.TakeString()));
     }
 
     // "return expression ;" or "return ;", depending on whether we
@@ -1529,29 +1615,26 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
     else if (SemaRef.getCurBlock() && 
              !SemaRef.getCurBlock()->ReturnType.isNull())
       isVoid = SemaRef.getCurBlock()->ReturnType->isVoidType();
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("return");
+    Builder.AddTypedTextChunk("return");
     if (!isVoid) {
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("expression");
     }
-    Results.AddResult(Result(Pattern));
+    Results.AddResult(Result(Builder.TakeString()));
 
     // goto identifier ;
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("goto");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("label");
-    Results.AddResult(Result(Pattern));    
+    Builder.AddTypedTextChunk("goto");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("label");
+    Results.AddResult(Result(Builder.TakeString()));    
 
     // Using directives
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("using");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddTextChunk("namespace");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("identifier");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk("using");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddTextChunk("namespace");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("identifier");
+    Results.AddResult(Result(Builder.TakeString()));
   }
 
   // Fall through (for statement expressions).
@@ -1560,8 +1643,8 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
     AddStorageSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     // Fall through: conditions and statements can have expressions.
 
+  case Sema::PCC_ParenthesizedExpression:
   case Sema::PCC_Expression: {
-    CodeCompletionString *Pattern = 0;
     if (SemaRef.getLangOptions().CPlusPlus) {
       // 'this', if we're in a non-static member function.
       if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(SemaRef.CurContext))
@@ -1573,103 +1656,93 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
       Results.AddResult(Result("false"));
 
       // dynamic_cast < type-id > ( expression )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("dynamic_cast");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftAngle);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_RightAngle);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("dynamic_cast");
+      Builder.AddChunk(CodeCompletionString::CK_LeftAngle);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_RightAngle);
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
       
       // static_cast < type-id > ( expression )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("static_cast");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftAngle);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_RightAngle);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("static_cast");
+      Builder.AddChunk(CodeCompletionString::CK_LeftAngle);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_RightAngle);
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // reinterpret_cast < type-id > ( expression )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("reinterpret_cast");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftAngle);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_RightAngle);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("reinterpret_cast");
+      Builder.AddChunk(CodeCompletionString::CK_LeftAngle);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_RightAngle);
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // const_cast < type-id > ( expression )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("const_cast");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftAngle);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_RightAngle);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("const_cast");
+      Builder.AddChunk(CodeCompletionString::CK_LeftAngle);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_RightAngle);
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // typeid ( expression-or-type )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("typeid");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expression-or-type");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("typeid");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expression-or-type");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // new T ( ... )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("new");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expressions");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("new");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expressions");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // new T [ ] ( ... )
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("new");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("type");
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBracket);
-      Pattern->AddPlaceholderChunk("size");
-      Pattern->AddChunk(CodeCompletionString::CK_RightBracket);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddPlaceholderChunk("expressions");
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("new");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("type");
+      Builder.AddChunk(CodeCompletionString::CK_LeftBracket);
+      Builder.AddPlaceholderChunk("size");
+      Builder.AddChunk(CodeCompletionString::CK_RightBracket);
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("expressions");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // delete expression
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("delete");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("expression");
-      Results.AddResult(Result(Pattern));      
+      Builder.AddTypedTextChunk("delete");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("expression");
+      Results.AddResult(Result(Builder.TakeString()));      
 
       // delete [] expression
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("delete");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBracket);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBracket);
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("expression");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("delete");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBracket);
+      Builder.AddChunk(CodeCompletionString::CK_RightBracket);
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("expression");
+      Results.AddResult(Result(Builder.TakeString()));
 
       // throw expression
-      Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk("throw");
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddPlaceholderChunk("expression");
-      Results.AddResult(Result(Pattern));
+      Builder.AddTypedTextChunk("throw");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddPlaceholderChunk("expression");
+      Results.AddResult(Result(Builder.TakeString()));
       
       // FIXME: Rethrow?
     }
@@ -1687,16 +1760,16 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
     }
 
     // sizeof expression
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("sizeof");
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("expression-or-type");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk("sizeof");
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("expression-or-type");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(Result(Builder.TakeString()));
     break;
   }
       
   case Sema::PCC_Type:
+  case Sema::PCC_LocalDeclarationSpecifiers:
     break;
   }
 
@@ -1707,16 +1780,56 @@ static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
     Results.AddResult(Result("operator"));
 }
 
+/// \brief Retrieve the string representation of the given type as a string
+/// that has the appropriate lifetime for code completion.
+///
+/// This routine provides a fast path where we provide constant strings for
+/// common type names.
+const char *GetCompletionTypeString(QualType T,
+                                    ASTContext &Context,
+                                    CodeCompletionAllocator &Allocator) {
+  PrintingPolicy Policy(Context.PrintingPolicy);
+  Policy.AnonymousTagLocations = false;
+
+  if (!T.getLocalQualifiers()) {
+    // Built-in type names are constant strings.
+    if (const BuiltinType *BT = dyn_cast<BuiltinType>(T))
+      return BT->getName(Context.getLangOptions());
+    
+    // Anonymous tag types are constant strings.
+    if (const TagType *TagT = dyn_cast<TagType>(T))
+      if (TagDecl *Tag = TagT->getDecl())
+        if (!Tag->getIdentifier() && !Tag->getTypedefForAnonDecl()) {
+          switch (Tag->getTagKind()) {
+          case TTK_Struct: return "struct <anonymous>";
+          case TTK_Class:  return "class <anonymous>";            
+          case TTK_Union:  return "union <anonymous>";
+          case TTK_Enum:   return "enum <anonymous>";
+          }
+        }
+  }
+  
+  // Slow path: format the type as a string.
+  std::string Result;
+  T.getAsStringInternal(Result, Policy);
+  return Allocator.CopyString(Result);
+}
+
 /// \brief If the given declaration has an associated type, add it as a result 
 /// type chunk.
 static void AddResultTypeChunk(ASTContext &Context,
                                NamedDecl *ND,
-                               CodeCompletionString *Result) {
+                               CodeCompletionBuilder &Result) {
   if (!ND)
     return;
-  
+
+  // Skip constructors and conversion functions, which have their return types
+  // built into their names.
+  if (isa<CXXConstructorDecl>(ND) || isa<CXXConversionDecl>(ND))
+    return;
+
   // Determine the type of the declaration (if it has a type).
-  QualType T;
+  QualType T;  
   if (FunctionDecl *Function = dyn_cast<FunctionDecl>(ND))
     T = Function->getResultType();
   else if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(ND))
@@ -1735,25 +1848,21 @@ static void AddResultTypeChunk(ASTContext &Context,
   if (T.isNull() || Context.hasSameType(T, Context.DependentTy))
     return;
   
-  PrintingPolicy Policy(Context.PrintingPolicy);
-  Policy.AnonymousTagLocations = false;
-  
-  std::string TypeStr;
-  T.getAsStringInternal(TypeStr, Policy);
-  Result->AddResultTypeChunk(TypeStr);
+  Result.AddResultTypeChunk(GetCompletionTypeString(T, Context, 
+                                                    Result.getAllocator()));
 }
 
 static void MaybeAddSentinel(ASTContext &Context, NamedDecl *FunctionOrMethod,
-                             CodeCompletionString *Result) {
+                             CodeCompletionBuilder &Result) {
   if (SentinelAttr *Sentinel = FunctionOrMethod->getAttr<SentinelAttr>())
     if (Sentinel->getSentinel() == 0) {
       if (Context.getLangOptions().ObjC1 &&
           Context.Idents.get("nil").hasMacroDefinition())
-        Result->AddTextChunk(", nil");
+        Result.AddTextChunk(", nil");
       else if (Context.Idents.get("NULL").hasMacroDefinition())
-        Result->AddTextChunk(", NULL");
+        Result.AddTextChunk(", NULL");
       else
-        Result->AddTextChunk(", (void*)0");
+        Result.AddTextChunk(", (void*)0");
     }
 }
 
@@ -1784,7 +1893,8 @@ static std::string FormatFunctionParameter(ASTContext &Context,
   
   // The argument for a block pointer parameter is a block literal with
   // the appropriate type.
-  FunctionProtoTypeLoc *Block = 0;
+  FunctionTypeLoc *Block = 0;
+  FunctionProtoTypeLoc *BlockProto = 0;
   TypeLoc TL;
   if (TypeSourceInfo *TSInfo = Param->getTypeSourceInfo()) {
     TL = TSInfo->getTypeLoc().getUnqualifiedLoc();
@@ -1808,8 +1918,9 @@ static std::string FormatFunctionParameter(ASTContext &Context,
       // then we're done.
       if (BlockPointerTypeLoc *BlockPtr
           = dyn_cast<BlockPointerTypeLoc>(&TL)) {
-        TL = BlockPtr->getPointeeLoc();
-        Block = dyn_cast<FunctionProtoTypeLoc>(&TL);
+        TL = BlockPtr->getPointeeLoc().IgnoreParens();
+        Block = dyn_cast<FunctionTypeLoc>(&TL);
+        BlockProto = dyn_cast<FunctionProtoTypeLoc>(&TL);
       }
       break;
     }
@@ -1834,47 +1945,65 @@ static std::string FormatFunctionParameter(ASTContext &Context,
   
   // We have the function prototype behind the block pointer type, as it was
   // written in the source.
-  std::string Result = "(^)(";
-  for (unsigned I = 0, N = Block->getNumArgs(); I != N; ++I) {
-    if (I)
-      Result += ", ";
-    Result += FormatFunctionParameter(Context, Block->getArg(I));
-    
-    if (I == N - 1 && Block->getTypePtr()->isVariadic())
-      Result += ", ...";
-  }
-  if (Block->getTypePtr()->isVariadic() && Block->getNumArgs() == 0)
-    Result += "...";
-  else if (Block->getNumArgs() == 0 && !Context.getLangOptions().CPlusPlus)
-    Result += "void";
-             
-  Result += ")";
-  Block->getTypePtr()->getResultType().getAsStringInternal(Result, 
-                                                        Context.PrintingPolicy);
+  std::string Result;
+  QualType ResultType = Block->getTypePtr()->getResultType();
+  if (!ResultType->isVoidType())
+    ResultType.getAsStringInternal(Result, Context.PrintingPolicy);
+  
+  Result = '^' + Result;
+  if (!BlockProto || Block->getNumArgs() == 0) {
+    if (BlockProto && BlockProto->getTypePtr()->isVariadic())
+      Result += "(...)";
+    else
+      Result += "(void)";
+  } else {
+    Result += "(";
+    for (unsigned I = 0, N = Block->getNumArgs(); I != N; ++I) {
+      if (I)
+        Result += ", ";
+      Result += FormatFunctionParameter(Context, Block->getArg(I));
+      
+      if (I == N - 1 && BlockProto->getTypePtr()->isVariadic())
+        Result += ", ...";
+    }
+    Result += ")";
+  }
+  
+  if (Param->getIdentifier())
+    Result += Param->getIdentifier()->getName();
+
   return Result;
 }
 
 /// \brief Add function parameter chunks to the given code completion string.
 static void AddFunctionParameterChunks(ASTContext &Context,
                                        FunctionDecl *Function,
-                                       CodeCompletionString *Result) {
+                                       CodeCompletionBuilder &Result,
+                                       unsigned Start = 0,
+                                       bool InOptional = false) {
   typedef CodeCompletionString::Chunk Chunk;
+  bool FirstParameter = true;
   
-  CodeCompletionString *CCStr = Result;
-  
-  for (unsigned P = 0, N = Function->getNumParams(); P != N; ++P) {
+  for (unsigned P = Start, N = Function->getNumParams(); P != N; ++P) {
     ParmVarDecl *Param = Function->getParamDecl(P);
     
-    if (Param->hasDefaultArg()) {
+    if (Param->hasDefaultArg() && !InOptional) {
       // When we see an optional default argument, put that argument and
       // the remaining default arguments into a new, optional string.
-      CodeCompletionString *Opt = new CodeCompletionString;
-      CCStr->AddOptionalChunk(std::auto_ptr<CodeCompletionString>(Opt));
-      CCStr = Opt;
+      CodeCompletionBuilder Opt(Result.getAllocator());
+      if (!FirstParameter)
+        Opt.AddChunk(Chunk(CodeCompletionString::CK_Comma));        
+      AddFunctionParameterChunks(Context, Function, Opt, P, true);
+      Result.AddOptionalChunk(Opt.TakeString());
+      break;
     }
     
-    if (P != 0)
-      CCStr->AddChunk(Chunk(CodeCompletionString::CK_Comma));
+    if (FirstParameter)
+      FirstParameter = false;
+    else
+      Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
+    
+    InOptional = false;
     
     // Format the placeholder string.
     std::string PlaceholderStr = FormatFunctionParameter(Context, Param);
@@ -1883,34 +2012,36 @@ static void AddFunctionParameterChunks(ASTContext &Context,
       PlaceholderStr += ", ...";
 
     // Add the placeholder string.
-    CCStr->AddPlaceholderChunk(PlaceholderStr);
+    Result.AddPlaceholderChunk(
+                             Result.getAllocator().CopyString(PlaceholderStr));
   }
   
   if (const FunctionProtoType *Proto 
         = Function->getType()->getAs<FunctionProtoType>())
     if (Proto->isVariadic()) {
       if (Proto->getNumArgs() == 0)
-        CCStr->AddPlaceholderChunk("...");
+        Result.AddPlaceholderChunk("...");
 
-      MaybeAddSentinel(Context, Function, CCStr);
+      MaybeAddSentinel(Context, Function, Result);
     }
 }
 
 /// \brief Add template parameter chunks to the given code completion string.
 static void AddTemplateParameterChunks(ASTContext &Context,
                                        TemplateDecl *Template,
-                                       CodeCompletionString *Result,
-                                       unsigned MaxParameters = 0) {
+                                       CodeCompletionBuilder &Result,
+                                       unsigned MaxParameters = 0,
+                                       unsigned Start = 0,
+                                       bool InDefaultArg = false) {
   typedef CodeCompletionString::Chunk Chunk;
-  
-  CodeCompletionString *CCStr = Result;
   bool FirstParameter = true;
   
   TemplateParameterList *Params = Template->getTemplateParameters();
   TemplateParameterList::iterator PEnd = Params->end();
   if (MaxParameters)
     PEnd = Params->begin() + MaxParameters;
-  for (TemplateParameterList::iterator P = Params->begin(); P != PEnd; ++P) {
+  for (TemplateParameterList::iterator P = Params->begin() + Start; 
+       P != PEnd; ++P) {
     bool HasDefaultArg = false;
     std::string PlaceholderStr;
     if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*P)) {
@@ -1926,7 +2057,7 @@ static void AddTemplateParameterChunks(ASTContext &Context,
       
       HasDefaultArg = TTP->hasDefaultArgument();
     } else if (NonTypeTemplateParmDecl *NTTP 
-               = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
+                                    = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
       if (NTTP->getIdentifier())
         PlaceholderStr = NTTP->getIdentifier()->getName();
       NTTP->getType().getAsStringInternal(PlaceholderStr, 
@@ -1947,28 +2078,35 @@ static void AddTemplateParameterChunks(ASTContext &Context,
       HasDefaultArg = TTP->hasDefaultArgument();
     }
     
-    if (HasDefaultArg) {
+    if (HasDefaultArg && !InDefaultArg) {
       // When we see an optional default argument, put that argument and
       // the remaining default arguments into a new, optional string.
-      CodeCompletionString *Opt = new CodeCompletionString;
-      CCStr->AddOptionalChunk(std::auto_ptr<CodeCompletionString>(Opt));
-      CCStr = Opt;
+      CodeCompletionBuilder Opt(Result.getAllocator());
+      if (!FirstParameter)
+        Opt.AddChunk(Chunk(CodeCompletionString::CK_Comma));
+      AddTemplateParameterChunks(Context, Template, Opt, MaxParameters,
+                                 P - Params->begin(), true);
+      Result.AddOptionalChunk(Opt.TakeString());
+      break;
     }
     
+    InDefaultArg = false;
+    
     if (FirstParameter)
       FirstParameter = false;
     else
-      CCStr->AddChunk(Chunk(CodeCompletionString::CK_Comma));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
     
     // Add the placeholder string.
-    CCStr->AddPlaceholderChunk(PlaceholderStr);
+    Result.AddPlaceholderChunk(
+                              Result.getAllocator().CopyString(PlaceholderStr));
   }    
 }
 
 /// \brief Add a qualifier to the given code-completion string, if the
 /// provided nested-name-specifier is non-NULL.
 static void 
-AddQualifierToCompletionString(CodeCompletionString *Result, 
+AddQualifierToCompletionString(CodeCompletionBuilder &Result, 
                                NestedNameSpecifier *Qualifier, 
                                bool QualifierIsInformative,
                                ASTContext &Context) {
@@ -1981,18 +2119,38 @@ AddQualifierToCompletionString(CodeCompletionString *Result,
     Qualifier->print(OS, Context.PrintingPolicy);
   }
   if (QualifierIsInformative)
-    Result->AddInformativeChunk(PrintedNNS);
+    Result.AddInformativeChunk(Result.getAllocator().CopyString(PrintedNNS));
   else
-    Result->AddTextChunk(PrintedNNS);
+    Result.AddTextChunk(Result.getAllocator().CopyString(PrintedNNS));
 }
 
-static void AddFunctionTypeQualsToCompletionString(CodeCompletionString *Result,
-                                                   FunctionDecl *Function) {
+static void 
+AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
+                                       FunctionDecl *Function) {
   const FunctionProtoType *Proto
     = Function->getType()->getAs<FunctionProtoType>();
   if (!Proto || !Proto->getTypeQuals())
     return;
 
+  // FIXME: Add ref-qualifier!
+  
+  // Handle single qualifiers without copying
+  if (Proto->getTypeQuals() == Qualifiers::Const) {
+    Result.AddInformativeChunk(" const");
+    return;
+  }
+
+  if (Proto->getTypeQuals() == Qualifiers::Volatile) {
+    Result.AddInformativeChunk(" volatile");
+    return;
+  }
+
+  if (Proto->getTypeQuals() == Qualifiers::Restrict) {
+    Result.AddInformativeChunk(" restrict");
+    return;
+  }
+
+  // Handle multiple qualifiers.
   std::string QualsStr;
   if (Proto->getTypeQuals() & Qualifiers::Const)
     QualsStr += " const";
@@ -2000,7 +2158,82 @@ static void AddFunctionTypeQualsToCompletionString(CodeCompletionString *Result,
     QualsStr += " volatile";
   if (Proto->getTypeQuals() & Qualifiers::Restrict)
     QualsStr += " restrict";
-  Result->AddInformativeChunk(QualsStr);
+  Result.AddInformativeChunk(Result.getAllocator().CopyString(QualsStr));
+}
+
+/// \brief Add the name of the given declaration 
+static void AddTypedNameChunk(ASTContext &Context, NamedDecl *ND,
+                              CodeCompletionBuilder &Result) {
+  typedef CodeCompletionString::Chunk Chunk;
+  
+  DeclarationName Name = ND->getDeclName();
+  if (!Name)
+    return;
+  
+  switch (Name.getNameKind()) {
+    case DeclarationName::CXXOperatorName: {
+      const char *OperatorName = 0;
+      switch (Name.getCXXOverloadedOperator()) {
+      case OO_None: 
+      case OO_Conditional:
+      case NUM_OVERLOADED_OPERATORS:
+        OperatorName = "operator"; 
+        break;
+    
+#define OVERLOADED_OPERATOR(Name,Spelling,Token,Unary,Binary,MemberOnly) \
+      case OO_##Name: OperatorName = "operator" Spelling; break;
+#define OVERLOADED_OPERATOR_MULTI(Name,Spelling,Unary,Binary,MemberOnly)
+#include "clang/Basic/OperatorKinds.def"
+          
+      case OO_New:          OperatorName = "operator new"; break;
+      case OO_Delete:       OperatorName = "operator delete"; break;
+      case OO_Array_New:    OperatorName = "operator new[]"; break;
+      case OO_Array_Delete: OperatorName = "operator delete[]"; break;
+      case OO_Call:         OperatorName = "operator()"; break;
+      case OO_Subscript:    OperatorName = "operator[]"; break;
+      }
+      Result.AddTypedTextChunk(OperatorName);
+      break;
+    }
+      
+  case DeclarationName::Identifier:
+  case DeclarationName::CXXConversionFunctionName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXLiteralOperatorName:
+    Result.AddTypedTextChunk(
+                      Result.getAllocator().CopyString(ND->getNameAsString()));
+    break;
+      
+  case DeclarationName::CXXUsingDirective:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    break;
+      
+  case DeclarationName::CXXConstructorName: {
+    CXXRecordDecl *Record = 0;
+    QualType Ty = Name.getCXXNameType();
+    if (const RecordType *RecordTy = Ty->getAs<RecordType>())
+      Record = cast<CXXRecordDecl>(RecordTy->getDecl());
+    else if (const InjectedClassNameType *InjectedTy
+                                        = Ty->getAs<InjectedClassNameType>())
+      Record = InjectedTy->getDecl();
+    else {
+      Result.AddTypedTextChunk(
+                      Result.getAllocator().CopyString(ND->getNameAsString()));
+      break;
+    }
+    
+    Result.AddTypedTextChunk(
+                  Result.getAllocator().CopyString(Record->getNameAsString()));
+    if (ClassTemplateDecl *Template = Record->getDescribedClassTemplate()) {
+      Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
+      AddTemplateParameterChunks(Context, Template, Result);
+      Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
+    }
+    break;
+  }
+  }
 }
 
 /// \brief If possible, create a new code completion string for the given
@@ -2011,39 +2244,42 @@ static void AddFunctionTypeQualsToCompletionString(CodeCompletionString *Result,
 /// result is all that is needed.
 CodeCompletionString *
 CodeCompletionResult::CreateCodeCompletionString(Sema &S,
-                                               CodeCompletionString *Result) {
+                                           CodeCompletionAllocator &Allocator) {
   typedef CodeCompletionString::Chunk Chunk;
+  CodeCompletionBuilder Result(Allocator, Priority, Availability);
   
-  if (Kind == RK_Pattern)
-    return Pattern->Clone(Result);
+  if (Kind == RK_Pattern) {
+    Pattern->Priority = Priority;
+    Pattern->Availability = Availability;
+    return Pattern;
+  }
   
-  if (!Result)
-    Result = new CodeCompletionString;
-
   if (Kind == RK_Keyword) {
-    Result->AddTypedTextChunk(Keyword);
-    return Result;
+    Result.AddTypedTextChunk(Keyword);
+    return Result.TakeString();
   }
   
   if (Kind == RK_Macro) {
     MacroInfo *MI = S.PP.getMacroInfo(Macro);
     assert(MI && "Not a macro?");
 
-    Result->AddTypedTextChunk(Macro->getName());
+    Result.AddTypedTextChunk(
+                            Result.getAllocator().CopyString(Macro->getName()));
 
     if (!MI->isFunctionLike())
-      return Result;
+      return Result.TakeString();
     
     // Format a function-like macro with placeholders for the arguments.
-    Result->AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
     for (MacroInfo::arg_iterator A = MI->arg_begin(), AEnd = MI->arg_end();
          A != AEnd; ++A) {
       if (A != MI->arg_begin())
-        Result->AddChunk(Chunk(CodeCompletionString::CK_Comma));
+        Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
       
       if (!MI->isVariadic() || A != AEnd - 1) {
         // Non-variadic argument.
-        Result->AddPlaceholderChunk((*A)->getName());
+        Result.AddPlaceholderChunk(
+                            Result.getAllocator().CopyString((*A)->getName()));
         continue;
       }
       
@@ -2051,24 +2287,25 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
       // variadic macros, providing a single placeholder for the rest of the
       // arguments.
       if ((*A)->isStr("__VA_ARGS__"))
-        Result->AddPlaceholderChunk("...");
+        Result.AddPlaceholderChunk("...");
       else {
         std::string Arg = (*A)->getName();
         Arg += "...";
-        Result->AddPlaceholderChunk(Arg);
+        Result.AddPlaceholderChunk(Result.getAllocator().CopyString(Arg));
       }
     }
-    Result->AddChunk(Chunk(CodeCompletionString::CK_RightParen));
-    return Result;
+    Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    return Result.TakeString();
   }
   
   assert(Kind == RK_Declaration && "Missed a result kind?");
   NamedDecl *ND = Declaration;
   
   if (StartsNestedNameSpecifier) {
-    Result->AddTypedTextChunk(ND->getNameAsString());
-    Result->AddTextChunk("::");
-    return Result;
+    Result.AddTypedTextChunk(
+                      Result.getAllocator().CopyString(ND->getNameAsString()));
+    Result.AddTextChunk("::");
+    return Result.TakeString();
   }
   
   AddResultTypeChunk(S.Context, ND, Result);
@@ -2076,20 +2313,20 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
   if (FunctionDecl *Function = dyn_cast<FunctionDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
                                    S.Context);
-    Result->AddTypedTextChunk(Function->getNameAsString());
-    Result->AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+    AddTypedNameChunk(S.Context, ND, Result);
+    Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
     AddFunctionParameterChunks(S.Context, Function, Result);
-    Result->AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
     AddFunctionTypeQualsToCompletionString(Result, Function);
-    return Result;
+    return Result.TakeString();
   }
   
   if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
                                    S.Context);
     FunctionDecl *Function = FunTmpl->getTemplatedDecl();
-    Result->AddTypedTextChunk(Function->getNameAsString());
-    
+    AddTypedNameChunk(S.Context, Function, Result);
+
     // Figure out which template parameters are deduced (or have default
     // arguments).
     llvm::SmallVector<bool, 16> Deduced;
@@ -2103,7 +2340,7 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
         // FIXME: We need to abstract template parameters better!
         bool HasDefaultArg = false;
         NamedDecl *Param = FunTmpl->getTemplateParameters()->getParam(
-                                                                      LastDeducibleArgument - 1);
+                                                    LastDeducibleArgument - 1);
         if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
           HasDefaultArg = TTP->hasDefaultArgument();
         else if (NonTypeTemplateParmDecl *NTTP 
@@ -2124,48 +2361,50 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
       // Some of the function template arguments cannot be deduced from a
       // function call, so we introduce an explicit template argument list
       // containing all of the arguments up to the first deducible argument.
-      Result->AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
       AddTemplateParameterChunks(S.Context, FunTmpl, Result, 
                                  LastDeducibleArgument);
-      Result->AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
     }
     
     // Add the function parameters
-    Result->AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
     AddFunctionParameterChunks(S.Context, Function, Result);
-    Result->AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
     AddFunctionTypeQualsToCompletionString(Result, Function);
-    return Result;
+    return Result.TakeString();
   }
   
   if (TemplateDecl *Template = dyn_cast<TemplateDecl>(ND)) {
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
                                    S.Context);
-    Result->AddTypedTextChunk(Template->getNameAsString());
-    Result->AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
+    Result.AddTypedTextChunk(
+                Result.getAllocator().CopyString(Template->getNameAsString()));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_LeftAngle));
     AddTemplateParameterChunks(S.Context, Template, Result);
-    Result->AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
-    return Result;
+    Result.AddChunk(Chunk(CodeCompletionString::CK_RightAngle));
+    return Result.TakeString();
   }
   
   if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(ND)) {
     Selector Sel = Method->getSelector();
     if (Sel.isUnarySelector()) {
-      Result->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
-      return Result;
+      Result.AddTypedTextChunk(Result.getAllocator().CopyString(
+                                  Sel.getNameForSlot(0)));
+      return Result.TakeString();
     }
 
-    std::string SelName = Sel.getIdentifierInfoForSlot(0)->getName().str();
+    std::string SelName = Sel.getNameForSlot(0).str();
     SelName += ':';
     if (StartParameter == 0)
-      Result->AddTypedTextChunk(SelName);
+      Result.AddTypedTextChunk(Result.getAllocator().CopyString(SelName));
     else {
-      Result->AddInformativeChunk(SelName);
+      Result.AddInformativeChunk(Result.getAllocator().CopyString(SelName));
       
       // If there is only one parameter, and we're past it, add an empty
       // typed-text chunk since there is nothing to type.
       if (Method->param_size() == 1)
-        Result->AddTypedTextChunk("");
+        Result.AddTypedTextChunk("");
     }
     unsigned Idx = 0;
     for (ObjCMethodDecl::param_iterator P = Method->param_begin(),
@@ -2174,16 +2413,14 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
       if (Idx > 0) {
         std::string Keyword;
         if (Idx > StartParameter)
-          Result->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+          Result.AddChunk(CodeCompletionString::CK_HorizontalSpace);
         if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(Idx))
           Keyword += II->getName().str();
         Keyword += ":";
         if (Idx < StartParameter || AllParametersAreInformative)
-          Result->AddInformativeChunk(Keyword);
-        else if (Idx == StartParameter)
-          Result->AddTypedTextChunk(Keyword);
-        else
-          Result->AddTextChunk(Keyword);
+          Result.AddInformativeChunk(Result.getAllocator().CopyString(Keyword));
+        else 
+          Result.AddTypedTextChunk(Result.getAllocator().CopyString(Keyword));
       }
       
       // If we're before the starting parameter, skip the placeholder.
@@ -2206,44 +2443,47 @@ CodeCompletionResult::CreateCodeCompletionString(Sema &S,
         Arg += ", ...";
       
       if (DeclaringEntity)
-        Result->AddTextChunk(Arg);
+        Result.AddTextChunk(Result.getAllocator().CopyString(Arg));
       else if (AllParametersAreInformative)
-        Result->AddInformativeChunk(Arg);
+        Result.AddInformativeChunk(Result.getAllocator().CopyString(Arg));
       else
-        Result->AddPlaceholderChunk(Arg);
+        Result.AddPlaceholderChunk(Result.getAllocator().CopyString(Arg));
     }
 
     if (Method->isVariadic()) {
       if (Method->param_size() == 0) {
         if (DeclaringEntity)
-          Result->AddTextChunk(", ...");
+          Result.AddTextChunk(", ...");
         else if (AllParametersAreInformative)
-          Result->AddInformativeChunk(", ...");
+          Result.AddInformativeChunk(", ...");
         else
-          Result->AddPlaceholderChunk(", ...");
+          Result.AddPlaceholderChunk(", ...");
       }
       
       MaybeAddSentinel(S.Context, Method, Result);
     }
     
-    return Result;
+    return Result.TakeString();
   }
 
   if (Qualifier)
     AddQualifierToCompletionString(Result, Qualifier, QualifierIsInformative, 
                                    S.Context);
 
-  Result->AddTypedTextChunk(ND->getNameAsString());
-  return Result;
+  Result.AddTypedTextChunk(
+                       Result.getAllocator().CopyString(ND->getNameAsString()));
+  return Result.TakeString();
 }
 
 CodeCompletionString *
 CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
                                                           unsigned CurrentArg,
-                                                               Sema &S) const {
+                                                               Sema &S,
+                                     CodeCompletionAllocator &Allocator) const {
   typedef CodeCompletionString::Chunk Chunk;
   
-  CodeCompletionString *Result = new CodeCompletionString;
+  // FIXME: Set priority, availability appropriately.
+  CodeCompletionBuilder Result(Allocator, 1, CXAvailability_Available);
   FunctionDecl *FDecl = getFunction();
   AddResultTypeChunk(S.Context, FDecl, Result);
   const FunctionProtoType *Proto 
@@ -2252,25 +2492,28 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
     // Function without a prototype. Just give the return type and a 
     // highlighted ellipsis.
     const FunctionType *FT = getFunctionType();
-    Result->AddTextChunk(
-            FT->getResultType().getAsString(S.Context.PrintingPolicy));
-    Result->AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
-    Result->AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, "..."));
-    Result->AddChunk(Chunk(CodeCompletionString::CK_RightParen));
-    return Result;
+    Result.AddTextChunk(GetCompletionTypeString(FT->getResultType(),
+                                                S.Context, 
+                                                Result.getAllocator()));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, "..."));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    return Result.TakeString();
   }
   
   if (FDecl)
-    Result->AddTextChunk(FDecl->getNameAsString());
+    Result.AddTextChunk(
+                    Result.getAllocator().CopyString(FDecl->getNameAsString()));
   else
-    Result->AddTextChunk(
-         Proto->getResultType().getAsString(S.Context.PrintingPolicy));
+    Result.AddTextChunk(
+         Result.getAllocator().CopyString(
+                Proto->getResultType().getAsString(S.Context.PrintingPolicy)));
   
-  Result->AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+  Result.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
   unsigned NumParams = FDecl? FDecl->getNumParams() : Proto->getNumArgs();
   for (unsigned I = 0; I != NumParams; ++I) {
     if (I)
-      Result->AddChunk(Chunk(CodeCompletionString::CK_Comma));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
     
     std::string ArgString;
     QualType ArgType;
@@ -2285,34 +2528,44 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
     ArgType.getAsStringInternal(ArgString, S.Context.PrintingPolicy);
     
     if (I == CurrentArg)
-      Result->AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, 
-                             ArgString));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, 
+                             Result.getAllocator().CopyString(ArgString)));
     else
-      Result->AddTextChunk(ArgString);
+      Result.AddTextChunk(Result.getAllocator().CopyString(ArgString));
   }
   
   if (Proto && Proto->isVariadic()) {
-    Result->AddChunk(Chunk(CodeCompletionString::CK_Comma));
+    Result.AddChunk(Chunk(CodeCompletionString::CK_Comma));
     if (CurrentArg < NumParams)
-      Result->AddTextChunk("...");
+      Result.AddTextChunk("...");
     else
-      Result->AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, "..."));
+      Result.AddChunk(Chunk(CodeCompletionString::CK_CurrentParameter, "..."));
   }
-  Result->AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+  Result.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
   
-  return Result;
+  return Result.TakeString();
 }
 
 unsigned clang::getMacroUsagePriority(llvm::StringRef MacroName, 
+                                      const LangOptions &LangOpts,
                                       bool PreferredTypeIsPointer) {
   unsigned Priority = CCP_Macro;
   
-  // Treat the "nil" and "NULL" macros as null pointer constants.
-  if (MacroName.equals("nil") || MacroName.equals("NULL")) {
+  // Treat the "nil", "Nil" and "NULL" macros as null pointer constants.
+  if (MacroName.equals("nil") || MacroName.equals("NULL") || 
+      MacroName.equals("Nil")) {
     Priority = CCP_Constant;
     if (PreferredTypeIsPointer)
       Priority = Priority / CCF_SimilarTypeMatch;
-  }
+  } 
+  // Treat "YES", "NO", "true", and "false" as constants.
+  else if (MacroName.equals("YES") || MacroName.equals("NO") ||
+           MacroName.equals("true") || MacroName.equals("false"))
+    Priority = CCP_Constant;
+  // Treat "bool" as a type.
+  else if (MacroName.equals("bool"))
+    Priority = CCP_Type + (LangOpts.ObjC1? CCD_bool_in_ObjC : 0);
+    
   
   return Priority;
 }
@@ -2385,14 +2638,18 @@ static void AddMacroResults(Preprocessor &PP, ResultBuilder &Results,
   typedef CodeCompletionResult Result;
   
   Results.EnterNewScope();
+  
   for (Preprocessor::macro_iterator M = PP.macro_begin(), 
                                  MEnd = PP.macro_end();
        M != MEnd; ++M) {
     Results.AddResult(Result(M->first, 
                              getMacroUsagePriority(M->first->getName(),
+                                                   PP.getLangOptions(),
                                                    TargetTypeIsPointer)));
   }
+  
   Results.ExitScope();
+  
 }
 
 static void AddPrettyFunctionResults(const LangOptions &LangOpts, 
@@ -2400,6 +2657,7 @@ static void AddPrettyFunctionResults(const LangOptions &LangOpts,
   typedef CodeCompletionResult Result;
   
   Results.EnterNewScope();
+  
   Results.AddResult(Result("__PRETTY_FUNCTION__", CCP_Constant));
   Results.AddResult(Result("__FUNCTION__", CCP_Constant));
   if (LangOpts.C99 || LangOpts.CPlusPlus0x)
@@ -2414,9 +2672,6 @@ static void HandleCodeCompleteResults(Sema *S,
                                       unsigned NumResults) {
   if (CodeCompleter)
     CodeCompleter->ProcessCodeCompleteResults(*S, Context, Results, NumResults);
-  
-  for (unsigned I = 0; I != NumResults; ++I)
-    Results[I].Destroy();
 }
 
 static enum CodeCompletionContext::Kind mapCodeCompletionContext(Sema &S, 
@@ -2439,11 +2694,24 @@ static enum CodeCompletionContext::Kind mapCodeCompletionContext(Sema &S,
       
   case Sema::PCC_Template:
   case Sema::PCC_MemberTemplate:
-  case Sema::PCC_RecoveryInFunction:
-    return CodeCompletionContext::CCC_Other;
+    if (S.CurContext->isFileContext())
+      return CodeCompletionContext::CCC_TopLevel;
+    else if (S.CurContext->isRecord())
+      return CodeCompletionContext::CCC_ClassStructUnion;
+    else 
+      return CodeCompletionContext::CCC_Other;
       
-  case Sema::PCC_Expression:
+  case Sema::PCC_RecoveryInFunction:
+    return CodeCompletionContext::CCC_Recovery;
+
   case Sema::PCC_ForInit:
+    if (S.getLangOptions().CPlusPlus || S.getLangOptions().C99 ||
+        S.getLangOptions().ObjC1)
+      return CodeCompletionContext::CCC_ParenthesizedExpression;
+    else
+      return CodeCompletionContext::CCC_Expression;
+
+  case Sema::PCC_Expression:
   case Sema::PCC_Condition:
     return CodeCompletionContext::CCC_Expression;
       
@@ -2452,6 +2720,12 @@ static enum CodeCompletionContext::Kind mapCodeCompletionContext(Sema &S,
 
   case Sema::PCC_Type:
     return CodeCompletionContext::CCC_Type;
+
+  case Sema::PCC_ParenthesizedExpression:
+    return CodeCompletionContext::CCC_ParenthesizedExpression;
+      
+  case Sema::PCC_LocalDeclarationSpecifiers:
+    return CodeCompletionContext::CCC_Type;
   }
   
   return CodeCompletionContext::CCC_Other;
@@ -2490,7 +2764,7 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
   for (CXXMethodDecl::method_iterator M = Method->begin_overridden_methods(),
                                    MEnd = Method->end_overridden_methods();
        M != MEnd; ++M) {
-    CodeCompletionString *Pattern = new CodeCompletionString;
+    CodeCompletionBuilder Builder(Results.getAllocator());
     CXXMethodDecl *Overridden = const_cast<CXXMethodDecl *>(*M);
     if (Overridden->getCanonicalDecl() == Method->getCanonicalDecl())
       continue;
@@ -2504,13 +2778,14 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
         std::string Str;
         llvm::raw_string_ostream OS(Str);
         NNS->print(OS, S.Context.PrintingPolicy);
-        Pattern->AddTextChunk(OS.str());
+        Builder.AddTextChunk(Results.getAllocator().CopyString(OS.str()));
       }
     } else if (!InContext->Equals(Overridden->getDeclContext()))
       continue;
     
-    Pattern->AddTypedTextChunk(Overridden->getNameAsString());
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddTypedTextChunk(Results.getAllocator().CopyString( 
+                                         Overridden->getNameAsString()));
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
     bool FirstParam = true;
     for (CXXMethodDecl::param_iterator P = Method->param_begin(),
                                     PEnd = Method->param_end();
@@ -2518,12 +2793,13 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
       if (FirstParam)
         FirstParam = false;
       else
-        Pattern->AddChunk(CodeCompletionString::CK_Comma);
+        Builder.AddChunk(CodeCompletionString::CK_Comma);
 
-      Pattern->AddPlaceholderChunk((*P)->getIdentifier()->getName());
+      Builder.AddPlaceholderChunk(Results.getAllocator().CopyString( 
+                                        (*P)->getIdentifier()->getName()));
     }
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(CodeCompletionResult(Pattern,
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Builder.TakeString(),
                                            CCP_SuperCompletion,
                                            CXCursor_CXXMethod));
     Results.Ignore(Overridden);
@@ -2533,9 +2809,10 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
 void Sema::CodeCompleteOrdinaryName(Scope *S, 
                                     ParserCompletionContext CompletionContext) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);  
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        mapCodeCompletionContext(*this, CompletionContext));
   Results.EnterNewScope();
-
+  
   // Determine how to filter results, e.g., so that the names of
   // values (functions, enumerators, function templates, etc.) are
   // only allowed where we can have an expression.
@@ -2548,16 +2825,12 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
   case PCC_Template:
   case PCC_MemberTemplate:
   case PCC_Type:
+  case PCC_LocalDeclarationSpecifiers:
     Results.setFilter(&ResultBuilder::IsOrdinaryNonValueName);
     break;
 
   case PCC_Statement:
-    // For statements that are expressions, we prefer to call 'void' functions 
-    // rather than functions that return a result, since then the result would
-    // be ignored.
-    Results.setPreferredType(Context.VoidTy);
-    // Fall through
-      
+  case PCC_ParenthesizedExpression:
   case PCC_Expression:
   case PCC_ForInit:
   case PCC_Condition:
@@ -2590,6 +2863,7 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
   Results.ExitScope();
 
   switch (CompletionContext) {
+  case PCC_ParenthesizedExpression:
   case PCC_Expression:
   case PCC_Statement:
   case PCC_RecoveryInFunction:
@@ -2607,22 +2881,33 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
   case PCC_ForInit:
   case PCC_Condition:
   case PCC_Type:
+  case PCC_LocalDeclarationSpecifiers:
     break;
   }
   
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
   
-  HandleCodeCompleteResults(this, CodeCompleter,
-                            mapCodeCompletionContext(*this, CompletionContext),
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
-void Sema::CodeCompleteDeclarator(Scope *S,
-                                  bool AllowNonIdentifiers,
-                                  bool AllowNestedNameSpecifiers) {
+static void AddClassMessageCompletions(Sema &SemaRef, Scope *S, 
+                                       ParsedType Receiver,
+                                       IdentifierInfo **SelIdents,
+                                       unsigned NumSelIdents,
+                                       bool AtArgumentExpression,
+                                       bool IsSuper,
+                                       ResultBuilder &Results);
+
+void Sema::CodeCompleteDeclSpec(Scope *S, DeclSpec &DS,
+                                bool AllowNonIdentifiers,
+                                bool AllowNestedNameSpecifiers) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);    
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        AllowNestedNameSpecifiers
+                          ? CodeCompletionContext::CCC_PotentiallyQualifiedName
+                          : CodeCompletionContext::CCC_Name);
   Results.EnterNewScope();
   
   // Type qualifiers can come after names.
@@ -2639,20 +2924,41 @@ void Sema::CodeCompleteDeclarator(Scope *S,
     // Add nested-name-specifiers.
     if (AllowNestedNameSpecifiers) {
       Results.allowNestedNameSpecifiers();
+      Results.setFilter(&ResultBuilder::IsImpossibleToSatisfy);
       CodeCompletionDeclConsumer Consumer(Results, CurContext);
       LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer,
                          CodeCompleter->includeGlobals());
+      Results.setFilter(0);
     }
   }
   Results.ExitScope();
 
+  // If we're in a context where we might have an expression (rather than a
+  // declaration), and what we've seen so far is an Objective-C type that could
+  // be a receiver of a class message, this may be a class message send with
+  // the initial opening bracket '[' missing. Add appropriate completions.
+  if (AllowNonIdentifiers && !AllowNestedNameSpecifiers &&
+      DS.getTypeSpecType() == DeclSpec::TST_typename &&
+      DS.getStorageClassSpecAsWritten() == DeclSpec::SCS_unspecified &&
+      !DS.isThreadSpecified() && !DS.isExternInLinkageSpec() &&
+      DS.getTypeSpecComplex() == DeclSpec::TSC_unspecified &&
+      DS.getTypeSpecSign() == DeclSpec::TSS_unspecified &&
+      DS.getTypeQualifiers() == 0 &&
+      S && 
+      (S->getFlags() & Scope::DeclScope) != 0 &&
+      (S->getFlags() & (Scope::ClassScope | Scope::TemplateParamScope |
+                        Scope::FunctionPrototypeScope | 
+                        Scope::AtCatchScope)) == 0) {
+    ParsedType T = DS.getRepAsType();
+    if (!T.get().isNull() && T.get()->isObjCObjectOrInterfaceType())
+      AddClassMessageCompletions(*this, S, T, 0, 0, false, false, Results); 
+  }
+
   // Note that we intentionally suppress macro results here, since we do not
   // encourage using macros to produce the names of entities.
 
-  HandleCodeCompleteResults(this, CodeCompleter,
-                        AllowNestedNameSpecifiers
-                          ? CodeCompletionContext::CCC_PotentiallyQualifiedName
-                          : CodeCompletionContext::CCC_Name,
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            Results.getCompletionContext(),
                             Results.data(), Results.size());
 }
 
@@ -2675,8 +2981,8 @@ struct Sema::CodeCompleteExpressionData {
 void Sema::CodeCompleteExpression(Scope *S, 
                                   const CodeCompleteExpressionData &Data) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
-  
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Expression);
   if (Data.ObjCCollection)
     Results.setFilter(&ResultBuilder::IsObjCCollection);
   else if (Data.IntegralConstantExpression)
@@ -2720,10 +3026,21 @@ void Sema::CodeCompleteExpression(Scope *S,
                             Results.data(),Results.size());
 }
 
+void Sema::CodeCompletePostfixExpression(Scope *S, ExprResult E) {
+  if (E.isInvalid())
+    CodeCompleteOrdinaryName(S, PCC_RecoveryInFunction);
+  else if (getLangOptions().ObjC1)
+    CodeCompleteObjCInstanceMessage(S, E.take(), 0, 0, false);
+}
+
+/// \brief The set of properties that have already been added, referenced by
+/// property name.
+typedef llvm::SmallPtrSet<IdentifierInfo*, 16> AddedPropertiesSet;
 
 static void AddObjCProperties(ObjCContainerDecl *Container, 
                               bool AllowCategories,
                               DeclContext *CurContext,
+                              AddedPropertiesSet &AddedProperties,
                               ResultBuilder &Results) {
   typedef CodeCompletionResult Result;
 
@@ -2731,40 +3048,46 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
   for (ObjCContainerDecl::prop_iterator P = Container->prop_begin(),
                                      PEnd = Container->prop_end();
        P != PEnd;
-       ++P)
-    Results.MaybeAddResult(Result(*P, 0), CurContext);
+       ++P) {
+    if (AddedProperties.insert(P->getIdentifier()))
+      Results.MaybeAddResult(Result(*P, 0), CurContext);
+  }
   
   // Add properties in referenced protocols.
   if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Container)) {
     for (ObjCProtocolDecl::protocol_iterator P = Protocol->protocol_begin(),
                                           PEnd = Protocol->protocol_end();
          P != PEnd; ++P)
-      AddObjCProperties(*P, AllowCategories, CurContext, Results);
+      AddObjCProperties(*P, AllowCategories, CurContext, AddedProperties, 
+                        Results);
   } else if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(Container)){
     if (AllowCategories) {
       // Look through categories.
       for (ObjCCategoryDecl *Category = IFace->getCategoryList();
            Category; Category = Category->getNextClassCategory())
-        AddObjCProperties(Category, AllowCategories, CurContext, Results);
+        AddObjCProperties(Category, AllowCategories, CurContext, 
+                          AddedProperties, Results);
     }
     
     // Look through protocols.
     for (ObjCInterfaceDecl::all_protocol_iterator
          I = IFace->all_referenced_protocol_begin(),
          E = IFace->all_referenced_protocol_end(); I != E; ++I)
-      AddObjCProperties(*I, AllowCategories, CurContext, Results);
+      AddObjCProperties(*I, AllowCategories, CurContext, AddedProperties,
+                        Results);
     
     // Look in the superclass.
     if (IFace->getSuperClass())
       AddObjCProperties(IFace->getSuperClass(), AllowCategories, CurContext, 
-                        Results);
+                        AddedProperties, Results);
   } else if (const ObjCCategoryDecl *Category
                                     = dyn_cast<ObjCCategoryDecl>(Container)) {
     // Look through protocols.
     for (ObjCCategoryDecl::protocol_iterator P = Category->protocol_begin(),
                                           PEnd = Category->protocol_end(); 
          P != PEnd; ++P)
-      AddObjCProperties(*P, AllowCategories, CurContext, Results);
+      AddObjCProperties(*P, AllowCategories, CurContext, AddedProperties,
+                        Results);
   }
 }
 
@@ -2788,7 +3111,10 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
       return;
   }
   
-  ResultBuilder Results(*this, &ResultBuilder::IsMember);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                  CodeCompletionContext(CodeCompletionContext::CCC_MemberAccess,
+                                        BaseType),
+                        &ResultBuilder::IsMember);
   Results.EnterNewScope();
   if (const RecordType *Record = BaseType->getAs<RecordType>()) {
     // Indicate that we are performing a member access, and the cv-qualifiers
@@ -2821,18 +3147,20 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
     }
   } else if (!IsArrow && BaseType->getAsObjCInterfacePointerType()) {
     // Objective-C property reference.
+    AddedPropertiesSet AddedProperties;
     
     // Add property results based on our interface.
     const ObjCObjectPointerType *ObjCPtr
       = BaseType->getAsObjCInterfacePointerType();
     assert(ObjCPtr && "Non-NULL pointer guaranteed above!");
-    AddObjCProperties(ObjCPtr->getInterfaceDecl(), true, CurContext, Results);
+    AddObjCProperties(ObjCPtr->getInterfaceDecl(), true, CurContext, 
+                      AddedProperties, Results);
     
     // Add properties from the protocols in a qualified interface.
     for (ObjCObjectPointerType::qual_iterator I = ObjCPtr->qual_begin(),
                                               E = ObjCPtr->qual_end();
          I != E; ++I)
-      AddObjCProperties(*I, true, CurContext, Results);
+      AddObjCProperties(*I, true, CurContext, AddedProperties, Results);
   } else if ((IsArrow && BaseType->isObjCObjectPointerType()) ||
              (!IsArrow && BaseType->isObjCObjectType())) {
     // Objective-C instance variable access.
@@ -2858,8 +3186,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
 
   // Hand off the results found for code completion.
   HandleCodeCompleteResults(this, CodeCompleter, 
-            CodeCompletionContext(CodeCompletionContext::CCC_MemberAccess,
-                                              BaseType),
+                            Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
@@ -2893,7 +3220,7 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
     return;
   }
   
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(), ContextKind);
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
 
   // First pass: look for tags.
@@ -2907,12 +3234,13 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
     LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer);
   }
   
-  HandleCodeCompleteResults(this, CodeCompleter, ContextKind,
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteTypeQualifiers(DeclSpec &DS) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_TypeQualifiers);
   Results.EnterNewScope();
   if (!(DS.getTypeQualifiers() & DeclSpec::TQ_const))
     Results.AddResult("const");
@@ -2923,7 +3251,7 @@ void Sema::CodeCompleteTypeQualifiers(DeclSpec &DS) {
     Results.AddResult("restrict");
   Results.ExitScope();
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_TypeQualifiers,
+                            Results.getCompletionContext(),
                             Results.data(), Results.size());
 }
 
@@ -2993,7 +3321,8 @@ void Sema::CodeCompleteCase(Scope *S) {
   }
   
   // Add any enumerators that have not yet been mentioned.
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Expression);
   Results.EnterNewScope();
   for (EnumDecl::enumerator_iterator E = Enum->enumerator_begin(),
                                   EEnd = Enum->enumerator_end();
@@ -3001,15 +3330,16 @@ void Sema::CodeCompleteCase(Scope *S) {
     if (EnumeratorsSeen.count(*E))
       continue;
     
-    Results.AddResult(CodeCompletionResult(*E, Qualifier),
-                      CurContext, 0, false);
+    CodeCompletionResult R(*E, Qualifier);
+    R.Priority = CCP_EnumInCase;
+    Results.AddResult(R, CurContext, 0, false);
   }
   Results.ExitScope();
 
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Expression,
+                            CodeCompletionContext::CCC_OtherWithMacros,
                             Results.data(),Results.size());
 }
 
@@ -3196,9 +3526,10 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
   if (!isDependentScopeSpecifier(SS) && RequireCompleteDeclContext(SS, Ctx))
     return;
 
-  ResultBuilder Results(*this);
-  
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Name);
   Results.EnterNewScope();
+  
   // The "template" keyword can follow "::" in the grammar, but only
   // put it into the grammar if the nested-name-specifier is dependent.
   NestedNameSpecifier *NNS = (NestedNameSpecifier *)SS.getScopeRep();
@@ -3226,7 +3557,9 @@ void Sema::CodeCompleteUsing(Scope *S) {
   if (!CodeCompleter)
     return;
   
-  ResultBuilder Results(*this, &ResultBuilder::IsNestedNameSpecifier);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_PotentiallyQualifiedName,
+                        &ResultBuilder::IsNestedNameSpecifier);
   Results.EnterNewScope();
   
   // If we aren't in class scope, we could see the "namespace" keyword.
@@ -3241,7 +3574,7 @@ void Sema::CodeCompleteUsing(Scope *S) {
   Results.ExitScope();
   
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Other,
+                            CodeCompletionContext::CCC_PotentiallyQualifiedName,
                             Results.data(),Results.size());
 }
 
@@ -3251,7 +3584,9 @@ void Sema::CodeCompleteUsingDirective(Scope *S) {
   
   // After "using namespace", we expect to see a namespace name or namespace
   // alias.
-  ResultBuilder Results(*this, &ResultBuilder::IsNamespaceOrAlias);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Namespace,
+                        &ResultBuilder::IsNamespaceOrAlias);
   Results.EnterNewScope();
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
   LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
@@ -3266,12 +3601,20 @@ void Sema::CodeCompleteNamespaceDecl(Scope *S)  {
   if (!CodeCompleter)
     return;
   
-  ResultBuilder Results(*this, &ResultBuilder::IsNamespace);
   DeclContext *Ctx = (DeclContext *)S->getEntity();
   if (!S->getParent())
     Ctx = Context.getTranslationUnitDecl();
   
-  if (Ctx && Ctx->isFileContext()) {
+  bool SuppressedGlobalResults
+    = Ctx && !CodeCompleter->includeGlobals() && isa<TranslationUnitDecl>(Ctx);
+  
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        SuppressedGlobalResults
+                          ? CodeCompletionContext::CCC_Namespace
+                          : CodeCompletionContext::CCC_Other,
+                        &ResultBuilder::IsNamespace);
+  
+  if (Ctx && Ctx->isFileContext() && !SuppressedGlobalResults) {
     // We only want to see those namespaces that have already been defined
     // within this scope, because its likely that the user is creating an
     // extended namespace declaration. Keep track of the most recent 
@@ -3294,7 +3637,7 @@ void Sema::CodeCompleteNamespaceDecl(Scope *S)  {
   }
   
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Other,
+                            Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
@@ -3303,12 +3646,14 @@ void Sema::CodeCompleteNamespaceAliasDecl(Scope *S)  {
     return;
   
   // After "namespace", we expect to see a namespace or alias.
-  ResultBuilder Results(*this, &ResultBuilder::IsNamespaceOrAlias);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Namespace,
+                        &ResultBuilder::IsNamespaceOrAlias);
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
   LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
                      CodeCompleter->includeGlobals());
   HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Namespace,
+                            Results.getCompletionContext(),
                             Results.data(),Results.size());
 }
 
@@ -3317,7 +3662,9 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
     return;
 
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this, &ResultBuilder::IsType);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Type,
+                        &ResultBuilder::IsType);
   Results.EnterNewScope();
   
   // Add the names of overloadable operators.
@@ -3342,14 +3689,15 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
 }
 
 void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
-                                    CXXBaseOrMemberInitializer** Initializers,
+                                              CXXCtorInitializer** Initializers,
                                               unsigned NumInitializers) {
   CXXConstructorDecl *Constructor
     = static_cast<CXXConstructorDecl *>(ConstructorD);
   if (!Constructor)
     return;
   
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_PotentiallyQualifiedName);
   Results.EnterNewScope();
   
   // Fill in any already-initialized fields or base classes.
@@ -3360,10 +3708,12 @@ void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
       InitializedBases.insert(
         Context.getCanonicalType(QualType(Initializers[I]->getBaseClass(), 0)));
     else
-      InitializedFields.insert(cast<FieldDecl>(Initializers[I]->getMember()));
+      InitializedFields.insert(cast<FieldDecl>(
+                               Initializers[I]->getAnyMember()));
   }
   
   // Add completions for base classes.
+  CodeCompletionBuilder Builder(Results.getAllocator());
   bool SawLastInitializer = (NumInitializers == 0);
   CXXRecordDecl *ClassDecl = Constructor->getParent();
   for (CXXRecordDecl::base_class_iterator Base = ClassDecl->bases_begin(),
@@ -3378,13 +3728,13 @@ void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
       continue;
     }
     
-    CodeCompletionString *Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(
-                           Base->getType().getAsString(Context.PrintingPolicy));
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("args");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(CodeCompletionResult(Pattern, 
+    Builder.AddTypedTextChunk(
+               Results.getAllocator().CopyString(
+                          Base->getType().getAsString(Context.PrintingPolicy)));
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("args");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Builder.TakeString(), 
                                    SawLastInitializer? CCP_NextInitializer
                                                      : CCP_MemberDeclaration));
     SawLastInitializer = false;
@@ -3403,13 +3753,13 @@ void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
       continue;
     }
     
-    CodeCompletionString *Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(
-                           Base->getType().getAsString(Context.PrintingPolicy));
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("args");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(CodeCompletionResult(Pattern, 
+    Builder.AddTypedTextChunk(
+               Builder.getAllocator().CopyString(
+                          Base->getType().getAsString(Context.PrintingPolicy)));
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("args");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Builder.TakeString(), 
                                    SawLastInitializer? CCP_NextInitializer
                                                      : CCP_MemberDeclaration));
     SawLastInitializer = false;
@@ -3422,28 +3772,28 @@ void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
     if (!InitializedFields.insert(cast<FieldDecl>(Field->getCanonicalDecl()))) {
       SawLastInitializer
         = NumInitializers > 0 && 
-          Initializers[NumInitializers - 1]->isMemberInitializer() &&
-          Initializers[NumInitializers - 1]->getMember() == *Field;
+          Initializers[NumInitializers - 1]->isAnyMemberInitializer() &&
+          Initializers[NumInitializers - 1]->getAnyMember() == *Field;
       continue;
     }
     
     if (!Field->getDeclName())
       continue;
     
-    CodeCompletionString *Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(Field->getIdentifier()->getName());
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("args");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Results.AddResult(CodeCompletionResult(Pattern, 
+    Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                         Field->getIdentifier()->getName()));
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("args");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Builder.TakeString(), 
                                    SawLastInitializer? CCP_NextInitializer
-                                                     : CCP_MemberDeclaration));
+                                                     : CCP_MemberDeclaration,
+                                           CXCursor_MemberRef));
     SawLastInitializer = false;
   }
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_Name,
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(), Results.size());
 }
 
@@ -3457,21 +3807,19 @@ static void AddObjCImplementationResults(const LangOptions &LangOpts,
   // Since we have an implementation, we can end it.
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,end)));
   
-  CodeCompletionString *Pattern = 0;
+  CodeCompletionBuilder Builder(Results.getAllocator());
   if (LangOpts.ObjC2) {
     // @dynamic
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,dynamic));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("property");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,dynamic));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("property");
+    Results.AddResult(Result(Builder.TakeString()));
     
     // @synthesize
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,synthesize));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("property");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,synthesize));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("property");
+    Results.AddResult(Result(Builder.TakeString()));
   }  
 }
 
@@ -3497,54 +3845,50 @@ static void AddObjCInterfaceResults(const LangOptions &LangOpts,
 
 static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
   typedef CodeCompletionResult Result;
-  CodeCompletionString *Pattern = 0;
+  CodeCompletionBuilder Builder(Results.getAllocator());
   
   // @class name ;
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,class));
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("name");
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,class));
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("name");
+  Results.AddResult(Result(Builder.TakeString()));
   
   if (Results.includeCodePatterns()) {
     // @interface name 
     // FIXME: Could introduce the whole pattern, including superclasses and 
     // such.
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,interface));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("class");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,interface));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("class");
+    Results.AddResult(Result(Builder.TakeString()));
   
     // @protocol name
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,protocol));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("protocol");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,protocol));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("protocol");
+    Results.AddResult(Result(Builder.TakeString()));
     
     // @implementation name
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,implementation));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("class");
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,implementation));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("class");
+    Results.AddResult(Result(Builder.TakeString()));
   }
   
   // @compatibility_alias name
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,compatibility_alias));
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("alias");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("class");
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,compatibility_alias));
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("alias");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("class");
+  Results.AddResult(Result(Builder.TakeString()));
 }
 
 void Sema::CodeCompleteObjCAtDirective(Scope *S, Decl *ObjCImpDecl,
                                        bool InInterface) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   if (ObjCImpDecl)
     AddObjCImplementationResults(getLangOptions(), Results, false);
@@ -3560,78 +3904,72 @@ void Sema::CodeCompleteObjCAtDirective(Scope *S, Decl *ObjCImpDecl,
 
 static void AddObjCExpressionResults(ResultBuilder &Results, bool NeedAt) {
   typedef CodeCompletionResult Result;
-  CodeCompletionString *Pattern = 0;
+  CodeCompletionBuilder Builder(Results.getAllocator());
 
   // @encode ( type-name )
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,encode));
-  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-  Pattern->AddPlaceholderChunk("type-name");
-  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,encode));
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddPlaceholderChunk("type-name");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Result(Builder.TakeString()));
   
   // @protocol ( protocol-name )
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,protocol));
-  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-  Pattern->AddPlaceholderChunk("protocol-name");
-  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,protocol));
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddPlaceholderChunk("protocol-name");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Result(Builder.TakeString()));
 
   // @selector ( selector )
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,selector));
-  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-  Pattern->AddPlaceholderChunk("selector");
-  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,selector));
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddPlaceholderChunk("selector");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Result(Builder.TakeString()));
 }
 
 static void AddObjCStatementResults(ResultBuilder &Results, bool NeedAt) {
   typedef CodeCompletionResult Result;
-  CodeCompletionString *Pattern = 0;
+  CodeCompletionBuilder Builder(Results.getAllocator());
   
   if (Results.includeCodePatterns()) {
     // @try { statements } @catch ( declaration ) { statements } @finally
     //   { statements }
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,try));
-    Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-    Pattern->AddPlaceholderChunk("statements");
-    Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-    Pattern->AddTextChunk("@catch");
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("parameter");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-    Pattern->AddPlaceholderChunk("statements");
-    Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-    Pattern->AddTextChunk("@finally");
-    Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-    Pattern->AddPlaceholderChunk("statements");
-    Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,try));
+    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+    Builder.AddPlaceholderChunk("statements");
+    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    Builder.AddTextChunk("@catch");
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("parameter");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+    Builder.AddPlaceholderChunk("statements");
+    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    Builder.AddTextChunk("@finally");
+    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+    Builder.AddPlaceholderChunk("statements");
+    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    Results.AddResult(Result(Builder.TakeString()));
   }
   
   // @throw
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,throw));
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("expression");
-  Results.AddResult(Result(Pattern));
+  Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,throw));
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("expression");
+  Results.AddResult(Result(Builder.TakeString()));
   
   if (Results.includeCodePatterns()) {
     // @synchronized ( expression ) { statements }
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,synchronized));
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-    Pattern->AddPlaceholderChunk("expression");
-    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-    Pattern->AddPlaceholderChunk("statements");
-    Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
-    Results.AddResult(Result(Pattern));
+    Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,synchronized));
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+    Builder.AddPlaceholderChunk("expression");
+    Builder.AddChunk(CodeCompletionString::CK_RightParen);
+    Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+    Builder.AddPlaceholderChunk("statements");
+    Builder.AddChunk(CodeCompletionString::CK_RightBrace);
+    Results.AddResult(Result(Builder.TakeString()));
   }
 }
 
@@ -3647,7 +3985,8 @@ static void AddObjCVisibilityResults(const LangOptions &LangOpts,
 }
 
 void Sema::CodeCompleteObjCAtVisibility(Scope *S) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCVisibilityResults(getLangOptions(), Results, false);
   Results.ExitScope();
@@ -3657,7 +3996,8 @@ void Sema::CodeCompleteObjCAtVisibility(Scope *S) {
 }
 
 void Sema::CodeCompleteObjCAtStatement(Scope *S) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCStatementResults(Results, false);
   AddObjCExpressionResults(Results, false);
@@ -3668,7 +4008,8 @@ void Sema::CodeCompleteObjCAtStatement(Scope *S) {
 }
 
 void Sema::CodeCompleteObjCAtExpression(Scope *S) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   AddObjCExpressionResults(Results, false);
   Results.ExitScope();
@@ -3714,7 +4055,8 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
   unsigned Attributes = ODS.getPropertyAttributes();
   
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_readonly))
     Results.AddResult(CodeCompletionResult("readonly"));
@@ -3729,18 +4071,18 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_nonatomic))
     Results.AddResult(CodeCompletionResult("nonatomic"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_setter)) {
-    CodeCompletionString *Setter = new CodeCompletionString;
-    Setter->AddTypedTextChunk("setter");
-    Setter->AddTextChunk(" = ");
-    Setter->AddPlaceholderChunk("method");
-    Results.AddResult(CodeCompletionResult(Setter));
+    CodeCompletionBuilder Setter(Results.getAllocator());
+    Setter.AddTypedTextChunk("setter");
+    Setter.AddTextChunk(" = ");
+    Setter.AddPlaceholderChunk("method");
+    Results.AddResult(CodeCompletionResult(Setter.TakeString()));
   }
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_getter)) {
-    CodeCompletionString *Getter = new CodeCompletionString;
-    Getter->AddTypedTextChunk("getter");
-    Getter->AddTextChunk(" = ");
-    Getter->AddPlaceholderChunk("method");
-    Results.AddResult(CodeCompletionResult(Getter));
+    CodeCompletionBuilder Getter(Results.getAllocator());
+    Getter.AddTypedTextChunk("getter");
+    Getter.AddTextChunk(" = ");
+    Getter.AddPlaceholderChunk("method");
+    Results.AddResult(CodeCompletionResult(Getter.TakeString()));
   }
   Results.ExitScope();
   HandleCodeCompleteResults(this, CodeCompleter, 
@@ -3759,7 +4101,8 @@ enum ObjCMethodKind {
 static bool isAcceptableObjCSelector(Selector Sel,
                                      ObjCMethodKind WantKind,
                                      IdentifierInfo **SelIdents,
-                                     unsigned NumSelIdents) {
+                                     unsigned NumSelIdents,
+                                     bool AllowSameLength = true) {
   if (NumSelIdents > Sel.getNumArgs())
     return false;
   
@@ -3769,6 +4112,9 @@ static bool isAcceptableObjCSelector(Selector Sel,
     case MK_OneArgSelector:  return Sel.getNumArgs() == 1;
   }
   
+  if (!AllowSameLength && NumSelIdents && NumSelIdents == Sel.getNumArgs())
+    return false;
+  
   for (unsigned I = 0; I != NumSelIdents; ++I)
     if (SelIdents[I] != Sel.getIdentifierInfoForSlot(I))
       return false;
@@ -3779,11 +4125,18 @@ static bool isAcceptableObjCSelector(Selector Sel,
 static bool isAcceptableObjCMethod(ObjCMethodDecl *Method,
                                    ObjCMethodKind WantKind,
                                    IdentifierInfo **SelIdents,
-                                   unsigned NumSelIdents) {
+                                   unsigned NumSelIdents,
+                                   bool AllowSameLength = true) {
   return isAcceptableObjCSelector(Method->getSelector(), WantKind, SelIdents,
-                                  NumSelIdents);
+                                  NumSelIdents, AllowSameLength);
 }
-                                   
+
+namespace {
+  /// \brief A set of selectors, which is used to avoid introducing multiple 
+  /// completions with the same selector into the result set.
+  typedef llvm::SmallPtrSet<Selector, 16> VisitedSelectorSet;
+}
+
 /// \brief Add all of the Objective-C methods in the given Objective-C 
 /// container to the set of results.
 ///
@@ -3800,6 +4153,9 @@ static bool isAcceptableObjCMethod(ObjCMethodDecl *Method,
 /// \param CurContext the context in which we're performing the lookup that
 /// finds methods.
 ///
+/// \param AllowSameLength Whether we allow a method to be added to the list
+/// when it has the same number of parameters as we have selector identifiers.
+///
 /// \param Results the structure into which we'll add results.
 static void AddObjCMethods(ObjCContainerDecl *Container, 
                            bool WantInstanceMethods,
@@ -3807,6 +4163,8 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                            IdentifierInfo **SelIdents,
                            unsigned NumSelIdents,
                            DeclContext *CurContext,
+                           VisitedSelectorSet &Selectors,
+                           bool AllowSameLength,
                            ResultBuilder &Results,
                            bool InOriginalClass = true) {
   typedef CodeCompletionResult Result;
@@ -3816,9 +4174,13 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
     if ((*M)->isInstanceMethod() == WantInstanceMethods) {
       // Check whether the selector identifiers we've been given are a 
       // subset of the identifiers for this particular method.
-      if (!isAcceptableObjCMethod(*M, WantKind, SelIdents, NumSelIdents))
+      if (!isAcceptableObjCMethod(*M, WantKind, SelIdents, NumSelIdents,
+                                  AllowSameLength))
         continue;
 
+      if (!Selectors.insert((*M)->getSelector()))
+        continue;
+      
       Result R = Result(*M, 0);
       R.StartParameter = NumSelIdents;
       R.AllParametersAreInformative = (WantKind != MK_Any);
@@ -3828,6 +4190,17 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
     }
   }
   
+  // Visit the protocols of protocols.
+  if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Container)) {
+    const ObjCList<ObjCProtocolDecl> &Protocols
+      = Protocol->getReferencedProtocols();
+    for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
+                                              E = Protocols.end(); 
+         I != E; ++I)
+      AddObjCMethods(*I, WantInstanceMethods, WantKind, SelIdents, NumSelIdents, 
+                     CurContext, Selectors, AllowSameLength, Results, false);    
+  }
+  
   ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(Container);
   if (!IFace)
     return;
@@ -3838,13 +4211,14 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                                             E = Protocols.end(); 
        I != E; ++I)
     AddObjCMethods(*I, WantInstanceMethods, WantKind, SelIdents, NumSelIdents, 
-                   CurContext, Results, false);
+                   CurContext, Selectors, AllowSameLength, Results, false);
   
   // Add methods in categories.
   for (ObjCCategoryDecl *CatDecl = IFace->getCategoryList(); CatDecl;
        CatDecl = CatDecl->getNextClassCategory()) {
     AddObjCMethods(CatDecl, WantInstanceMethods, WantKind, SelIdents, 
-                   NumSelIdents, CurContext, Results, InOriginalClass);
+                   NumSelIdents, CurContext, Selectors, AllowSameLength, 
+                   Results, InOriginalClass);
     
     // Add a categories protocol methods.
     const ObjCList<ObjCProtocolDecl> &Protocols 
@@ -3853,29 +4227,31 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                                               E = Protocols.end();
          I != E; ++I)
       AddObjCMethods(*I, WantInstanceMethods, WantKind, SelIdents, 
-                     NumSelIdents, CurContext, Results, false);
+                     NumSelIdents, CurContext, Selectors, AllowSameLength, 
+                     Results, false);
     
     // Add methods in category implementations.
     if (ObjCCategoryImplDecl *Impl = CatDecl->getImplementation())
       AddObjCMethods(Impl, WantInstanceMethods, WantKind, SelIdents, 
-                     NumSelIdents, CurContext, Results, InOriginalClass);
+                     NumSelIdents, CurContext, Selectors, AllowSameLength, 
+                     Results, InOriginalClass);
   }
   
   // Add methods in superclass.
   if (IFace->getSuperClass())
     AddObjCMethods(IFace->getSuperClass(), WantInstanceMethods, WantKind, 
-                   SelIdents, NumSelIdents, CurContext, Results, false);
+                   SelIdents, NumSelIdents, CurContext, Selectors, 
+                   AllowSameLength, Results, false);
 
   // Add methods in our implementation, if any.
   if (ObjCImplementationDecl *Impl = IFace->getImplementation())
     AddObjCMethods(Impl, WantInstanceMethods, WantKind, SelIdents,
-                   NumSelIdents, CurContext, Results, InOriginalClass);
+                   NumSelIdents, CurContext, Selectors, AllowSameLength, 
+                   Results, InOriginalClass);
 }
 
 
-void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl,
-                                          Decl **Methods,
-                                          unsigned NumMethods) {
+void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl) {
   typedef CodeCompletionResult Result;
 
   // Try to find the interface where getters might live.
@@ -3890,32 +4266,20 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl,
   }
 
   // Find all of the potential getters.
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
-  // FIXME: We need to do this because Objective-C methods don't get
-  // pushed into DeclContexts early enough. Argh!
-  for (unsigned I = 0; I != NumMethods; ++I) { 
-    if (ObjCMethodDecl *Method
-            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I]))
-      if (Method->isInstanceMethod() &&
-          isAcceptableObjCMethod(Method, MK_ZeroArgSelector, 0, 0)) {
-        Result R = Result(Method, 0);
-        R.AllParametersAreInformative = true;
-        Results.MaybeAddResult(R, CurContext);
-      }
-  }
-
-  AddObjCMethods(Class, true, MK_ZeroArgSelector, 0, 0, CurContext, Results);
+  VisitedSelectorSet Selectors;
+  AddObjCMethods(Class, true, MK_ZeroArgSelector, 0, 0, CurContext, Selectors,
+                 /*AllowSameLength=*/true, Results);
   Results.ExitScope();
   HandleCodeCompleteResults(this, CodeCompleter,
                             CodeCompletionContext::CCC_Other,
                             Results.data(),Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl,
-                                          Decl **Methods,
-                                          unsigned NumMethods) {
+void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl) {
   typedef CodeCompletionResult Result;
 
   // Try to find the interface where setters might live.
@@ -3931,23 +4295,13 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl,
   }
 
   // Find all of the potential getters.
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
-  // FIXME: We need to do this because Objective-C methods don't get
-  // pushed into DeclContexts early enough. Argh!
-  for (unsigned I = 0; I != NumMethods; ++I) { 
-    if (ObjCMethodDecl *Method
-            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I]))
-      if (Method->isInstanceMethod() &&
-          isAcceptableObjCMethod(Method, MK_OneArgSelector, 0, 0)) {
-        Result R = Result(Method, 0);
-        R.AllParametersAreInformative = true;
-        Results.MaybeAddResult(R, CurContext);
-      }
-  }
-
-  AddObjCMethods(Class, true, MK_OneArgSelector, 0, 0, CurContext, Results);
+  VisitedSelectorSet Selectors;
+  AddObjCMethods(Class, true, MK_OneArgSelector, 0, 0, CurContext, 
+                 Selectors, /*AllowSameLength=*/true, Results);
 
   Results.ExitScope();
   HandleCodeCompleteResults(this, CodeCompleter,
@@ -3955,9 +4309,11 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl,
                             Results.data(),Results.size());
 }
 
-void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS) {
+void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
+                                       bool IsParameter) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Type);
   Results.EnterNewScope();
   
   // Add context-sensitive, Objective-C parameter-passing keywords.
@@ -3982,6 +4338,26 @@ void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS) {
      Results.AddResult("oneway");
   }
   
+  // If we're completing the return type of an Objective-C method and the 
+  // identifier IBAction refers to a macro, provide a completion item for
+  // an action, e.g.,
+  //   IBAction)<#selector#>:(id)sender
+  if (DS.getObjCDeclQualifier() == 0 && !IsParameter &&
+      Context.Idents.get("IBAction").hasMacroDefinition()) {
+    typedef CodeCompletionString::Chunk Chunk;
+    CodeCompletionBuilder Builder(Results.getAllocator(), CCP_CodePattern, 
+                                  CXAvailability_Available);
+    Builder.AddTypedTextChunk("IBAction");
+    Builder.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    Builder.AddPlaceholderChunk("selector");
+    Builder.AddChunk(Chunk(CodeCompletionString::CK_Colon));
+    Builder.AddChunk(Chunk(CodeCompletionString::CK_LeftParen));
+    Builder.AddTextChunk("id");
+    Builder.AddChunk(Chunk(CodeCompletionString::CK_RightParen));
+    Builder.AddTextChunk("sender");
+    Results.AddResult(CodeCompletionResult(Builder.TakeString()));
+  }
+  
   // Add various builtin type names and specifiers.
   AddOrdinaryNameResults(PCC_Type, S, *this, Results);
   Results.ExitScope();
@@ -4005,7 +4381,7 @@ void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS) {
 /// common uses of Objective-C. This routine returns that class type,
 /// or NULL if no better result could be determined.
 static ObjCInterfaceDecl *GetAssumedMessageSendExprType(Expr *E) {
-  ObjCMessageExpr *Msg = dyn_cast<ObjCMessageExpr>(E);
+  ObjCMessageExpr *Msg = dyn_cast_or_null<ObjCMessageExpr>(E);
   if (!Msg)
     return 0;
 
@@ -4102,10 +4478,21 @@ static ObjCMethodDecl *AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
   
   // Try to find a superclass method with the same selector.
   ObjCMethodDecl *SuperMethod = 0;
-  while ((Class = Class->getSuperClass()) && !SuperMethod)
+  while ((Class = Class->getSuperClass()) && !SuperMethod) {
+    // Check in the class
     SuperMethod = Class->getMethod(CurMethod->getSelector(), 
                                    CurMethod->isInstanceMethod());
 
+    // Check in categories or class extensions.
+    if (!SuperMethod) {
+      for (ObjCCategoryDecl *Category = Class->getCategoryList(); Category;
+           Category = Category->getNextClassCategory())
+        if ((SuperMethod = Category->getMethod(CurMethod->getSelector(), 
+                                               CurMethod->isInstanceMethod())))
+          break;
+    }
+  }
+
   if (!SuperMethod)
     return 0;
   
@@ -4129,45 +4516,52 @@ static ObjCMethodDecl *AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
   }
   
   // We have a superclass method. Now, form the send-to-super completion.
-  CodeCompletionString *Pattern = new CodeCompletionString;
+  CodeCompletionBuilder Builder(Results.getAllocator());
   
   // Give this completion a return type.
-  AddResultTypeChunk(S.Context, SuperMethod, Pattern);
+  AddResultTypeChunk(S.Context, SuperMethod, Builder);
 
   // If we need the "super" keyword, add it (plus some spacing).
   if (NeedSuperKeyword) {
-    Pattern->AddTypedTextChunk("super");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddTypedTextChunk("super");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
   }
   
   Selector Sel = CurMethod->getSelector();
   if (Sel.isUnarySelector()) {
     if (NeedSuperKeyword)
-      Pattern->AddTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+      Builder.AddTextChunk(Builder.getAllocator().CopyString(
+                                  Sel.getNameForSlot(0)));
     else
-      Pattern->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+      Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                   Sel.getNameForSlot(0)));
   } else {
     ObjCMethodDecl::param_iterator CurP = CurMethod->param_begin();
     for (unsigned I = 0, N = Sel.getNumArgs(); I != N; ++I, ++CurP) {
       if (I > NumSelIdents)
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
       
       if (I < NumSelIdents)
-        Pattern->AddInformativeChunk(
-                       Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
+        Builder.AddInformativeChunk(
+                   Builder.getAllocator().CopyString(
+                                                 Sel.getNameForSlot(I) + ":"));
       else if (NeedSuperKeyword || I > NumSelIdents) {
-        Pattern->AddTextChunk(
-                        Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
-        Pattern->AddPlaceholderChunk((*CurP)->getIdentifier()->getName());
+        Builder.AddTextChunk(
+                 Builder.getAllocator().CopyString(
+                                                  Sel.getNameForSlot(I) + ":"));
+        Builder.AddPlaceholderChunk(Builder.getAllocator().CopyString(
+                                         (*CurP)->getIdentifier()->getName()));
       } else {
-        Pattern->AddTypedTextChunk(
-                              Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
-        Pattern->AddPlaceholderChunk((*CurP)->getIdentifier()->getName());        
+        Builder.AddTypedTextChunk(
+                  Builder.getAllocator().CopyString(
+                                                  Sel.getNameForSlot(I) + ":"));
+        Builder.AddPlaceholderChunk(Builder.getAllocator().CopyString(
+                                         (*CurP)->getIdentifier()->getName())); 
       }
     }
   }
   
-  Results.AddResult(CodeCompletionResult(Pattern, CCP_SuperCompletion,
+  Results.AddResult(CodeCompletionResult(Builder.TakeString(), CCP_SuperCompletion,
                                          SuperMethod->isInstanceMethod()
                                            ? CXCursor_ObjCInstanceMethodDecl
                                            : CXCursor_ObjCClassMethodDecl));
@@ -4176,10 +4570,10 @@ static ObjCMethodDecl *AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
                                    
 void Sema::CodeCompleteObjCMessageReceiver(Scope *S) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_ObjCMessageReceiver,
+                        &ResultBuilder::IsObjCMessageReceiver);
   
-  // Find anything that looks like it could be a message receiver.
-  Results.setFilter(&ResultBuilder::IsObjCMessageReceiver);
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
   Results.EnterNewScope();
   LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
@@ -4199,15 +4593,15 @@ void Sema::CodeCompleteObjCMessageReceiver(Scope *S) {
   
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
-  HandleCodeCompleteResults(this, CodeCompleter, 
-                            CodeCompletionContext::CCC_ObjCMessageReceiver,
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(), Results.size());
   
 }
 
 void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
                                         IdentifierInfo **SelIdents,
-                                        unsigned NumSelIdents) {
+                                        unsigned NumSelIdents,
+                                        bool AtArgumentExpression) {
   ObjCInterfaceDecl *CDecl = 0;
   if (ObjCMethodDecl *CurMethod = getCurMethodDecl()) {
     // Figure out which interface we're in.
@@ -4223,15 +4617,11 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
     if (CurMethod->isInstanceMethod()) {
       // We are inside an instance method, which means that the message
       // send [super ...] is actually calling an instance method on the
-      // current object. Build the super expression and handle this like
-      // an instance method.
-      QualType SuperTy = Context.getObjCInterfaceType(CDecl);
-      SuperTy = Context.getObjCObjectPointerType(SuperTy);
-      ExprResult Super
-        = Owned(new (Context) ObjCSuperExpr(SuperLoc, SuperTy));
-      return CodeCompleteObjCInstanceMessage(S, (Expr *)Super.get(),
+      // current object.
+      return CodeCompleteObjCInstanceMessage(S, 0,
                                              SelIdents, NumSelIdents,
-                                             /*IsSuper=*/true);
+                                             AtArgumentExpression,
+                                             CDecl);
     }
 
     // Fall through to send to the superclass in CDecl.
@@ -4256,7 +4646,8 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
       id.setIdentifier(Super, SuperLoc);
       ExprResult SuperExpr = ActOnIdExpression(S, SS, id, false, false);
       return CodeCompleteObjCInstanceMessage(S, (Expr *)SuperExpr.get(),
-                                             SelIdents, NumSelIdents);
+                                             SelIdents, NumSelIdents,
+                                             AtArgumentExpression);
     }
 
     // Fall through
@@ -4266,71 +4657,105 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
   if (CDecl)
     Receiver = ParsedType::make(Context.getObjCInterfaceType(CDecl));
   return CodeCompleteObjCClassMessage(S, Receiver, SelIdents, 
-                                      NumSelIdents, /*IsSuper=*/true);
+                                      NumSelIdents, AtArgumentExpression,
+                                      /*IsSuper=*/true);
 }
 
-void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
-                                        IdentifierInfo **SelIdents,
-                                        unsigned NumSelIdents) {
-  CodeCompleteObjCClassMessage(S, Receiver, SelIdents, NumSelIdents, false);
+/// \brief Given a set of code-completion results for the argument of a message
+/// send, determine the preferred type (if any) for that argument expression.
+static QualType getPreferredArgumentTypeForMessageSend(ResultBuilder &Results,
+                                                       unsigned NumSelIdents) {
+  typedef CodeCompletionResult Result;  
+  ASTContext &Context = Results.getSema().Context;
+  
+  QualType PreferredType;
+  unsigned BestPriority = CCP_Unlikely * 2;
+  Result *ResultsData = Results.data();
+  for (unsigned I = 0, N = Results.size(); I != N; ++I) {
+    Result &R = ResultsData[I];
+    if (R.Kind == Result::RK_Declaration && 
+        isa<ObjCMethodDecl>(R.Declaration)) {
+      if (R.Priority <= BestPriority) {
+        ObjCMethodDecl *Method = cast<ObjCMethodDecl>(R.Declaration);
+        if (NumSelIdents <= Method->param_size()) {
+          QualType MyPreferredType = Method->param_begin()[NumSelIdents - 1]
+                                       ->getType();
+          if (R.Priority < BestPriority || PreferredType.isNull()) {
+            BestPriority = R.Priority;
+            PreferredType = MyPreferredType;
+          } else if (!Context.hasSameUnqualifiedType(PreferredType,
+                                                     MyPreferredType)) {
+            PreferredType = QualType();
+          }
+        }
+      }
+    }
+  }
+
+  return PreferredType;
 }
 
-void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
-                                        IdentifierInfo **SelIdents,
-                                        unsigned NumSelIdents,
-                                        bool IsSuper) {
+static void AddClassMessageCompletions(Sema &SemaRef, Scope *S, 
+                                       ParsedType Receiver,
+                                       IdentifierInfo **SelIdents,
+                                       unsigned NumSelIdents,
+                                       bool AtArgumentExpression,
+                                       bool IsSuper,
+                                       ResultBuilder &Results) {
   typedef CodeCompletionResult Result;
   ObjCInterfaceDecl *CDecl = 0;
-
+  
   // If the given name refers to an interface type, retrieve the
   // corresponding declaration.
   if (Receiver) {
-    QualType T = GetTypeFromParser(Receiver, 0);
+    QualType T = SemaRef.GetTypeFromParser(Receiver, 0);
     if (!T.isNull()) 
       if (const ObjCObjectType *Interface = T->getAs<ObjCObjectType>())
         CDecl = Interface->getInterface();
   }
-
+  
   // Add all of the factory methods in this Objective-C class, its protocols,
   // superclasses, categories, implementation, etc.
-  ResultBuilder Results(*this);
   Results.EnterNewScope();
-
+  
   // If this is a send-to-super, try to add the special "super" send 
   // completion.
   if (IsSuper) {
     if (ObjCMethodDecl *SuperMethod
-          = AddSuperSendCompletion(*this, false, SelIdents, NumSelIdents, 
-                                   Results))
+        = AddSuperSendCompletion(SemaRef, false, SelIdents, NumSelIdents, 
+                                 Results))
       Results.Ignore(SuperMethod);
   }
-
+  
   // If we're inside an Objective-C method definition, prefer its selector to
   // others.
-  if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
+  if (ObjCMethodDecl *CurMethod = SemaRef.getCurMethodDecl())
     Results.setPreferredSelector(CurMethod->getSelector());
-
+  
+  VisitedSelectorSet Selectors;
   if (CDecl) 
-    AddObjCMethods(CDecl, false, MK_Any, SelIdents, NumSelIdents, CurContext, 
+    AddObjCMethods(CDecl, false, MK_Any, SelIdents, NumSelIdents, 
+                   SemaRef.CurContext, Selectors, AtArgumentExpression,
                    Results);  
   else {
     // We're messaging "id" as a type; provide all class/factory methods.
-
+    
     // If we have an external source, load the entire class method
     // pool from the AST file.
-    if (ExternalSource) {
-      for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
+    if (SemaRef.ExternalSource) {
+      for (uint32_t I = 0, 
+                    N = SemaRef.ExternalSource->GetNumExternalSelectors();
            I != N; ++I) {
-        Selector Sel = ExternalSource->GetExternalSelector(I);
-        if (Sel.isNull() || MethodPool.count(Sel))
+        Selector Sel = SemaRef.ExternalSource->GetExternalSelector(I);
+        if (Sel.isNull() || SemaRef.MethodPool.count(Sel))
           continue;
-
-        ReadMethodPool(Sel);
+        
+        SemaRef.ReadMethodPool(Sel);
       }
     }
-
-    for (GlobalMethodPool::iterator M = MethodPool.begin(),
-                                    MEnd = MethodPool.end();
+    
+    for (Sema::GlobalMethodPool::iterator M = SemaRef.MethodPool.begin(),
+                                       MEnd = SemaRef.MethodPool.end();
          M != MEnd; ++M) {
       for (ObjCMethodList *MethList = &M->second.second;
            MethList && MethList->Method; 
@@ -4338,16 +4763,43 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
         if (!isAcceptableObjCMethod(MethList->Method, MK_Any, SelIdents, 
                                     NumSelIdents))
           continue;
-
+        
         Result R(MethList->Method, 0);
         R.StartParameter = NumSelIdents;
         R.AllParametersAreInformative = false;
-        Results.MaybeAddResult(R, CurContext);
+        Results.MaybeAddResult(R, SemaRef.CurContext);
       }
     }
   }
+  
+  Results.ExitScope();  
+}
+
+void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
+                                        IdentifierInfo **SelIdents,
+                                        unsigned NumSelIdents,
+                                        bool AtArgumentExpression,
+                                        bool IsSuper) {
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
+  AddClassMessageCompletions(*this, S, Receiver, SelIdents, NumSelIdents, 
+                             AtArgumentExpression, IsSuper, Results);
+  
+  // If we're actually at the argument expression (rather than prior to the 
+  // selector), we're actually performing code completion for an expression.
+  // Determine whether we have a single, best method. If so, we can 
+  // code-complete the expression using the corresponding parameter type as
+  // our preferred type, improving completion results.
+  if (AtArgumentExpression) {
+    QualType PreferredType = getPreferredArgumentTypeForMessageSend(Results, 
+                                                                    NumSelIdents);
+    if (PreferredType.isNull())
+      CodeCompleteOrdinaryName(S, PCC_Expression);
+    else
+      CodeCompleteExpression(S, PreferredType);
+    return;
+  }
 
-  Results.ExitScope();
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_Other,
                             Results.data(), Results.size());
@@ -4355,30 +4807,45 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
 
 void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
                                            IdentifierInfo **SelIdents,
-                                           unsigned NumSelIdents) {
-  CodeCompleteObjCInstanceMessage(S, Receiver, SelIdents, NumSelIdents, false);
-}
-
-void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
-                                           IdentifierInfo **SelIdents,
                                            unsigned NumSelIdents,
-                                           bool IsSuper) {
+                                           bool AtArgumentExpression,
+                                           ObjCInterfaceDecl *Super) {
   typedef CodeCompletionResult Result;
   
   Expr *RecExpr = static_cast<Expr *>(Receiver);
   
   // If necessary, apply function/array conversion to the receiver.
   // C99 6.7.5.3p[7,8].
-  DefaultFunctionArrayLvalueConversion(RecExpr);
-  QualType ReceiverType = RecExpr->getType();
+  if (RecExpr)
+    DefaultFunctionArrayLvalueConversion(RecExpr);
+  QualType ReceiverType = RecExpr? RecExpr->getType() 
+                          : Super? Context.getObjCObjectPointerType(
+                                            Context.getObjCInterfaceType(Super))
+                                 : Context.getObjCIdType();
   
+  // If we're messaging an expression with type "id" or "Class", check
+  // whether we know something special about the receiver that allows
+  // us to assume a more-specific receiver type.
+  if (ReceiverType->isObjCIdType() || ReceiverType->isObjCClassType())
+    if (ObjCInterfaceDecl *IFace = GetAssumedMessageSendExprType(RecExpr)) {
+      if (ReceiverType->isObjCClassType())
+        return CodeCompleteObjCClassMessage(S, 
+                       ParsedType::make(Context.getObjCInterfaceType(IFace)),
+                                            SelIdents, NumSelIdents,
+                                            AtArgumentExpression, Super);
+
+      ReceiverType = Context.getObjCObjectPointerType(
+                                          Context.getObjCInterfaceType(IFace));
+    }
+
   // Build the set of methods we can see.
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
   // If this is a send-to-super, try to add the special "super" send 
   // completion.
-  if (IsSuper) {
+  if (Super) {
     if (ObjCMethodDecl *SuperMethod
           = AddSuperSendCompletion(*this, false, SelIdents, NumSelIdents, 
                                    Results))
@@ -4389,14 +4856,9 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
   // others.
   if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
     Results.setPreferredSelector(CurMethod->getSelector());
-
-  // If we're messaging an expression with type "id" or "Class", check
-  // whether we know something special about the receiver that allows
-  // us to assume a more-specific receiver type.
-  if (ReceiverType->isObjCIdType() || ReceiverType->isObjCClassType())
-    if (ObjCInterfaceDecl *IFace = GetAssumedMessageSendExprType(RecExpr))
-      ReceiverType = Context.getObjCObjectPointerType(
-                                          Context.getObjCInterfaceType(IFace));
+  
+  // Keep track of the selectors we've already added.
+  VisitedSelectorSet Selectors;
   
   // Handle messages to Class. This really isn't a message to an instance
   // method, so we treat it the same way we would treat a message send to a
@@ -4406,7 +4868,7 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
     if (ObjCMethodDecl *CurMethod = getCurMethodDecl()) {
       if (ObjCInterfaceDecl *ClassDecl = CurMethod->getClassInterface())
         AddObjCMethods(ClassDecl, false, MK_Any, SelIdents, NumSelIdents, 
-                       CurContext, Results);
+                       CurContext, Selectors, AtArgumentExpression, Results);
     }
   } 
   // Handle messages to a qualified ID ("id<foo>").
@@ -4417,21 +4879,22 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
                                               E = QualID->qual_end(); 
          I != E; ++I)
       AddObjCMethods(*I, true, MK_Any, SelIdents, NumSelIdents, CurContext, 
-                     Results);
+                     Selectors, AtArgumentExpression, Results);
   }
   // Handle messages to a pointer to interface type.
   else if (const ObjCObjectPointerType *IFacePtr
                               = ReceiverType->getAsObjCInterfacePointerType()) {
     // Search the class, its superclasses, etc., for instance methods.
     AddObjCMethods(IFacePtr->getInterfaceDecl(), true, MK_Any, SelIdents,
-                   NumSelIdents, CurContext, Results);
+                   NumSelIdents, CurContext, Selectors, AtArgumentExpression, 
+                   Results);
     
     // Search protocols for instance methods.
     for (ObjCObjectPointerType::qual_iterator I = IFacePtr->qual_begin(),
          E = IFacePtr->qual_end(); 
          I != E; ++I)
       AddObjCMethods(*I, true, MK_Any, SelIdents, NumSelIdents, CurContext, 
-                     Results);
+                     Selectors, AtArgumentExpression, Results);
   }
   // Handle messages to "id".
   else if (ReceiverType->isObjCIdType()) {
@@ -4460,7 +4923,10 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
         if (!isAcceptableObjCMethod(MethList->Method, MK_Any, SelIdents, 
                                     NumSelIdents))
           continue;
-
+        
+        if (!Selectors.insert(MethList->Method->getSelector()))
+          continue;
+        
         Result R(MethList->Method, 0);
         R.StartParameter = NumSelIdents;
         R.AllParametersAreInformative = false;
@@ -4468,8 +4934,24 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
       }
     }
   }
-
   Results.ExitScope();
+  
+  
+  // If we're actually at the argument expression (rather than prior to the 
+  // selector), we're actually performing code completion for an expression.
+  // Determine whether we have a single, best method. If so, we can 
+  // code-complete the expression using the corresponding parameter type as
+  // our preferred type, improving completion results.
+  if (AtArgumentExpression) {
+    QualType PreferredType = getPreferredArgumentTypeForMessageSend(Results, 
+                                                                  NumSelIdents);
+    if (PreferredType.isNull())
+      CodeCompleteOrdinaryName(S, PCC_Expression);
+    else
+      CodeCompleteExpression(S, PreferredType);
+    return;
+  }
+  
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_Other,
                             Results.data(),Results.size());
@@ -4506,7 +4988,8 @@ void Sema::CodeCompleteObjCSelector(Scope *S, IdentifierInfo **SelIdents,
     }
   }
   
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_SelectorName);
   Results.EnterNewScope();
   for (GlobalMethodPool::iterator M = MethodPool.begin(),
                                MEnd = MethodPool.end();
@@ -4516,10 +4999,11 @@ void Sema::CodeCompleteObjCSelector(Scope *S, IdentifierInfo **SelIdents,
     if (!isAcceptableObjCSelector(Sel, MK_Any, SelIdents, NumSelIdents))
       continue;
 
-    CodeCompletionString *Pattern = new CodeCompletionString;
+    CodeCompletionBuilder Builder(Results.getAllocator());
     if (Sel.isUnarySelector()) {
-      Pattern->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
-      Results.AddResult(Pattern);
+      Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                                       Sel.getNameForSlot(0)));
+      Results.AddResult(Builder.TakeString());
       continue;
     }
     
@@ -4527,16 +5011,17 @@ void Sema::CodeCompleteObjCSelector(Scope *S, IdentifierInfo **SelIdents,
     for (unsigned I = 0, N = Sel.getNumArgs(); I != N; ++I) {
       if (I == NumSelIdents) {
         if (!Accumulator.empty()) {
-          Pattern->AddInformativeChunk(Accumulator);
+          Builder.AddInformativeChunk(Builder.getAllocator().CopyString(
+                                                 Accumulator));
           Accumulator.clear();
         }
       }
       
-      Accumulator += Sel.getIdentifierInfoForSlot(I)->getName().str();
+      Accumulator += Sel.getNameForSlot(I).str();
       Accumulator += ':';
     }
-    Pattern->AddTypedTextChunk(Accumulator);
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk(Builder.getAllocator().CopyString( Accumulator));
+    Results.AddResult(Builder.TakeString());
   }
   Results.ExitScope();
   
@@ -4575,35 +5060,46 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
 
 void Sema::CodeCompleteObjCProtocolReferences(IdentifierLocPair *Protocols,
                                               unsigned NumProtocols) {
-  ResultBuilder Results(*this);
-  Results.EnterNewScope();
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_ObjCProtocolName);
   
-  // Tell the result set to ignore all of the protocols we have
-  // already seen.
-  for (unsigned I = 0; I != NumProtocols; ++I)
-    if (ObjCProtocolDecl *Protocol = LookupProtocol(Protocols[I].first,
-                                                    Protocols[I].second))
-      Results.Ignore(Protocol);
-
-  // Add all protocols.
-  AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, false,
-                     Results);
+  if (CodeCompleter && CodeCompleter->includeGlobals()) {
+    Results.EnterNewScope();
+    
+    // Tell the result set to ignore all of the protocols we have
+    // already seen.
+    // FIXME: This doesn't work when caching code-completion results.
+    for (unsigned I = 0; I != NumProtocols; ++I)
+      if (ObjCProtocolDecl *Protocol = LookupProtocol(Protocols[I].first,
+                                                      Protocols[I].second))
+        Results.Ignore(Protocol);
+
+    // Add all protocols.
+    AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, false,
+                       Results);
 
-  Results.ExitScope();
+    Results.ExitScope();
+  }
+  
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_ObjCProtocolName,
                             Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCProtocolDecl(Scope *) {
-  ResultBuilder Results(*this);
-  Results.EnterNewScope();
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_ObjCProtocolName);
   
-  // Add all protocols.
-  AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, true,
-                     Results);
+  if (CodeCompleter && CodeCompleter->includeGlobals()) {
+    Results.EnterNewScope();
+    
+    // Add all protocols.
+    AddProtocolResults(Context.getTranslationUnitDecl(), CurContext, true,
+                       Results);
 
-  Results.ExitScope();
+    Results.ExitScope();
+  }
+  
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_ObjCProtocolName,
                             Results.data(),Results.size());
@@ -4639,7 +5135,8 @@ static void AddInterfaceResults(DeclContext *Ctx, DeclContext *CurContext,
 }
 
 void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) { 
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   
   // Add all classes.
@@ -4647,6 +5144,8 @@ void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) {
                       false, Results);
 
   Results.ExitScope();
+  // FIXME: Add a special context for this, use cached global completion 
+  // results.
   HandleCodeCompleteResults(this, CodeCompleter,
                             CodeCompletionContext::CCC_Other,
                             Results.data(),Results.size());
@@ -4654,7 +5153,8 @@ void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) {
 
 void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
                                       SourceLocation ClassNameLoc) { 
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   
   // Make sure that we ignore the class we're currently defining.
@@ -4668,13 +5168,16 @@ void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
                       false, Results);
 
   Results.ExitScope();
+  // FIXME: Add a special context for this, use cached global completion 
+  // results.
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_Other,
                             Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCImplementationDecl(Scope *S) { 
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
 
   // Add all unimplemented classes.
@@ -4682,6 +5185,8 @@ void Sema::CodeCompleteObjCImplementationDecl(Scope *S) {
                       true, Results);
 
   Results.ExitScope();
+  // FIXME: Add a special context for this, use cached global completion 
+  // results.
   HandleCodeCompleteResults(this, CodeCompleter, 
                             CodeCompletionContext::CCC_Other,
                             Results.data(),Results.size());
@@ -4692,7 +5197,8 @@ void Sema::CodeCompleteObjCInterfaceCategory(Scope *S,
                                              SourceLocation ClassNameLoc) {
   typedef CodeCompletionResult Result;
   
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   
   // Ignore any categories we find that have already been implemented by this
   // interface.
@@ -4734,7 +5240,8 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
   if (!Class)
     return CodeCompleteObjCInterfaceCategory(S, ClassName, ClassNameLoc);
     
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   
   // Add all of the categories that have have corresponding interface 
   // declarations in this class and any of its superclasses, except for
@@ -4761,7 +5268,8 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
 
 void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
@@ -4779,14 +5287,15 @@ void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl) {
       Results.Ignore(PropertyImpl->getPropertyDecl());
   
   // Add any properties that we find.
+  AddedPropertiesSet AddedProperties;
   Results.EnterNewScope();
   if (ObjCImplementationDecl *ClassImpl
         = dyn_cast<ObjCImplementationDecl>(Container))
     AddObjCProperties(ClassImpl->getClassInterface(), false, CurContext, 
-                      Results);
+                      AddedProperties, Results);
   else
     AddObjCProperties(cast<ObjCCategoryImplDecl>(Container)->getCategoryDecl(),
-                      false, CurContext, Results);
+                      false, CurContext, AddedProperties, Results);
   Results.ExitScope();
   
   HandleCodeCompleteResults(this, CodeCompleter, 
@@ -4798,7 +5307,8 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
                                                   IdentifierInfo *PropertyName,
                                                   Decl *ObjCImpDecl) {
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
@@ -4847,7 +5357,6 @@ static void FindImplementableMethods(ASTContext &Context,
                                      ObjCContainerDecl *Container,
                                      bool WantInstanceMethods,
                                      QualType ReturnType,
-                                     bool IsInImplementation,
                                      KnownMethodsMap &KnownMethods,
                                      bool InOriginalClass = true) {
   if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(Container)) {
@@ -4855,28 +5364,23 @@ static void FindImplementableMethods(ASTContext &Context,
     const ObjCList<ObjCProtocolDecl> &Protocols
       = IFace->getReferencedProtocols();
     for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
-           E = Protocols.end(); 
+                                              E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods,
-                               InOriginalClass);
+                               KnownMethods, InOriginalClass);
 
-    // If we're not in the implementation of a class, also visit the
-    // superclass.
-    if (!IsInImplementation && IFace->getSuperClass())
-      FindImplementableMethods(Context, IFace->getSuperClass(), 
-                               WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods,
-                               false);
-
-    // Add methods from any class extensions (but not from categories;
-    // those should go into category implementations).
-    for (const ObjCCategoryDecl *Cat = IFace->getFirstClassExtension(); Cat;
-         Cat = Cat->getNextClassExtension())
+    // Add methods from any class extensions and categories.
+    for (const ObjCCategoryDecl *Cat = IFace->getCategoryList(); Cat;
+         Cat = Cat->getNextClassCategory())
       FindImplementableMethods(Context, const_cast<ObjCCategoryDecl*>(Cat), 
                                WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods,
-                               InOriginalClass);      
+                               KnownMethods, false);      
+    
+    // Visit the superclass.
+    if (IFace->getSuperClass())
+      FindImplementableMethods(Context, IFace->getSuperClass(), 
+                               WantInstanceMethods, ReturnType,
+                               KnownMethods, false);
   }
 
   if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(Container)) {
@@ -4884,11 +5388,16 @@ static void FindImplementableMethods(ASTContext &Context,
     const ObjCList<ObjCProtocolDecl> &Protocols
       = Category->getReferencedProtocols();
     for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
-           E = Protocols.end(); 
+                                              E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods,
-                               InOriginalClass);
+                               KnownMethods, InOriginalClass);
+    
+    // If this category is the original class, jump to the interface.
+    if (InOriginalClass && Category->getClassInterface())
+      FindImplementableMethods(Context, Category->getClassInterface(), 
+                               WantInstanceMethods, ReturnType, KnownMethods,
+                               false);
   }
 
   if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Container)) {
@@ -4899,7 +5408,7 @@ static void FindImplementableMethods(ASTContext &Context,
            E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods, false);
+                               KnownMethods, false);
   }
 
   // Add methods in this container. This operation occurs last because
@@ -4918,6 +5427,632 @@ static void FindImplementableMethods(ASTContext &Context,
   }
 }
 
+/// \brief Add the parenthesized return or parameter type chunk to a code 
+/// completion string.
+static void AddObjCPassingTypeChunk(QualType Type,
+                                    ASTContext &Context,
+                                    CodeCompletionBuilder &Builder) {
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddTextChunk(GetCompletionTypeString(Type, Context, 
+                                               Builder.getAllocator()));
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+}
+
+/// \brief Determine whether the given class is or inherits from a class by
+/// the given name.
+static bool InheritsFromClassNamed(ObjCInterfaceDecl *Class, 
+                                   llvm::StringRef Name) {
+  if (!Class)
+    return false;
+  
+  if (Class->getIdentifier() && Class->getIdentifier()->getName() == Name)
+    return true;
+  
+  return InheritsFromClassNamed(Class->getSuperClass(), Name);
+}
+                  
+/// \brief Add code completions for Objective-C Key-Value Coding (KVC) and
+/// Key-Value Observing (KVO).
+static void AddObjCKeyValueCompletions(ObjCPropertyDecl *Property,
+                                       bool IsInstanceMethod,
+                                       QualType ReturnType,
+                                       ASTContext &Context,
+                                       const KnownMethodsMap &KnownMethods,
+                                       ResultBuilder &Results) {
+  IdentifierInfo *PropName = Property->getIdentifier();
+  if (!PropName || PropName->getLength() == 0)
+    return;
+  
+  
+  // Builder that will create each code completion.
+  typedef CodeCompletionResult Result;
+  CodeCompletionAllocator &Allocator = Results.getAllocator();
+  CodeCompletionBuilder Builder(Allocator);
+  
+  // The selector table.
+  SelectorTable &Selectors = Context.Selectors;
+  
+  // The property name, copied into the code completion allocation region
+  // on demand.
+  struct KeyHolder {
+    CodeCompletionAllocator &Allocator;
+    llvm::StringRef Key;
+    const char *CopiedKey;
+    
+    KeyHolder(CodeCompletionAllocator &Allocator, llvm::StringRef Key)
+    : Allocator(Allocator), Key(Key), CopiedKey(0) { }
+    
+    operator const char *() {
+      if (CopiedKey)
+        return CopiedKey;
+      
+      return CopiedKey = Allocator.CopyString(Key);
+    }
+  } Key(Allocator, PropName->getName());
+  
+  // The uppercased name of the property name.
+  std::string UpperKey = PropName->getName();
+  if (!UpperKey.empty())
+    UpperKey[0] = toupper(UpperKey[0]);      
+  
+  bool ReturnTypeMatchesProperty = ReturnType.isNull() ||
+    Context.hasSameUnqualifiedType(ReturnType.getNonReferenceType(), 
+                                   Property->getType());
+  bool ReturnTypeMatchesVoid 
+    = ReturnType.isNull() || ReturnType->isVoidType();
+  
+  // Add the normal accessor -(type)key.
+  if (IsInstanceMethod &&
+      !KnownMethods.count(Selectors.getNullarySelector(PropName)) &&
+      ReturnTypeMatchesProperty && !Property->getGetterMethodDecl()) {
+    if (ReturnType.isNull())
+      AddObjCPassingTypeChunk(Property->getType(), Context, Builder);
+    
+    Builder.AddTypedTextChunk(Key);
+    Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
+                             CXCursor_ObjCInstanceMethodDecl));
+  }
+  
+  // If we have an integral or boolean property (or the user has provided
+  // an integral or boolean return type), add the accessor -(type)isKey.
+  if (IsInstanceMethod &&
+      ((!ReturnType.isNull() && 
+        (ReturnType->isIntegerType() || ReturnType->isBooleanType())) ||
+       (ReturnType.isNull() && 
+        (Property->getType()->isIntegerType() || 
+         Property->getType()->isBooleanType())))) {
+    std::string SelectorName = (llvm::Twine("is") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getNullarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("BOOL");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(
+                                Allocator.CopyString(SelectorId->getName()));
+      Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Add the normal mutator.
+  if (IsInstanceMethod && ReturnTypeMatchesVoid && 
+      !Property->getSetterMethodDecl()) {
+    std::string SelectorName = (llvm::Twine("set") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(
+                                Allocator.CopyString(SelectorId->getName()));
+      Builder.AddTypedTextChunk(":");
+      AddObjCPassingTypeChunk(Property->getType(), Context, Builder);
+      Builder.AddTextChunk(Key);
+      Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Indexed and unordered accessors
+  unsigned IndexedGetterPriority = CCP_CodePattern;
+  unsigned IndexedSetterPriority = CCP_CodePattern;
+  unsigned UnorderedGetterPriority = CCP_CodePattern;
+  unsigned UnorderedSetterPriority = CCP_CodePattern;
+  if (const ObjCObjectPointerType *ObjCPointer 
+                    = Property->getType()->getAs<ObjCObjectPointerType>()) {
+    if (ObjCInterfaceDecl *IFace = ObjCPointer->getInterfaceDecl()) {
+      // If this interface type is not provably derived from a known
+      // collection, penalize the corresponding completions.
+      if (!InheritsFromClassNamed(IFace, "NSMutableArray")) {
+        IndexedSetterPriority += CCD_ProbablyNotObjCCollection;            
+        if (!InheritsFromClassNamed(IFace, "NSArray"))
+          IndexedGetterPriority += CCD_ProbablyNotObjCCollection;
+      }
+
+      if (!InheritsFromClassNamed(IFace, "NSMutableSet")) {
+        UnorderedSetterPriority += CCD_ProbablyNotObjCCollection;            
+        if (!InheritsFromClassNamed(IFace, "NSSet"))
+          UnorderedGetterPriority += CCD_ProbablyNotObjCCollection;
+      }
+    }
+  } else {
+    IndexedGetterPriority += CCD_ProbablyNotObjCCollection;
+    IndexedSetterPriority += CCD_ProbablyNotObjCCollection;
+    UnorderedGetterPriority += CCD_ProbablyNotObjCCollection;
+    UnorderedSetterPriority += CCD_ProbablyNotObjCCollection;
+  }
+  
+  // Add -(NSUInteger)countOf<key>
+  if (IsInstanceMethod &&  
+      (ReturnType.isNull() || ReturnType->isIntegerType())) {
+    std::string SelectorName = (llvm::Twine("countOf") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getNullarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("NSUInteger");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(
+                                Allocator.CopyString(SelectorId->getName()));
+      Results.AddResult(Result(Builder.TakeString(), 
+                               std::min(IndexedGetterPriority, 
+                                        UnorderedGetterPriority),
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Indexed getters
+  // Add -(id)objectInKeyAtIndex:(NSUInteger)index
+  if (IsInstanceMethod &&
+      (ReturnType.isNull() || ReturnType->isObjCObjectPointerType())) {
+    std::string SelectorName
+      = (llvm::Twine("objectIn") + UpperKey + "AtIndex").str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("id");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSUInteger");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("index");
+      Results.AddResult(Result(Builder.TakeString(), IndexedGetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Add -(NSArray *)keyAtIndexes:(NSIndexSet *)indexes
+  if (IsInstanceMethod &&
+      (ReturnType.isNull() || 
+       (ReturnType->isObjCObjectPointerType() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
+                                                ->getName() == "NSArray"))) {
+    std::string SelectorName
+      = (llvm::Twine(Property->getName()) + "AtIndexes").str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("NSArray *");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+       
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSIndexSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("indexes");
+      Results.AddResult(Result(Builder.TakeString(), IndexedGetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Add -(void)getKey:(type **)buffer range:(NSRange)inRange
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("get") + UpperKey).str();
+    IdentifierInfo *SelectorIds[2] = {
+      &Context.Idents.get(SelectorName),
+      &Context.Idents.get("range")
+    };
+    
+    if (!KnownMethods.count(Selectors.getSelector(2, SelectorIds))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("object-type");
+      Builder.AddTextChunk(" **");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("buffer");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTypedTextChunk("range:");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSRange");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("inRange");
+      Results.AddResult(Result(Builder.TakeString(), IndexedGetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Mutable indexed accessors
+  
+  // - (void)insertObject:(type *)object inKeyAtIndex:(NSUInteger)index
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("in") + UpperKey + "AtIndex").str();
+    IdentifierInfo *SelectorIds[2] = {
+      &Context.Idents.get("insertObject"),
+      &Context.Idents.get(SelectorName)
+    };
+    
+    if (!KnownMethods.count(Selectors.getSelector(2, SelectorIds))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk("insertObject:");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("object-type");
+      Builder.AddTextChunk(" *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("object");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("NSUInteger");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("index");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // - (void)insertKey:(NSArray *)array atIndexes:(NSIndexSet *)indexes
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("insert") + UpperKey).str();
+    IdentifierInfo *SelectorIds[2] = {
+      &Context.Idents.get(SelectorName),
+      &Context.Idents.get("atIndexes")
+    };
+    
+    if (!KnownMethods.count(Selectors.getSelector(2, SelectorIds))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSArray *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("array");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTypedTextChunk("atIndexes:");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("NSIndexSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("indexes");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // -(void)removeObjectFromKeyAtIndex:(NSUInteger)index
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName
+      = (llvm::Twine("removeObjectFrom") + UpperKey + "AtIndex").str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);        
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSUInteger");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("index");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // -(void)removeKeyAtIndexes:(NSIndexSet *)indexes
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName
+      = (llvm::Twine("remove") + UpperKey + "AtIndexes").str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);        
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSIndexSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("indexes");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // - (void)replaceObjectInKeyAtIndex:(NSUInteger)index withObject:(id)object
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName
+      = (llvm::Twine("replaceObjectIn") + UpperKey + "AtIndex").str();
+    IdentifierInfo *SelectorIds[2] = {
+      &Context.Idents.get(SelectorName),
+      &Context.Idents.get("withObject")
+    };
+    
+    if (!KnownMethods.count(Selectors.getSelector(2, SelectorIds))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("NSUInteger");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("index");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTypedTextChunk("withObject:");
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("id");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("object");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // - (void)replaceKeyAtIndexes:(NSIndexSet *)indexes withKey:(NSArray *)array
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName1 
+      = (llvm::Twine("replace") + UpperKey + "AtIndexes").str();
+    std::string SelectorName2 = (llvm::Twine("with") + UpperKey).str();
+    IdentifierInfo *SelectorIds[2] = {
+      &Context.Idents.get(SelectorName1),
+      &Context.Idents.get(SelectorName2)
+    };
+    
+    if (!KnownMethods.count(Selectors.getSelector(2, SelectorIds))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName1 + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("NSIndexSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("indexes");
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName2 + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSArray *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("array");
+      Results.AddResult(Result(Builder.TakeString(), IndexedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }  
+  
+  // Unordered getters
+  // - (NSEnumerator *)enumeratorOfKey
+  if (IsInstanceMethod && 
+      (ReturnType.isNull() || 
+       (ReturnType->isObjCObjectPointerType() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
+          ->getName() == "NSEnumerator"))) {
+    std::string SelectorName = (llvm::Twine("enumeratorOf") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getNullarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("NSEnumerator *");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+       
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName));
+      Results.AddResult(Result(Builder.TakeString(), UnorderedGetterPriority, 
+                              CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+
+  // - (type *)memberOfKey:(type *)object
+  if (IsInstanceMethod && 
+      (ReturnType.isNull() || ReturnType->isObjCObjectPointerType())) {
+    std::string SelectorName = (llvm::Twine("memberOf") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddPlaceholderChunk("object-type");
+        Builder.AddTextChunk(" *");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      if (ReturnType.isNull()) {
+        Builder.AddPlaceholderChunk("object-type");
+        Builder.AddTextChunk(" *");
+      } else {
+        Builder.AddTextChunk(GetCompletionTypeString(ReturnType, Context, 
+                                                     Builder.getAllocator()));
+      }
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("object");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedGetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+  
+  // Mutable unordered accessors
+  // - (void)addKeyObject:(type *)object
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName
+      = (llvm::Twine("add") + UpperKey + llvm::Twine("Object")).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("object-type");
+      Builder.AddTextChunk(" *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("object");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }  
+
+  // - (void)addKey:(NSSet *)objects
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("add") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("objects");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }  
+  
+  // - (void)removeKeyObject:(type *)object
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName
+      = (llvm::Twine("remove") + UpperKey + llvm::Twine("Object")).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddPlaceholderChunk("object-type");
+      Builder.AddTextChunk(" *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("object");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }  
+  
+  // - (void)removeKey:(NSSet *)objects
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("remove") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("objects");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }    
+
+  // - (void)intersectKey:(NSSet *)objects
+  if (IsInstanceMethod && ReturnTypeMatchesVoid) {
+    std::string SelectorName = (llvm::Twine("intersect") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getUnarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("void");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+      
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName + ":"));
+      Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+      Builder.AddTextChunk("NSSet *");
+      Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      Builder.AddTextChunk("objects");
+      Results.AddResult(Result(Builder.TakeString(), UnorderedSetterPriority, 
+                               CXCursor_ObjCInstanceMethodDecl));
+    }
+  }  
+  
+  // Key-Value Observing
+  // + (NSSet *)keyPathsForValuesAffectingKey
+  if (!IsInstanceMethod && 
+      (ReturnType.isNull() || 
+       (ReturnType->isObjCObjectPointerType() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl() &&
+        ReturnType->getAs<ObjCObjectPointerType>()->getInterfaceDecl()
+                                                    ->getName() == "NSSet"))) {
+    std::string SelectorName 
+      = (llvm::Twine("keyPathsForValuesAffecting") + UpperKey).str();
+    IdentifierInfo *SelectorId = &Context.Idents.get(SelectorName);
+    if (!KnownMethods.count(Selectors.getNullarySelector(SelectorId))) {
+      if (ReturnType.isNull()) {
+        Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+        Builder.AddTextChunk("NSSet *");
+        Builder.AddChunk(CodeCompletionString::CK_RightParen);
+      }
+       
+      Builder.AddTypedTextChunk(Allocator.CopyString(SelectorName));
+      Results.AddResult(Result(Builder.TakeString(), CCP_CodePattern, 
+                              CXCursor_ObjCInstanceMethodDecl));
+    }
+  }
+}
+
 void Sema::CodeCompleteObjCMethodDecl(Scope *S, 
                                       bool IsInstanceMethod,
                                       ParsedType ReturnTy,
@@ -4926,33 +6061,27 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   // provided.
   QualType ReturnType = GetTypeFromParser(ReturnTy);
 
-  // Determine where we should start searching for methods, and where we 
-  ObjCContainerDecl *SearchDecl = 0, *CurrentDecl = 0;
+  // Determine where we should start searching for methods.
+  ObjCContainerDecl *SearchDecl = 0;
   bool IsInImplementation = false;
   if (Decl *D = IDecl) {
     if (ObjCImplementationDecl *Impl = dyn_cast<ObjCImplementationDecl>(D)) {
       SearchDecl = Impl->getClassInterface();
-      CurrentDecl = Impl;
       IsInImplementation = true;
     } else if (ObjCCategoryImplDecl *CatImpl 
-                                       = dyn_cast<ObjCCategoryImplDecl>(D)) {
+                                         = dyn_cast<ObjCCategoryImplDecl>(D)) {
       SearchDecl = CatImpl->getCategoryDecl();
-      CurrentDecl = CatImpl;
       IsInImplementation = true;
-    } else {
+    } else
       SearchDecl = dyn_cast<ObjCContainerDecl>(D);
-      CurrentDecl = SearchDecl;
-    }
   }
 
   if (!SearchDecl && S) {
-    if (DeclContext *DC = static_cast<DeclContext *>(S->getEntity())) {
+    if (DeclContext *DC = static_cast<DeclContext *>(S->getEntity()))
       SearchDecl = dyn_cast<ObjCContainerDecl>(DC);
-      CurrentDecl = SearchDecl;
-    }
   }
 
-  if (!SearchDecl || !CurrentDecl) {
+  if (!SearchDecl) {
     HandleCodeCompleteResults(this, CodeCompleter, 
                               CodeCompletionContext::CCC_Other,
                               0, 0);
@@ -4962,24 +6091,12 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   // Find all of the methods that we could declare/implement here.
   KnownMethodsMap KnownMethods;
   FindImplementableMethods(Context, SearchDecl, IsInstanceMethod, 
-                           ReturnType, IsInImplementation, KnownMethods);
+                           ReturnType, KnownMethods);
   
-  // Erase any methods that have already been declared or
-  // implemented here.
-  for (ObjCContainerDecl::method_iterator M = CurrentDecl->meth_begin(),
-                                       MEnd = CurrentDecl->meth_end();
-       M != MEnd; ++M) {
-    if ((*M)->isInstanceMethod() != IsInstanceMethod)
-      continue;
-    
-    KnownMethodsMap::iterator Pos = KnownMethods.find((*M)->getSelector());
-    if (Pos != KnownMethods.end())
-      KnownMethods.erase(Pos);
-  }
-
   // Add declarations or definitions for each of the known methods.
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   Results.EnterNewScope();
   PrintingPolicy Policy(Context.PrintingPolicy);
   Policy.AnonymousTagLocations = false;
@@ -4987,22 +6104,18 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
                               MEnd = KnownMethods.end();
        M != MEnd; ++M) {
     ObjCMethodDecl *Method = M->second.first;
-    CodeCompletionString *Pattern = new CodeCompletionString;
+    CodeCompletionBuilder Builder(Results.getAllocator());
     
     // If the result type was not already provided, add it to the
     // pattern as (type).
-    if (ReturnType.isNull()) {
-      std::string TypeStr;
-      Method->getResultType().getAsStringInternal(TypeStr, Policy);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddTextChunk(TypeStr);
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-    }
+    if (ReturnType.isNull())
+      AddObjCPassingTypeChunk(Method->getResultType(), Context, Builder);
 
     Selector Sel = Method->getSelector();
 
     // Add the first part of the selector to the pattern.
-    Pattern->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+    Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                                       Sel.getNameForSlot(0)));
 
     // Add parameters to the pattern.
     unsigned I = 0;
@@ -5011,59 +6124,82 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
          P != PEnd; (void)++P, ++I) {
       // Add the part of the selector name.
       if (I == 0)
-        Pattern->AddChunk(CodeCompletionString::CK_Colon);
+        Builder.AddTypedTextChunk(":");
       else if (I < Sel.getNumArgs()) {
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddTextChunk(Sel.getIdentifierInfoForSlot(I)->getName());
-        Pattern->AddChunk(CodeCompletionString::CK_Colon);
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddTypedTextChunk(
+                Builder.getAllocator().CopyString(Sel.getNameForSlot(I) + ":"));
       } else
         break;
 
       // Add the parameter type.
-      std::string TypeStr;
-      (*P)->getOriginalType().getAsStringInternal(TypeStr, Policy);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-      Pattern->AddTextChunk(TypeStr);
-      Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+      AddObjCPassingTypeChunk((*P)->getOriginalType(), Context, Builder);
       
       if (IdentifierInfo *Id = (*P)->getIdentifier())
-        Pattern->AddTextChunk(Id->getName());      
+        Builder.AddTextChunk(Builder.getAllocator().CopyString( Id->getName())); 
     }
 
     if (Method->isVariadic()) {
       if (Method->param_size() > 0)
-        Pattern->AddChunk(CodeCompletionString::CK_Comma);
-      Pattern->AddTextChunk("...");
+        Builder.AddChunk(CodeCompletionString::CK_Comma);
+      Builder.AddTextChunk("...");
     }        
 
     if (IsInImplementation && Results.includeCodePatterns()) {
       // We will be defining the method here, so add a compound statement.
-      Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_LeftBrace);
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_LeftBrace);
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
       if (!Method->getResultType()->isVoidType()) {
         // If the result type is not void, add a return clause.
-        Pattern->AddTextChunk("return");
-        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddPlaceholderChunk("expression");
-        Pattern->AddChunk(CodeCompletionString::CK_SemiColon);
+        Builder.AddTextChunk("return");
+        Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+        Builder.AddPlaceholderChunk("expression");
+        Builder.AddChunk(CodeCompletionString::CK_SemiColon);
       } else
-        Pattern->AddPlaceholderChunk("statements");
+        Builder.AddPlaceholderChunk("statements");
         
-      Pattern->AddChunk(CodeCompletionString::CK_VerticalSpace);
-      Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
+      Builder.AddChunk(CodeCompletionString::CK_VerticalSpace);
+      Builder.AddChunk(CodeCompletionString::CK_RightBrace);
     }
 
     unsigned Priority = CCP_CodePattern;
     if (!M->second.second)
       Priority += CCD_InBaseClass;
     
-    Results.AddResult(Result(Pattern, Priority, 
+    Results.AddResult(Result(Builder.TakeString(), Priority, 
                              Method->isInstanceMethod()
                                ? CXCursor_ObjCInstanceMethodDecl
                                : CXCursor_ObjCClassMethodDecl));
   }
 
+  // Add Key-Value-Coding and Key-Value-Observing accessor methods for all of 
+  // the properties in this class and its categories.
+  if (Context.getLangOptions().ObjC2) {
+    llvm::SmallVector<ObjCContainerDecl *, 4> Containers;
+    Containers.push_back(SearchDecl);
+    
+    ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(SearchDecl);
+    if (!IFace)
+      if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(SearchDecl))
+        IFace = Category->getClassInterface();
+    
+    if (IFace) {
+      for (ObjCCategoryDecl *Category = IFace->getCategoryList(); Category;
+           Category = Category->getNextClassCategory())
+        Containers.push_back(Category);
+    }
+    
+    for (unsigned I = 0, N = Containers.size(); I != N; ++I) {
+      for (ObjCContainerDecl::prop_iterator P = Containers[I]->prop_begin(),
+                                         PEnd = Containers[I]->prop_end(); 
+           P != PEnd; ++P) {
+        AddObjCKeyValueCompletions(*P, IsInstanceMethod, ReturnType, Context, 
+                                   KnownMethods, Results);
+      }
+    }
+  }
+  
   Results.ExitScope();
   
   HandleCodeCompleteResults(this, CodeCompleter, 
@@ -5092,7 +6228,8 @@ void Sema::CodeCompleteObjCMethodDeclSelector(Scope *S,
 
   // Build the set of methods we can see.
   typedef CodeCompletionResult Result;
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_Other);
   
   if (ReturnTy)
     Results.setPreferredType(GetTypeFromParser(ReturnTy).getNonReferenceType());
@@ -5114,9 +6251,10 @@ void Sema::CodeCompleteObjCMethodDeclSelector(Scope *S,
         if (NumSelIdents && NumSelIdents <= MethList->Method->param_size()) {
           ParmVarDecl *Param = MethList->Method->param_begin()[NumSelIdents-1];
           if (Param->getIdentifier()) {
-            CodeCompletionString *Pattern = new CodeCompletionString;
-            Pattern->AddTypedTextChunk(Param->getIdentifier()->getName());
-            Results.AddResult(Pattern);
+            CodeCompletionBuilder Builder(Results.getAllocator());
+            Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                           Param->getIdentifier()->getName()));
+            Results.AddResult(Builder.TakeString());
           }
         }
         
@@ -5138,167 +6276,149 @@ void Sema::CodeCompleteObjCMethodDeclSelector(Scope *S,
 }
 
 void Sema::CodeCompletePreprocessorDirective(bool InConditional) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_PreprocessorDirective);
   Results.EnterNewScope();
   
   // #if <condition>
-  CodeCompletionString *Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("if");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("condition");
-  Results.AddResult(Pattern);
+  CodeCompletionBuilder Builder(Results.getAllocator());
+  Builder.AddTypedTextChunk("if");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("condition");
+  Results.AddResult(Builder.TakeString());
   
   // #ifdef <macro>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("ifdef");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("macro");
-  Results.AddResult(Pattern);
-
+  Builder.AddTypedTextChunk("ifdef");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("macro");
+  Results.AddResult(Builder.TakeString());
+  
   // #ifndef <macro>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("ifndef");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("macro");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("ifndef");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("macro");
+  Results.AddResult(Builder.TakeString());
 
   if (InConditional) {
     // #elif <condition>
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("elif");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddPlaceholderChunk("condition");
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk("elif");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddPlaceholderChunk("condition");
+    Results.AddResult(Builder.TakeString());
 
     // #else
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("else");
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk("else");
+    Results.AddResult(Builder.TakeString());
 
     // #endif
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("endif");
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk("endif");
+    Results.AddResult(Builder.TakeString());
   }
   
   // #include "header"
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("include");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddTextChunk("\"");
-  Pattern->AddPlaceholderChunk("header");
-  Pattern->AddTextChunk("\"");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("include");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("\"");
+  Builder.AddPlaceholderChunk("header");
+  Builder.AddTextChunk("\"");
+  Results.AddResult(Builder.TakeString());
 
   // #include <header>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("include");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddTextChunk("<");
-  Pattern->AddPlaceholderChunk("header");
-  Pattern->AddTextChunk(">");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("include");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("<");
+  Builder.AddPlaceholderChunk("header");
+  Builder.AddTextChunk(">");
+  Results.AddResult(Builder.TakeString());
   
   // #define <macro>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("define");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("macro");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("define");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("macro");
+  Results.AddResult(Builder.TakeString());
   
   // #define <macro>(<args>)
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("define");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("macro");
-  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-  Pattern->AddPlaceholderChunk("args");
-  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("define");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("macro");
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddPlaceholderChunk("args");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Builder.TakeString());
   
   // #undef <macro>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("undef");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("macro");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("undef");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("macro");
+  Results.AddResult(Builder.TakeString());
 
   // #line <number>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("line");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("number");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("line");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("number");
+  Results.AddResult(Builder.TakeString());
   
   // #line <number> "filename"
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("line");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("number");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddTextChunk("\"");
-  Pattern->AddPlaceholderChunk("filename");
-  Pattern->AddTextChunk("\"");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("line");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("number");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("\"");
+  Builder.AddPlaceholderChunk("filename");
+  Builder.AddTextChunk("\"");
+  Results.AddResult(Builder.TakeString());
   
   // #error <message>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("error");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("message");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("error");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("message");
+  Results.AddResult(Builder.TakeString());
 
   // #pragma <arguments>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("pragma");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("arguments");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("pragma");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("arguments");
+  Results.AddResult(Builder.TakeString());
 
   if (getLangOptions().ObjC1) {
     // #import "header"
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("import");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddTextChunk("\"");
-    Pattern->AddPlaceholderChunk("header");
-    Pattern->AddTextChunk("\"");
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk("import");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddTextChunk("\"");
+    Builder.AddPlaceholderChunk("header");
+    Builder.AddTextChunk("\"");
+    Results.AddResult(Builder.TakeString());
     
     // #import <header>
-    Pattern = new CodeCompletionString;
-    Pattern->AddTypedTextChunk("import");
-    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-    Pattern->AddTextChunk("<");
-    Pattern->AddPlaceholderChunk("header");
-    Pattern->AddTextChunk(">");
-    Results.AddResult(Pattern);
+    Builder.AddTypedTextChunk("import");
+    Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Builder.AddTextChunk("<");
+    Builder.AddPlaceholderChunk("header");
+    Builder.AddTextChunk(">");
+    Results.AddResult(Builder.TakeString());
   }
   
   // #include_next "header"
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("include_next");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddTextChunk("\"");
-  Pattern->AddPlaceholderChunk("header");
-  Pattern->AddTextChunk("\"");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("include_next");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("\"");
+  Builder.AddPlaceholderChunk("header");
+  Builder.AddTextChunk("\"");
+  Results.AddResult(Builder.TakeString());
   
   // #include_next <header>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("include_next");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddTextChunk("<");
-  Pattern->AddPlaceholderChunk("header");
-  Pattern->AddTextChunk(">");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("include_next");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddTextChunk("<");
+  Builder.AddPlaceholderChunk("header");
+  Builder.AddTextChunk(">");
+  Results.AddResult(Builder.TakeString());
 
   // #warning <message>
-  Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("warning");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddPlaceholderChunk("message");
-  Results.AddResult(Pattern);
+  Builder.AddTypedTextChunk("warning");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddPlaceholderChunk("message");
+  Results.AddResult(Builder.TakeString());
 
   // Note: #ident and #sccs are such crazy anachronisms that we don't provide
   // completions for them. And __include_macros is a Clang-internal extension
@@ -5319,43 +6439,45 @@ void Sema::CodeCompleteInPreprocessorConditionalExclusion(Scope *S) {
 }
 
 void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        IsDefinition? CodeCompletionContext::CCC_MacroName
+                                    : CodeCompletionContext::CCC_MacroNameUse);
   if (!IsDefinition && (!CodeCompleter || CodeCompleter->includeMacros())) {
     // Add just the names of macros, not their arguments.    
+    CodeCompletionBuilder Builder(Results.getAllocator());
     Results.EnterNewScope();
     for (Preprocessor::macro_iterator M = PP.macro_begin(), 
                                    MEnd = PP.macro_end();
          M != MEnd; ++M) {
-      CodeCompletionString *Pattern = new CodeCompletionString;
-      Pattern->AddTypedTextChunk(M->first->getName());
-      Results.AddResult(Pattern);
+      Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
+                                           M->first->getName()));
+      Results.AddResult(Builder.TakeString());
     }
     Results.ExitScope();
   } else if (IsDefinition) {
     // FIXME: Can we detect when the user just wrote an include guard above?
   }
   
-  HandleCodeCompleteResults(this, CodeCompleter, 
-                      IsDefinition? CodeCompletionContext::CCC_MacroName
-                                  : CodeCompletionContext::CCC_MacroNameUse,
+  HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(), Results.size()); 
 }
 
 void Sema::CodeCompletePreprocessorExpression() {
-  ResultBuilder Results(*this);
+  ResultBuilder Results(*this, CodeCompleter->getAllocator(),
+                        CodeCompletionContext::CCC_PreprocessorExpression);
   
   if (!CodeCompleter || CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
   
     // defined (<macro>)
   Results.EnterNewScope();
-  CodeCompletionString *Pattern = new CodeCompletionString;
-  Pattern->AddTypedTextChunk("defined");
-  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
-  Pattern->AddPlaceholderChunk("macro");
-  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
-  Results.AddResult(Pattern);
+  CodeCompletionBuilder Builder(Results.getAllocator());
+  Builder.AddTypedTextChunk("defined");
+  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Builder.AddChunk(CodeCompletionString::CK_LeftParen);
+  Builder.AddPlaceholderChunk("macro");
+  Builder.AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Builder.TakeString());
   Results.ExitScope();
   
   HandleCodeCompleteResults(this, CodeCompleter, 
@@ -5381,10 +6503,9 @@ void Sema::CodeCompleteNaturalLanguage() {
                             0, 0);
 }
 
-void Sema::GatherGlobalCodeCompletions(
+void Sema::GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
                  llvm::SmallVectorImpl<CodeCompletionResult> &Results) {
-  ResultBuilder Builder(*this);
-
+  ResultBuilder Builder(*this, Allocator, CodeCompletionContext::CCC_Recovery);
   if (!CodeCompleter || CodeCompleter->includeGlobals()) {
     CodeCompletionDeclConsumer Consumer(Builder, 
                                         Context.getTranslationUnitDecl());
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index f5e045a722a0..dd30c1261ed7 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -26,6 +26,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/CharUnits.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Parse/ParseDiagnostic.h"
@@ -59,7 +60,7 @@ Sema::DeclGroupPtrTy Sema::ConvertDeclToDeclGroup(Decl *Ptr) {
 /// and then return NULL.
 ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
                              Scope *S, CXXScopeSpec *SS,
-                             bool isClassName,
+                             bool isClassName, bool HasTrailingDot,
                              ParsedType ObjectTypePtr) {
   // Determine where we will perform name lookup.
   DeclContext *LookupCtx = 0;
@@ -192,13 +193,15 @@ ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
       T = getElaboratedType(ETK_None, *SS, T);
     
   } else if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(IIDecl)) {
-    T = Context.getObjCInterfaceType(IDecl);
-  } else {
+    if (!HasTrailingDot)
+      T = Context.getObjCInterfaceType(IDecl);
+  }
+
+  if (T.isNull()) {
     // If it's not plausibly a type, suppress diagnostics.
     Result.suppressDiagnostics();
     return ParsedType();
   }
-
   return ParsedType::make(T);
 }
 
@@ -544,7 +547,9 @@ static bool IsDisallowedCopyOrAssign(const CXXMethodDecl *D) {
 
   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(D))
     return CD->isCopyConstructor();
-  return D->isCopyAssignment();
+  if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
+    return Method->isCopyAssignmentOperator();
+  return false;
 }
 
 bool Sema::ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const {
@@ -554,11 +559,8 @@ bool Sema::ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const {
     return false;
 
   // Ignore class templates.
-  if (D->getDeclContext()->isDependentContext())
-    return false;
-
-  // We warn for unused decls internal to the translation unit.
-  if (D->getLinkage() == ExternalLinkage)
+  if (D->getDeclContext()->isDependentContext() ||
+      D->getLexicalDeclContext()->isDependentContext())
     return false;
 
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
@@ -575,25 +577,32 @@ bool Sema::ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const {
         return false;
     }
 
-    if (FD->isThisDeclarationADefinition())
-      return !Context.DeclMustBeEmitted(FD);
-    return true;
-  }
+    if (FD->isThisDeclarationADefinition() &&
+        Context.DeclMustBeEmitted(FD))
+      return false;
+
+  } else if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    if (!VD->isFileVarDecl() ||
+        VD->getType().isConstant(Context) ||
+        Context.DeclMustBeEmitted(VD))
+      return false;
 
-  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
     if (VD->isStaticDataMember() &&
         VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       return false;
 
-    if ( VD->isFileVarDecl() &&
-        !VD->getType().isConstant(Context))
-      return !Context.DeclMustBeEmitted(VD);
+  } else {
+    return false;
   }
 
-   return false;
- }
+  // Only warn for unused decls internal to the translation unit.
+  if (D->getLinkage() == ExternalLinkage)
+    return false;
+
+  return true;
+}
 
- void Sema::MarkUnusedFileScopedDecl(const DeclaratorDecl *D) {
+void Sema::MarkUnusedFileScopedDecl(const DeclaratorDecl *D) {
   if (!D)
     return;
 
@@ -620,6 +629,9 @@ static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
   if (D->isUsed() || D->hasAttr<UnusedAttr>())
     return false;
 
+  if (isa<LabelDecl>(D))
+    return true;
+  
   // White-list anything that isn't a local variable.
   if (!isa<VarDecl>(D) || isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D) ||
       !D->getDeclContext()->isFunctionOrMethod())
@@ -662,16 +674,29 @@ static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
   return true;
 }
 
+/// DiagnoseUnusedDecl - Emit warnings about declarations that are not used
+/// unless they are marked attr(unused).
 void Sema::DiagnoseUnusedDecl(const NamedDecl *D) {
   if (!ShouldDiagnoseUnusedDecl(D))
     return;
   
+  unsigned DiagID;
   if (isa<VarDecl>(D) && cast<VarDecl>(D)->isExceptionVariable())
-    Diag(D->getLocation(), diag::warn_unused_exception_param)
-    << D->getDeclName();
+    DiagID = diag::warn_unused_exception_param;
+  else if (isa<LabelDecl>(D))
+    DiagID = diag::warn_unused_label;
   else
-    Diag(D->getLocation(), diag::warn_unused_variable) 
-    << D->getDeclName();
+    DiagID = diag::warn_unused_variable;
+
+  Diag(D->getLocation(), DiagID) << D->getDeclName();
+}
+
+static void CheckPoppedLabel(LabelDecl *L, Sema &S) {
+  // Verify that we have no forward references left.  If so, there was a goto
+  // or address of a label taken, but no definition of it.  Label fwd
+  // definitions are indicated with a null substmt.
+  if (L->getStmt() == 0)
+    S.Diag(L->getLocation(), diag::err_undeclared_label_use) <<L->getDeclName();
 }
 
 void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
@@ -690,9 +715,13 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
     if (!D->getDeclName()) continue;
 
     // Diagnose unused variables in this scope.
-    if (S->getNumErrorsAtStart() == getDiagnostics().getNumErrors())
+    if (!S->hasErrorOccurred())
       DiagnoseUnusedDecl(D);
     
+    // If this was a forward reference to a label, verify it was defined.
+    if (LabelDecl *LD = dyn_cast<LabelDecl>(D))
+      CheckPoppedLabel(LD, *this);
+    
     // Remove this name from our lexical scope.
     IdResolver.RemoveDecl(D);
   }
@@ -769,17 +798,6 @@ Scope *Sema::getNonFieldDeclScope(Scope *S) {
   return S;
 }
 
-void Sema::InitBuiltinVaListType() {
-  if (!Context.getBuiltinVaListType().isNull())
-    return;
-
-  IdentifierInfo *VaIdent = &Context.Idents.get("__builtin_va_list");
-  NamedDecl *VaDecl = LookupSingleName(TUScope, VaIdent, SourceLocation(),
-                                       LookupOrdinaryName, ForRedeclaration);
-  TypedefDecl *VaTypedef = cast<TypedefDecl>(VaDecl);
-  Context.setBuiltinVaListType(Context.getTypedefType(VaTypedef));
-}
-
 /// LazilyCreateBuiltin - The specified Builtin-ID was first used at
 /// file scope.  lazily create a decl for it. ForRedeclaration is true
 /// if we're creating this built-in in anticipation of redeclaring the
@@ -789,9 +807,6 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
                                      SourceLocation Loc) {
   Builtin::ID BID = (Builtin::ID)bid;
 
-  if (Context.BuiltinInfo.hasVAListUse(BID))
-    InitBuiltinVaListType();
-
   ASTContext::GetBuiltinTypeError Error;
   QualType R = Context.GetBuiltinType(BID, Error);
   switch (Error) {
@@ -801,13 +816,13 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
 
   case ASTContext::GE_Missing_stdio:
     if (ForRedeclaration)
-      Diag(Loc, diag::err_implicit_decl_requires_stdio)
+      Diag(Loc, diag::warn_implicit_decl_requires_stdio)
         << Context.BuiltinInfo.GetName(BID);
     return 0;
 
   case ASTContext::GE_Missing_setjmp:
     if (ForRedeclaration)
-      Diag(Loc, diag::err_implicit_decl_requires_setjmp)
+      Diag(Loc, diag::warn_implicit_decl_requires_setjmp)
         << Context.BuiltinInfo.GetName(BID);
     return 0;
   }
@@ -817,7 +832,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
       << Context.BuiltinInfo.GetName(BID)
       << R;
     if (Context.BuiltinInfo.getHeaderName(BID) &&
-        Diags.getDiagnosticLevel(diag::ext_implicit_lib_function_decl)
+        Diags.getDiagnosticLevel(diag::ext_implicit_lib_function_decl, Loc)
           != Diagnostic::Ignored)
       Diag(Loc, diag::note_please_include_header)
         << Context.BuiltinInfo.getHeaderName(BID)
@@ -834,7 +849,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
 
   // Create Decl objects for each parameter, adding them to the
   // FunctionDecl.
-  if (FunctionProtoType *FT = dyn_cast<FunctionProtoType>(R)) {
+  if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(R)) {
     llvm::SmallVector<ParmVarDecl*, 16> Params;
     for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i)
       Params.push_back(ParmVarDecl::Create(Context, New, SourceLocation(), 0,
@@ -1026,11 +1041,11 @@ static void MergeDeclAttributes(Decl *New, Decl *Old, ASTContext &C) {
   // we process them.
   if (!New->hasAttrs())
     New->setAttrs(AttrVec());
-  for (Decl::attr_iterator i = Old->attr_begin(), e = Old->attr_end(); i != e;
-       ++i) {
-    // FIXME: Make this more general than just checking for Overloadable.
-    if (!DeclHasAttr(New, *i) && (*i)->getKind() != attr::Overloadable) {
-      Attr *NewAttr = (*i)->clone(C);
+  for (specific_attr_iterator<InheritableAttr>
+       i = Old->specific_attr_begin<InheritableAttr>(),
+       e = Old->specific_attr_end<InheritableAttr>(); i != e; ++i) {
+    if (!DeclHasAttr(New, *i)) {
+      InheritableAttr *NewAttr = cast<InheritableAttr>((*i)->clone(C));
       NewAttr->setInherited(true);
       New->addAttr(NewAttr);
     }
@@ -1061,7 +1076,8 @@ Sema::CXXSpecialMember Sema::getSpecialMember(const CXXMethodDecl *MD) {
   if (isa<CXXDestructorDecl>(MD))
     return Sema::CXXDestructor;
   
-  assert(MD->isCopyAssignment() && "Must have copy assignment operator");
+  assert(MD->isCopyAssignmentOperator() && 
+         "Must have copy assignment operator");
   return Sema::CXXCopyAssignment;
 }
 
@@ -1147,15 +1163,15 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
   //
   // Note also that we DO NOT return at this point, because we still have
   // other tests to run.
-  const FunctionType *OldType = OldQType->getAs<FunctionType>();
+  const FunctionType *OldType = cast<FunctionType>(OldQType);
   const FunctionType *NewType = New->getType()->getAs<FunctionType>();
-  const FunctionType::ExtInfo OldTypeInfo = OldType->getExtInfo();
-  const FunctionType::ExtInfo NewTypeInfo = NewType->getExtInfo();
+  FunctionType::ExtInfo OldTypeInfo = OldType->getExtInfo();
+  FunctionType::ExtInfo NewTypeInfo = NewType->getExtInfo();
+  bool RequiresAdjustment = false;
   if (OldTypeInfo.getCC() != CC_Default &&
       NewTypeInfo.getCC() == CC_Default) {
-    NewQType = Context.getCallConvType(NewQType, OldTypeInfo.getCC());
-    New->setType(NewQType);
-    NewQType = Context.getCanonicalType(NewQType);
+    NewTypeInfo = NewTypeInfo.withCallingConv(OldTypeInfo.getCC());
+    RequiresAdjustment = true;
   } else if (!Context.isSameCallConv(OldTypeInfo.getCC(),
                                      NewTypeInfo.getCC())) {
     // Calling conventions really aren't compatible, so complain.
@@ -1169,25 +1185,29 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
   }
 
   // FIXME: diagnose the other way around?
-  if (OldType->getNoReturnAttr() && !NewType->getNoReturnAttr()) {
-    NewQType = Context.getNoReturnType(NewQType);
-    New->setType(NewQType);
-    assert(NewQType.isCanonical());
+  if (OldTypeInfo.getNoReturn() && !NewTypeInfo.getNoReturn()) {
+    NewTypeInfo = NewTypeInfo.withNoReturn(true);
+    RequiresAdjustment = true;
   }
 
   // Merge regparm attribute.
-  if (OldType->getRegParmType() != NewType->getRegParmType()) {
-    if (NewType->getRegParmType()) {
+  if (OldTypeInfo.getRegParm() != NewTypeInfo.getRegParm()) {
+    if (NewTypeInfo.getRegParm()) {
       Diag(New->getLocation(), diag::err_regparm_mismatch)
         << NewType->getRegParmType()
         << OldType->getRegParmType();
       Diag(Old->getLocation(), diag::note_previous_declaration);      
       return true;
     }
-    
-    NewQType = Context.getRegParmType(NewQType, OldType->getRegParmType());
-    New->setType(NewQType);
-    assert(NewQType.isCanonical());    
+
+    NewTypeInfo = NewTypeInfo.withRegParm(OldTypeInfo.getRegParm());
+    RequiresAdjustment = true;
+  }
+
+  if (RequiresAdjustment) {
+    NewType = Context.adjustFunctionType(NewType, NewTypeInfo);
+    New->setType(QualType(NewType, 0));
+    NewQType = Context.getCanonicalType(New->getType());
   }
   
   if (getLangOptions().CPlusPlus) {
@@ -1195,17 +1215,19 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
     //   Certain function declarations cannot be overloaded:
     //     -- Function declarations that differ only in the return type
     //        cannot be overloaded.
-    QualType OldReturnType
-      = cast<FunctionType>(OldQType.getTypePtr())->getResultType();
-    QualType NewReturnType
-      = cast<FunctionType>(NewQType.getTypePtr())->getResultType();
+    QualType OldReturnType = OldType->getResultType();
+    QualType NewReturnType = cast<FunctionType>(NewQType)->getResultType();
     QualType ResQT;
     if (OldReturnType != NewReturnType) {
       if (NewReturnType->isObjCObjectPointerType()
           && OldReturnType->isObjCObjectPointerType())
         ResQT = Context.mergeObjCGCQualifiers(NewQType, OldQType);
       if (ResQT.isNull()) {
-        Diag(New->getLocation(), diag::err_ovl_diff_return_type);
+        if (New->isCXXClassMember() && New->isOutOfLine())
+          Diag(New->getLocation(),
+               diag::err_member_def_does_not_match_ret_type) << New;
+        else
+          Diag(New->getLocation(), diag::err_ovl_diff_return_type);
         Diag(Old->getLocation(), PrevDiag) << Old << Old->getType();
         return true;
       }
@@ -1268,9 +1290,19 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
     // (C++98 8.3.5p3):
     //   All declarations for a function shall agree exactly in both the
     //   return type and the parameter-type-list.
-    // attributes should be ignored when comparing.
-    if (Context.getNoReturnType(OldQType, false) ==
-        Context.getNoReturnType(NewQType, false))
+    // We also want to respect all the extended bits except noreturn.
+
+    // noreturn should now match unless the old type info didn't have it.
+    QualType OldQTypeForComparison = OldQType;
+    if (!OldTypeInfo.getNoReturn() && NewTypeInfo.getNoReturn()) {
+      assert(OldQType == QualType(OldType, 0));
+      const FunctionType *OldTypeForComparison
+        = Context.adjustFunctionType(OldType, OldTypeInfo.withNoReturn(true));
+      OldQTypeForComparison = QualType(OldTypeForComparison, 0);
+      assert(OldQTypeForComparison.isCanonical());
+    }
+
+    if (OldQTypeForComparison == NewQType)
       return MergeCompatibleFunctionDecls(New, Old);
 
     // Fall through for conflicting redeclarations and redefinitions.
@@ -1292,10 +1324,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
                                                  OldProto->arg_type_end());
       NewQType = Context.getFunctionType(NewFuncType->getResultType(),
                                          ParamTypes.data(), ParamTypes.size(),
-                                         OldProto->isVariadic(),
-                                         OldProto->getTypeQuals(),
-                                         false, false, 0, 0,
-                                         OldProto->getExtInfo());
+                                         OldProto->getExtProtoInfo());
       New->setType(NewQType);
       New->setHasInheritedPrototype();
 
@@ -1377,9 +1406,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
 
       New->setType(Context.getFunctionType(MergedReturn, &ArgTypes[0],
                                            ArgTypes.size(),
-                                           OldProto->isVariadic(), 0,
-                                           false, false, 0, 0,
-                                           OldProto->getExtInfo()));
+                                           OldProto->getExtProtoInfo()));
       return MergeCompatibleFunctionDecls(New, Old);
     }
 
@@ -1444,48 +1471,26 @@ bool Sema::MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old) {
   return false;
 }
 
-/// MergeVarDecl - We just parsed a variable 'New' which has the same name
-/// and scope as a previous declaration 'Old'.  Figure out how to resolve this
-/// situation, merging decls or emitting diagnostics as appropriate.
+/// MergeVarDecl - We parsed a variable 'New' which has the same name and scope
+/// as a previous declaration 'Old'.  Figure out how to merge their types,
+/// emitting diagnostics as appropriate.
 ///
-/// Tentative definition rules (C99 6.9.2p2) are checked by
-/// FinalizeDeclaratorGroup. Unfortunately, we can't analyze tentative
-/// definitions here, since the initializer hasn't been attached.
+/// Declarations using the auto type specifier (C++ [decl.spec.auto]) call back
+/// to here in AddInitializerToDecl and AddCXXDirectInitializerToDecl. We can't
+/// check them before the initializer is attached.
 ///
-void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
-  // If the new decl is already invalid, don't do any other checking.
-  if (New->isInvalidDecl())
+void Sema::MergeVarDeclTypes(VarDecl *New, VarDecl *Old) {
+  if (New->isInvalidDecl() || Old->isInvalidDecl())
     return;
 
-  // Verify the old decl was also a variable.
-  VarDecl *Old = 0;
-  if (!Previous.isSingleResult() ||
-      !(Old = dyn_cast<VarDecl>(Previous.getFoundDecl()))) {
-    Diag(New->getLocation(), diag::err_redefinition_different_kind)
-      << New->getDeclName();
-    Diag(Previous.getRepresentativeDecl()->getLocation(),
-         diag::note_previous_definition);
-    return New->setInvalidDecl();
-  }
-
-  // C++ [class.mem]p1:
-  //   A member shall not be declared twice in the member-specification [...]
-  // 
-  // Here, we need only consider static data members.
-  if (Old->isStaticDataMember() && !New->isOutOfLine()) {
-    Diag(New->getLocation(), diag::err_duplicate_member) 
-      << New->getIdentifier();
-    Diag(Old->getLocation(), diag::note_previous_declaration);
-    New->setInvalidDecl();
-  }
-  
-  MergeDeclAttributes(New, Old, Context);
-
-  // Merge the types
   QualType MergedT;
   if (getLangOptions().CPlusPlus) {
-    if (Context.hasSameType(New->getType(), Old->getType()))
-      MergedT = New->getType();
+    AutoType *AT = New->getType()->getContainedAutoType();
+    if (AT && !AT->isDeduced()) {
+      // We don't know what the new type is until the initializer is attached.
+      return;
+    } else if (Context.hasSameType(New->getType(), Old->getType()))
+      return;
     // C++ [basic.link]p10:
     //   [...] the types specified by all declarations referring to a given
     //   object or function shall be identical, except that declarations for an
@@ -1509,7 +1514,8 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
         MergedT = Old->getType();
     } else if (New->getType()->isObjCObjectPointerType()
                && Old->getType()->isObjCObjectPointerType()) {
-        MergedT = Context.mergeObjCGCQualifiers(New->getType(), Old->getType());
+        MergedT = Context.mergeObjCGCQualifiers(New->getType(),
+                                                        Old->getType());
     }
   } else {
     MergedT = Context.mergeTypes(New->getType(), Old->getType());
@@ -1521,6 +1527,49 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
     return New->setInvalidDecl();
   }
   New->setType(MergedT);
+}
+
+/// MergeVarDecl - We just parsed a variable 'New' which has the same name
+/// and scope as a previous declaration 'Old'.  Figure out how to resolve this
+/// situation, merging decls or emitting diagnostics as appropriate.
+///
+/// Tentative definition rules (C99 6.9.2p2) are checked by
+/// FinalizeDeclaratorGroup. Unfortunately, we can't analyze tentative
+/// definitions here, since the initializer hasn't been attached.
+///
+void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
+  // If the new decl is already invalid, don't do any other checking.
+  if (New->isInvalidDecl())
+    return;
+
+  // Verify the old decl was also a variable.
+  VarDecl *Old = 0;
+  if (!Previous.isSingleResult() ||
+      !(Old = dyn_cast<VarDecl>(Previous.getFoundDecl()))) {
+    Diag(New->getLocation(), diag::err_redefinition_different_kind)
+      << New->getDeclName();
+    Diag(Previous.getRepresentativeDecl()->getLocation(),
+         diag::note_previous_definition);
+    return New->setInvalidDecl();
+  }
+
+  // C++ [class.mem]p1:
+  //   A member shall not be declared twice in the member-specification [...]
+  // 
+  // Here, we need only consider static data members.
+  if (Old->isStaticDataMember() && !New->isOutOfLine()) {
+    Diag(New->getLocation(), diag::err_duplicate_member) 
+      << New->getIdentifier();
+    Diag(Old->getLocation(), diag::note_previous_declaration);
+    New->setInvalidDecl();
+  }
+  
+  MergeDeclAttributes(New, Old, Context);
+
+  // Merge the types.
+  MergeVarDeclTypes(New, Old);
+  if (New->isInvalidDecl())
+    return;
 
   // C99 6.2.2p4: Check if we have a static decl followed by a non-static.
   if (New->getStorageClass() == SC_Static &&
@@ -1547,6 +1596,20 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
     return New->setInvalidDecl();
   }
 
+  // Check if extern is followed by non-extern and vice-versa.
+  if (New->hasExternalStorage() &&
+      !Old->hasLinkage() && Old->isLocalVarDecl()) {
+    Diag(New->getLocation(), diag::err_extern_non_extern) << New->getDeclName();
+    Diag(Old->getLocation(), diag::note_previous_definition);
+    return New->setInvalidDecl();
+  }
+  if (Old->hasExternalStorage() &&
+      !New->hasLinkage() && New->isLocalVarDecl()) {
+    Diag(New->getLocation(), diag::err_non_extern_extern) << New->getDeclName();
+    Diag(Old->getLocation(), diag::note_previous_definition);
+    return New->setInvalidDecl();
+  }
+
   // Variables with external linkage are analyzed in FinalizeDeclaratorGroup.
 
   // FIXME: The test for external storage here seems wrong? We still
@@ -1602,7 +1665,6 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
 /// no declarator (e.g. "struct foo;") is parsed.
 Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
                                             DeclSpec &DS) {
-  // FIXME: Error on auto/register at file scope
   // FIXME: Error on inline/virtual/explicit
   // FIXME: Warn on useless __thread
   // FIXME: Warn on useless const/volatile
@@ -1635,15 +1697,15 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   }
 
   if (DS.isFriendSpecified()) {
-    // If we're dealing with a class template decl, assume that the
-    // template routines are handling it.
-    if (TagD && isa<ClassTemplateDecl>(TagD))
+    // If we're dealing with a decl but not a TagDecl, assume that
+    // whatever routines created it handled the friendship aspect.
+    if (TagD && !Tag)
       return 0;
     return ActOnFriendTypeDecl(S, DS, MultiTemplateParamsArg(*this, 0, 0));
   }
          
   if (RecordDecl *Record = dyn_cast_or_null<RecordDecl>(Tag)) {
-    ProcessDeclAttributeList(S, Record, DS.getAttributes());
+    ProcessDeclAttributeList(S, Record, DS.getAttributes().getList());
     
     if (!Record->getDeclName() && Record->isDefinition() &&
         DS.getStorageClassSpec() != DeclSpec::SCS_typedef) {
@@ -1654,12 +1716,24 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
       Diag(DS.getSourceRange().getBegin(), diag::ext_no_declarators)
         << DS.getSourceRange();
     }
+  }
 
-    // Microsoft allows unnamed struct/union fields. Don't complain
-    // about them.
-    // FIXME: Should we support Microsoft's extensions in this area?
-    if (Record->getDeclName() && getLangOptions().Microsoft)
-      return Tag;
+  // Check for Microsoft C extension: anonymous struct.
+  if (getLangOptions().Microsoft && !getLangOptions().CPlusPlus &&
+      CurContext->isRecord() &&
+      DS.getStorageClassSpec() == DeclSpec::SCS_unspecified) {
+    // Handle 2 kinds of anonymous struct:
+    //   struct STRUCT;
+    // and
+    //   STRUCT_TYPE;  <- where STRUCT_TYPE is a typedef struct.
+    RecordDecl *Record = dyn_cast_or_null<RecordDecl>(Tag);
+    if ((Record && Record->getDeclName() && !Record->isDefinition()) ||
+        (DS.getTypeSpecType() == DeclSpec::TST_typename &&
+         DS.getRepAsType().get()->isStructureType())) {
+      Diag(DS.getSourceRange().getBegin(), diag::ext_ms_anonymous_struct)
+        << DS.getSourceRange();
+      return BuildMicrosoftCAnonymousStruct(S, DS, Record);
+    }
   }
   
   if (getLangOptions().CPlusPlus && 
@@ -1688,6 +1762,25 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   return TagD;
 }
 
+/// ActOnVlaStmt - This rouine if finds a vla expression in a decl spec.
+/// builds a statement for it and returns it so it is evaluated.
+StmtResult Sema::ActOnVlaStmt(const DeclSpec &DS) {
+  StmtResult R;
+  if (DS.getTypeSpecType() == DeclSpec::TST_typeofExpr) {
+    Expr *Exp = DS.getRepAsExpr();
+    QualType Ty = Exp->getType();
+    if (Ty->isPointerType()) {
+      do
+        Ty = Ty->getAs<PointerType>()->getPointeeType();
+      while (Ty->isPointerType());
+    }
+    if (Ty->isVariableArrayType()) {
+      R = ActOnExprStmt(MakeFullExpr(Exp));
+    }
+  }
+  return R;
+}
+
 /// We are trying to inject an anonymous member into the given scope;
 /// check if there's an existing declaration that can't be overloaded.
 ///
@@ -1707,11 +1800,10 @@ static bool CheckAnonMemberRedeclaration(Sema &SemaRef,
 
   // Pick a representative declaration.
   NamedDecl *PrevDecl = R.getRepresentativeDecl()->getUnderlyingDecl();
-  if (PrevDecl && Owner->isRecord()) {
-    RecordDecl *Record = cast<RecordDecl>(Owner);
-    if (!SemaRef.isDeclInScope(PrevDecl, Record, S))
-      return false;
-  }
+  assert(PrevDecl && "Expected a non-null Decl");
+
+  if (!SemaRef.isDeclInScope(PrevDecl, Owner, S))
+    return false;
 
   SemaRef.Diag(NameLoc, diagnostic) << Name;
   SemaRef.Diag(PrevDecl->getLocation(), diag::note_previous_declaration);
@@ -1738,18 +1830,24 @@ static bool CheckAnonMemberRedeclaration(Sema &SemaRef,
 static bool InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S,
                                                 DeclContext *Owner,
                                                 RecordDecl *AnonRecord,
-                                                AccessSpecifier AS) {
+                                                AccessSpecifier AS,
+                              llvm::SmallVector<NamedDecl*, 2> &Chaining,
+                                                      bool MSAnonStruct) {
   unsigned diagKind
     = AnonRecord->isUnion() ? diag::err_anonymous_union_member_redecl
                             : diag::err_anonymous_struct_member_redecl;
 
   bool Invalid = false;
-  for (RecordDecl::field_iterator F = AnonRecord->field_begin(),
-                               FEnd = AnonRecord->field_end();
-       F != FEnd; ++F) {
-    if ((*F)->getDeclName()) {
-      if (CheckAnonMemberRedeclaration(SemaRef, S, Owner, (*F)->getDeclName(),
-                                       (*F)->getLocation(), diagKind)) {
+
+  // Look every FieldDecl and IndirectFieldDecl with a name.
+  for (RecordDecl::decl_iterator D = AnonRecord->decls_begin(),
+                               DEnd = AnonRecord->decls_end();
+       D != DEnd; ++D) {
+    if ((isa<FieldDecl>(*D) || isa<IndirectFieldDecl>(*D)) &&
+        cast<NamedDecl>(*D)->getDeclName()) {
+      ValueDecl *VD = cast<ValueDecl>(*D);
+      if (CheckAnonMemberRedeclaration(SemaRef, S, Owner, VD->getDeclName(),
+                                       VD->getLocation(), diagKind)) {
         // C++ [class.union]p2:
         //   The names of the members of an anonymous union shall be
         //   distinct from the names of any other entity in the
@@ -1761,20 +1859,34 @@ static bool InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S,
         //   definition, the members of the anonymous union are
         //   considered to have been defined in the scope in which the
         //   anonymous union is declared.
-        Owner->makeDeclVisibleInContext(*F);
-        S->AddDecl(*F);
-        SemaRef.IdResolver.AddDecl(*F);
+        unsigned OldChainingSize = Chaining.size();
+        if (IndirectFieldDecl *IF = dyn_cast<IndirectFieldDecl>(VD))
+          for (IndirectFieldDecl::chain_iterator PI = IF->chain_begin(),
+               PE = IF->chain_end(); PI != PE; ++PI)
+            Chaining.push_back(*PI);
+        else
+          Chaining.push_back(VD);
+
+        assert(Chaining.size() >= 2);
+        NamedDecl **NamedChain =
+          new (SemaRef.Context)NamedDecl*[Chaining.size()];
+        for (unsigned i = 0; i < Chaining.size(); i++)
+          NamedChain[i] = Chaining[i];
+
+        IndirectFieldDecl* IndirectField =
+          IndirectFieldDecl::Create(SemaRef.Context, Owner, VD->getLocation(),
+                                    VD->getIdentifier(), VD->getType(),
+                                    NamedChain, Chaining.size());
+
+        IndirectField->setAccess(AS);
+        IndirectField->setImplicit();
+        SemaRef.PushOnScopeChains(IndirectField, S);
 
         // That includes picking up the appropriate access specifier.
-        if (AS != AS_none) (*F)->setAccess(AS);
+        if (AS != AS_none) IndirectField->setAccess(AS);
+
+        Chaining.resize(OldChainingSize);
       }
-    } else if (const RecordType *InnerRecordType
-                 = (*F)->getType()->getAs<RecordType>()) {
-      RecordDecl *InnerRecord = InnerRecordType->getDecl();
-      if (InnerRecord->isAnonymousStructOrUnion())
-        Invalid = Invalid ||
-          InjectAnonymousStructOrUnionMembers(SemaRef, S, Owner,
-                                              InnerRecord, AS);
     }
   }
 
@@ -1819,7 +1931,7 @@ StorageClassSpecToFunctionDeclStorageClass(DeclSpec::SCS StorageClassSpec) {
   llvm_unreachable("unknown storage class specifier");
 }
 
-/// ActOnAnonymousStructOrUnion - Handle the declaration of an
+/// BuildAnonymousStructOrUnion - Handle the declaration of an
 /// anonymous structure or union. Anonymous unions are a C++ feature
 /// (C++ [class.union]) and a GNU C extension; anonymous structures
 /// are a GNU C and GNU C++ extension.
@@ -1852,7 +1964,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
 
       // Recover by adding 'static'.
       DS.SetStorageClassSpec(DeclSpec::SCS_static, SourceLocation(),
-                             PrevSpec, DiagID);
+                             PrevSpec, DiagID, getLangOptions());
     }
     // C++ [class.union]p3:
     //   A storage class is not allowed in a declaration of an
@@ -1865,7 +1977,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
 
       // Recover by removing the storage specifier.
       DS.SetStorageClassSpec(DeclSpec::SCS_unspecified, SourceLocation(),
-                             PrevSpec, DiagID);
+                             PrevSpec, DiagID, getLangOptions());
     }
 
     // C++ [class.union]p2:
@@ -1898,10 +2010,16 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
       } else if (RecordDecl *MemRecord = dyn_cast<RecordDecl>(*Mem)) {
         if (!MemRecord->isAnonymousStructOrUnion() &&
             MemRecord->getDeclName()) {
-          // This is a nested type declaration.
-          Diag(MemRecord->getLocation(), diag::err_anonymous_record_with_type)
-            << (int)Record->isUnion();
-          Invalid = true;
+          // Visual C++ allows type definition in anonymous struct or union.
+          if (getLangOptions().Microsoft)
+            Diag(MemRecord->getLocation(), diag::ext_anonymous_record_with_type)
+              << (int)Record->isUnion();
+          else {
+            // This is a nested type declaration.
+            Diag(MemRecord->getLocation(), diag::err_anonymous_record_with_type)
+              << (int)Record->isUnion();
+            Invalid = true;
+          }
         }
       } else if (isa<AccessSpecDecl>(*Mem)) {
         // Any access specifier is fine.
@@ -1915,9 +2033,17 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
           DK = diag::err_anonymous_record_with_function;
         else if (isa<VarDecl>(*Mem))
           DK = diag::err_anonymous_record_with_static;
-        Diag((*Mem)->getLocation(), DK)
+        
+        // Visual C++ allows type definition in anonymous struct or union.
+        if (getLangOptions().Microsoft &&
+            DK == diag::err_anonymous_record_with_type)
+          Diag((*Mem)->getLocation(), diag::ext_anonymous_record_with_type)
             << (int)Record->isUnion();
+        else {
+          Diag((*Mem)->getLocation(), DK)
+              << (int)Record->isUnion();
           Invalid = true;
+        }
       }
     }
   }
@@ -1942,11 +2068,8 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
                              TInfo,
                              /*BitWidth=*/0, /*Mutable=*/false);
     Anon->setAccess(AS);
-    if (getLangOptions().CPlusPlus) {
+    if (getLangOptions().CPlusPlus)
       FieldCollector->Add(cast<FieldDecl>(Anon));
-      if (!cast<CXXRecordDecl>(Record)->isEmpty())
-        cast<CXXRecordDecl>(OwningClass)->setEmpty(false);
-    }
   } else {
     DeclSpec::SCS SCSpec = DS.getStorageClassSpec();
     assert(SCSpec != DeclSpec::SCS_typedef &&
@@ -1978,7 +2101,11 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   // Inject the members of the anonymous struct/union into the owning
   // context and into the identifier resolver chain for name lookup
   // purposes.
-  if (InjectAnonymousStructOrUnionMembers(*this, S, Owner, Record, AS))
+  llvm::SmallVector<NamedDecl*, 2> Chain;
+  Chain.push_back(Anon);
+
+  if (InjectAnonymousStructOrUnionMembers(*this, S, Owner, Record, AS,
+                                          Chain, false))
     Invalid = true;
 
   // Mark this as an anonymous struct/union type. Note that we do not
@@ -1995,6 +2122,57 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   return Anon;
 }
 
+/// BuildMicrosoftCAnonymousStruct - Handle the declaration of an
+/// Microsoft C anonymous structure.
+/// Ref: http://msdn.microsoft.com/en-us/library/z2cx9y4f.aspx
+/// Example:
+///
+/// struct A { int a; };
+/// struct B { struct A; int b; };
+///
+/// void foo() {
+///   B var;
+///   var.a = 3; 
+/// }
+///
+Decl *Sema::BuildMicrosoftCAnonymousStruct(Scope *S, DeclSpec &DS,
+                                           RecordDecl *Record) {
+  
+  // If there is no Record, get the record via the typedef.
+  if (!Record)
+    Record = DS.getRepAsType().get()->getAsStructureType()->getDecl();
+
+  // Mock up a declarator.
+  Declarator Dc(DS, Declarator::TypeNameContext);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(Dc, S);
+  assert(TInfo && "couldn't build declarator info for anonymous struct");
+
+  // Create a declaration for this anonymous struct.
+  NamedDecl* Anon = FieldDecl::Create(Context,
+                             cast<RecordDecl>(CurContext),
+                             DS.getSourceRange().getBegin(),
+                             /*IdentifierInfo=*/0,
+                             Context.getTypeDeclType(Record),
+                             TInfo,
+                             /*BitWidth=*/0, /*Mutable=*/false);
+  Anon->setImplicit();
+
+  // Add the anonymous struct object to the current context.
+  CurContext->addDecl(Anon);
+
+  // Inject the members of the anonymous struct into the current
+  // context and into the identifier resolver chain for name lookup
+  // purposes.
+  llvm::SmallVector<NamedDecl*, 2> Chain;
+  Chain.push_back(Anon);
+
+  if (InjectAnonymousStructOrUnionMembers(*this, S, CurContext,
+                                          Record->getDefinition(),
+                                          AS_none, Chain, true))
+    Anon->setInvalidDecl();
+
+  return Anon;
+}
 
 /// GetNameForDeclarator - Determine the full declaration name for the
 /// given Declarator.
@@ -2216,7 +2394,8 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
            diag::err_declarator_need_ident)
         << D.getDeclSpec().getSourceRange() << D.getSourceRange();
     return 0;
-  }
+  } else if (DiagnoseUnexpandedParameterPack(NameInfo, UPPC_DeclarationType))
+    return 0;
 
   // The scope passed in may not be a decl scope.  Zip up the scope tree until
   // we find one that is.
@@ -2228,6 +2407,10 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
   if (D.getCXXScopeSpec().isInvalid())
     D.setInvalidType();
   else if (D.getCXXScopeSpec().isSet()) {
+    if (DiagnoseUnexpandedParameterPack(D.getCXXScopeSpec(), 
+                                        UPPC_DeclarationQualifier))
+      return 0;
+
     bool EnteringContext = !D.getDeclSpec().isFriendSpecified();
     DC = computeDeclContext(D.getCXXScopeSpec(), EnteringContext);
     if (!DC) {
@@ -2248,11 +2431,30 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
         RequireCompleteDeclContext(D.getCXXScopeSpec(), DC))
       return 0;
 
-    if (isa<CXXRecordDecl>(DC) && !cast<CXXRecordDecl>(DC)->hasDefinition()) {
-      Diag(D.getIdentifierLoc(),
-           diag::err_member_def_undefined_record)
-        << Name << DC << D.getCXXScopeSpec().getRange();
-      D.setInvalidType();
+    if (isa<CXXRecordDecl>(DC)) {
+      if (!cast<CXXRecordDecl>(DC)->hasDefinition()) {
+        Diag(D.getIdentifierLoc(),
+             diag::err_member_def_undefined_record)
+          << Name << DC << D.getCXXScopeSpec().getRange();
+        D.setInvalidType();
+      } else if (isa<CXXRecordDecl>(CurContext) && 
+                 !D.getDeclSpec().isFriendSpecified()) {
+        // The user provided a superfluous scope specifier inside a class
+        // definition:
+        //
+        // class X {
+        //   void X::f();
+        // };
+        if (CurContext->Equals(DC))
+          Diag(D.getIdentifierLoc(), diag::warn_member_extra_qualification)
+            << Name << FixItHint::CreateRemoval(D.getCXXScopeSpec().getRange());
+        else
+          Diag(D.getIdentifierLoc(), diag::err_member_qualification)
+            << Name << D.getCXXScopeSpec().getRange();
+        
+        // Pretend that this qualifier was not here.
+        D.getCXXScopeSpec().clear();
+      }
     }
 
     // Check whether we need to rebuild the type of the given
@@ -2264,12 +2466,33 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
         D.setInvalidType();
     }
   }
-
+  
+  // C++ [class.mem]p13:
+  //   If T is the name of a class, then each of the following shall have a 
+  //   name different from T:
+  //     - every static data member of class T;
+  //     - every member function of class T
+  //     - every member of class T that is itself a type;
+  if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC)) 
+    if (Record->getIdentifier() && Record->getDeclName() == Name) {
+      Diag(D.getIdentifierLoc(), diag::err_member_name_of_class)
+        << Name;
+      
+      // If this is a typedef, we'll end up spewing multiple diagnostics.
+      // Just return early; it's safer.
+      if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef)
+        return 0;
+    }
+  
   NamedDecl *New;
 
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
   QualType R = TInfo->getType();
 
+  if (DiagnoseUnexpandedParameterPack(D.getIdentifierLoc(), TInfo,
+                                      UPPC_DeclarationType))
+    D.setInvalidType();
+
   LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
                         ForRedeclaration);
 
@@ -2425,7 +2648,16 @@ static QualType TryToFixInvalidVariablyModifiedType(QualType T,
                                             Oversized);
     if (FixedType.isNull()) return FixedType;
     FixedType = Context.getPointerType(FixedType);
-    return Qs.apply(FixedType);
+    return Qs.apply(Context, FixedType);
+  }
+  if (const ParenType* PTy = dyn_cast<ParenType>(Ty)) {
+    QualType Inner = PTy->getInnerType();
+    QualType FixedType =
+        TryToFixInvalidVariablyModifiedType(Inner, Context, SizeIsNegative,
+                                            Oversized);
+    if (FixedType.isNull()) return FixedType;
+    FixedType = Context.getParenType(FixedType);
+    return Qs.apply(Context, FixedType);
   }
 
   const VariableArrayType* VLATy = dyn_cast<VariableArrayType>(T);
@@ -2595,6 +2827,8 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
         Context.setjmp_bufDecl(NewTD);
       else if (II->isStr("sigjmp_buf"))
         Context.setsigjmp_bufDecl(NewTD);
+      else if (II->isStr("__builtin_va_list"))
+        Context.setBuiltinVaListType(Context.getTypedefType(NewTD));
     }
 
   return NewTD;
@@ -2666,7 +2900,7 @@ static void SetNestedNameSpecifier(DeclaratorDecl *DD, Declarator &D) {
 }
 
 NamedDecl*
-Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
+Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                               QualType R, TypeSourceInfo *TInfo,
                               LookupResult &Previous,
                               MultiTemplateParamsArg TemplateParamLists,
@@ -2715,73 +2949,99 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       D.setInvalidType();
     }
   }
-  if (DC->isRecord() && !CurContext->isRecord()) {
-    // This is an out-of-line definition of a static data member.
+  
+  bool isExplicitSpecialization = false;
+  VarDecl *NewVD;
+  if (!getLangOptions().CPlusPlus) {
+      NewVD = VarDecl::Create(Context, DC, D.getIdentifierLoc(),
+                              II, R, TInfo, SC, SCAsWritten);
+  
+    if (D.isInvalidType())
+      NewVD->setInvalidDecl();
+  } else {
+    if (DC->isRecord() && !CurContext->isRecord()) {
+      // This is an out-of-line definition of a static data member.
+      if (SC == SC_Static) {
+        Diag(D.getDeclSpec().getStorageClassSpecLoc(),
+             diag::err_static_out_of_line)
+          << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
+      } else if (SC == SC_None)
+        SC = SC_Static;
+    }
     if (SC == SC_Static) {
-      Diag(D.getDeclSpec().getStorageClassSpecLoc(),
-           diag::err_static_out_of_line)
-        << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
-    } else if (SC == SC_None)
-      SC = SC_Static;
-  }
-  if (SC == SC_Static) {
-    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC)) {
-      if (RD->isLocalClass())
-        Diag(D.getIdentifierLoc(),
-             diag::err_static_data_member_not_allowed_in_local_class)
-          << Name << RD->getDeclName();
+      if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC)) {
+        if (RD->isLocalClass())
+          Diag(D.getIdentifierLoc(),
+               diag::err_static_data_member_not_allowed_in_local_class)
+            << Name << RD->getDeclName();
+
+        // C++ [class.union]p1: If a union contains a static data member,
+        // the program is ill-formed.
+        //
+        // We also disallow static data members in anonymous structs.
+        if (CurContext->isRecord() && (RD->isUnion() || !RD->getDeclName()))
+          Diag(D.getIdentifierLoc(),
+               diag::err_static_data_member_not_allowed_in_union_or_anon_struct)
+            << Name << RD->isUnion();
+      }
     }
-  }
 
-  // Match up the template parameter lists with the scope specifier, then
-  // determine whether we have a template or a template specialization.
-  bool isExplicitSpecialization = false;
-  unsigned NumMatchedTemplateParamLists = TemplateParamLists.size();
-  bool Invalid = false;
-  if (TemplateParameterList *TemplateParams
+    // Match up the template parameter lists with the scope specifier, then
+    // determine whether we have a template or a template specialization.
+    isExplicitSpecialization = false;
+    unsigned NumMatchedTemplateParamLists = TemplateParamLists.size();
+    bool Invalid = false;
+    if (TemplateParameterList *TemplateParams
         = MatchTemplateParametersToScopeSpecifier(
-                                  D.getDeclSpec().getSourceRange().getBegin(),
+                                                  D.getDeclSpec().getSourceRange().getBegin(),
                                                   D.getCXXScopeSpec(),
-                        (TemplateParameterList**)TemplateParamLists.get(),
-                                                   TemplateParamLists.size(),
+                                                  TemplateParamLists.get(),
+                                                  TemplateParamLists.size(),
                                                   /*never a friend*/ false,
                                                   isExplicitSpecialization,
                                                   Invalid)) {
-    // All but one template parameter lists have been matching.
-    --NumMatchedTemplateParamLists;
-
-    if (TemplateParams->size() > 0) {
-      // There is no such thing as a variable template.
-      Diag(D.getIdentifierLoc(), diag::err_template_variable)
-        << II
-        << SourceRange(TemplateParams->getTemplateLoc(),
-                       TemplateParams->getRAngleLoc());
-      return 0;
-    } else {
-      // There is an extraneous 'template<>' for this variable. Complain
-      // about it, but allow the declaration of the variable.
-      Diag(TemplateParams->getTemplateLoc(),
-           diag::err_template_variable_noparams)
-        << II
-        << SourceRange(TemplateParams->getTemplateLoc(),
-                       TemplateParams->getRAngleLoc());
+      // All but one template parameter lists have been matching.
+      --NumMatchedTemplateParamLists;
+
+      if (TemplateParams->size() > 0) {
+        // There is no such thing as a variable template.
+        Diag(D.getIdentifierLoc(), diag::err_template_variable)
+          << II
+          << SourceRange(TemplateParams->getTemplateLoc(),
+                         TemplateParams->getRAngleLoc());
+        return 0;
+      } else {
+        // There is an extraneous 'template<>' for this variable. Complain
+        // about it, but allow the declaration of the variable.
+        Diag(TemplateParams->getTemplateLoc(),
+             diag::err_template_variable_noparams)
+          << II
+          << SourceRange(TemplateParams->getTemplateLoc(),
+                         TemplateParams->getRAngleLoc());
       
-      isExplicitSpecialization = true;
+        isExplicitSpecialization = true;
+      }
     }
-  }
 
-  VarDecl *NewVD = VarDecl::Create(Context, DC, D.getIdentifierLoc(),
-                                   II, R, TInfo, SC, SCAsWritten);
+    NewVD = VarDecl::Create(Context, DC, D.getIdentifierLoc(),
+                            II, R, TInfo, SC, SCAsWritten);
 
-  if (D.isInvalidType() || Invalid)
-    NewVD->setInvalidDecl();
+    // If this decl has an auto type in need of deduction, mark the VarDecl so
+    // we can diagnose uses of it in its own initializer.
+    if (D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto) {
+      NewVD->setParsingAutoInit(R->getContainedAutoType());
+    }
+
+    if (D.isInvalidType() || Invalid)
+      NewVD->setInvalidDecl();
 
-  SetNestedNameSpecifier(NewVD, D);
+    SetNestedNameSpecifier(NewVD, D);
 
-  if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
-    NewVD->setTemplateParameterListsInfo(Context,
-                                         NumMatchedTemplateParamLists,
-                        (TemplateParameterList**)TemplateParamLists.release());
+    if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
+      NewVD->setTemplateParameterListsInfo(Context,
+                                           NumMatchedTemplateParamLists,
+                                           TemplateParamLists.release());
+    }
   }
 
   if (D.getDeclSpec().isThreadSpecified()) {
@@ -2801,11 +3061,29 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   ProcessDeclAttributes(S, NewVD, D);
 
   // Handle GNU asm-label extension (encoded as an attribute).
-  if (Expr *E = (Expr*) D.getAsmLabel()) {
+  if (Expr *E = (Expr*)D.getAsmLabel()) {
     // The parser guarantees this is a string.
     StringLiteral *SE = cast<StringLiteral>(E);
-    NewVD->addAttr(::new (Context) AsmLabelAttr(SE->getStrTokenLoc(0), 
-                                                Context, SE->getString()));
+    llvm::StringRef Label = SE->getString();
+    if (S->getFnParent() != 0) {
+      switch (SC) {
+      case SC_None:
+      case SC_Auto:
+        Diag(E->getExprLoc(), diag::warn_asm_label_on_auto_decl) << Label;
+        break;
+      case SC_Register:
+        if (!Context.Target.isValidGCCRegisterName(Label))
+          Diag(E->getExprLoc(), diag::err_asm_unknown_register_name) << Label;
+        break;
+      case SC_Static:
+      case SC_Extern:
+      case SC_PrivateExtern:
+        break;
+      }
+    }
+
+    NewVD->addAttr(::new (Context) AsmLabelAttr(SE->getStrTokenLoc(0),
+                                                Context, Label));
   }
 
   // Diagnose shadowed variables before filtering for scope.
@@ -2817,33 +3095,37 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // declaration has linkage).
   FilterLookupForScope(*this, Previous, DC, S, NewVD->hasLinkage());
   
-  // Merge the decl with the existing one if appropriate.
-  if (!Previous.empty()) {
-    if (Previous.isSingleResult() &&
-        isa<FieldDecl>(Previous.getFoundDecl()) &&
-        D.getCXXScopeSpec().isSet()) {
-      // The user tried to define a non-static data member
-      // out-of-line (C++ [dcl.meaning]p1).
-      Diag(NewVD->getLocation(), diag::err_nonstatic_member_out_of_line)
+  if (!getLangOptions().CPlusPlus)
+    CheckVariableDeclaration(NewVD, Previous, Redeclaration);
+  else {
+    // Merge the decl with the existing one if appropriate.
+    if (!Previous.empty()) {
+      if (Previous.isSingleResult() &&
+          isa<FieldDecl>(Previous.getFoundDecl()) &&
+          D.getCXXScopeSpec().isSet()) {
+        // The user tried to define a non-static data member
+        // out-of-line (C++ [dcl.meaning]p1).
+        Diag(NewVD->getLocation(), diag::err_nonstatic_member_out_of_line)
+          << D.getCXXScopeSpec().getRange();
+        Previous.clear();
+        NewVD->setInvalidDecl();
+      }
+    } else if (D.getCXXScopeSpec().isSet()) {
+      // No previous declaration in the qualifying scope.
+      Diag(D.getIdentifierLoc(), diag::err_no_member)
+        << Name << computeDeclContext(D.getCXXScopeSpec(), true)
         << D.getCXXScopeSpec().getRange();
-      Previous.clear();
       NewVD->setInvalidDecl();
     }
-  } else if (D.getCXXScopeSpec().isSet()) {
-    // No previous declaration in the qualifying scope.
-    Diag(D.getIdentifierLoc(), diag::err_no_member)
-      << Name << computeDeclContext(D.getCXXScopeSpec(), true)
-      << D.getCXXScopeSpec().getRange();
-    NewVD->setInvalidDecl();
-  }
-
-  CheckVariableDeclaration(NewVD, Previous, Redeclaration);
 
-  // This is an explicit specialization of a static data member. Check it.
-  if (isExplicitSpecialization && !NewVD->isInvalidDecl() &&
-      CheckMemberSpecialization(NewVD, Previous))
-    NewVD->setInvalidDecl();
+    CheckVariableDeclaration(NewVD, Previous, Redeclaration);
 
+    // This is an explicit specialization of a static data member. Check it.
+    if (isExplicitSpecialization && !NewVD->isInvalidDecl() &&
+        CheckMemberSpecialization(NewVD, Previous))
+      NewVD->setInvalidDecl();
+  }
+  
   // attributes declared post-definition are currently ignored
   // FIXME: This should be handled in attribute merging, not
   // here.
@@ -2866,7 +3148,7 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // member, set the visibility of this variable.
   if (NewVD->getLinkage() == ExternalLinkage && !DC->isRecord())
     AddPushedVisibilityAttribute(NewVD);
-
+  
   MarkUnusedFileScopedDecl(NewVD);
 
   return NewVD;
@@ -2883,13 +3165,13 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 ///
 void Sema::CheckShadow(Scope *S, VarDecl *D, const LookupResult& R) {
   // Return if warning is ignored.
-  if (Diags.getDiagnosticLevel(diag::warn_decl_shadow) == Diagnostic::Ignored)
+  if (Diags.getDiagnosticLevel(diag::warn_decl_shadow, R.getNameLoc()) ==
+        Diagnostic::Ignored)
     return;
 
-  // Don't diagnose declarations at file scope.  The scope might not
-  // have a DeclContext if (e.g.) we're parsing a function prototype.
-  DeclContext *NewDC = static_cast<DeclContext*>(S->getEntity());
-  if (NewDC && NewDC->isFileContext())
+  // Don't diagnose declarations at file scope.
+  DeclContext *NewDC = D->getDeclContext();
+  if (NewDC->isFileContext())
     return;
   
   // Only diagnose if we're shadowing an unambiguous field or variable.
@@ -2900,6 +3182,36 @@ void Sema::CheckShadow(Scope *S, VarDecl *D, const LookupResult& R) {
   if (!isa<VarDecl>(ShadowedDecl) && !isa<FieldDecl>(ShadowedDecl))
     return;
 
+  // Fields are not shadowed by variables in C++ static methods.
+  if (isa<FieldDecl>(ShadowedDecl))
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(NewDC))
+      if (MD->isStatic())
+        return;
+
+  if (VarDecl *shadowedVar = dyn_cast<VarDecl>(ShadowedDecl))
+    if (shadowedVar->isExternC()) {
+      // Don't warn for this case:
+      //
+      // @code
+      // extern int bob;
+      // void f() {
+      //   extern int bob;
+      // }
+      // @endcode
+      if (D->isExternC())
+        return;
+
+      // For shadowing external vars, make sure that we point to the global
+      // declaration, not a locally scoped extern declaration.
+      for (VarDecl::redecl_iterator
+             I = shadowedVar->redecls_begin(), E = shadowedVar->redecls_end();
+           I != E; ++I)
+        if (I->isFileVarDecl()) {
+          ShadowedDecl = *I;
+          break;
+        }
+    }
+
   DeclContext *OldDC = ShadowedDecl->getDeclContext();
 
   // Only warn about certain kinds of shadowing for class members.
@@ -2937,6 +3249,10 @@ void Sema::CheckShadow(Scope *S, VarDecl *D, const LookupResult& R) {
 
 /// \brief Check -Wshadow without the advantage of a previous lookup.
 void Sema::CheckShadow(Scope *S, VarDecl *D) {
+  if (Diags.getDiagnosticLevel(diag::warn_decl_shadow, D->getLocation()) ==
+        Diagnostic::Ignored)
+    return;
+
   LookupResult R(*this, D->getDeclName(), D->getLocation(),
                  Sema::LookupOrdinaryName, Sema::ForRedeclaration);
   LookupName(R, S);
@@ -2971,7 +3287,7 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
   // This includes arrays of objects with address space qualifiers, but not
   // automatic variables that point to other address spaces.
   // ISO/IEC TR 18037 S5.1.2
-  if (NewVD->hasLocalStorage() && (T.getAddressSpace() != 0)) {
+  if (NewVD->hasLocalStorage() && T.getAddressSpace() != 0) {
     Diag(NewVD->getLocation(), diag::err_as_qualified_auto_decl);
     return NewVD->setInvalidDecl();
   }
@@ -2979,7 +3295,7 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
   if (NewVD->hasLocalStorage() && T.isObjCGCWeak()
       && !NewVD->hasAttr<BlocksAttr>())
     Diag(NewVD->getLocation(), diag::warn_attribute_weak_on_local);
-
+  
   bool isVM = T->isVariablyModifiedType();
   if (isVM || NewVD->hasAttr<CleanupAttr>() ||
       NewVD->hasAttr<BlocksAttr>())
@@ -3115,23 +3431,42 @@ static bool FindOverriddenMethod(const CXXBaseSpecifier *Specifier,
 
 /// AddOverriddenMethods - See if a method overrides any in the base classes,
 /// and if so, check that it's a valid override and remember it.
-void Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
+bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
   // Look for virtual methods in base classes that this method might override.
   CXXBasePaths Paths;
   FindOverriddenMethodData Data;
   Data.Method = MD;
   Data.S = this;
+  bool AddedAny = false;
   if (DC->lookupInBases(&FindOverriddenMethod, &Data, Paths)) {
     for (CXXBasePaths::decl_iterator I = Paths.found_decls_begin(),
          E = Paths.found_decls_end(); I != E; ++I) {
       if (CXXMethodDecl *OldMD = dyn_cast<CXXMethodDecl>(*I)) {
         if (!CheckOverridingFunctionReturnType(MD, OldMD) &&
             !CheckOverridingFunctionExceptionSpec(MD, OldMD) &&
-            !CheckOverridingFunctionAttributes(MD, OldMD))
+            !CheckIfOverriddenFunctionIsMarkedFinal(MD, OldMD)) {
           MD->addOverriddenMethod(OldMD->getCanonicalDecl());
+          AddedAny = true;
+        }
       }
     }
   }
+  
+  return AddedAny;
+}
+
+static void DiagnoseInvalidRedeclaration(Sema &S, FunctionDecl *NewFD) {
+  LookupResult Prev(S, NewFD->getDeclName(), NewFD->getLocation(),
+                    Sema::LookupOrdinaryName, Sema::ForRedeclaration);
+  S.LookupQualifiedName(Prev, NewFD->getDeclContext());
+  assert(!Prev.isAmbiguous() &&
+         "Cannot have an ambiguity in previous-declaration lookup");
+  for (LookupResult::iterator Func = Prev.begin(), FuncEnd = Prev.end();
+       Func != FuncEnd; ++Func) {
+    if (isa<FunctionDecl>(*Func) &&
+        isNearlyMatchingFunction(S.Context, cast<FunctionDecl>(*Func), NewFD))
+      S.Diag((*Func)->getLocation(), diag::note_member_def_close_match);
+  }
 }
 
 NamedDecl*
@@ -3171,315 +3506,336 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       SC = SC_Static;
     break;
   }
-  case DeclSpec::SCS_private_extern: SC = SC_PrivateExtern;break;
+  case DeclSpec::SCS_private_extern: SC = SC_PrivateExtern; break;
   }
 
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
 
-  bool isFriend = D.getDeclSpec().isFriendSpecified();
-  bool isInline = D.getDeclSpec().isInlineSpecified();
-  bool isVirtual = D.getDeclSpec().isVirtualSpecified();
-  bool isExplicit = D.getDeclSpec().isExplicitSpecified();
-
-  DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
-  FunctionDecl::StorageClass SCAsWritten
-    = StorageClassSpecToFunctionDeclStorageClass(SCSpec);
-
-  // Check that the return type is not an abstract class type.
-  // For record types, this is done by the AbstractClassUsageDiagnoser once
-  // the class has been completely parsed.
-  if (!DC->isRecord() &&
-      RequireNonAbstractType(D.getIdentifierLoc(),
-                             R->getAs<FunctionType>()->getResultType(),
-                             diag::err_abstract_type_in_decl,
-                             AbstractReturnType))
-    D.setInvalidType();
-
   // Do not allow returning a objc interface by-value.
   if (R->getAs<FunctionType>()->getResultType()->isObjCObjectType()) {
     Diag(D.getIdentifierLoc(),
          diag::err_object_cannot_be_passed_returned_by_value) << 0
-      << R->getAs<FunctionType>()->getResultType();
+    << R->getAs<FunctionType>()->getResultType();
     D.setInvalidType();
   }
-
-  bool isVirtualOkay = false;
+  
   FunctionDecl *NewFD;
-
-  if (isFriend) {
-    // C++ [class.friend]p5
-    //   A function can be defined in a friend declaration of a
-    //   class . . . . Such a function is implicitly inline.
-    isInline |= IsFunctionDefinition;
-  }
-
-  if (Name.getNameKind() == DeclarationName::CXXConstructorName) {
-    // This is a C++ constructor declaration.
-    assert(DC->isRecord() &&
-           "Constructors can only be declared in a member context");
-
-    R = CheckConstructorDeclarator(D, R, SC);
-
-    // Create the new declaration
-    NewFD = CXXConstructorDecl::Create(Context,
-                                       cast<CXXRecordDecl>(DC),
-                                       NameInfo, R, TInfo,
-                                       isExplicit, isInline,
-                                       /*isImplicitlyDeclared=*/false);
-  } else if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
-    // This is a C++ destructor declaration.
-    if (DC->isRecord()) {
-      R = CheckDestructorDeclarator(D, R, SC);
-
-      NewFD = CXXDestructorDecl::Create(Context,
-                                        cast<CXXRecordDecl>(DC),
-                                        NameInfo, R,
-                                        isInline,
-                                        /*isImplicitlyDeclared=*/false);
-      NewFD->setTypeSourceInfo(TInfo);
-
-      isVirtualOkay = true;
-    } else {
-      Diag(D.getIdentifierLoc(), diag::err_destructor_not_member);
-
-      // Create a FunctionDecl to satisfy the function definition parsing
-      // code path.
-      NewFD = FunctionDecl::Create(Context, DC, D.getIdentifierLoc(),
-                                   Name, R, TInfo, SC, SCAsWritten, isInline,
-                                   /*hasPrototype=*/true);
-      D.setInvalidType();
-    }
-  } else if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName) {
-    if (!DC->isRecord()) {
-      Diag(D.getIdentifierLoc(),
-           diag::err_conv_function_not_member);
-      return 0;
-    }
-
-    CheckConversionDeclarator(D, R, SC);
-    NewFD = CXXConversionDecl::Create(Context, cast<CXXRecordDecl>(DC),
-                                      NameInfo, R, TInfo,
-                                      isInline, isExplicit);
-
-    isVirtualOkay = true;
-  } else if (DC->isRecord()) {
-    // If the of the function is the same as the name of the record, then this
-    // must be an invalid constructor that has a return type.
-    // (The parser checks for a return type and makes the declarator a
-    // constructor if it has no return type).
-    // must have an invalid constructor that has a return type
-    if (Name.getAsIdentifierInfo() &&
-        Name.getAsIdentifierInfo() == cast<CXXRecordDecl>(DC)->getIdentifier()){
-      Diag(D.getIdentifierLoc(), diag::err_constructor_return_type)
-        << SourceRange(D.getDeclSpec().getTypeSpecTypeLoc())
-        << SourceRange(D.getIdentifierLoc());
-      return 0;
-    }
-
-    bool isStatic = SC == SC_Static;
-    
-    // [class.free]p1:
-    // Any allocation function for a class T is a static member
-    // (even if not explicitly declared static).
-    if (Name.getCXXOverloadedOperator() == OO_New ||
-        Name.getCXXOverloadedOperator() == OO_Array_New)
-      isStatic = true;
-
-    // [class.free]p6 Any deallocation function for a class X is a static member
-    // (even if not explicitly declared static).
-    if (Name.getCXXOverloadedOperator() == OO_Delete ||
-        Name.getCXXOverloadedOperator() == OO_Array_Delete)
-      isStatic = true;
-    
-    // This is a C++ method declaration.
-    NewFD = CXXMethodDecl::Create(Context, cast<CXXRecordDecl>(DC),
-                                  NameInfo, R, TInfo,
-                                  isStatic, SCAsWritten, isInline);
-
-    isVirtualOkay = !isStatic;
-  } else {
+  bool isInline = D.getDeclSpec().isInlineSpecified();
+  bool isFriend = false;
+  DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
+  FunctionDecl::StorageClass SCAsWritten
+    = StorageClassSpecToFunctionDeclStorageClass(SCSpec);
+  FunctionTemplateDecl *FunctionTemplate = 0;
+  bool isExplicitSpecialization = false;
+  bool isFunctionTemplateSpecialization = false;
+  unsigned NumMatchedTemplateParamLists = 0;
+  
+  if (!getLangOptions().CPlusPlus) {
     // Determine whether the function was written with a
     // prototype. This true when:
-    //   - we're in C++ (where every function has a prototype),
     //   - there is a prototype in the declarator, or
     //   - the type R of the function is some kind of typedef or other reference
     //     to a type name (which eventually refers to a function type).
     bool HasPrototype =
-       getLangOptions().CPlusPlus ||
-       (D.getNumTypeObjects() && D.getTypeObject(0).Fun.hasPrototype) ||
-       (!isa<FunctionType>(R.getTypePtr()) && R->isFunctionProtoType());
-
+    (D.isFunctionDeclarator() && D.getFunctionTypeInfo().hasPrototype) ||
+    (!isa<FunctionType>(R.getTypePtr()) && R->isFunctionProtoType());
+  
     NewFD = FunctionDecl::Create(Context, DC,
                                  NameInfo, R, TInfo, SC, SCAsWritten, isInline,
                                  HasPrototype);
-  }
-
-  if (D.isInvalidType())
-    NewFD->setInvalidDecl();
+    if (D.isInvalidType())
+      NewFD->setInvalidDecl();
+    
+    // Set the lexical context.
+    NewFD->setLexicalDeclContext(CurContext);
+    // Filter out previous declarations that don't match the scope.
+    FilterLookupForScope(*this, Previous, DC, S, NewFD->hasLinkage());
+  } else {
+    isFriend = D.getDeclSpec().isFriendSpecified();
+    bool isVirtual = D.getDeclSpec().isVirtualSpecified();
+    bool isExplicit = D.getDeclSpec().isExplicitSpecified();
+    bool isVirtualOkay = false;
+
+    // Check that the return type is not an abstract class type.
+    // For record types, this is done by the AbstractClassUsageDiagnoser once
+    // the class has been completely parsed.
+    if (!DC->isRecord() &&
+      RequireNonAbstractType(D.getIdentifierLoc(),
+                             R->getAs<FunctionType>()->getResultType(),
+                             diag::err_abstract_type_in_decl,
+                             AbstractReturnType))
+      D.setInvalidType();
 
-  SetNestedNameSpecifier(NewFD, D);
 
-  // Set the lexical context. If the declarator has a C++
-  // scope specifier, or is the object of a friend declaration, the
-  // lexical context will be different from the semantic context.
-  NewFD->setLexicalDeclContext(CurContext);
+    if (isFriend) {
+      // C++ [class.friend]p5
+      //   A function can be defined in a friend declaration of a
+      //   class . . . . Such a function is implicitly inline.
+      isInline |= IsFunctionDefinition;
+    }
+
+    if (Name.getNameKind() == DeclarationName::CXXConstructorName) {
+      // This is a C++ constructor declaration.
+      assert(DC->isRecord() &&
+             "Constructors can only be declared in a member context");
+
+      R = CheckConstructorDeclarator(D, R, SC);
+
+      // Create the new declaration
+      NewFD = CXXConstructorDecl::Create(Context,
+                                         cast<CXXRecordDecl>(DC),
+                                         NameInfo, R, TInfo,
+                                         isExplicit, isInline,
+                                         /*isImplicitlyDeclared=*/false);
+    } else if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
+      // This is a C++ destructor declaration.
+      if (DC->isRecord()) {
+        R = CheckDestructorDeclarator(D, R, SC);
+
+        NewFD = CXXDestructorDecl::Create(Context,
+                                          cast<CXXRecordDecl>(DC),
+                                          NameInfo, R, TInfo,
+                                          isInline,
+                                          /*isImplicitlyDeclared=*/false);
+        isVirtualOkay = true;
+      } else {
+        Diag(D.getIdentifierLoc(), diag::err_destructor_not_member);
 
-  // Match up the template parameter lists with the scope specifier, then
-  // determine whether we have a template or a template specialization.
-  FunctionTemplateDecl *FunctionTemplate = 0;
-  bool isExplicitSpecialization = false;
-  bool isFunctionTemplateSpecialization = false;
-  unsigned NumMatchedTemplateParamLists = TemplateParamLists.size();
-  bool Invalid = false;
-  if (TemplateParameterList *TemplateParams
-        = MatchTemplateParametersToScopeSpecifier(
-                                  D.getDeclSpec().getSourceRange().getBegin(),
-                                  D.getCXXScopeSpec(),
-                           (TemplateParameterList**)TemplateParamLists.get(),
-                                                  TemplateParamLists.size(),
-                                                  isFriend,
-                                                  isExplicitSpecialization,
-                                                  Invalid)) {
-    // All but one template parameter lists have been matching.
-    --NumMatchedTemplateParamLists;
+        // Create a FunctionDecl to satisfy the function definition parsing
+        // code path.
+        NewFD = FunctionDecl::Create(Context, DC, D.getIdentifierLoc(),
+                                     Name, R, TInfo, SC, SCAsWritten, isInline,
+                                     /*hasPrototype=*/true);
+        D.setInvalidType();
+      }
+    } else if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName) {
+      if (!DC->isRecord()) {
+        Diag(D.getIdentifierLoc(),
+             diag::err_conv_function_not_member);
+        return 0;
+      }
 
-    if (TemplateParams->size() > 0) {
-      // This is a function template
+      CheckConversionDeclarator(D, R, SC);
+      NewFD = CXXConversionDecl::Create(Context, cast<CXXRecordDecl>(DC),
+                                        NameInfo, R, TInfo,
+                                        isInline, isExplicit);
 
-      // Check that we can declare a template here.
-      if (CheckTemplateDeclScope(S, TemplateParams))
+      isVirtualOkay = true;
+    } else if (DC->isRecord()) {
+      // If the of the function is the same as the name of the record, then this
+      // must be an invalid constructor that has a return type.
+      // (The parser checks for a return type and makes the declarator a
+      // constructor if it has no return type).
+      // must have an invalid constructor that has a return type
+      if (Name.getAsIdentifierInfo() &&
+          Name.getAsIdentifierInfo() == cast<CXXRecordDecl>(DC)->getIdentifier()){
+        Diag(D.getIdentifierLoc(), diag::err_constructor_return_type)
+          << SourceRange(D.getDeclSpec().getTypeSpecTypeLoc())
+          << SourceRange(D.getIdentifierLoc());
         return 0;
+      }
+
+      bool isStatic = SC == SC_Static;
+    
+      // [class.free]p1:
+      // Any allocation function for a class T is a static member
+      // (even if not explicitly declared static).
+      if (Name.getCXXOverloadedOperator() == OO_New ||
+          Name.getCXXOverloadedOperator() == OO_Array_New)
+        isStatic = true;
+
+      // [class.free]p6 Any deallocation function for a class X is a static member
+      // (even if not explicitly declared static).
+      if (Name.getCXXOverloadedOperator() == OO_Delete ||
+          Name.getCXXOverloadedOperator() == OO_Array_Delete)
+        isStatic = true;
+    
+      // This is a C++ method declaration.
+      NewFD = CXXMethodDecl::Create(Context, cast<CXXRecordDecl>(DC),
+                                    NameInfo, R, TInfo,
+                                    isStatic, SCAsWritten, isInline);
 
-      FunctionTemplate = FunctionTemplateDecl::Create(Context, DC,
+      isVirtualOkay = !isStatic;
+    } else {
+      // Determine whether the function was written with a
+      // prototype. This true when:
+      //   - we're in C++ (where every function has a prototype),
+      NewFD = FunctionDecl::Create(Context, DC,
+                                   NameInfo, R, TInfo, SC, SCAsWritten, isInline,
+                                   true/*HasPrototype*/);
+    }
+    SetNestedNameSpecifier(NewFD, D);
+    isExplicitSpecialization = false;
+    isFunctionTemplateSpecialization = false;
+    NumMatchedTemplateParamLists = TemplateParamLists.size();
+    if (D.isInvalidType())
+      NewFD->setInvalidDecl();
+    
+    // Set the lexical context. If the declarator has a C++
+    // scope specifier, or is the object of a friend declaration, the
+    // lexical context will be different from the semantic context.
+    NewFD->setLexicalDeclContext(CurContext);
+    
+    // Match up the template parameter lists with the scope specifier, then
+    // determine whether we have a template or a template specialization.
+    bool Invalid = false;
+    if (TemplateParameterList *TemplateParams
+        = MatchTemplateParametersToScopeSpecifier(
+                                  D.getDeclSpec().getSourceRange().getBegin(),
+                                  D.getCXXScopeSpec(),
+                                  TemplateParamLists.get(),
+                                  TemplateParamLists.size(),
+                                  isFriend,
+                                  isExplicitSpecialization,
+                                  Invalid)) {
+          // All but one template parameter lists have been matching.
+          --NumMatchedTemplateParamLists;
+
+          if (TemplateParams->size() > 0) {
+            // This is a function template
+
+            // Check that we can declare a template here.
+            if (CheckTemplateDeclScope(S, TemplateParams))
+              return 0;
+
+            FunctionTemplate = FunctionTemplateDecl::Create(Context, DC,
                                                       NewFD->getLocation(),
                                                       Name, TemplateParams,
                                                       NewFD);
-      FunctionTemplate->setLexicalDeclContext(CurContext);
-      NewFD->setDescribedFunctionTemplate(FunctionTemplate);
-    } else {
-      // This is a function template specialization.
-      isFunctionTemplateSpecialization = true;
-
-      // C++0x [temp.expl.spec]p20 forbids "template<> friend void foo(int);".
-      if (isFriend && isFunctionTemplateSpecialization) {
-        // We want to remove the "template<>", found here.
-        SourceRange RemoveRange = TemplateParams->getSourceRange();
-
-        // If we remove the template<> and the name is not a
-        // template-id, we're actually silently creating a problem:
-        // the friend declaration will refer to an untemplated decl,
-        // and clearly the user wants a template specialization.  So
-        // we need to insert '<>' after the name.
-        SourceLocation InsertLoc;
-        if (D.getName().getKind() != UnqualifiedId::IK_TemplateId) {
-          InsertLoc = D.getName().getSourceRange().getEnd();
-          InsertLoc = PP.getLocForEndOfToken(InsertLoc);
+            FunctionTemplate->setLexicalDeclContext(CurContext);
+            NewFD->setDescribedFunctionTemplate(FunctionTemplate);
+          } else {
+            // This is a function template specialization.
+            isFunctionTemplateSpecialization = true;
+
+            // C++0x [temp.expl.spec]p20 forbids "template<> friend void foo(int);".
+            if (isFriend && isFunctionTemplateSpecialization) {
+              // We want to remove the "template<>", found here.
+              SourceRange RemoveRange = TemplateParams->getSourceRange();
+
+              // If we remove the template<> and the name is not a
+              // template-id, we're actually silently creating a problem:
+              // the friend declaration will refer to an untemplated decl,
+              // and clearly the user wants a template specialization.  So
+              // we need to insert '<>' after the name.
+              SourceLocation InsertLoc;
+              if (D.getName().getKind() != UnqualifiedId::IK_TemplateId) {
+                InsertLoc = D.getName().getSourceRange().getEnd();
+                InsertLoc = PP.getLocForEndOfToken(InsertLoc);
+              }
+
+              Diag(D.getIdentifierLoc(), diag::err_template_spec_decl_friend)
+              << Name << RemoveRange
+              << FixItHint::CreateRemoval(RemoveRange)
+              << FixItHint::CreateInsertion(InsertLoc, "<>");
+            }
+          }
         }
 
-        Diag(D.getIdentifierLoc(), diag::err_template_spec_decl_friend)
-          << Name << RemoveRange
-          << FixItHint::CreateRemoval(RemoveRange)
-          << FixItHint::CreateInsertion(InsertLoc, "<>");
-      }
+    if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
+      NewFD->setTemplateParameterListsInfo(Context,
+                                           NumMatchedTemplateParamLists,
+                                           TemplateParamLists.release());
     }
-  }
 
-  if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
-    NewFD->setTemplateParameterListsInfo(Context,
-                                         NumMatchedTemplateParamLists,
-                        (TemplateParameterList**)TemplateParamLists.release());
-  }
-
-  if (Invalid) {
-    NewFD->setInvalidDecl();
-    if (FunctionTemplate)
-      FunctionTemplate->setInvalidDecl();
-  }
+    if (Invalid) {
+      NewFD->setInvalidDecl();
+      if (FunctionTemplate)
+        FunctionTemplate->setInvalidDecl();
+    }
   
-  // C++ [dcl.fct.spec]p5:
-  //   The virtual specifier shall only be used in declarations of
-  //   nonstatic class member functions that appear within a
-  //   member-specification of a class declaration; see 10.3.
-  //
-  if (isVirtual && !NewFD->isInvalidDecl()) {
-    if (!isVirtualOkay) {
-       Diag(D.getDeclSpec().getVirtualSpecLoc(),
-           diag::err_virtual_non_function);
-    } else if (!CurContext->isRecord()) {
-      // 'virtual' was specified outside of the class.
-      Diag(D.getDeclSpec().getVirtualSpecLoc(), diag::err_virtual_out_of_class)
-        << FixItHint::CreateRemoval(D.getDeclSpec().getVirtualSpecLoc());
-    } else {
-      // Okay: Add virtual to the method.
-      CXXRecordDecl *CurClass = cast<CXXRecordDecl>(DC);
-      CurClass->setMethodAsVirtual(NewFD);
+    // C++ [dcl.fct.spec]p5:
+    //   The virtual specifier shall only be used in declarations of
+    //   nonstatic class member functions that appear within a
+    //   member-specification of a class declaration; see 10.3.
+    //
+    if (isVirtual && !NewFD->isInvalidDecl()) {
+      if (!isVirtualOkay) {
+        Diag(D.getDeclSpec().getVirtualSpecLoc(),
+             diag::err_virtual_non_function);
+      } else if (!CurContext->isRecord()) {
+        // 'virtual' was specified outside of the class.
+        Diag(D.getDeclSpec().getVirtualSpecLoc(), 
+             diag::err_virtual_out_of_class)
+          << FixItHint::CreateRemoval(D.getDeclSpec().getVirtualSpecLoc());
+      } else if (NewFD->getDescribedFunctionTemplate()) {
+        // C++ [temp.mem]p3:
+        //  A member function template shall not be virtual.
+        Diag(D.getDeclSpec().getVirtualSpecLoc(),
+             diag::err_virtual_member_function_template)
+          << FixItHint::CreateRemoval(D.getDeclSpec().getVirtualSpecLoc());
+      } else {
+        // Okay: Add virtual to the method.
+        NewFD->setVirtualAsWritten(true);
+      }
     }
-  }
 
-  // C++ [dcl.fct.spec]p3:
-  //  The inline specifier shall not appear on a block scope function declaration.
-  if (isInline && !NewFD->isInvalidDecl() && getLangOptions().CPlusPlus) {
-    if (CurContext->isFunctionOrMethod()) {
-      // 'inline' is not allowed on block scope function declaration.
-      Diag(D.getDeclSpec().getInlineSpecLoc(), 
-           diag::err_inline_declaration_block_scope) << Name
-        << FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
+    // C++ [dcl.fct.spec]p3:
+    //  The inline specifier shall not appear on a block scope function declaration.
+    if (isInline && !NewFD->isInvalidDecl()) {
+      if (CurContext->isFunctionOrMethod()) {
+        // 'inline' is not allowed on block scope function declaration.
+        Diag(D.getDeclSpec().getInlineSpecLoc(), 
+             diag::err_inline_declaration_block_scope) << Name
+          << FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
+      }
     }
-  }
  
-  // C++ [dcl.fct.spec]p6:
-  //  The explicit specifier shall be used only in the declaration of a
-  //  constructor or conversion function within its class definition; see 12.3.1
-  //  and 12.3.2.
-  if (isExplicit && !NewFD->isInvalidDecl()) {
-    if (!CurContext->isRecord()) {
-      // 'explicit' was specified outside of the class.
-      Diag(D.getDeclSpec().getExplicitSpecLoc(), 
-           diag::err_explicit_out_of_class)
-        << FixItHint::CreateRemoval(D.getDeclSpec().getExplicitSpecLoc());
-    } else if (!isa<CXXConstructorDecl>(NewFD) && 
-               !isa<CXXConversionDecl>(NewFD)) {
-      // 'explicit' was specified on a function that wasn't a constructor
-      // or conversion function.
-      Diag(D.getDeclSpec().getExplicitSpecLoc(),
-           diag::err_explicit_non_ctor_or_conv_function)
-        << FixItHint::CreateRemoval(D.getDeclSpec().getExplicitSpecLoc());
-    }      
-  }
-
-  // Filter out previous declarations that don't match the scope.
-  FilterLookupForScope(*this, Previous, DC, S, NewFD->hasLinkage());
-
-  if (isFriend) {
-    // DC is the namespace in which the function is being declared.
-    assert((DC->isFileContext() || !Previous.empty()) &&
-           "previously-undeclared friend function being created "
-           "in a non-namespace context");
-
-    // For now, claim that the objects have no previous declaration.
-    if (FunctionTemplate) {
-      FunctionTemplate->setObjectOfFriendDecl(false);
-      FunctionTemplate->setAccess(AS_public);
+    // C++ [dcl.fct.spec]p6:
+    //  The explicit specifier shall be used only in the declaration of a
+    //  constructor or conversion function within its class definition; see 12.3.1
+    //  and 12.3.2.
+    if (isExplicit && !NewFD->isInvalidDecl()) {
+      if (!CurContext->isRecord()) {
+        // 'explicit' was specified outside of the class.
+        Diag(D.getDeclSpec().getExplicitSpecLoc(), 
+             diag::err_explicit_out_of_class)
+          << FixItHint::CreateRemoval(D.getDeclSpec().getExplicitSpecLoc());
+      } else if (!isa<CXXConstructorDecl>(NewFD) && 
+                 !isa<CXXConversionDecl>(NewFD)) {
+        // 'explicit' was specified on a function that wasn't a constructor
+        // or conversion function.
+        Diag(D.getDeclSpec().getExplicitSpecLoc(),
+             diag::err_explicit_non_ctor_or_conv_function)
+          << FixItHint::CreateRemoval(D.getDeclSpec().getExplicitSpecLoc());
+      }      
+    }
+
+    // Filter out previous declarations that don't match the scope.
+    FilterLookupForScope(*this, Previous, DC, S, NewFD->hasLinkage());
+
+    if (isFriend) {
+      // For now, claim that the objects have no previous declaration.
+      if (FunctionTemplate) {
+        FunctionTemplate->setObjectOfFriendDecl(false);
+        FunctionTemplate->setAccess(AS_public);
+      }
+      NewFD->setObjectOfFriendDecl(false);
+      NewFD->setAccess(AS_public);
     }
-    NewFD->setObjectOfFriendDecl(false);
-    NewFD->setAccess(AS_public);
-  }
 
-  if (SC == SC_Static && isa<CXXMethodDecl>(NewFD) &&
-      !CurContext->isRecord()) {
-    // C++ [class.static]p1:
-    //   A data or function member of a class may be declared static
-    //   in a class definition, in which case it is a static member of
-    //   the class.
+    if (isa<CXXMethodDecl>(NewFD) && DC == CurContext && IsFunctionDefinition) {
+      // A method is implicitly inline if it's defined in its class
+      // definition.
+      NewFD->setImplicitlyInline();
+    }
 
-    // Complain about the 'static' specifier if it's on an out-of-line
-    // member function definition.
-    Diag(D.getDeclSpec().getStorageClassSpecLoc(),
-         diag::err_static_out_of_line)
-      << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
-  }
+    if (SC == SC_Static && isa<CXXMethodDecl>(NewFD) &&
+        !CurContext->isRecord()) {
+      // C++ [class.static]p1:
+      //   A data or function member of a class may be declared static
+      //   in a class definition, in which case it is a static member of
+      //   the class.
 
+      // Complain about the 'static' specifier if it's on an out-of-line
+      // member function definition.
+      Diag(D.getDeclSpec().getStorageClassSpecLoc(),
+           diag::err_static_out_of_line)
+        << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
+    }
+  }
+  
   // Handle GNU asm-label extension (encoded as an attribute).
   if (Expr *E = (Expr*) D.getAsmLabel()) {
     // The parser guarantees this is a string.
@@ -3491,8 +3847,8 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // Copy the parameter declarations from the declarator D to the function
   // declaration NewFD, if they are available.  First scavenge them into Params.
   llvm::SmallVector<ParmVarDecl*, 16> Params;
-  if (D.getNumTypeObjects() > 0) {
-    DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  if (D.isFunctionDeclarator()) {
+    DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
     // Check for C99 6.7.5.3p10 - foo(void) is a non-varargs
     // function that takes no arguments, not a function that takes a
@@ -3510,7 +3866,6 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       if (getLangOptions().CPlusPlus &&
           Param->getType().getUnqualifiedType() != Context.VoidTy)
         Diag(Param->getLocation(), diag::err_param_typedef_of_void);
-      // FIXME: Leaks decl?
     } else if (FTI.NumArgs > 0 && FTI.ArgInfo[0].Param != 0) {
       for (unsigned i = 0, e = FTI.NumArgs; i != e; ++i) {
         ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
@@ -3547,168 +3902,209 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // Finally, we know we have the right number of parameters, install them.
   NewFD->setParams(Params.data(), Params.size());
 
-  // If the declarator is a template-id, translate the parser's template 
-  // argument list into our AST format.
-  bool HasExplicitTemplateArgs = false;
-  TemplateArgumentListInfo TemplateArgs;
-  if (D.getName().getKind() == UnqualifiedId::IK_TemplateId) {
-    TemplateIdAnnotation *TemplateId = D.getName().TemplateId;
-    TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
-    TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
-    ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                       TemplateId->getTemplateArgs(),
-                                       TemplateId->NumArgs);
-    translateTemplateArguments(TemplateArgsPtr,
-                               TemplateArgs);
-    TemplateArgsPtr.release();
+  // Process the non-inheritable attributes on this declaration.
+  ProcessDeclAttributes(S, NewFD, D,
+                        /*NonInheritable=*/true, /*Inheritable=*/false);
+
+  if (!getLangOptions().CPlusPlus) {
+    // Perform semantic checking on the function declaration.
+    bool isExplctSpecialization=false;
+    CheckFunctionDeclaration(S, NewFD, Previous, isExplctSpecialization,
+                             Redeclaration);
+    assert((NewFD->isInvalidDecl() || !Redeclaration ||
+            Previous.getResultKind() != LookupResult::FoundOverloaded) &&
+           "previous declaration set still overloaded");
+  } else {
+    // If the declarator is a template-id, translate the parser's template 
+    // argument list into our AST format.
+    bool HasExplicitTemplateArgs = false;
+    TemplateArgumentListInfo TemplateArgs;
+    if (D.getName().getKind() == UnqualifiedId::IK_TemplateId) {
+      TemplateIdAnnotation *TemplateId = D.getName().TemplateId;
+      TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
+      TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
+      ASTTemplateArgsPtr TemplateArgsPtr(*this,
+                                         TemplateId->getTemplateArgs(),
+                                         TemplateId->NumArgs);
+      translateTemplateArguments(TemplateArgsPtr,
+                                 TemplateArgs);
+      TemplateArgsPtr.release();
     
-    HasExplicitTemplateArgs = true;
+      HasExplicitTemplateArgs = true;
     
-    if (FunctionTemplate) {
-      // FIXME: Diagnose function template with explicit template
-      // arguments.
-      HasExplicitTemplateArgs = false;
-    } else if (!isFunctionTemplateSpecialization && 
-               !D.getDeclSpec().isFriendSpecified()) {
-      // We have encountered something that the user meant to be a 
-      // specialization (because it has explicitly-specified template
-      // arguments) but that was not introduced with a "template<>" (or had
-      // too few of them).
-      Diag(D.getIdentifierLoc(), diag::err_template_spec_needs_header)
-        << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc)
-        << FixItHint::CreateInsertion(
-                                   D.getDeclSpec().getSourceRange().getBegin(),
-                                                 "template<> ");
-      isFunctionTemplateSpecialization = true;
-    } else {
-      // "friend void foo<>(int);" is an implicit specialization decl.
-      isFunctionTemplateSpecialization = true;
-    }
-  } else if (isFriend && isFunctionTemplateSpecialization) {
-    // This combination is only possible in a recovery case;  the user
-    // wrote something like:
-    //   template <> friend void foo(int);
-    // which we're recovering from as if the user had written:
-    //   friend void foo<>(int);
-    // Go ahead and fake up a template id.
-    HasExplicitTemplateArgs = true;
-    TemplateArgs.setLAngleLoc(D.getIdentifierLoc());
-    TemplateArgs.setRAngleLoc(D.getIdentifierLoc());
-  }
-
-  // If it's a friend (and only if it's a friend), it's possible
-  // that either the specialized function type or the specialized
-  // template is dependent, and therefore matching will fail.  In
-  // this case, don't check the specialization yet.
-  if (isFunctionTemplateSpecialization && isFriend &&
-      (NewFD->getType()->isDependentType() || DC->isDependentContext())) {
-    assert(HasExplicitTemplateArgs &&
-           "friend function specialization without template args");
-    if (CheckDependentFunctionTemplateSpecialization(NewFD, TemplateArgs,
-                                                     Previous))
-      NewFD->setInvalidDecl();
-  } else if (isFunctionTemplateSpecialization) {
-    if (CheckFunctionTemplateSpecialization(NewFD,
-                               (HasExplicitTemplateArgs ? &TemplateArgs : 0),
-                                            Previous))
-      NewFD->setInvalidDecl();
-  } else if (isExplicitSpecialization && isa<CXXMethodDecl>(NewFD)) {
-    if (CheckMemberSpecialization(NewFD, Previous))
-      NewFD->setInvalidDecl();
-  }
+      if (FunctionTemplate) {
+        // Function template with explicit template arguments.
+        Diag(D.getIdentifierLoc(), diag::err_function_template_partial_spec)
+          << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc);
+
+        HasExplicitTemplateArgs = false;
+      } else if (!isFunctionTemplateSpecialization && 
+                 !D.getDeclSpec().isFriendSpecified()) {
+        // We have encountered something that the user meant to be a 
+        // specialization (because it has explicitly-specified template
+        // arguments) but that was not introduced with a "template<>" (or had
+        // too few of them).
+        Diag(D.getIdentifierLoc(), diag::err_template_spec_needs_header)
+          << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc)
+          << FixItHint::CreateInsertion(
+                                        D.getDeclSpec().getSourceRange().getBegin(),
+                                                  "template<> ");
+        isFunctionTemplateSpecialization = true;
+      } else {
+        // "friend void foo<>(int);" is an implicit specialization decl.
+        isFunctionTemplateSpecialization = true;
+      }
+    } else if (isFriend && isFunctionTemplateSpecialization) {
+      // This combination is only possible in a recovery case;  the user
+      // wrote something like:
+      //   template <> friend void foo(int);
+      // which we're recovering from as if the user had written:
+      //   friend void foo<>(int);
+      // Go ahead and fake up a template id.
+      HasExplicitTemplateArgs = true;
+        TemplateArgs.setLAngleLoc(D.getIdentifierLoc());
+      TemplateArgs.setRAngleLoc(D.getIdentifierLoc());
+    }
+
+    // If it's a friend (and only if it's a friend), it's possible
+    // that either the specialized function type or the specialized
+    // template is dependent, and therefore matching will fail.  In
+    // this case, don't check the specialization yet.
+    if (isFunctionTemplateSpecialization && isFriend &&
+        (NewFD->getType()->isDependentType() || DC->isDependentContext())) {
+      assert(HasExplicitTemplateArgs &&
+             "friend function specialization without template args");
+      if (CheckDependentFunctionTemplateSpecialization(NewFD, TemplateArgs,
+                                                       Previous))
+        NewFD->setInvalidDecl();
+    } else if (isFunctionTemplateSpecialization) {
+      if (CheckFunctionTemplateSpecialization(NewFD,
+                                              (HasExplicitTemplateArgs ? &TemplateArgs : 0),
+                                              Previous))
+        NewFD->setInvalidDecl();
+    } else if (isExplicitSpecialization && isa<CXXMethodDecl>(NewFD)) {
+      if (CheckMemberSpecialization(NewFD, Previous))
+          NewFD->setInvalidDecl();
+    }
 
-  // Perform semantic checking on the function declaration.
-  bool OverloadableAttrRequired = false; // FIXME: HACK!
-  CheckFunctionDeclaration(S, NewFD, Previous, isExplicitSpecialization,
-                           Redeclaration, /*FIXME:*/OverloadableAttrRequired);
+    // Perform semantic checking on the function declaration.
+    CheckFunctionDeclaration(S, NewFD, Previous, isExplicitSpecialization,
+                             Redeclaration);
 
-  assert((NewFD->isInvalidDecl() || !Redeclaration ||
-          Previous.getResultKind() != LookupResult::FoundOverloaded) &&
-         "previous declaration set still overloaded");
+    assert((NewFD->isInvalidDecl() || !Redeclaration ||
+            Previous.getResultKind() != LookupResult::FoundOverloaded) &&
+           "previous declaration set still overloaded");
 
-  NamedDecl *PrincipalDecl = (FunctionTemplate
-                              ? cast<NamedDecl>(FunctionTemplate)
-                              : NewFD);
+    NamedDecl *PrincipalDecl = (FunctionTemplate
+                                ? cast<NamedDecl>(FunctionTemplate)
+                                : NewFD);
 
-  if (isFriend && Redeclaration) {
-    AccessSpecifier Access = AS_public;
-    if (!NewFD->isInvalidDecl())
-      Access = NewFD->getPreviousDeclaration()->getAccess();
+    if (isFriend && Redeclaration) {
+      AccessSpecifier Access = AS_public;
+      if (!NewFD->isInvalidDecl())
+        Access = NewFD->getPreviousDeclaration()->getAccess();
 
-    NewFD->setAccess(Access);
-    if (FunctionTemplate) FunctionTemplate->setAccess(Access);
+      NewFD->setAccess(Access);
+      if (FunctionTemplate) FunctionTemplate->setAccess(Access);
 
-    PrincipalDecl->setObjectOfFriendDecl(true);
-  }
+      PrincipalDecl->setObjectOfFriendDecl(true);
+    }
 
-  if (NewFD->isOverloadedOperator() && !DC->isRecord() &&
-      PrincipalDecl->isInIdentifierNamespace(Decl::IDNS_Ordinary))
-    PrincipalDecl->setNonMemberOperator();
+    if (NewFD->isOverloadedOperator() && !DC->isRecord() &&
+        PrincipalDecl->isInIdentifierNamespace(Decl::IDNS_Ordinary))
+      PrincipalDecl->setNonMemberOperator();
 
-  // If we have a function template, check the template parameter
-  // list. This will check and merge default template arguments.
-  if (FunctionTemplate) {
-    FunctionTemplateDecl *PrevTemplate = FunctionTemplate->getPreviousDeclaration();
-    CheckTemplateParameterList(FunctionTemplate->getTemplateParameters(),
-                      PrevTemplate? PrevTemplate->getTemplateParameters() : 0,
-             D.getDeclSpec().isFriendSpecified()? TPC_FriendFunctionTemplate
-                                                : TPC_FunctionTemplate);
-  }
+    // If we have a function template, check the template parameter
+    // list. This will check and merge default template arguments.
+    if (FunctionTemplate) {
+      FunctionTemplateDecl *PrevTemplate = FunctionTemplate->getPreviousDeclaration();
+      CheckTemplateParameterList(FunctionTemplate->getTemplateParameters(),
+                                 PrevTemplate? PrevTemplate->getTemplateParameters() : 0,
+                            D.getDeclSpec().isFriendSpecified()
+                              ? (IsFunctionDefinition 
+                                   ? TPC_FriendFunctionTemplateDefinition
+                                   : TPC_FriendFunctionTemplate)
+                              : (D.getCXXScopeSpec().isSet() && 
+                                 DC && DC->isRecord() && 
+                                 DC->isDependentContext())
+                                  ? TPC_ClassTemplateMember
+                                  : TPC_FunctionTemplate);
+    }
+
+    if (NewFD->isInvalidDecl()) {
+      // Ignore all the rest of this.
+    } else if (!Redeclaration) {
+      // Fake up an access specifier if it's supposed to be a class member.
+      if (isa<CXXRecordDecl>(NewFD->getDeclContext()))
+        NewFD->setAccess(AS_public);
+
+      // Qualified decls generally require a previous declaration.
+      if (D.getCXXScopeSpec().isSet()) {
+        // ...with the major exception of templated-scope or
+        // dependent-scope friend declarations.
+
+        // TODO: we currently also suppress this check in dependent
+        // contexts because (1) the parameter depth will be off when
+        // matching friend templates and (2) we might actually be
+        // selecting a friend based on a dependent factor.  But there
+        // are situations where these conditions don't apply and we
+        // can actually do this check immediately.
+        if (isFriend &&
+            (NumMatchedTemplateParamLists ||
+             D.getCXXScopeSpec().getScopeRep()->isDependent() ||
+             CurContext->isDependentContext())) {
+              // ignore these
+            } else {
+              // The user tried to provide an out-of-line definition for a
+              // function that is a member of a class or namespace, but there
+              // was no such member function declared (C++ [class.mfct]p2,
+              // C++ [namespace.memdef]p2). For example:
+              //
+              // class X {
+              //   void f() const;
+              // };
+              //
+              // void X::f() { } // ill-formed
+              //
+              // Complain about this problem, and attempt to suggest close
+              // matches (e.g., those that differ only in cv-qualifiers and
+              // whether the parameter types are references).
+              Diag(D.getIdentifierLoc(), diag::err_member_def_does_not_match)
+              << Name << DC << D.getCXXScopeSpec().getRange();
+              NewFD->setInvalidDecl();
+
+              DiagnoseInvalidRedeclaration(*this, NewFD);
+            }
 
-  if (D.getCXXScopeSpec().isSet() && !NewFD->isInvalidDecl()) {
-    // Fake up an access specifier if it's supposed to be a class member.
-    if (!Redeclaration && isa<CXXRecordDecl>(NewFD->getDeclContext()))
-      NewFD->setAccess(AS_public);
+        // Unqualified local friend declarations are required to resolve
+        // to something.
+        } else if (isFriend && cast<CXXRecordDecl>(CurContext)->isLocalClass()) {
+          Diag(D.getIdentifierLoc(), diag::err_no_matching_local_friend);
+          NewFD->setInvalidDecl();
+          DiagnoseInvalidRedeclaration(*this, NewFD);
+        }
 
-    // An out-of-line member function declaration must also be a
-    // definition (C++ [dcl.meaning]p1).
-    // Note that this is not the case for explicit specializations of
-    // function templates or member functions of class templates, per
-    // C++ [temp.expl.spec]p2. We also allow these declarations as an extension
-    // for compatibility with old SWIG code which likes to generate them.
-    if (!IsFunctionDefinition && !isFriend &&
-        !isFunctionTemplateSpecialization && !isExplicitSpecialization) {
+    } else if (!IsFunctionDefinition && D.getCXXScopeSpec().isSet() &&
+               !isFriend && !isFunctionTemplateSpecialization &&
+               !isExplicitSpecialization) {
+      // An out-of-line member function declaration must also be a
+      // definition (C++ [dcl.meaning]p1).
+      // Note that this is not the case for explicit specializations of
+      // function templates or member functions of class templates, per
+      // C++ [temp.expl.spec]p2. We also allow these declarations as an extension
+      // for compatibility with old SWIG code which likes to generate them.
       Diag(NewFD->getLocation(), diag::ext_out_of_line_declaration)
         << D.getCXXScopeSpec().getRange();
     }
-    if (!Redeclaration && !(isFriend && CurContext->isDependentContext())) {
-      // The user tried to provide an out-of-line definition for a
-      // function that is a member of a class or namespace, but there
-      // was no such member function declared (C++ [class.mfct]p2,
-      // C++ [namespace.memdef]p2). For example:
-      //
-      // class X {
-      //   void f() const;
-      // };
-      //
-      // void X::f() { } // ill-formed
-      //
-      // Complain about this problem, and attempt to suggest close
-      // matches (e.g., those that differ only in cv-qualifiers and
-      // whether the parameter types are references).
-      Diag(D.getIdentifierLoc(), diag::err_member_def_does_not_match)
-        << Name << DC << D.getCXXScopeSpec().getRange();
-      NewFD->setInvalidDecl();
-
-      LookupResult Prev(*this, Name, D.getIdentifierLoc(), LookupOrdinaryName,
-                        ForRedeclaration);
-      LookupQualifiedName(Prev, DC);
-      assert(!Prev.isAmbiguous() &&
-             "Cannot have an ambiguity in previous-declaration lookup");
-      for (LookupResult::iterator Func = Prev.begin(), FuncEnd = Prev.end();
-           Func != FuncEnd; ++Func) {
-        if (isa<FunctionDecl>(*Func) &&
-            isNearlyMatchingFunction(Context, cast<FunctionDecl>(*Func), NewFD))
-          Diag((*Func)->getLocation(), diag::note_member_def_close_match);
-      }
-    }
   }
-
+  
+  
   // Handle attributes. We need to have merged decls when handling attributes
   // (for example to check for conflicts, etc).
   // FIXME: This needs to happen before we merge declarations. Then,
   // let attribute merging cope with attribute conflicts.
-  ProcessDeclAttributes(S, NewFD, D);
+  ProcessDeclAttributes(S, NewFD, D,
+                        /*NonInheritable=*/false, /*Inheritable=*/true);
 
   // attributes declared post-definition are currently ignored
   // FIXME: This should happen during attribute merging
@@ -3723,17 +4119,6 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   AddKnownFunctionAttributes(NewFD);
 
-  if (OverloadableAttrRequired && !NewFD->hasAttr<OverloadableAttr>()) {
-    // If a function name is overloadable in C, then every function
-    // with that name must be marked "overloadable".
-    Diag(NewFD->getLocation(), diag::err_attribute_overloadable_missing)
-      << Redeclaration << NewFD;
-    if (!Previous.empty())
-      Diag(Previous.getRepresentativeDecl()->getLocation(),
-           diag::note_attribute_overloadable_prev_overload);
-    NewFD->addAttr(::new (Context) OverloadableAttr(SourceLocation(), Context));
-  }
-
   if (NewFD->hasAttr<OverloadableAttr>() && 
       !NewFD->getType()->getAs<FunctionProtoType>()) {
     Diag(NewFD->getLocation(),
@@ -3742,9 +4127,11 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
     // Turn this into a variadic function with no parameters.
     const FunctionType *FT = NewFD->getType()->getAs<FunctionType>();
-    QualType R = Context.getFunctionType(FT->getResultType(),
-                                         0, 0, true, 0, false, false, 0, 0,
-                                         FT->getExtInfo());
+    FunctionProtoType::ExtProtoInfo EPI;
+    EPI.Variadic = true;
+    EPI.ExtInfo = FT->getExtInfo();
+
+    QualType R = Context.getFunctionType(FT->getResultType(), 0, 0, EPI);
     NewFD->setType(R);
   }
 
@@ -3762,14 +4149,28 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // Set this FunctionDecl's range up to the right paren.
   NewFD->setLocEnd(D.getSourceRange().getEnd());
 
-  if (FunctionTemplate && NewFD->isInvalidDecl())
-    FunctionTemplate->setInvalidDecl();
-
-  if (FunctionTemplate)
-    return FunctionTemplate;
+  if (getLangOptions().CPlusPlus) {
+    if (FunctionTemplate) {
+      if (NewFD->isInvalidDecl())
+        FunctionTemplate->setInvalidDecl();
+      return FunctionTemplate;
+    }
+  }
 
   MarkUnusedFileScopedDecl(NewFD);
 
+  if (getLangOptions().CUDA)
+    if (IdentifierInfo *II = NewFD->getIdentifier())
+      if (!NewFD->isInvalidDecl() &&
+          NewFD->getDeclContext()->getRedeclContext()->isTranslationUnit()) {
+        if (II->isStr("cudaConfigureCall")) {
+          if (!R->getAs<FunctionType>()->getResultType()->isScalarType())
+            Diag(NewFD->getLocation(), diag::err_config_scalar_return);
+
+          Context.setcudaConfigureCallDecl(NewFD);
+        }
+      }
+
   return NewFD;
 }
 
@@ -3790,8 +4191,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
                                     LookupResult &Previous,
                                     bool IsExplicitSpecialization,
-                                    bool &Redeclaration,
-                                    bool &OverloadableAttrRequired) {
+                                    bool &Redeclaration) {
   // If NewFD is already known erroneous, don't do any of this checking.
   if (NewFD->isInvalidDecl()) {
     // If this is a class member, mark the class invalid immediately.
@@ -3836,9 +4236,6 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
       Redeclaration = true;
       OldDecl = Previous.getFoundDecl();
     } else {
-      if (!getLangOptions().CPlusPlus)
-        OverloadableAttrRequired = true;
-
       switch (CheckOverload(S, NewFD, Previous, OldDecl,
                             /*NewIsUsingDecl*/ false)) {
       case Ovl_Match:
@@ -3853,6 +4250,23 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
         Redeclaration = false;
         break;
       }
+
+      if (!getLangOptions().CPlusPlus && !NewFD->hasAttr<OverloadableAttr>()) {
+        // If a function name is overloadable in C, then every function
+        // with that name must be marked "overloadable".
+        Diag(NewFD->getLocation(), diag::err_attribute_overloadable_missing)
+          << Redeclaration << NewFD;
+        NamedDecl *OverloadedDecl = 0;
+        if (Redeclaration)
+          OverloadedDecl = OldDecl;
+        else if (!Previous.empty())
+          OverloadedDecl = Previous.getRepresentativeDecl();
+        if (OverloadedDecl)
+          Diag(OverloadedDecl->getLocation(),
+               diag::note_attribute_overloadable_prev_overload);
+        NewFD->addAttr(::new (Context) OverloadableAttr(SourceLocation(),
+                                                        Context));
+      }
     }
 
     if (Redeclaration) {
@@ -3907,23 +4321,11 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
         DeclarationName Name
           = Context.DeclarationNames.getCXXDestructorName(
                                         Context.getCanonicalType(ClassType));
-//         NewFD->getDeclName().dump();
-//         Name.dump();
         if (NewFD->getDeclName() != Name) {
           Diag(NewFD->getLocation(), diag::err_destructor_name);
           return NewFD->setInvalidDecl();
         }
       }
-
-      Record->setUserDeclaredDestructor(true);
-      // C++ [class]p4: A POD-struct is an aggregate class that has [...] no
-      // user-defined destructor.
-      Record->setPOD(false);
-
-      // C++ [class.dtor]p3: A destructor is trivial if it is an implicitly-
-      // declared destructor.
-      // FIXME: C++0x: don't do this for "= default" destructors
-      Record->setHasTrivialDestructor(false);
     } else if (CXXConversionDecl *Conversion
                = dyn_cast<CXXConversionDecl>(NewFD)) {
       ActOnConversionDeclarator(Conversion);
@@ -3932,8 +4334,23 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     // Find any virtual functions that this function overrides.
     if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(NewFD)) {
       if (!Method->isFunctionTemplateSpecialization() && 
-          !Method->getDescribedFunctionTemplate())
-        AddOverriddenMethods(Method->getParent(), Method);
+          !Method->getDescribedFunctionTemplate()) {
+        if (AddOverriddenMethods(Method->getParent(), Method)) {
+          // If the function was marked as "static", we have a problem.
+          if (NewFD->getStorageClass() == SC_Static) {
+            Diag(NewFD->getLocation(), diag::err_static_overrides_virtual)
+              << NewFD->getDeclName();
+            for (CXXMethodDecl::method_iterator 
+                      Overridden = Method->begin_overridden_methods(),
+                   OverriddenEnd = Method->end_overridden_methods();
+                 Overridden != OverriddenEnd;
+                 ++Overridden) {
+              Diag((*Overridden)->getLocation(), 
+                   diag::note_overridden_virtual_function);
+            }
+          }
+        }
+      }
     }
 
     // Extra checking for C++ overloaded operators (C++ [over.oper]).
@@ -3951,6 +4368,18 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     // during delayed parsing anyway.
     if (!CurContext->isRecord())
       CheckCXXDefaultArguments(NewFD);
+    
+    // If this function declares a builtin function, check the type of this
+    // declaration against the expected type for the builtin. 
+    if (unsigned BuiltinID = NewFD->getBuiltinID()) {
+      ASTContext::GetBuiltinTypeError Error;
+      QualType T = Context.GetBuiltinType(BuiltinID, Error);
+      if (!T.isNull() && !Context.hasSameType(T, NewFD->getType())) {
+        // The type of this function differs from the type of the builtin,
+        // so forget about the builtin entirely.
+        Context.BuiltinInfo.ForgetBuiltin(BuiltinID, Context.Idents);
+      }
+    }
   }
 }
 
@@ -3976,7 +4405,6 @@ void Sema::CheckMain(FunctionDecl* FD) {
   const FunctionType* FT = T->getAs<FunctionType>();
 
   if (!Context.hasSameUnqualifiedType(FT->getResultType(), Context.IntTy)) {
-    // TODO: add a replacement fixit to turn the return type into 'int'.
     Diag(FD->getTypeSpecStartLoc(), diag::err_main_returns_nonint);
     FD->setInvalidDecl(true);
   }
@@ -4043,6 +4471,11 @@ void Sema::CheckMain(FunctionDecl* FD) {
   if (nparams == 1 && !FD->isInvalidDecl()) {
     Diag(FD->getLocation(), diag::warn_main_one_arg);
   }
+  
+  if (!FD->isInvalidDecl() && FD->getDescribedFunctionTemplate()) {
+    Diag(FD->getLocation(), diag::err_main_template_decl);
+    FD->setInvalidDecl();
+  }
 }
 
 bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
@@ -4061,17 +4494,14 @@ bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   return true;
 }
 
-void Sema::AddInitializerToDecl(Decl *dcl, Expr *init) {
-  AddInitializerToDecl(dcl, init, /*DirectInit=*/false);
-}
-
 /// AddInitializerToDecl - Adds the initializer Init to the
 /// declaration dcl. If DirectInit is true, this is C++ direct
 /// initialization rather than copy initialization.
-void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
+void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
+                                bool DirectInit, bool TypeMayContainAuto) {
   // If there is no declaration, there was an error parsing it.  Just ignore
   // the initializer.
-  if (RealDecl == 0)
+  if (RealDecl == 0 || RealDecl->isInvalidDecl())
     return;
 
   if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(RealDecl)) {
@@ -4095,14 +4525,32 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     if (getLangOptions().CPlusPlus &&
         RealDecl->getLexicalDeclContext()->isRecord() &&
         isa<NamedDecl>(RealDecl))
-      Diag(RealDecl->getLocation(), diag::err_member_initialization)
-        << cast<NamedDecl>(RealDecl)->getDeclName();
+      Diag(RealDecl->getLocation(), diag::err_member_initialization);
     else
       Diag(RealDecl->getLocation(), diag::err_illegal_initializer);
     RealDecl->setInvalidDecl();
     return;
   }
 
+  // C++0x [decl.spec.auto]p6. Deduce the type which 'auto' stands in for.
+  if (TypeMayContainAuto && VDecl->getType()->getContainedAutoType()) {
+    VDecl->setParsingAutoInit(false);
+
+    QualType DeducedType;
+    if (!DeduceAutoType(VDecl->getType(), Init, DeducedType)) {
+      Diag(VDecl->getLocation(), diag::err_auto_var_deduction_failure)
+        << VDecl->getDeclName() << VDecl->getType() << Init->getType()
+        << Init->getSourceRange();
+      RealDecl->setInvalidDecl();
+      return;
+    }
+    VDecl->setType(DeducedType);
+
+    // If this is a redeclaration, check that the type we just deduced matches
+    // the previously declared type.
+    if (VarDecl *Old = VDecl->getPreviousDeclaration())
+      MergeVarDeclTypes(VDecl, Old);
+  }
   
 
   // A definition must end up with a complete type, which means it must be
@@ -4132,27 +4580,34 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     return;
   }
   
-  // C++ [class.static.data]p4
-  //   If a static data member is of const integral or const
-  //   enumeration type, its declaration in the class definition can
-  //   specify a constant-initializer which shall be an integral
-  //   constant expression (5.19). In that case, the member can appear
-  //   in integral constant expressions. The member shall still be
-  //   defined in a namespace scope if it is used in the program and the
-  //   namespace scope definition shall not contain an initializer.
-  //
-  // We already performed a redefinition check above, but for static
-  // data members we also need to check whether there was an in-class
-  // declaration with an initializer.
   const VarDecl* PrevInit = 0;
-  if (VDecl->isStaticDataMember() && VDecl->getAnyInitializer(PrevInit)) {
-    Diag(VDecl->getLocation(), diag::err_redefinition) << VDecl->getDeclName();
-    Diag(PrevInit->getLocation(), diag::note_previous_definition);
-    return;
-  }  
+  if (getLangOptions().CPlusPlus) {
+    // C++ [class.static.data]p4
+    //   If a static data member is of const integral or const
+    //   enumeration type, its declaration in the class definition can
+    //   specify a constant-initializer which shall be an integral
+    //   constant expression (5.19). In that case, the member can appear
+    //   in integral constant expressions. The member shall still be
+    //   defined in a namespace scope if it is used in the program and the
+    //   namespace scope definition shall not contain an initializer.
+    //
+    // We already performed a redefinition check above, but for static
+    // data members we also need to check whether there was an in-class
+    // declaration with an initializer.
+    if (VDecl->isStaticDataMember() && VDecl->getAnyInitializer(PrevInit)) {
+      Diag(VDecl->getLocation(), diag::err_redefinition) << VDecl->getDeclName();
+      Diag(PrevInit->getLocation(), diag::note_previous_definition);
+      return;
+    }  
 
-  if (getLangOptions().CPlusPlus && VDecl->hasLocalStorage())
-    getCurFunction()->setHasBranchProtectedScope();
+    if (VDecl->hasLocalStorage())
+      getCurFunction()->setHasBranchProtectedScope();
+
+    if (DiagnoseUnexpandedParameterPack(Init, UPPC_Initializer)) {
+      VDecl->setInvalidDecl();
+      return;
+    }
+  }
 
   // Capture the variable that is being initialized and the style of
   // initialization.
@@ -4169,7 +4624,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   // Get the decls type and save a reference for later, since
   // CheckInitializerTypes may change it.
   QualType DclT = VDecl->getType(), SavT = DclT;
-  if (VDecl->isBlockVarDecl()) {
+  if (VDecl->isLocalVarDecl()) {
     if (VDecl->hasExternalStorage()) { // C99 6.7.8p5
       Diag(VDecl->getLocation(), diag::err_block_extern_cant_init);
       VDecl->setInvalidDecl();
@@ -4200,34 +4655,38 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     //   static const int value = 17;
     // };
 
-    // Attach the initializer
-    VDecl->setInit(Init);
+    // Try to perform the initialization regardless.
+    if (!VDecl->isInvalidDecl()) {
+      InitializationSequence InitSeq(*this, Entity, Kind, &Init, 1);
+      ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+                                          MultiExprArg(*this, &Init, 1),
+                                          &DclT);
+      if (Result.isInvalid()) {
+        VDecl->setInvalidDecl();
+        return;
+      }
+
+      Init = Result.takeAs<Expr>();
+    }
 
     // C++ [class.mem]p4:
     //   A member-declarator can contain a constant-initializer only
     //   if it declares a static member (9.4) of const integral or
     //   const enumeration type, see 9.4.2.
     QualType T = VDecl->getType();
-    if (!T->isDependentType() &&
-        (!Context.getCanonicalType(T).isConstQualified() ||
-         !T->isIntegralOrEnumerationType())) {
-      Diag(VDecl->getLocation(), diag::err_member_initialization)
-        << VDecl->getDeclName() << Init->getSourceRange();
+
+    // Do nothing on dependent types.
+    if (T->isDependentType()) {
+
+    // Require constness.
+    } else if (!T.isConstQualified()) {
+      Diag(VDecl->getLocation(), diag::err_in_class_initializer_non_const)
+        << Init->getSourceRange();
       VDecl->setInvalidDecl();
-    } else {
-      // C++ [class.static.data]p4:
-      //   If a static data member is of const integral or const
-      //   enumeration type, its declaration in the class definition
-      //   can specify a constant-initializer which shall be an
-      //   integral constant expression (5.19).
-      if (!Init->isTypeDependent() &&
-          !Init->getType()->isIntegralOrEnumerationType()) {
-        // We have a non-dependent, non-integral or enumeration type.
-        Diag(Init->getSourceRange().getBegin(),
-             diag::err_in_class_initializer_non_integral_type)
-          << Init->getType() << Init->getSourceRange();
-        VDecl->setInvalidDecl();
-      } else if (!Init->isTypeDependent() && !Init->isValueDependent()) {
+
+    // We allow integer constant expressions in all cases.
+    } else if (T->isIntegralOrEnumerationType()) {
+      if (!Init->isValueDependent()) {
         // Check whether the expression is a constant expression.
         llvm::APSInt Value;
         SourceLocation Loc;
@@ -4235,12 +4694,37 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
           Diag(Loc, diag::err_in_class_initializer_non_constant)
             << Init->getSourceRange();
           VDecl->setInvalidDecl();
-        } else if (!VDecl->getType()->isDependentType())
-          ImpCastExprToType(Init, VDecl->getType(), CK_IntegralCast);
+        }
+      }
+
+    // We allow floating-point constants as an extension in C++03, and
+    // C++0x has far more complicated rules that we don't really
+    // implement fully.
+    } else {
+      bool Allowed = false;
+      if (getLangOptions().CPlusPlus0x) {
+        Allowed = T->isLiteralType();
+      } else if (T->isFloatingType()) { // also permits complex, which is ok
+        Diag(VDecl->getLocation(), diag::ext_in_class_initializer_float_type)
+          << T << Init->getSourceRange();
+        Allowed = true;
+      }
+
+      if (!Allowed) {
+        Diag(VDecl->getLocation(), diag::err_in_class_initializer_bad_type)
+          << T << Init->getSourceRange();
+        VDecl->setInvalidDecl();
+
+      // TODO: there are probably expressions that pass here that shouldn't.
+      } else if (!Init->isValueDependent() &&
+                 !Init->isConstantInitializer(Context, false)) {
+        Diag(Init->getExprLoc(), diag::err_in_class_initializer_non_constant)
+          << Init->getSourceRange();
+        VDecl->setInvalidDecl();
       }
     }
   } else if (VDecl->isFileVarDecl()) {
-    if (VDecl->getStorageClass() == SC_Extern && 
+    if (VDecl->getStorageClassAsWritten() == SC_Extern && 
         (!getLangOptions().CPlusPlus || 
          !Context.getBaseElementType(VDecl->getType()).isConstQualified()))
       Diag(VDecl->getLocation(), diag::warn_extern_init);
@@ -4273,28 +4757,35 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     Init->setType(DclT);
   }
 
-  Init = MaybeCreateCXXExprWithTemporaries(Init);
+  
+  // If this variable is a local declaration with record type, make sure it
+  // doesn't have a flexible member initialization.  We only support this as a
+  // global/static definition.
+  if (VDecl->hasLocalStorage())
+    if (const RecordType *RT = VDecl->getType()->getAs<RecordType>())
+      if (RT->getDecl()->hasFlexibleArrayMember()) {
+        // Check whether the initializer tries to initialize the flexible
+        // array member itself to anything other than an empty initializer list.
+        if (InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) {
+          unsigned Index = std::distance(RT->getDecl()->field_begin(),
+                                         RT->getDecl()->field_end()) - 1;
+          if (Index < ILE->getNumInits() &&
+              !(isa<InitListExpr>(ILE->getInit(Index)) &&
+                cast<InitListExpr>(ILE->getInit(Index))->getNumInits() == 0)) {
+            Diag(VDecl->getLocation(), diag::err_nonstatic_flexible_variable);
+            VDecl->setInvalidDecl();
+          }
+        }
+      }
+  
+  // Check any implicit conversions within the expression.
+  CheckImplicitConversions(Init, VDecl->getLocation());
+
+  Init = MaybeCreateExprWithCleanups(Init);
   // Attach the initializer to the decl.
   VDecl->setInit(Init);
 
-  if (getLangOptions().CPlusPlus) {
-    if (!VDecl->isInvalidDecl() &&
-        !VDecl->getDeclContext()->isDependentContext() &&
-        VDecl->hasGlobalStorage() && !VDecl->isStaticLocal() &&
-        !Init->isConstantInitializer(Context,
-                                     VDecl->getType()->isReferenceType()))
-      Diag(VDecl->getLocation(), diag::warn_global_constructor)
-        << Init->getSourceRange();
-
-    // Make sure we mark the destructor as used if necessary.
-    QualType InitType = VDecl->getType();
-    while (const ArrayType *Array = Context.getAsArrayType(InitType))
-      InitType = Context.getBaseElementType(Array);
-    if (const RecordType *Record = InitType->getAs<RecordType>())
-      FinalizeVarWithDestructor(VDecl, Record);
-  }
-
-  return;
+  CheckCompleteVariableDeclaration(VDecl);
 }
 
 /// ActOnInitializerError - Given that there was an error parsing an
@@ -4308,6 +4799,13 @@ void Sema::ActOnInitializerError(Decl *D) {
   VarDecl *VD = dyn_cast<VarDecl>(D);
   if (!VD) return;
 
+  // Auto types are meaningless if we can't make sense of the initializer.
+  if (VD->isParsingAutoInit()) {
+    VD->setParsingAutoInit(false);
+    VD->setInvalidDecl();
+    return;
+  }
+
   QualType Ty = VD->getType();
   if (Ty->isDependentType()) return;
 
@@ -4332,7 +4830,7 @@ void Sema::ActOnInitializerError(Decl *D) {
 }
 
 void Sema::ActOnUninitializedDecl(Decl *RealDecl,
-                                  bool TypeContainsUndeducedAuto) {
+                                  bool TypeMayContainAuto) {
   // If there is no declaration, there was an error parsing it. Just ignore it.
   if (RealDecl == 0)
     return;
@@ -4341,7 +4839,9 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
     QualType Type = Var->getType();
 
     // C++0x [dcl.spec.auto]p3
-    if (TypeContainsUndeducedAuto) {
+    if (TypeMayContainAuto && Type->getContainedAutoType()) {
+      Var->setParsingAutoInit(false);
+
       Diag(Var->getLocation(), diag::err_auto_var_requires_init)
         << Var->getDeclName() << Type;
       Var->setInvalidDecl();
@@ -4365,7 +4865,7 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
       // Block scope. C99 6.7p7: If an identifier for an object is
       // declared with no linkage (C99 6.2.2p6), the type for the
       // object shall be complete.
-      if (!Type->isDependentType() && Var->isBlockVarDecl() && 
+      if (!Type->isDependentType() && Var->isLocalVarDecl() && 
           !Var->getLinkage() && !Var->isInvalidDecl() &&
           RequireCompleteType(Var->getLocation(), Type,
                               diag::err_typecheck_decl_incomplete_type))
@@ -4488,22 +4988,62 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
                                         MultiExprArg(*this, 0, 0));
       if (Init.isInvalid())
         Var->setInvalidDecl();
-      else if (Init.get()) {
-        Var->setInit(MaybeCreateCXXExprWithTemporaries(Init.takeAs<Expr>()));
-
-        if (getLangOptions().CPlusPlus && !Var->isInvalidDecl() && 
-            Var->hasGlobalStorage() && !Var->isStaticLocal() &&
-            !Var->getDeclContext()->isDependentContext() &&
-            !Var->getInit()->isConstantInitializer(Context, false))
-          Diag(Var->getLocation(), diag::warn_global_constructor);
-      }
+      else if (Init.get())
+        Var->setInit(MaybeCreateExprWithCleanups(Init.get()));
     }
 
-    if (!Var->isInvalidDecl() && getLangOptions().CPlusPlus && Record)
-      FinalizeVarWithDestructor(Var, Record);
+    CheckCompleteVariableDeclaration(Var);
   }
 }
 
+void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
+  if (var->isInvalidDecl()) return;
+
+  // All the following checks are C++ only.
+  if (!getLangOptions().CPlusPlus) return;
+
+  QualType baseType = Context.getBaseElementType(var->getType());
+  if (baseType->isDependentType()) return;
+
+  // __block variables might require us to capture a copy-initializer.
+  if (var->hasAttr<BlocksAttr>()) {
+    // It's currently invalid to ever have a __block variable with an
+    // array type; should we diagnose that here?
+
+    // Regardless, we don't want to ignore array nesting when
+    // constructing this copy.
+    QualType type = var->getType();
+
+    if (type->isStructureOrClassType()) {
+      SourceLocation poi = var->getLocation();
+      Expr *varRef = new (Context) DeclRefExpr(var, type, VK_LValue, poi);
+      ExprResult result =
+        PerformCopyInitialization(
+                        InitializedEntity::InitializeBlock(poi, type, false),
+                                  poi, Owned(varRef));
+      if (!result.isInvalid()) {
+        result = MaybeCreateExprWithCleanups(result);
+        Expr *init = result.takeAs<Expr>();
+        Context.setBlockVarCopyInits(var, init);
+      }
+    }
+  }
+
+  // Check for global constructors.
+  if (!var->getDeclContext()->isDependentContext() &&
+      var->hasGlobalStorage() &&
+      !var->isStaticLocal() &&
+      var->getInit() &&
+      !var->getInit()->isConstantInitializer(Context,
+                                             baseType->isReferenceType()))
+    Diag(var->getLocation(), diag::warn_global_constructor)
+      << var->getInit()->getSourceRange();
+
+  // Require the destructor.
+  if (const RecordType *recordType = baseType->getAs<RecordType>())
+    FinalizeVarWithDestructor(var, recordType);
+}
+
 Sema::DeclGroupPtrTy
 Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
                               Decl **Group, unsigned NumDecls) {
@@ -4512,6 +5052,41 @@ Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
   if (DS.isTypeSpecOwned())
     Decls.push_back(DS.getRepAsDecl());
 
+  // C++0x [dcl.spec.auto]p7:
+  //   If the type deduced for the template parameter U is not the same in each
+  //   deduction, the program is ill-formed.
+  // FIXME: When initializer-list support is added, a distinction is needed
+  // between the deduced type U and the deduced type which 'auto' stands for.
+  //   auto a = 0, b = { 1, 2, 3 };
+  // is legal because the deduced type U is 'int' in both cases.
+  bool TypeContainsAuto = DS.getTypeSpecType() == DeclSpec::TST_auto;
+  if (TypeContainsAuto && NumDecls > 1) {
+    QualType Deduced;
+    CanQualType DeducedCanon;
+    VarDecl *DeducedDecl = 0;
+    for (unsigned i = 0; i != NumDecls; ++i) {
+      if (VarDecl *D = dyn_cast<VarDecl>(Group[i])) {
+        AutoType *AT = D->getType()->getContainedAutoType();
+        if (AT && AT->isDeduced()) {
+          QualType U = AT->getDeducedType();
+          CanQualType UCanon = Context.getCanonicalType(U);
+          if (Deduced.isNull()) {
+            Deduced = U;
+            DeducedCanon = UCanon;
+            DeducedDecl = D;
+          } else if (DeducedCanon != UCanon) {
+            Diag(DS.getTypeSpecTypeLoc(), diag::err_auto_different_deductions)
+              << Deduced << DeducedDecl->getDeclName()
+              << U << D->getDeclName()
+              << DeducedDecl->getInit()->getSourceRange()
+              << D->getInit()->getSourceRange();
+            break;
+          }
+        }
+      }
+    }
+  }
+
   for (unsigned i = 0; i != NumDecls; ++i)
     if (Decl *D = Group[i])
       Decls.push_back(D);
@@ -4543,24 +5118,42 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
 
   DiagnoseFunctionSpecifiers(D);
 
-  // Check that there are no default arguments inside the type of this
-  // parameter (C++ only).
-  if (getLangOptions().CPlusPlus)
-    CheckExtraCXXDefaultArguments(D);
-
   TagDecl *OwnedDecl = 0;
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S, &OwnedDecl);
   QualType parmDeclType = TInfo->getType();
 
-  if (getLangOptions().CPlusPlus && OwnedDecl && OwnedDecl->isDefinition()) {
-    // C++ [dcl.fct]p6:
-    //   Types shall not be defined in return or parameter types.
-    Diag(OwnedDecl->getLocation(), diag::err_type_defined_in_param_type)
-      << Context.getTypeDeclType(OwnedDecl);
+  if (getLangOptions().CPlusPlus) {
+    // Check that there are no default arguments inside the type of this
+    // parameter.
+    CheckExtraCXXDefaultArguments(D);
+     
+    if (OwnedDecl && OwnedDecl->isDefinition()) {
+      // C++ [dcl.fct]p6:
+      //   Types shall not be defined in return or parameter types.
+      Diag(OwnedDecl->getLocation(), diag::err_type_defined_in_param_type)
+        << Context.getTypeDeclType(OwnedDecl);
+    }
+    
+    // Parameter declarators cannot be qualified (C++ [dcl.meaning]p1).
+    if (D.getCXXScopeSpec().isSet()) {
+      Diag(D.getIdentifierLoc(), diag::err_qualified_param_declarator)
+        << D.getCXXScopeSpec().getRange();
+      D.getCXXScopeSpec().clear();
+    }
+  }
+
+  // Ensure we have a valid name
+  IdentifierInfo *II = 0;
+  if (D.hasName()) {
+    II = D.getIdentifier();
+    if (!II) {
+      Diag(D.getIdentifierLoc(), diag::err_bad_parameter_name)
+        << GetNameForDeclarator(D).getName().getAsString();
+      D.setInvalidType(true);
+    }
   }
 
   // Check for redeclaration of parameters, e.g. int foo(int x, int x);
-  IdentifierInfo *II = D.getIdentifier();
   if (II) {
     LookupResult R(*this, II, D.getIdentifierLoc(), LookupOrdinaryName,
                    ForRedeclaration);
@@ -4595,13 +5188,6 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
   if (D.isInvalidType())
     New->setInvalidDecl();  
   
-  // Parameter declarators cannot be qualified (C++ [dcl.meaning]p1).
-  if (D.getCXXScopeSpec().isSet()) {
-    Diag(D.getIdentifierLoc(), diag::err_qualified_param_declarator)
-      << D.getCXXScopeSpec().getRange();
-    New->setInvalidDecl();
-  }
-
   // Add the parameter declaration into this scope.
   S->AddDecl(New);
   if (II)
@@ -4629,10 +5215,6 @@ ParmVarDecl *Sema::BuildParmVarDeclForTypedef(DeclContext *DC,
 
 void Sema::DiagnoseUnusedParameters(ParmVarDecl * const *Param,
                                     ParmVarDecl * const *ParamEnd) {
-  if (Diags.getDiagnosticLevel(diag::warn_unused_parameter) ==
-        Diagnostic::Ignored)
-    return;
-
   // Don't diagnose unused-parameter errors in template instantiations; we
   // will already have done so in the template itself.
   if (!ActiveTemplateInstantiations.empty())
@@ -4647,6 +5229,35 @@ void Sema::DiagnoseUnusedParameters(ParmVarDecl * const *Param,
   }
 }
 
+void Sema::DiagnoseSizeOfParametersAndReturnValue(ParmVarDecl * const *Param,
+                                                  ParmVarDecl * const *ParamEnd,
+                                                  QualType ReturnTy,
+                                                  NamedDecl *D) {
+  if (LangOpts.NumLargeByValueCopy == 0) // No check.
+    return;
+
+  // Warn if the return value is pass-by-value and larger than the specified
+  // threshold.
+  if (ReturnTy->isPODType()) {
+    unsigned Size = Context.getTypeSizeInChars(ReturnTy).getQuantity();
+    if (Size > LangOpts.NumLargeByValueCopy)
+      Diag(D->getLocation(), diag::warn_return_value_size)
+          << D->getDeclName() << Size;
+  }
+
+  // Warn if any parameter is pass-by-value and larger than the specified
+  // threshold.
+  for (; Param != ParamEnd; ++Param) {
+    QualType T = (*Param)->getType();
+    if (!T->isPODType())
+      continue;
+    unsigned Size = Context.getTypeSizeInChars(T).getQuantity();
+    if (Size > LangOpts.NumLargeByValueCopy)
+      Diag((*Param)->getLocation(), diag::warn_parameter_size)
+          << (*Param)->getDeclName() << Size;
+  }
+}
+
 ParmVarDecl *Sema::CheckParameter(DeclContext *DC, 
                                   TypeSourceInfo *TSInfo, QualType T,
                                   IdentifierInfo *Name,
@@ -4688,9 +5299,7 @@ ParmVarDecl *Sema::CheckParameter(DeclContext *DC,
 
 void Sema::ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D,
                                            SourceLocation LocAfterDecls) {
-  assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
-         "Not a function declarator!");
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
   // Verify 6.9.1p6: 'every identifier in the identifier list shall be declared'
   // for a K&R function.
@@ -4724,14 +5333,7 @@ void Sema::ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D,
 Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope,
                                          Declarator &D) {
   assert(getCurFunctionDecl() == 0 && "Function parsing confused");
-  assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
-         "Not a function declarator!");
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
-
-  if (FTI.hasPrototype) {
-    // FIXME: Diagnose arguments without names in C.
-  }
-
+  assert(D.isFunctionDeclarator() && "Not a function declarator!");
   Scope *ParentScope = FnBodyScope->getParent();
 
   Decl *DP = HandleDeclarator(ParentScope, D,
@@ -4806,7 +5408,12 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
   const FunctionDecl *Definition;
   if (FD->hasBody(Definition) &&
       !canRedefineFunction(Definition, getLangOptions())) {
-    Diag(FD->getLocation(), diag::err_redefinition) << FD->getDeclName();
+    if (getLangOptions().GNUMode && Definition->isInlineSpecified() &&
+        Definition->getStorageClass() == SC_Extern)
+      Diag(FD->getLocation(), diag::err_redefinition_extern_inline)
+        << FD->getDeclName() << getLangOptions().CPlusPlus;
+    else
+      Diag(FD->getLocation(), diag::err_redefinition) << FD->getDeclName();
     Diag(Definition->getLocation(), diag::note_previous_definition);
   }
 
@@ -4839,10 +5446,8 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
     PushDeclContext(FnBodyScope, FD);
 
   // Check the validity of our function parameters
-  CheckParmsForFunctionDef(FD);
-
-  bool ShouldCheckShadow =
-    Diags.getDiagnosticLevel(diag::warn_decl_shadow) != Diagnostic::Ignored;
+  CheckParmsForFunctionDef(FD->param_begin(), FD->param_end(),
+                           /*CheckParameterNames=*/true);
 
   // Introduce our parameters into the function scope
   for (unsigned p = 0, NumParams = FD->getNumParams(); p < NumParams; ++p) {
@@ -4851,8 +5456,7 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
 
     // If this has an identifier, add it to the scope stack.
     if (Param->getIdentifier() && FnBodyScope) {
-      if (ShouldCheckShadow)
-        CheckShadow(FnBodyScope, Param);
+      CheckShadow(FnBodyScope, Param);
 
       PushOnScopeChains(Param, FnBodyScope);
     }
@@ -4945,6 +5549,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
 
     if (!FD->isInvalidDecl()) {
       DiagnoseUnusedParameters(FD->param_begin(), FD->param_end());
+      DiagnoseSizeOfParametersAndReturnValue(FD->param_begin(), FD->param_end(),
+                                             FD->getResultType(), FD);
       
       // If this is a constructor, we need a vtable.
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(FD))
@@ -4957,55 +5563,18 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
   } else if (ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(dcl)) {
     assert(MD == getCurMethodDecl() && "Method parsing confused");
     MD->setBody(Body);
-    MD->setEndLoc(Body->getLocEnd());
-    if (!MD->isInvalidDecl())
+    if (Body)
+      MD->setEndLoc(Body->getLocEnd());
+    if (!MD->isInvalidDecl()) {
       DiagnoseUnusedParameters(MD->param_begin(), MD->param_end());
+      DiagnoseSizeOfParametersAndReturnValue(MD->param_begin(), MD->param_end(),
+                                             MD->getResultType(), MD);
+    }
   } else {
     return 0;
   }
 
   // Verify and clean out per-function state.
-
-  // Check goto/label use.
-  FunctionScopeInfo *CurFn = getCurFunction();
-  for (llvm::DenseMap<IdentifierInfo*, LabelStmt*>::iterator
-         I = CurFn->LabelMap.begin(), E = CurFn->LabelMap.end(); I != E; ++I) {
-    LabelStmt *L = I->second;
-
-    // Verify that we have no forward references left.  If so, there was a goto
-    // or address of a label taken, but no definition of it.  Label fwd
-    // definitions are indicated with a null substmt.
-    if (L->getSubStmt() != 0)
-      continue;
-
-    // Emit error.
-    Diag(L->getIdentLoc(), diag::err_undeclared_label_use) << L->getName();
-
-    // At this point, we have gotos that use the bogus label.  Stitch it into
-    // the function body so that they aren't leaked and that the AST is well
-    // formed.
-    if (Body == 0) {
-      // The whole function wasn't parsed correctly.
-      continue;
-    }
-
-    // Otherwise, the body is valid: we want to stitch the label decl into the
-    // function somewhere so that it is properly owned and so that the goto
-    // has a valid target.  Do this by creating a new compound stmt with the
-    // label in it.
-
-    // Give the label a sub-statement.
-    L->setSubStmt(new (Context) NullStmt(L->getIdentLoc()));
-
-    CompoundStmt *Compound = isa<CXXTryStmt>(Body) ?
-                               cast<CXXTryStmt>(Body)->getTryBlock() :
-                               cast<CompoundStmt>(Body);
-    llvm::SmallVector<Stmt*, 64> Elements(Compound->body_begin(),
-                                          Compound->body_end());
-    Elements.push_back(L);
-    Compound->setStmts(Context, Elements.data(), Elements.size());
-  }
-
   if (Body) {
     // C++ constructors that have function-try-blocks can't have return
     // statements in the handlers of that block. (C++ [except.handle]p14)
@@ -5091,11 +5660,13 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
   DeclSpec DS;
   unsigned DiagID;
   bool Error = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, Dummy, DiagID);
-  Error = Error; // Silence warning.
+  (void)Error; // Silence warning.
   assert(!Error && "Error setting up implicit decl!");
   Declarator D(DS, Declarator::BlockContext);
-  D.AddTypeInfo(DeclaratorChunk::getFunction(false, false, SourceLocation(), 0,
-                                             0, 0, false, SourceLocation(),
+  D.AddTypeInfo(DeclaratorChunk::getFunction(ParsedAttributes(),
+                                             false, false, SourceLocation(), 0,
+                                             0, 0, true, SourceLocation(),
+                                             false, SourceLocation(),
                                              false, 0,0,0, Loc, Loc, D),
                 SourceLocation());
   D.SetIdentifier(&II, Loc);
@@ -5154,8 +5725,6 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) {
         FD->addAttr(::new (Context) ConstAttr(FD->getLocation(), Context));
     }
 
-    if (Context.BuiltinInfo.isNoReturn(BuiltinID))
-      FD->setType(Context.getNoReturnType(FD->getType()));
     if (Context.BuiltinInfo.isNoThrow(BuiltinID))
       FD->addAttr(::new (Context) NoThrowAttr(FD->getLocation(), Context));
     if (Context.BuiltinInfo.isConst(BuiltinID))
@@ -5210,17 +5779,46 @@ TypedefDecl *Sema::ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
                                            D.getIdentifier(),
                                            TInfo);
 
-  if (const TagType *TT = T->getAs<TagType>()) {
-    TagDecl *TD = TT->getDecl();
+  // Bail out immediately if we have an invalid declaration.
+  if (D.isInvalidType()) {
+    NewTD->setInvalidDecl();
+    return NewTD;
+  }
+
+  // C++ [dcl.typedef]p8:
+  //   If the typedef declaration defines an unnamed class (or
+  //   enum), the first typedef-name declared by the declaration
+  //   to be that class type (or enum type) is used to denote the
+  //   class type (or enum type) for linkage purposes only.
+  // We need to check whether the type was declared in the declaration.
+  switch (D.getDeclSpec().getTypeSpecType()) {
+  case TST_enum:
+  case TST_struct:
+  case TST_union:
+  case TST_class: {
+    TagDecl *tagFromDeclSpec = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
+
+    // Do nothing if the tag is not anonymous or already has an
+    // associated typedef (from an earlier typedef in this decl group).
+    if (tagFromDeclSpec->getIdentifier()) break;
+    if (tagFromDeclSpec->getTypedefForAnonDecl()) break;
+
+    // A well-formed anonymous tag must always be a TUK_Definition.
+    assert(tagFromDeclSpec->isThisDeclarationADefinition());
+
+    // The type must match the tag exactly;  no qualifiers allowed.
+    if (!Context.hasSameType(T, Context.getTagDeclType(tagFromDeclSpec)))
+      break;
 
-    // If the TagDecl that the TypedefDecl points to is an anonymous decl
-    // keep track of the TypedefDecl.
-    if (!TD->getIdentifier() && !TD->getTypedefForAnonDecl())
-      TD->setTypedefForAnonDecl(NewTD);
+    // Otherwise, set this is the anon-decl typedef for the tag.
+    tagFromDeclSpec->setTypedefForAnonDecl(NewTD);
+    break;
+  }
+    
+  default:
+    break;
   }
 
-  if (D.isInvalidType())
-    NewTD->setInvalidDecl();
   return NewTD;
 }
 
@@ -5273,14 +5871,17 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
 /// TagSpec indicates what kind of tag this is. TUK indicates whether this is a
 /// reference/declaration/definition of a tag.
 Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
-                               SourceLocation KWLoc, CXXScopeSpec &SS,
-                               IdentifierInfo *Name, SourceLocation NameLoc,
-                               AttributeList *Attr, AccessSpecifier AS,
-                               MultiTemplateParamsArg TemplateParameterLists,
-                               bool &OwnedDecl, bool &IsDependent) {
+                     SourceLocation KWLoc, CXXScopeSpec &SS,
+                     IdentifierInfo *Name, SourceLocation NameLoc,
+                     AttributeList *Attr, AccessSpecifier AS,
+                     MultiTemplateParamsArg TemplateParameterLists,
+                     bool &OwnedDecl, bool &IsDependent,
+                     bool ScopedEnum, bool ScopedEnumUsesClassTag,
+                     TypeResult UnderlyingType) {
   // If this is not a definition, it must have a name.
   assert((Name != 0 || TUK == TUK_Definition) &&
          "Nameless record must be a definition!");
+  assert(TemplateParameterLists.size() == 0 || TUK != TUK_Reference);
 
   OwnedDecl = false;
   TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForTypeSpec(TagSpec);
@@ -5289,11 +5890,16 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   bool isExplicitSpecialization = false;
   unsigned NumMatchedTemplateParamLists = TemplateParameterLists.size();
   bool Invalid = false;
-  if (TUK != TUK_Reference) {
+
+  // We only need to do this matching if we have template parameters
+  // or a scope specifier, which also conveniently avoids this work
+  // for non-C++ cases.
+  if (NumMatchedTemplateParamLists ||
+      (SS.isNotEmpty() && TUK != TUK_Reference)) {
     if (TemplateParameterList *TemplateParams
           = MatchTemplateParametersToScopeSpecifier(KWLoc, SS,
-                        (TemplateParameterList**)TemplateParameterLists.get(),
-                                              TemplateParameterLists.size(),
+                                                TemplateParameterLists.get(),
+                                               TemplateParameterLists.size(),
                                                     TUK == TUK_Friend,
                                                     isExplicitSpecialization,
                                                     Invalid)) {
@@ -5322,6 +5928,41 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
     }
   }
 
+  // Figure out the underlying type if this a enum declaration. We need to do
+  // this early, because it's needed to detect if this is an incompatible
+  // redeclaration.
+  llvm::PointerUnion<const Type*, TypeSourceInfo*> EnumUnderlying;
+
+  if (Kind == TTK_Enum) {
+    if (UnderlyingType.isInvalid() || (!UnderlyingType.get() && ScopedEnum))
+      // No underlying type explicitly specified, or we failed to parse the
+      // type, default to int.
+      EnumUnderlying = Context.IntTy.getTypePtr();
+    else if (UnderlyingType.get()) {
+      // C++0x 7.2p2: The type-specifier-seq of an enum-base shall name an
+      // integral type; any cv-qualification is ignored.
+      TypeSourceInfo *TI = 0;
+      QualType T = GetTypeFromParser(UnderlyingType.get(), &TI);
+      EnumUnderlying = TI;
+
+      SourceLocation UnderlyingLoc = TI->getTypeLoc().getBeginLoc();
+
+      if (!T->isDependentType() && !T->isIntegralType(Context)) {
+        Diag(UnderlyingLoc, diag::err_enum_invalid_underlying)
+          << T;
+        // Recover by falling back to int.
+        EnumUnderlying = Context.IntTy.getTypePtr();
+      }
+
+      if (DiagnoseUnexpandedParameterPack(UnderlyingLoc, TI, 
+                                          UPPC_FixedUnderlyingType))
+        EnumUnderlying = Context.IntTy.getTypePtr();
+
+    } else if (getLangOptions().Microsoft)
+      // Microsoft enums are always of int type.
+      EnumUnderlying = Context.IntTy.getTypePtr();
+  }
+
   DeclContext *SearchDC = CurContext;
   DeclContext *DC = CurContext;
   bool isStdBadAlloc = false;
@@ -5374,7 +6015,9 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       // and that current instantiation has any dependent base
       // classes, we might find something at instantiation time: treat
       // this as a dependent elaborated-type-specifier.
-      if (Previous.wasNotFoundInCurrentInstantiation()) {
+      // But this only makes any sense for reference-like lookups.
+      if (Previous.wasNotFoundInCurrentInstantiation() &&
+          (TUK == TUK_Reference || TUK == TUK_Friend)) {
         IsDependent = true;
         return 0;
       }
@@ -5406,6 +6049,10 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       while (isa<RecordDecl>(SearchDC) || isa<EnumDecl>(SearchDC))
         SearchDC = SearchDC->getParent();
     }
+  } else if (S->isFunctionPrototypeScope()) {
+    // If this is an enum declaration in function prototype scope, set its
+    // initial context to the translation unit.
+    SearchDC = Context.getTranslationUnitDecl();
   }
 
   if (Previous.isSingleResult() &&
@@ -5467,7 +6114,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       // Find the context where we'll be declaring the tag.
       // FIXME: We would like to maintain the current DeclContext as the
       // lexical context,
-      while (SearchDC->isRecord())
+      while (SearchDC->isRecord() || SearchDC->isTransparentContext())
         SearchDC = SearchDC->getParent();
 
       // Find the scope where we'll be declaring the tag.
@@ -5554,6 +6201,41 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
           }
         }
 
+        if (Kind == TTK_Enum && PrevTagDecl->getTagKind() == TTK_Enum) {
+          const EnumDecl *PrevEnum = cast<EnumDecl>(PrevTagDecl);
+
+          // All conflicts with previous declarations are recovered by
+          // returning the previous declaration.
+          if (ScopedEnum != PrevEnum->isScoped()) {
+            Diag(KWLoc, diag::err_enum_redeclare_scoped_mismatch)
+              << PrevEnum->isScoped();
+            Diag(PrevTagDecl->getLocation(), diag::note_previous_use);
+            return PrevTagDecl;
+          }
+          else if (EnumUnderlying && PrevEnum->isFixed()) {
+            QualType T;
+            if (TypeSourceInfo *TI = EnumUnderlying.dyn_cast<TypeSourceInfo*>())
+                T = TI->getType();
+            else
+                T = QualType(EnumUnderlying.get<const Type*>(), 0);
+
+            if (!Context.hasSameUnqualifiedType(T, PrevEnum->getIntegerType())) {
+              Diag(NameLoc.isValid() ? NameLoc : KWLoc, 
+                   diag::err_enum_redeclare_type_mismatch)
+                << T
+                << PrevEnum->getIntegerType();
+              Diag(PrevTagDecl->getLocation(), diag::note_previous_use);
+              return PrevTagDecl;
+            }
+          }
+          else if (!EnumUnderlying.isNull() != PrevEnum->isFixed()) {
+            Diag(KWLoc, diag::err_enum_redeclare_fixed_mismatch)
+              << PrevEnum->isFixed();
+            Diag(PrevTagDecl->getLocation(), diag::note_previous_use);
+            return PrevTagDecl;
+          }
+        }
+
         if (!Invalid) {
           // If this is a use, just return the declaration we found.
 
@@ -5587,7 +6269,8 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
             } else {
               // If the type is currently being defined, complain
               // about a nested redefinition.
-              TagType *Tag = cast<TagType>(Context.getTagDeclType(PrevTagDecl));
+              const TagType *Tag
+                = cast<TagType>(Context.getTagDeclType(PrevTagDecl));
               if (Tag->isBeingDefined()) {
                 Diag(NameLoc, diag::err_nested_redefinition) << Name;
                 Diag(PrevTagDecl->getLocation(),
@@ -5689,24 +6372,49 @@ CreateNewDecl:
   // PrevDecl.
   TagDecl *New;
 
+  bool IsForwardReference = false;
   if (Kind == TTK_Enum) {
     // FIXME: Tag decls should be chained to any simultaneous vardecls, e.g.:
     // enum X { A, B, C } D;    D should chain to X.
     New = EnumDecl::Create(Context, SearchDC, Loc, Name, KWLoc,
-                           cast_or_null<EnumDecl>(PrevDecl));
+                           cast_or_null<EnumDecl>(PrevDecl), ScopedEnum,
+                           ScopedEnumUsesClassTag, !EnumUnderlying.isNull());
     // If this is an undefined enum, warn.
     if (TUK != TUK_Definition && !Invalid) {
       TagDecl *Def;
-      if (PrevDecl && (Def = cast<EnumDecl>(PrevDecl)->getDefinition())) {
+      if (getLangOptions().CPlusPlus0x && cast<EnumDecl>(New)->isFixed()) {
+        // C++0x: 7.2p2: opaque-enum-declaration.
+        // Conflicts are diagnosed above. Do nothing.
+      }
+      else if (PrevDecl && (Def = cast<EnumDecl>(PrevDecl)->getDefinition())) {
         Diag(Loc, diag::ext_forward_ref_enum_def)
           << New;
         Diag(Def->getLocation(), diag::note_previous_definition);
       } else {
-        Diag(Loc, 
-             getLangOptions().CPlusPlus? diag::err_forward_ref_enum
-                                       : diag::ext_forward_ref_enum);
+        unsigned DiagID = diag::ext_forward_ref_enum;
+        if (getLangOptions().Microsoft)
+          DiagID = diag::ext_ms_forward_ref_enum;
+        else if (getLangOptions().CPlusPlus)
+          DiagID = diag::err_forward_ref_enum;
+        Diag(Loc, DiagID);
+        
+        // If this is a forward-declared reference to an enumeration, make a 
+        // note of it; we won't actually be introducing the declaration into
+        // the declaration context.
+        if (TUK == TUK_Reference)
+          IsForwardReference = true;
       }
     }
+
+    if (EnumUnderlying) {
+      EnumDecl *ED = cast<EnumDecl>(New);
+      if (TypeSourceInfo *TI = EnumUnderlying.dyn_cast<TypeSourceInfo*>())
+        ED->setIntegerTypeSourceInfo(TI);
+      else
+        ED->setIntegerType(QualType(EnumUnderlying.get<const Type*>(), 0));
+      ED->setPromotionType(ED->getIntegerType());
+    }
+
   } else {
     // struct/union/class
 
@@ -5798,7 +6506,10 @@ CreateNewDecl:
         PushOnScopeChains(New, EnclosingScope, /* AddToContext = */ false);
   } else if (Name) {
     S = getNonFieldDeclScope(S);
-    PushOnScopeChains(New, S);
+    PushOnScopeChains(New, S, !IsForwardReference);
+    if (IsForwardReference)
+      SearchDC->makeDeclVisibleInContext(New, /* Recoverable = */ false);
+
   } else {
     CurContext->addDecl(New);
   }
@@ -5823,6 +6534,7 @@ void Sema::ActOnTagStartDefinition(Scope *S, Decl *TagD) {
 }
 
 void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
+                                           ClassVirtSpecifiers &CVS,
                                            SourceLocation LBraceLoc) {
   AdjustDeclIfTemplate(TagD);
   CXXRecordDecl *Record = cast<CXXRecordDecl>(TagD);
@@ -5832,6 +6544,11 @@ void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
   if (!Record->getIdentifier())
     return;
 
+  if (CVS.isFinalSpecified())
+    Record->addAttr(new (Context) FinalAttr(CVS.getFinalLoc(), Context));
+  if (CVS.isExplicitSpecified())
+    Record->addAttr(new (Context) ExplicitAttr(CVS.getExplicitLoc(), Context));
+    
   // C++ [class]p2:
   //   [...] The class-name is also inserted into the scope of the
   //   class itself; this is known as the injected-class-name. For
@@ -5842,7 +6559,9 @@ void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
                             CurContext, Record->getLocation(),
                             Record->getIdentifier(),
                             Record->getTagKeywordLoc(),
-                            Record);
+                            /*PrevDecl=*/0,
+                            /*DelayTypeCreation=*/true);
+  Context.getTypeDeclType(InjectedClassName, Record);
   InjectedClassName->setImplicit();
   InjectedClassName->setAccess(AS_public);
   if (ClassTemplateDecl *Template = Record->getDescribedClassTemplate())
@@ -5899,7 +6618,9 @@ bool Sema::VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName,
         << FieldName << FieldTy << BitWidth->getSourceRange();
     return Diag(FieldLoc, diag::err_not_integral_type_anon_bitfield)
       << FieldTy << BitWidth->getSourceRange();
-  }
+  } else if (DiagnoseUnexpandedParameterPack(const_cast<Expr *>(BitWidth),
+                                             UPPC_BitFieldWidth))
+    return true;
 
   // If the bit-width is type- or value-dependent, don't try to check
   // it now.
@@ -5974,9 +6695,17 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record,
 
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
   QualType T = TInfo->getType();
-  if (getLangOptions().CPlusPlus)
+  if (getLangOptions().CPlusPlus) {
     CheckExtraCXXDefaultArguments(D);
 
+    if (DiagnoseUnexpandedParameterPack(D.getIdentifierLoc(), TInfo,
+                                        UPPC_DataMemberType)) {
+      D.setInvalidType();
+      T = Context.IntTy;
+      TInfo = Context.getTrivialTypeSourceInfo(T, Loc);
+    }
+  }
+
   DiagnoseFunctionSpecifiers(D);
 
   if (D.getDeclSpec().isThreadSpecified())
@@ -6128,35 +6857,27 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   }
 
   if (!InvalidDecl && getLangOptions().CPlusPlus) {
-    CXXRecordDecl* CXXRecord = cast<CXXRecordDecl>(Record);
+    if (Record->isUnion()) {
+      if (const RecordType *RT = EltTy->getAs<RecordType>()) {
+        CXXRecordDecl* RDecl = cast<CXXRecordDecl>(RT->getDecl());
+        if (RDecl->getDefinition()) {
+          // C++ [class.union]p1: An object of a class with a non-trivial
+          // constructor, a non-trivial copy constructor, a non-trivial
+          // destructor, or a non-trivial copy assignment operator
+          // cannot be a member of a union, nor can an array of such
+          // objects.
+          // TODO: C++0x alters this restriction significantly.
+          if (CheckNontrivialField(NewFD))
+            NewFD->setInvalidDecl();
+        }
+      }
 
-    if (!T->isPODType())
-      CXXRecord->setPOD(false);
-    if (!ZeroWidth)
-      CXXRecord->setEmpty(false);
-    if (T->isReferenceType())
-      CXXRecord->setHasTrivialConstructor(false);
-
-    if (const RecordType *RT = EltTy->getAs<RecordType>()) {
-      CXXRecordDecl* RDecl = cast<CXXRecordDecl>(RT->getDecl());
-      if (RDecl->getDefinition()) {
-        if (!RDecl->hasTrivialConstructor())
-          CXXRecord->setHasTrivialConstructor(false);
-        if (!RDecl->hasTrivialCopyConstructor())
-          CXXRecord->setHasTrivialCopyConstructor(false);
-        if (!RDecl->hasTrivialCopyAssignment())
-          CXXRecord->setHasTrivialCopyAssignment(false);
-        if (!RDecl->hasTrivialDestructor())
-          CXXRecord->setHasTrivialDestructor(false);
-
-        // C++ 9.5p1: An object of a class with a non-trivial
-        // constructor, a non-trivial copy constructor, a non-trivial
-        // destructor, or a non-trivial copy assignment operator
-        // cannot be a member of a union, nor can an array of such
-        // objects.
-        // TODO: C++0x alters this restriction significantly.
-        if (Record->isUnion() && CheckNontrivialField(NewFD))
-          NewFD->setInvalidDecl();
+      // C++ [class.union]p1: If a union contains a member of reference type,
+      // the program is ill-formed.
+      if (EltTy->isReferenceType()) {
+        Diag(NewFD->getLocation(), diag::err_union_member_of_reference_type)
+          << NewFD->getDeclName() << EltTy;
+        NewFD->setInvalidDecl();
       }
     }
   }
@@ -6171,18 +6892,6 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
     Diag(Loc, diag::warn_attribute_weak_on_field);
 
   NewFD->setAccess(AS);
-
-  // C++ [dcl.init.aggr]p1:
-  //   An aggregate is an array or a class (clause 9) with [...] no
-  //   private or protected non-static data members (clause 11).
-  // A POD must be an aggregate.
-  if (getLangOptions().CPlusPlus &&
-      (AS == AS_private || AS == AS_protected)) {
-    CXXRecordDecl *CXXRecord = cast<CXXRecordDecl>(Record);
-    CXXRecord->setAggregate(false);
-    CXXRecord->setPOD(false);
-  }
-
   return NewFD;
 }
 
@@ -6532,7 +7241,7 @@ void Sema::ActOnFields(Scope* S,
     FieldDecl *FD = cast<FieldDecl>(Fields[i]);
 
     // Get the type for the field.
-    Type *FDTy = FD->getType().getTypePtr();
+    const Type *FDTy = FD->getType().getTypePtr();
 
     if (!FD->isAnonymousStructOrUnion()) {
       // Remember all fields written by the user.
@@ -6563,10 +7272,22 @@ void Sema::ActOnFields(Scope* S,
       FD->setInvalidDecl();
       EnclosingDecl->setInvalidDecl();
       continue;
-    } else if (FDTy->isIncompleteArrayType() && i == NumFields - 1 &&
-               Record && !Record->isUnion()) {
+    } else if (FDTy->isIncompleteArrayType() && Record && 
+               ((i == NumFields - 1 && !Record->isUnion()) ||
+                (getLangOptions().Microsoft &&
+                 (i == NumFields - 1 || Record->isUnion())))) {
       // Flexible array member.
-      if (NumNamedMembers < 1) {
+      // Microsoft is more permissive regarding flexible array.
+      // It will accept flexible array in union and also
+      // as the sole element of a struct/class.
+      if (getLangOptions().Microsoft) {
+        if (Record->isUnion()) 
+          Diag(FD->getLocation(), diag::ext_flexible_array_union)
+            << FD->getDeclName();
+        else if (NumFields == 1) 
+          Diag(FD->getLocation(), diag::ext_flexible_array_empty_aggregate)
+            << FD->getDeclName() << Record->getTagKind();
+      } else  if (NumNamedMembers < 1) {
         Diag(FD->getLocation(), diag::err_flexible_array_empty_struct)
           << FD->getDeclName();
         FD->setInvalidDecl();
@@ -6581,7 +7302,6 @@ void Sema::ActOnFields(Scope* S,
         EnclosingDecl->setInvalidDecl();
         continue;
       }
-      
       // Okay, we have a legal flexible array member at the end of the struct.
       if (Record)
         Record->setHasFlexibleArrayMember(true);
@@ -6641,7 +7361,64 @@ void Sema::ActOnFields(Scope* S,
 
   // Okay, we successfully defined 'Record'.
   if (Record) {
-    Record->completeDefinition();
+    bool Completed = false;
+    if (CXXRecordDecl *CXXRecord = dyn_cast<CXXRecordDecl>(Record)) {
+      if (!CXXRecord->isInvalidDecl()) {
+        // Set access bits correctly on the directly-declared conversions.
+        UnresolvedSetImpl *Convs = CXXRecord->getConversionFunctions();
+        for (UnresolvedSetIterator I = Convs->begin(), E = Convs->end(); 
+             I != E; ++I)
+          Convs->setAccess(I, (*I)->getAccess());
+        
+        if (!CXXRecord->isDependentType()) {
+          // Add any implicitly-declared members to this class.
+          AddImplicitlyDeclaredMembersToClass(CXXRecord);
+
+          // If we have virtual base classes, we may end up finding multiple 
+          // final overriders for a given virtual function. Check for this 
+          // problem now.
+          if (CXXRecord->getNumVBases()) {
+            CXXFinalOverriderMap FinalOverriders;
+            CXXRecord->getFinalOverriders(FinalOverriders);
+            
+            for (CXXFinalOverriderMap::iterator M = FinalOverriders.begin(), 
+                                             MEnd = FinalOverriders.end();
+                 M != MEnd; ++M) {
+              for (OverridingMethods::iterator SO = M->second.begin(), 
+                                            SOEnd = M->second.end();
+                   SO != SOEnd; ++SO) {
+                assert(SO->second.size() > 0 && 
+                       "Virtual function without overridding functions?");
+                if (SO->second.size() == 1)
+                  continue;
+                
+                // C++ [class.virtual]p2:
+                //   In a derived class, if a virtual member function of a base
+                //   class subobject has more than one final overrider the
+                //   program is ill-formed.
+                Diag(Record->getLocation(), diag::err_multiple_final_overriders)
+                  << (NamedDecl *)M->first << Record;
+                Diag(M->first->getLocation(), 
+                     diag::note_overridden_virtual_function);
+                for (OverridingMethods::overriding_iterator 
+                          OM = SO->second.begin(), 
+                       OMEnd = SO->second.end();
+                     OM != OMEnd; ++OM)
+                  Diag(OM->Method->getLocation(), diag::note_final_overrider)
+                    << (NamedDecl *)M->first << OM->Method->getParent();
+                
+                Record->setInvalidDecl();
+              }
+            }
+            CXXRecord->completeDefinition(&FinalOverriders);
+            Completed = true;
+          }
+        }
+      }
+    }
+    
+    if (!Completed)
+      Record->completeDefinition();
   } else {
     ObjCIvarDecl **ClsFields =
       reinterpret_cast<ObjCIvarDecl**>(RecFields.data());
@@ -6695,9 +7472,11 @@ static bool isRepresentableIntegerValue(ASTContext &Context,
   assert(T->isIntegralType(Context) && "Integral type required!");
   unsigned BitWidth = Context.getIntWidth(T);
   
-  if (Value.isUnsigned() || Value.isNonNegative())
-    return Value.getActiveBits() < BitWidth;
-  
+  if (Value.isUnsigned() || Value.isNonNegative()) {
+    if (T->isSignedIntegerType()) 
+      --BitWidth;
+    return Value.getActiveBits() <= BitWidth;
+  }  
   return Value.getMinSignedBits() <= BitWidth;
 }
 
@@ -6734,6 +7513,10 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
   unsigned IntWidth = Context.Target.getIntWidth();
   llvm::APSInt EnumVal(IntWidth);
   QualType EltTy;
+
+  if (Val && DiagnoseUnexpandedParameterPack(Val, UPPC_EnumeratorValue))
+    Val = 0;
+
   if (Val) {
     if (Enum->isDependentType() || Val->isTypeDependent())
       EltTy = Context.DependentTy;
@@ -6761,12 +7544,30 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
           }
         }
         
-        // C++0x [dcl.enum]p5:
-        //   If the underlying type is not fixed, the type of each enumerator
-        //   is the type of its initializing value:
-        //     - If an initializer is specified for an enumerator, the 
-        //       initializing value has the same type as the expression.
-        EltTy = Val->getType();
+        if (Enum->isFixed()) {
+          EltTy = Enum->getIntegerType();
+
+          // C++0x [dcl.enum]p5:
+          //   ... if the initializing value of an enumerator cannot be
+          //   represented by the underlying type, the program is ill-formed.
+          if (!isRepresentableIntegerValue(Context, EnumVal, EltTy)) {
+            if (getLangOptions().Microsoft) {
+              Diag(IdLoc, diag::ext_enumerator_too_large) << EltTy;
+              ImpCastExprToType(Val, EltTy, CK_IntegralCast);
+            } else 
+              Diag(IdLoc, diag::err_enumerator_too_large)
+                << EltTy;
+          } else
+            ImpCastExprToType(Val, EltTy, CK_IntegralCast);
+        }
+        else {
+          // C++0x [dcl.enum]p5:
+          //   If the underlying type is not fixed, the type of each enumerator
+          //   is the type of its initializing value:
+          //     - If an initializer is specified for an enumerator, the 
+          //       initializing value has the same type as the expression.
+          EltTy = Val->getType();
+        }
       }
     }
   }
@@ -6783,7 +7584,12 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
       //
       // GCC uses 'int' for its unspecified integral type, as does 
       // C99 6.7.2.2p3.
-      EltTy = Context.IntTy;
+      if (Enum->isFixed()) {
+        EltTy = Enum->getIntegerType();
+      }
+      else {
+        EltTy = Context.IntTy;
+      }
     } else {
       // Assign the last value + 1.
       EnumVal = LastEnumConst->getInitVal();
@@ -6803,13 +7609,20 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
         //       sufficient to contain the incremented value. If no such type
         //       exists, the program is ill-formed.
         QualType T = getNextLargerIntegralType(Context, EltTy);
-        if (T.isNull()) {
+        if (T.isNull() || Enum->isFixed()) {
           // There is no integral type larger enough to represent this 
           // value. Complain, then allow the value to wrap around.
           EnumVal = LastEnumConst->getInitVal();
-          EnumVal.zext(EnumVal.getBitWidth() * 2);
-          Diag(IdLoc, diag::warn_enumerator_too_large)
-            << EnumVal.toString(10);
+          EnumVal = EnumVal.zext(EnumVal.getBitWidth() * 2);
+          ++EnumVal;
+          if (Enum->isFixed())
+            // When the underlying type is fixed, this is ill-formed.
+            Diag(IdLoc, diag::err_enumerator_wrapped)
+              << EnumVal.toString(10)
+              << EltTy;
+          else
+            Diag(IdLoc, diag::warn_enumerator_too_large)
+              << EnumVal.toString(10);
         } else {
           EltTy = T;
         }
@@ -6819,7 +7632,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
         // value, then increment.
         EnumVal = LastEnumConst->getInitVal();
         EnumVal.setIsSigned(EltTy->isSignedIntegerType());
-        EnumVal.zextOrTrunc(Context.getIntWidth(EltTy));
+        EnumVal = EnumVal.zextOrTrunc(Context.getIntWidth(EltTy));
         ++EnumVal;        
         
         // If we're not in C++, diagnose the overflow of enumerator values,
@@ -6841,7 +7654,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
   if (!EltTy->isDependentType()) {
     // Make the enumerator value match the signedness and size of the 
     // enumerator's type.
-    EnumVal.zextOrTrunc(Context.getIntWidth(EltTy));
+    EnumVal = EnumVal.zextOrTrunc(Context.getIntWidth(EltTy));
     EnumVal.setIsSigned(EltTy->isSignedIntegerType());
   }
   
@@ -6850,11 +7663,10 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
 }
 
 
-Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl,
-                                        Decl *lastEnumConst,
-                                        SourceLocation IdLoc,
-                                        IdentifierInfo *Id,
-                                        SourceLocation EqualLoc, ExprTy *val) {
+Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst,
+                              SourceLocation IdLoc, IdentifierInfo *Id,
+                              AttributeList *Attr,
+                              SourceLocation EqualLoc, ExprTy *val) {
   EnumDecl *TheEnumDecl = cast<EnumDecl>(theEnumDecl);
   EnumConstantDecl *LastEnumConst =
     cast_or_null<EnumConstantDecl>(lastEnumConst);
@@ -6890,11 +7702,25 @@ Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl,
     }
   }
 
-  EnumConstantDecl *New = CheckEnumConstant(TheEnumDecl, LastEnumConst,
-                                            IdLoc, Id, Val);
+  // C++ [class.mem]p13:
+  //   If T is the name of a class, then each of the following shall have a 
+  //   name different from T:
+  //     - every enumerator of every member of class T that is an enumerated 
+  //       type
+  if (CXXRecordDecl *Record
+                      = dyn_cast<CXXRecordDecl>(
+                             TheEnumDecl->getDeclContext()->getRedeclContext()))
+    if (Record->getIdentifier() && Record->getIdentifier() == Id)
+      Diag(IdLoc, diag::err_member_name_of_class) << Id;
+  
+  EnumConstantDecl *New =
+    CheckEnumConstant(TheEnumDecl, LastEnumConst, IdLoc, Id, Val);
 
-  // Register this decl in the current scope stack.
   if (New) {
+    // Process attributes.
+    if (Attr) ProcessDeclAttributeList(S, New, Attr);
+
+    // Register this decl in the current scope stack.
     New->setAccess(TheEnumDecl->getAccess());
     PushOnScopeChains(New, S);
   }
@@ -6961,7 +7787,6 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   }
 
   // Figure out the type that should be used for this enum.
-  // FIXME: Support -fshort-enums.
   QualType BestType;
   unsigned BestWidth;
 
@@ -6977,8 +7802,19 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   QualType BestPromotionType;
 
   bool Packed = Enum->getAttr<PackedAttr>() ? true : false;
-
-  if (NumNegativeBits) {
+  // -fshort-enums is the equivalent to specifying the packed attribute on all
+  // enum definitions.
+  if (LangOpts.ShortEnums)
+    Packed = true;
+
+  if (Enum->isFixed()) {
+    BestType = BestPromotionType = Enum->getIntegerType();
+    // We don't need to set BestWidth, because BestType is going to be the type
+    // of the enumerators, but we do anyway because otherwise some compilers
+    // warn that it might be used uninitialized.
+    BestWidth = CharWidth;
+  }
+  else if (NumNegativeBits) {
     // If there is a negative value, figure out the smallest integer type (of
     // int/long/longlong) that fits.
     // If it's packed, check also if it fits a char or a short.
@@ -7081,12 +7917,13 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
     }
 
     // Adjust the APSInt value.
-    InitVal.extOrTrunc(NewWidth);
+    InitVal = InitVal.extOrTrunc(NewWidth);
     InitVal.setIsSigned(NewSign);
     ECD->setInitVal(InitVal);
 
     // Adjust the Expr initializer and type.
-    if (ECD->getInitExpr())
+    if (ECD->getInitExpr() &&
+	!Context.hasSameType(NewTy, ECD->getInitExpr()->getType()))
       ECD->setInitExpr(ImplicitCastExpr::Create(Context, NewTy,
                                                 CK_IntegralCast,
                                                 ECD->getInitExpr(),
diff --git a/lib/Sema/SemaDeclAttr.cpp b/lib/Sema/SemaDeclAttr.cpp
index 25af73ae13d3..cbc940f2f0ac 100644
--- a/lib/Sema/SemaDeclAttr.cpp
+++ b/lib/Sema/SemaDeclAttr.cpp
@@ -78,6 +78,13 @@ static bool isFunctionOrMethodOrBlock(const Decl *d) {
   return isa<BlockDecl>(d);
 }
 
+/// Return true if the given decl has a declarator that should have
+/// been processed by Sema::GetTypeForDeclarator.
+static bool hasDeclarator(const Decl *d) {
+  // In some sense, TypedefDecl really *ought* to be a DeclaratorDecl.
+  return isa<DeclaratorDecl>(d) || isa<BlockDecl>(d) || isa<TypedefDecl>(d);
+}
+
 /// hasFunctionProto - Return true if the given decl has a argument
 /// information. This decl should have already passed
 /// isFunctionOrMethod or isFunctionOrMethodOrBlock.
@@ -127,6 +134,12 @@ static bool isFunctionOrMethodVariadic(const Decl *d) {
   }
 }
 
+static bool isInstanceMethod(const Decl *d) {
+  if (const CXXMethodDecl *MethodDecl = dyn_cast<CXXMethodDecl>(d))
+    return MethodDecl->isInstance();
+  return false;
+}
+
 static inline bool isNSStringType(QualType T, ASTContext &Ctx) {
   const ObjCObjectPointerType *PT = T->getAs<ObjCObjectPointerType>();
   if (!PT)
@@ -191,7 +204,7 @@ static void HandleExtVectorTypeAttr(Scope *scope, Decl *d,
       S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
       return;
     }
-    sizeExpr = static_cast<Expr *>(Attr.getArg(0));
+    sizeExpr = Attr.getArg(0);
   }
 
   // Instantiate/Install the vector type, and let Sema build the type for us.
@@ -242,7 +255,7 @@ static void HandleIBAction(Decl *d, const AttributeList &Attr, Sema &S) {
       return;
     }
 
-  S.Diag(Attr.getLoc(), diag::err_attribute_ibaction) << Attr.getName();
+  S.Diag(Attr.getLoc(), diag::warn_attribute_ibaction) << Attr.getName();
 }
 
 static void HandleIBOutlet(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -259,7 +272,7 @@ static void HandleIBOutlet(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  S.Diag(Attr.getLoc(), diag::err_attribute_iboutlet) << Attr.getName();
+  S.Diag(Attr.getLoc(), diag::warn_attribute_iboutlet) << Attr.getName();
 }
 
 static void HandleIBOutletCollection(Decl *d, const AttributeList &Attr,
@@ -274,7 +287,7 @@ static void HandleIBOutletCollection(Decl *d, const AttributeList &Attr,
   // The IBOutletCollection attributes only apply to instance variables of
   // Objective-C classes.
   if (!(isa<ObjCIvarDecl>(d) || isa<ObjCPropertyDecl>(d))) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_iboutlet) << Attr.getName();
+    S.Diag(Attr.getLoc(), diag::warn_attribute_iboutlet) << Attr.getName();
     return;
   }
   if (const ValueDecl *VD = dyn_cast<ValueDecl>(d))
@@ -323,7 +336,10 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  unsigned NumArgs = getFunctionOrMethodNumArgs(d);
+  // In C++ the implicit 'this' function parameter also counts, and they are
+  // counted from one.
+  bool HasImplicitThisParam = isInstanceMethod(d);
+  unsigned NumArgs  = getFunctionOrMethodNumArgs(d) + HasImplicitThisParam;
 
   // The nonnull attribute only applies to pointers.
   llvm::SmallVector<unsigned, 10> NonNullArgs;
@@ -333,7 +349,7 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
 
     // The argument must be an integer constant expression.
-    Expr *Ex = static_cast<Expr *>(*I);
+    Expr *Ex = *I;
     llvm::APSInt ArgNum(32);
     if (Ex->isTypeDependent() || Ex->isValueDependent() ||
         !Ex->isIntegerConstantExpr(ArgNum, S.Context)) {
@@ -351,9 +367,18 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     }
 
     --x;
+    if (HasImplicitThisParam) {
+      if (x == 0) {
+        S.Diag(Attr.getLoc(),
+               diag::err_attribute_invalid_implicit_this_argument)
+          << "nonnull" << Ex->getSourceRange();
+        return;
+      }
+      --x;
+    }
 
     // Is the function argument a pointer type?
-    QualType T = getFunctionOrMethodArgType(d, x);
+    QualType T = getFunctionOrMethodArgType(d, x).getNonReferenceType();
     if (!T->isAnyPointerType() && !T->isBlockPointerType()) {
       // FIXME: Should also highlight argument in decl.
       S.Diag(Attr.getLoc(), diag::warn_nonnull_pointers_only)
@@ -368,13 +393,30 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // arguments have a nonnull attribute.
   if (NonNullArgs.empty()) {
     for (unsigned I = 0, E = getFunctionOrMethodNumArgs(d); I != E; ++I) {
-      QualType T = getFunctionOrMethodArgType(d, I);
+      QualType T = getFunctionOrMethodArgType(d, I).getNonReferenceType();
       if (T->isAnyPointerType() || T->isBlockPointerType())
         NonNullArgs.push_back(I);
+      else if (const RecordType *UT = T->getAsUnionType()) {
+        if (UT && UT->getDecl()->hasAttr<TransparentUnionAttr>()) {
+          RecordDecl *UD = UT->getDecl();
+          for (RecordDecl::field_iterator it = UD->field_begin(),
+               itend = UD->field_end(); it != itend; ++it) {
+            T = it->getType();
+            if (T->isAnyPointerType() || T->isBlockPointerType()) {
+              NonNullArgs.push_back(I);
+              break;
+            }
+          }
+        }
+      }
     }
 
+    // No pointer arguments?
     if (NonNullArgs.empty()) {
-      S.Diag(Attr.getLoc(), diag::warn_attribute_nonnull_no_pointers);
+      // Warn the trivial case only if attribute is not coming from a
+      // macro instantiation.
+      if (Attr.getLoc().isFileID())
+        S.Diag(Attr.getLoc(), diag::warn_attribute_nonnull_no_pointers);
       return;
     }
   }
@@ -437,7 +479,10 @@ static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
     return;
   }
 
-  unsigned NumArgs = getFunctionOrMethodNumArgs(d);
+  // In C++ the implicit 'this' function parameter also counts, and they are
+  // counted from one.
+  bool HasImplicitThisParam = isInstanceMethod(d);
+  unsigned NumArgs  = getFunctionOrMethodNumArgs(d) + HasImplicitThisParam;
 
   llvm::StringRef Module = AL.getParameterName()->getName();
 
@@ -450,7 +495,7 @@ static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
   for (AttributeList::arg_iterator I = AL.arg_begin(), E = AL.arg_end(); I != E;
        ++I) {
 
-    Expr *IdxExpr = static_cast<Expr *>(*I);
+    Expr *IdxExpr = *I;
     llvm::APSInt ArgNum(32);
     if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent()
         || !IdxExpr->isIntegerConstantExpr(ArgNum, S.Context)) {
@@ -467,6 +512,15 @@ static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
       continue;
     }
     --x;
+    if (HasImplicitThisParam) {
+      if (x == 0) {
+        S.Diag(AL.getLoc(), diag::err_attribute_invalid_implicit_this_argument)
+          << "ownership" << IdxExpr->getSourceRange();
+        return;
+      }
+      --x;
+    }
+
     switch (K) {
     case OwnershipAttr::Takes:
     case OwnershipAttr::Holds: {
@@ -485,7 +539,7 @@ static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
     case OwnershipAttr::Returns: {
       if (AL.getNumArgs() > 1) {
           // Is the function argument an integer type?
-          Expr *IdxExpr = static_cast<Expr *>(AL.getArg(0));
+          Expr *IdxExpr = AL.getArg(0);
           llvm::APSInt ArgNum(32);
           if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent()
               || !IdxExpr->isIntegerConstantExpr(ArgNum, S.Context)) {
@@ -532,11 +586,23 @@ static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
                                              start, size));
 }
 
-static bool isStaticVarOrStaticFunciton(Decl *D) {
-  if (VarDecl *VD = dyn_cast<VarDecl>(D))
-    return VD->getStorageClass() == SC_Static;
-  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
-    return FD->getStorageClass() == SC_Static;
+/// Whether this declaration has internal linkage for the purposes of
+/// things that want to complain about things not have internal linkage.
+static bool hasEffectivelyInternalLinkage(NamedDecl *D) {
+  switch (D->getLinkage()) {
+  case NoLinkage:
+  case InternalLinkage:
+    return true;
+
+  // Template instantiations that go from external to unique-external
+  // shouldn't get diagnosed.
+  case UniqueExternalLinkage:
+    return true;
+
+  case ExternalLinkage:
+    return false;
+  }
+  llvm_unreachable("unknown linkage kind!");
   return false;
 }
 
@@ -547,6 +613,14 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
+  if (!isa<VarDecl>(d) && !isa<FunctionDecl>(d)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_decl_type)
+      << Attr.getName() << 2 /*variables and functions*/;
+    return;
+  }
+
+  NamedDecl *nd = cast<NamedDecl>(d);
+
   // gcc rejects
   // class c {
   //   static int a __attribute__((weakref ("v2")));
@@ -560,7 +634,7 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   const DeclContext *Ctx = d->getDeclContext()->getRedeclContext();
   if (!Ctx->isFileContext()) {
     S.Diag(Attr.getLoc(), diag::err_attribute_weakref_not_global_context) <<
-        dyn_cast<NamedDecl>(d)->getNameAsString();
+        nd->getNameAsString();
     return;
   }
 
@@ -582,9 +656,8 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // This looks like a bug in gcc. We reject that for now. We should revisit
   // it if this behaviour is actually used.
 
-  if (!isStaticVarOrStaticFunciton(d)) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_weakref_not_static) <<
-      dyn_cast<NamedDecl>(d)->getNameAsString();
+  if (!hasEffectivelyInternalLinkage(nd)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_weakref_not_static);
     return;
   }
 
@@ -593,7 +666,7 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // Should we? How to check that weakref is before or after alias?
 
   if (Attr.getNumArgs() == 1) {
-    Expr *Arg = static_cast<Expr*>(Attr.getArg(0));
+    Expr *Arg = Attr.getArg(0);
     Arg = Arg->IgnoreParenCasts();
     StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
@@ -604,7 +677,8 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     }
     // GCC will accept anything as the argument of weakref. Should we
     // check for an existing decl?
-    d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context, Str->getString()));
+    d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context,
+                                           Str->getString()));
   }
 
   d->addAttr(::new (S.Context) WeakRefAttr(Attr.getLoc(), S.Context));
@@ -617,7 +691,7 @@ static void HandleAliasAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  Expr *Arg = static_cast<Expr*>(Attr.getArg(0));
+  Expr *Arg = Attr.getArg(0);
   Arg = Arg->IgnoreParenCasts();
   StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
@@ -627,14 +701,37 @@ static void HandleAliasAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
+  if (S.Context.Target.getTriple().getOS() == llvm::Triple::Darwin) {
+    S.Diag(Attr.getLoc(), diag::err_alias_not_supported_on_darwin);
+    return;
+  }
+
   // FIXME: check if target symbol exists in current file
 
-  d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context, Str->getString()));
+  d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context,
+                                         Str->getString()));
+}
+
+static void HandleNakedAttr(Decl *d, const AttributeList &Attr,
+                                   Sema &S) {
+  // Check the attribute arguments.
+  if (Attr.getNumArgs() != 0) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+    return;
+  }
+
+  if (!isa<FunctionDecl>(d)) {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << 0 /*function*/;
+    return;
+  }
+
+  d->addAttr(::new (S.Context) NakedAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleAlwaysInlineAttr(Decl *d, const AttributeList &Attr,
                                    Sema &S) {
-  // check the attribute arguments.
+  // Check the attribute arguments.
   if (Attr.getNumArgs() != 0) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
     return;
@@ -642,7 +739,7 @@ static void HandleAlwaysInlineAttr(Decl *d, const AttributeList &Attr,
 
   if (!isa<FunctionDecl>(d)) {
     S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
-    << Attr.getName() << 0 /*function*/;
+      << Attr.getName() << 0 /*function*/;
     return;
   }
 
@@ -650,7 +747,7 @@ static void HandleAlwaysInlineAttr(Decl *d, const AttributeList &Attr,
 }
 
 static void HandleMallocAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  // check the attribute arguments.
+  // Check the attribute arguments.
   if (Attr.getNumArgs() != 0) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
     return;
@@ -667,10 +764,56 @@ static void HandleMallocAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   S.Diag(Attr.getLoc(), diag::warn_attribute_malloc_pointer_only);
 }
 
-static void HandleNoReturnAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  /* Diagnostics (if any) was emitted by Sema::ProcessFnAttr(). */
+static void HandleMayAliasAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  // check the attribute arguments.
+  if (Attr.getNumArgs() != 0) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+    return;
+  }
+
+  d->addAttr(::new (S.Context) MayAliasAttr(Attr.getLoc(), S.Context));
+}
+
+static void HandleNoCommonAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   assert(Attr.isInvalid() == false);
-  d->addAttr(::new (S.Context) NoReturnAttr(Attr.getLoc(), S.Context));
+  if (isa<VarDecl>(d))
+    d->addAttr(::new (S.Context) NoCommonAttr(Attr.getLoc(), S.Context));
+  else
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << 12 /* variable */;
+}
+
+static void HandleCommonAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  assert(Attr.isInvalid() == false);
+  if (isa<VarDecl>(d))
+    d->addAttr(::new (S.Context) CommonAttr(Attr.getLoc(), S.Context));
+  else
+    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << 12 /* variable */;
+}
+
+static void HandleNoReturnAttr(Decl *d, const AttributeList &attr, Sema &S) {
+  if (hasDeclarator(d)) return;
+
+  if (S.CheckNoReturnAttr(attr)) return;
+
+  if (!isa<ObjCMethodDecl>(d)) {
+    S.Diag(attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << attr.getName() << 0 /*function*/;
+    return;
+  }
+
+  d->addAttr(::new (S.Context) NoReturnAttr(attr.getLoc(), S.Context));
+}
+
+bool Sema::CheckNoReturnAttr(const AttributeList &attr) {
+  if (attr.getNumArgs() != 0) {
+    Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+    attr.setInvalid();
+    return true;
+  }
+
+  return false;
 }
 
 static void HandleAnalyzerNoReturnAttr(Decl *d, const AttributeList &Attr,
@@ -705,10 +848,11 @@ static void HandleVecReturnAttr(Decl *d, const AttributeList &Attr,
 /*
   Returning a Vector Class in Registers
   
-  According to the PPU ABI specifications, a class with a single member of vector type is returned in
-  memory when used as the return value of a function. This results in inefficient code when implementing
-  vector classes. To return the value in a single vector register, add the vecreturn attribute to the class
-  definition. This attribute is also applicable to struct types.
+  According to the PPU ABI specifications, a class with a single member of 
+  vector type is returned in memory when used as the return value of a function.
+  This results in inefficient code when implementing vector classes. To return
+  the value in a single vector register, add the vecreturn attribute to the
+  class definition. This attribute is also applicable to struct types.
   
   Example:
   
@@ -724,7 +868,7 @@ static void HandleVecReturnAttr(Decl *d, const AttributeList &Attr,
     return result; // This will be returned in a register
   }
 */
-  if (!isa<CXXRecordDecl>(d)) {
+  if (!isa<RecordDecl>(d)) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_decl_type)
       << Attr.getName() << 9 /*class*/;
     return;
@@ -735,6 +879,28 @@ static void HandleVecReturnAttr(Decl *d, const AttributeList &Attr,
     return;
   }
 
+  RecordDecl *record = cast<RecordDecl>(d);
+  int count = 0;
+
+  if (!isa<CXXRecordDecl>(record)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_vecreturn_only_vector_member);
+    return;
+  }
+
+  if (!cast<CXXRecordDecl>(record)->isPOD()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_vecreturn_only_pod_record);
+    return;
+  }
+
+  for (RecordDecl::field_iterator iter = record->field_begin();
+       iter != record->field_end(); iter++) {
+    if ((count == 1) || !iter->getType()->isVectorType()) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_vecreturn_only_vector_member);
+      return;
+    }
+    count++;
+  }
+
   d->addAttr(::new (S.Context) VecReturnAttr(Attr.getLoc(), S.Context));
 }
 
@@ -755,9 +921,9 @@ static void HandleUnusedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 
   if (!isa<VarDecl>(d) && !isa<ObjCIvarDecl>(d) && !isFunctionOrMethod(d) &&
-      !isa<TypeDecl>(d)) {
+      !isa<TypeDecl>(d) && !isa<LabelDecl>(d)) {
     S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 2 /*variable and function*/;
+      << Attr.getName() << 14 /*variable, function, labels*/;
     return;
   }
 
@@ -795,7 +961,7 @@ static void HandleConstructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   int priority = 65535; // FIXME: Do not hardcode such constants.
   if (Attr.getNumArgs() > 0) {
-    Expr *E = static_cast<Expr *>(Attr.getArg(0));
+    Expr *E = Attr.getArg(0);
     llvm::APSInt Idx(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
         !E->isIntegerConstantExpr(Idx, S.Context)) {
@@ -812,7 +978,8 @@ static void HandleConstructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) ConstructorAttr(Attr.getLoc(), S.Context, priority));
+  d->addAttr(::new (S.Context) ConstructorAttr(Attr.getLoc(), S.Context,
+                                               priority));
 }
 
 static void HandleDestructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -825,7 +992,7 @@ static void HandleDestructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   int priority = 65535; // FIXME: Do not hardcode such constants.
   if (Attr.getNumArgs() > 0) {
-    Expr *E = static_cast<Expr *>(Attr.getArg(0));
+    Expr *E = Attr.getArg(0);
     llvm::APSInt Idx(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
         !E->isIntegerConstantExpr(Idx, S.Context)) {
@@ -842,27 +1009,59 @@ static void HandleDestructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) DestructorAttr(Attr.getLoc(), S.Context, priority));
+  d->addAttr(::new (S.Context) DestructorAttr(Attr.getLoc(), S.Context,
+                                              priority));
 }
 
 static void HandleDeprecatedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // check the attribute arguments.
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    return;
+  int noArgs = Attr.getNumArgs();
+  if (noArgs > 1) {
+    S.Diag(Attr.getLoc(), 
+           diag::err_attribute_wrong_number_arguments) << "0 or 1";
+    return;
+  }
+  // Handle the case where deprecated attribute has a text message.
+  StringLiteral *SE;
+  if (noArgs == 1) {
+    Expr *ArgExpr = Attr.getArg(0);
+    SE = dyn_cast<StringLiteral>(ArgExpr);
+    if (!SE) {
+      S.Diag(ArgExpr->getLocStart(), 
+             diag::err_attribute_not_string) << "deprecated";
+      return;
+    }
   }
+  else
+    SE = StringLiteral::CreateEmpty(S.Context, 1);
 
-  d->addAttr(::new (S.Context) DeprecatedAttr(Attr.getLoc(), S.Context));
+  d->addAttr(::new (S.Context) DeprecatedAttr(Attr.getLoc(), S.Context,
+                                              SE->getString()));
 }
 
 static void HandleUnavailableAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // check the attribute arguments.
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+  int noArgs = Attr.getNumArgs();
+  if (noArgs > 1) {
+    S.Diag(Attr.getLoc(),
+           diag::err_attribute_wrong_number_arguments) << "0 or 1";
     return;
   }
-
-  d->addAttr(::new (S.Context) UnavailableAttr(Attr.getLoc(), S.Context));
+  // Handle the case where unavailable attribute has a text message.
+  StringLiteral *SE;
+  if (noArgs == 1) {
+    Expr *ArgExpr = Attr.getArg(0);
+    SE = dyn_cast<StringLiteral>(ArgExpr);
+    if (!SE) {
+      S.Diag(ArgExpr->getLocStart(), 
+             diag::err_attribute_not_string) << "unavailable";
+      return;
+    }
+  }
+  else
+    SE = StringLiteral::CreateEmpty(S.Context, 1);
+  d->addAttr(::new (S.Context) UnavailableAttr(Attr.getLoc(), S.Context,
+                                               SE->getString()));
 }
 
 static void HandleVisibilityAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -872,7 +1071,7 @@ static void HandleVisibilityAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  Expr *Arg = static_cast<Expr*>(Attr.getArg(0));
+  Expr *Arg = Attr.getArg(0);
   Arg = Arg->IgnoreParenCasts();
   StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
 
@@ -982,7 +1181,7 @@ static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   int sentinel = 0;
   if (Attr.getNumArgs() > 0) {
-    Expr *E = static_cast<Expr *>(Attr.getArg(0));
+    Expr *E = Attr.getArg(0);
     llvm::APSInt Idx(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
         !E->isIntegerConstantExpr(Idx, S.Context)) {
@@ -1001,7 +1200,7 @@ static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   int nullPos = 0;
   if (Attr.getNumArgs() > 1) {
-    Expr *E = static_cast<Expr *>(Attr.getArg(1));
+    Expr *E = Attr.getArg(1);
     llvm::APSInt Idx(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
         !E->isIntegerConstantExpr(Idx, S.Context)) {
@@ -1046,7 +1245,7 @@ static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     QualType Ty = V->getType();
     if (Ty->isBlockPointerType() || Ty->isFunctionPointerType()) {
       const FunctionType *FT = Ty->isFunctionPointerType() ? getFunctionType(d)
-        : Ty->getAs<BlockPointerType>()->getPointeeType()->getAs<FunctionType>();
+       : Ty->getAs<BlockPointerType>()->getPointeeType()->getAs<FunctionType>();
       if (!cast<FunctionProtoType>(FT)->isVariadic()) {
         int m = Ty->isFunctionPointerType() ? 0 : 1;
         S.Diag(Attr.getLoc(), diag::warn_attribute_sentinel_not_variadic) << m;
@@ -1062,7 +1261,8 @@ static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
       << Attr.getName() << 6 /*function, method or block */;
     return;
   }
-  d->addAttr(::new (S.Context) SentinelAttr(Attr.getLoc(), S.Context, sentinel, nullPos));
+  d->addAttr(::new (S.Context) SentinelAttr(Attr.getLoc(), S.Context, sentinel,
+                                            nullPos));
 }
 
 static void HandleWarnUnusedResult(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -1093,28 +1293,28 @@ static void HandleWarnUnusedResult(Decl *D, const AttributeList &Attr, Sema &S)
   D->addAttr(::new (S.Context) WarnUnusedResultAttr(Attr.getLoc(), S.Context));
 }
 
-static void HandleWeakAttr(Decl *D, const AttributeList &Attr, Sema &S) {
+static void HandleWeakAttr(Decl *d, const AttributeList &attr, Sema &S) {
   // check the attribute arguments.
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+  if (attr.getNumArgs() != 0) {
+    S.Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
     return;
   }
 
-  /* weak only applies to non-static declarations */
-  if (isStaticVarOrStaticFunciton(D)) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_weak_static) <<
-      dyn_cast<NamedDecl>(D)->getNameAsString();
+  if (!isa<VarDecl>(d) && !isa<FunctionDecl>(d)) {
+    S.Diag(attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << attr.getName() << 2 /*variables and functions*/;
     return;
   }
 
-  // TODO: could also be applied to methods?
-  if (!isa<FunctionDecl>(D) && !isa<VarDecl>(D)) {
-    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 2 /*variable and function*/;
+  NamedDecl *nd = cast<NamedDecl>(d);
+
+  // 'weak' only applies to declarations with external linkage.
+  if (hasEffectivelyInternalLinkage(nd)) {
+    S.Diag(attr.getLoc(), diag::err_attribute_weak_static);
     return;
   }
 
-  D->addAttr(::new (S.Context) WeakAttr(Attr.getLoc(), S.Context));
+  nd->addAttr(::new (S.Context) WeakAttr(attr.getLoc(), S.Context));
 }
 
 static void HandleWeakImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -1163,7 +1363,7 @@ static void HandleReqdWorkGroupSize(Decl *D, const AttributeList &Attr,
 
   unsigned WGSize[3];
   for (unsigned i = 0; i < 3; ++i) {
-    Expr *E = static_cast<Expr *>(Attr.getArg(i));
+    Expr *E = Attr.getArg(i);
     llvm::APSInt ArgNum(32);
     if (E->isTypeDependent() || E->isValueDependent() ||
         !E->isIntegerConstantExpr(ArgNum, S.Context)) {
@@ -1187,7 +1387,7 @@ static void HandleSectionAttr(Decl *D, const AttributeList &Attr, Sema &S) {
 
   // Make sure that there is a string literal as the sections's single
   // argument.
-  Expr *ArgExpr = static_cast<Expr *>(Attr.getArg(0));
+  Expr *ArgExpr = Attr.getArg(0);
   StringLiteral *SE = dyn_cast<StringLiteral>(ArgExpr);
   if (!SE) {
     S.Diag(ArgExpr->getLocStart(), diag::err_attribute_not_string) << "section";
@@ -1208,7 +1408,8 @@ static void HandleSectionAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
   
-  D->addAttr(::new (S.Context) SectionAttr(Attr.getLoc(), S.Context, SE->getString()));
+  D->addAttr(::new (S.Context) SectionAttr(Attr.getLoc(), S.Context,
+                                           SE->getString()));
 }
 
 
@@ -1262,27 +1463,27 @@ static void HandleCleanupAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   // Look up the function
   // FIXME: Lookup probably isn't looking in the right place
-  // FIXME: The lookup source location should be in the attribute, not the
-  // start of the attribute.
   NamedDecl *CleanupDecl
-    = S.LookupSingleName(S.TUScope, Attr.getParameterName(), Attr.getLoc(),
-                         Sema::LookupOrdinaryName);
+    = S.LookupSingleName(S.TUScope, Attr.getParameterName(),
+                         Attr.getParameterLoc(), Sema::LookupOrdinaryName);
   if (!CleanupDecl) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_cleanup_arg_not_found) <<
+    S.Diag(Attr.getParameterLoc(), diag::err_attribute_cleanup_arg_not_found) <<
       Attr.getParameterName();
     return;
   }
 
   FunctionDecl *FD = dyn_cast<FunctionDecl>(CleanupDecl);
   if (!FD) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_cleanup_arg_not_function) <<
-      Attr.getParameterName();
+    S.Diag(Attr.getParameterLoc(),
+           diag::err_attribute_cleanup_arg_not_function)
+      << Attr.getParameterName();
     return;
   }
 
   if (FD->getNumParams() != 1) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_cleanup_func_must_take_one_arg) <<
-      Attr.getParameterName();
+    S.Diag(Attr.getParameterLoc(),
+           diag::err_attribute_cleanup_func_must_take_one_arg)
+      << Attr.getParameterName();
     return;
   }
 
@@ -1290,14 +1491,16 @@ static void HandleCleanupAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   // If this ever proves to be a problem it should be easy to fix.
   QualType Ty = S.Context.getPointerType(VD->getType());
   QualType ParamTy = FD->getParamDecl(0)->getType();
-  if (S.CheckAssignmentConstraints(ParamTy, Ty) != Sema::Compatible) {
-    S.Diag(Attr.getLoc(),
+  if (S.CheckAssignmentConstraints(FD->getParamDecl(0)->getLocation(),
+                                   ParamTy, Ty) != Sema::Compatible) {
+    S.Diag(Attr.getParameterLoc(),
            diag::err_attribute_cleanup_func_arg_incompatible_type) <<
       Attr.getParameterName() << ParamTy << Ty;
     return;
   }
 
   d->addAttr(::new (S.Context) CleanupAttr(Attr.getLoc(), S.Context, FD));
+  S.MarkDeclarationReferenced(Attr.getParameterLoc(), FD);
 }
 
 /// Handle __attribute__((format_arg((idx)))) attribute based on
@@ -1312,12 +1515,15 @@ static void HandleFormatArgAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     << Attr.getName() << 0 /*function*/;
     return;
   }
-  // FIXME: in C++ the implicit 'this' function parameter also counts.  this is
-  // needed in order to be compatible with GCC the index must start with 1.
-  unsigned NumArgs  = getFunctionOrMethodNumArgs(d);
+
+  // In C++ the implicit 'this' function parameter also counts, and they are
+  // counted from one.
+  bool HasImplicitThisParam = isInstanceMethod(d);
+  unsigned NumArgs  = getFunctionOrMethodNumArgs(d) + HasImplicitThisParam;
   unsigned FirstIdx = 1;
+
   // checks for the 2nd argument
-  Expr *IdxExpr = static_cast<Expr *>(Attr.getArg(0));
+  Expr *IdxExpr = Attr.getArg(0);
   llvm::APSInt Idx(32);
   if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent() ||
       !IdxExpr->isIntegerConstantExpr(Idx, S.Context)) {
@@ -1334,6 +1540,15 @@ static void HandleFormatArgAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   unsigned ArgIdx = Idx.getZExtValue() - 1;
 
+  if (HasImplicitThisParam) {
+    if (ArgIdx == 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_invalid_implicit_this_argument)
+        << "format_arg" << IdxExpr->getSourceRange();
+      return;
+    }
+    ArgIdx--;
+  }
+
   // make sure the format string is really a string
   QualType Ty = getFunctionOrMethodArgType(d, ArgIdx);
 
@@ -1360,7 +1575,8 @@ static void HandleFormatArgAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) FormatArgAttr(Attr.getLoc(), S.Context, Idx.getZExtValue()));
+  d->addAttr(::new (S.Context) FormatArgAttr(Attr.getLoc(), S.Context,
+                                             Idx.getZExtValue()));
 }
 
 enum FormatAttrKind {
@@ -1387,7 +1603,8 @@ static FormatAttrKind getFormatAttrKind(llvm::StringRef Format) {
   if (Format == "scanf" || Format == "printf" || Format == "printf0" ||
       Format == "strfmon" || Format == "cmn_err" || Format == "strftime" ||
       Format == "NSString" || Format == "CFString" || Format == "vcmn_err" ||
-      Format == "zcmn_err")
+      Format == "zcmn_err" ||
+      Format == "kprintf")  // OpenBSD.
     return SupportedFormat;
 
   if (Format == "gcc_diag" || Format == "gcc_cdiag" ||
@@ -1425,7 +1642,7 @@ static void HandleInitPriorityAttr(Decl *d, const AttributeList &Attr,
     Attr.setInvalid();
     return;
   }
-  Expr *priorityExpr = static_cast<Expr *>(Attr.getArg(0));
+  Expr *priorityExpr = Attr.getArg(0);
   
   llvm::APSInt priority(32);
   if (priorityExpr->isTypeDependent() || priorityExpr->isValueDependent() ||
@@ -1442,7 +1659,8 @@ static void HandleInitPriorityAttr(Decl *d, const AttributeList &Attr,
     Attr.setInvalid();
     return;
   }
-  d->addAttr(::new (S.Context) InitPriorityAttr(Attr.getLoc(), S.Context, prioritynum));
+  d->addAttr(::new (S.Context) InitPriorityAttr(Attr.getLoc(), S.Context,
+                                                prioritynum));
 }
 
 /// Handle __attribute__((format(type,idx,firstarg))) attributes based on
@@ -1466,7 +1684,10 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  unsigned NumArgs  = getFunctionOrMethodNumArgs(d);
+  // In C++ the implicit 'this' function parameter also counts, and they are
+  // counted from one.
+  bool HasImplicitThisParam = isInstanceMethod(d);
+  unsigned NumArgs  = getFunctionOrMethodNumArgs(d) + HasImplicitThisParam;
   unsigned FirstIdx = 1;
 
   llvm::StringRef Format = Attr.getParameterName()->getName();
@@ -1488,7 +1709,7 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 
   // checks for the 2nd argument
-  Expr *IdxExpr = static_cast<Expr *>(Attr.getArg(0));
+  Expr *IdxExpr = Attr.getArg(0);
   llvm::APSInt Idx(32);
   if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent() ||
       !IdxExpr->isIntegerConstantExpr(Idx, S.Context)) {
@@ -1497,16 +1718,6 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  // FIXME: We should handle the implicit 'this' parameter in a more generic
-  // way that can be used for other arguments.
-  bool HasImplicitThisParam = false;
-  if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(d)) {
-    if (MD->isInstance()) {
-      HasImplicitThisParam = true;
-      NumArgs++;
-    }
-  }
-
   if (Idx.getZExtValue() < FirstIdx || Idx.getZExtValue() > NumArgs) {
     S.Diag(Attr.getLoc(), diag::err_attribute_argument_out_of_bounds)
       << "format" << 2 << IdxExpr->getSourceRange();
@@ -1518,8 +1729,9 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   if (HasImplicitThisParam) {
     if (ArgIdx == 0) {
-      S.Diag(Attr.getLoc(), diag::err_format_attribute_not)
-        << "a string type" << IdxExpr->getSourceRange();
+      S.Diag(Attr.getLoc(),
+             diag::err_format_attribute_implicit_this_format_string)
+        << IdxExpr->getSourceRange();
       return;
     }
     ArgIdx--;
@@ -1552,7 +1764,7 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 
   // check the 3rd argument
-  Expr *FirstArgExpr = static_cast<Expr *>(Attr.getArg(1));
+  Expr *FirstArgExpr = Attr.getArg(1);
   llvm::APSInt FirstArg(32);
   if (FirstArgExpr->isTypeDependent() || FirstArgExpr->isValueDependent() ||
       !FirstArgExpr->isIntegerConstantExpr(FirstArg, S.Context)) {
@@ -1665,7 +1877,7 @@ static void HandleAnnotateAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
     return;
   }
-  Expr *ArgExpr = static_cast<Expr *>(Attr.getArg(0));
+  Expr *ArgExpr = Attr.getArg(0);
   StringLiteral *SE = dyn_cast<StringLiteral>(ArgExpr);
 
   // Make sure that there is a string literal as the annotation's single
@@ -1674,7 +1886,8 @@ static void HandleAnnotateAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     S.Diag(ArgExpr->getLocStart(), diag::err_attribute_not_string) <<"annotate";
     return;
   }
-  d->addAttr(::new (S.Context) AnnotateAttr(Attr.getLoc(), S.Context, SE->getString()));
+  d->addAttr(::new (S.Context) AnnotateAttr(Attr.getLoc(), S.Context,
+                                            SE->getString()));
 }
 
 static void HandleAlignedAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -1693,7 +1906,7 @@ static void HandleAlignedAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  S.AddAlignedAttr(Attr.getLoc(), D, static_cast<Expr *>(Attr.getArg(0)));
+  S.AddAlignedAttr(Attr.getLoc(), D, Attr.getArg(0));
 }
 
 void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E) {
@@ -1943,7 +2156,123 @@ static void HandleNoInstrumentFunctionAttr(Decl *d, const AttributeList &Attr,
     return;
   }
 
-  d->addAttr(::new (S.Context) NoInstrumentFunctionAttr(Attr.getLoc(), S.Context));
+  d->addAttr(::new (S.Context) NoInstrumentFunctionAttr(Attr.getLoc(),
+                                                        S.Context));
+}
+
+static void HandleConstantAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+      return;
+    }
+
+    if (!isa<VarDecl>(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attr.getName() << 12 /*variable*/;
+      return;
+    }
+
+    d->addAttr(::new (S.Context) CUDAConstantAttr(Attr.getLoc(), S.Context));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "constant";
+  }
+}
+
+static void HandleDeviceAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+      return;
+    }
+
+    if (!isa<FunctionDecl>(d) && !isa<VarDecl>(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attr.getName() << 2 /*variable and function*/;
+      return;
+    }
+
+    d->addAttr(::new (S.Context) CUDADeviceAttr(Attr.getLoc(), S.Context));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "device";
+  }
+}
+
+static void HandleGlobalAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+      return;
+    }
+
+    if (!isa<FunctionDecl>(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attr.getName() << 0 /*function*/;
+      return;
+    }
+
+    FunctionDecl *FD = cast<FunctionDecl>(d);
+    if (!FD->getResultType()->isVoidType()) {
+      TypeLoc TL = FD->getTypeSourceInfo()->getTypeLoc().IgnoreParens();
+      if (FunctionTypeLoc* FTL = dyn_cast<FunctionTypeLoc>(&TL)) {
+        S.Diag(FD->getTypeSpecStartLoc(), diag::err_kern_type_not_void_return)
+          << FD->getType()
+          << FixItHint::CreateReplacement(FTL->getResultLoc().getSourceRange(),
+                                          "void");
+      } else {
+        S.Diag(FD->getTypeSpecStartLoc(), diag::err_kern_type_not_void_return)
+          << FD->getType();
+      }
+      return;
+    }
+
+    d->addAttr(::new (S.Context) CUDAGlobalAttr(Attr.getLoc(), S.Context));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "global";
+  }
+}
+
+static void HandleHostAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+      return;
+    }
+
+    if (!isa<FunctionDecl>(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attr.getName() << 0 /*function*/;
+      return;
+    }
+
+    d->addAttr(::new (S.Context) CUDAHostAttr(Attr.getLoc(), S.Context));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "host";
+  }
+}
+
+static void HandleSharedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 0) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+      return;
+    }
+
+    if (!isa<VarDecl>(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+        << Attr.getName() << 12 /*variable*/;
+      return;
+    }
+
+    d->addAttr(::new (S.Context) CUDASharedAttr(Attr.getLoc(), S.Context));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "shared";
+  }
 }
 
 static void HandleGNUInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1968,26 +2297,36 @@ static void HandleGNUInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   d->addAttr(::new (S.Context) GNUInlineAttr(Attr.getLoc(), S.Context));
 }
 
-static void HandleCallConvAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  // Diagnostic is emitted elsewhere: here we store the (valid) Attr
+static void HandleCallConvAttr(Decl *d, const AttributeList &attr, Sema &S) {
+  if (hasDeclarator(d)) return;
+
+  // Diagnostic is emitted elsewhere: here we store the (valid) attr
   // in the Decl node for syntactic reasoning, e.g., pretty-printing.
-  assert(Attr.isInvalid() == false);
+  CallingConv CC;
+  if (S.CheckCallingConvAttr(attr, CC))
+    return;
 
-  switch (Attr.getKind()) {
+  if (!isa<ObjCMethodDecl>(d)) {
+    S.Diag(attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << attr.getName() << 0 /*function*/;
+    return;
+  }
+
+  switch (attr.getKind()) {
   case AttributeList::AT_fastcall:
-    d->addAttr(::new (S.Context) FastCallAttr(Attr.getLoc(), S.Context));
+    d->addAttr(::new (S.Context) FastCallAttr(attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_stdcall:
-    d->addAttr(::new (S.Context) StdCallAttr(Attr.getLoc(), S.Context));
+    d->addAttr(::new (S.Context) StdCallAttr(attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_thiscall:
-    d->addAttr(::new (S.Context) ThisCallAttr(Attr.getLoc(), S.Context));
+    d->addAttr(::new (S.Context) ThisCallAttr(attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_cdecl:
-    d->addAttr(::new (S.Context) CDeclAttr(Attr.getLoc(), S.Context));
+    d->addAttr(::new (S.Context) CDeclAttr(attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_pascal:
-    d->addAttr(::new (S.Context) PascalAttr(Attr.getLoc(), S.Context));
+    d->addAttr(::new (S.Context) PascalAttr(attr.getLoc(), S.Context));
     return;
   default:
     llvm_unreachable("unexpected attribute kind");
@@ -1995,214 +2334,336 @@ static void HandleCallConvAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 }
 
-static void HandleRegparmAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  // check the attribute arguments.
-  if (Attr.getNumArgs() != 1) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
-    return;
+static void HandleOpenCLKernelAttr(Decl *d, const AttributeList &Attr, Sema &S){
+  assert(Attr.isInvalid() == false);
+  d->addAttr(::new (S.Context) OpenCLKernelAttr(Attr.getLoc(), S.Context));
+}
+
+bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
+  if (attr.isInvalid())
+    return true;
+
+  if (attr.getNumArgs() != 0) {
+    Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+    attr.setInvalid();
+    return true;
   }
 
-  if (!isFunctionOrMethod(d)) {
-    S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
-    << Attr.getName() << 0 /*function*/;
+  // TODO: diagnose uses of these conventions on the wrong target.
+  switch (attr.getKind()) {
+  case AttributeList::AT_cdecl: CC = CC_C; break;
+  case AttributeList::AT_fastcall: CC = CC_X86FastCall; break;
+  case AttributeList::AT_stdcall: CC = CC_X86StdCall; break;
+  case AttributeList::AT_thiscall: CC = CC_X86ThisCall; break;
+  case AttributeList::AT_pascal: CC = CC_X86Pascal; break;
+  default: llvm_unreachable("unexpected attribute kind"); return true;
+  }
+
+  return false;
+}
+
+static void HandleRegparmAttr(Decl *d, const AttributeList &attr, Sema &S) {
+  if (hasDeclarator(d)) return;
+
+  unsigned numParams;
+  if (S.CheckRegparmAttr(attr, numParams))
+    return;
+
+  if (!isa<ObjCMethodDecl>(d)) {
+    S.Diag(attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << attr.getName() << 0 /*function*/;
     return;
   }
 
-  Expr *NumParamsExpr = static_cast<Expr *>(Attr.getArg(0));
+  d->addAttr(::new (S.Context) RegparmAttr(attr.getLoc(), S.Context, numParams));
+}
+
+/// Checks a regparm attribute, returning true if it is ill-formed and
+/// otherwise setting numParams to the appropriate value.
+bool Sema::CheckRegparmAttr(const AttributeList &attr, unsigned &numParams) {
+  if (attr.isInvalid())
+    return true;
+
+  if (attr.getNumArgs() != 1) {
+    Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+    attr.setInvalid();
+    return true;
+  }
+
+  Expr *NumParamsExpr = attr.getArg(0);
   llvm::APSInt NumParams(32);
   if (NumParamsExpr->isTypeDependent() || NumParamsExpr->isValueDependent() ||
-      !NumParamsExpr->isIntegerConstantExpr(NumParams, S.Context)) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_argument_not_int)
+      !NumParamsExpr->isIntegerConstantExpr(NumParams, Context)) {
+    Diag(attr.getLoc(), diag::err_attribute_argument_not_int)
       << "regparm" << NumParamsExpr->getSourceRange();
-    return;
+    attr.setInvalid();
+    return true;
   }
 
-  if (S.Context.Target.getRegParmMax() == 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_regparm_wrong_platform)
+  if (Context.Target.getRegParmMax() == 0) {
+    Diag(attr.getLoc(), diag::err_attribute_regparm_wrong_platform)
       << NumParamsExpr->getSourceRange();
-    return;
+    attr.setInvalid();
+    return true;
   }
 
-  if (NumParams.getLimitedValue(255) > S.Context.Target.getRegParmMax()) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_regparm_invalid_number)
-      << S.Context.Target.getRegParmMax() << NumParamsExpr->getSourceRange();
-    return;
+  numParams = NumParams.getZExtValue();
+  if (numParams > Context.Target.getRegParmMax()) {
+    Diag(attr.getLoc(), diag::err_attribute_regparm_invalid_number)
+      << Context.Target.getRegParmMax() << NumParamsExpr->getSourceRange();
+    attr.setInvalid();
+    return true;
   }
 
-  d->addAttr(::new (S.Context) RegparmAttr(Attr.getLoc(), S.Context,
-                                           NumParams.getZExtValue()));
+  return false;
 }
 
-static void HandleFinalAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  // check the attribute arguments.
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    return;
-  }
+static void HandleLaunchBoundsAttr(Decl *d, const AttributeList &Attr, Sema &S){
+  if (S.LangOpts.CUDA) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 1 && Attr.getNumArgs() != 2) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+        << "1 or 2";
+      return;
+    }
 
-  if (!isa<CXXRecordDecl>(d)
-   && (!isa<CXXMethodDecl>(d) || !cast<CXXMethodDecl>(d)->isVirtual())) {
-    S.Diag(Attr.getLoc(),
-           Attr.isCXX0XAttribute() ? diag::err_attribute_wrong_decl_type
-                                   : diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 7 /*virtual method or class*/;
-    return;
-  }
-  
-  // FIXME: Conform to C++0x redeclaration rules.
-  
-  if (d->getAttr<FinalAttr>()) {
-    S.Diag(Attr.getLoc(), diag::err_repeat_attribute) << "final";
-    return;
-  }
+    if (!isFunctionOrMethod(d)) {
+      S.Diag(Attr.getLoc(), diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << 0 /*function*/;
+      return;
+    }
+
+    Expr *MaxThreadsExpr = Attr.getArg(0);
+    llvm::APSInt MaxThreads(32);
+    if (MaxThreadsExpr->isTypeDependent() ||
+        MaxThreadsExpr->isValueDependent() ||
+        !MaxThreadsExpr->isIntegerConstantExpr(MaxThreads, S.Context)) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_int)
+        << "launch_bounds" << 1 << MaxThreadsExpr->getSourceRange();
+      return;
+    }
 
-  d->addAttr(::new (S.Context) FinalAttr(Attr.getLoc(), S.Context));
+    llvm::APSInt MinBlocks(32);
+    if (Attr.getNumArgs() > 1) {
+      Expr *MinBlocksExpr = Attr.getArg(1);
+      if (MinBlocksExpr->isTypeDependent() ||
+          MinBlocksExpr->isValueDependent() ||
+          !MinBlocksExpr->isIntegerConstantExpr(MinBlocks, S.Context)) {
+        S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_int)
+          << "launch_bounds" << 2 << MinBlocksExpr->getSourceRange();
+        return;
+      }
+    }
+
+    d->addAttr(::new (S.Context) CUDALaunchBoundsAttr(Attr.getLoc(), S.Context,
+                                                      MaxThreads.getZExtValue(),
+                                                     MinBlocks.getZExtValue()));
+  } else {
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "launch_bounds";
+  }
 }
 
 //===----------------------------------------------------------------------===//
-// C++0x member checking attributes
+// Checker-specific attribute handlers.
 //===----------------------------------------------------------------------===//
 
-static void HandleBaseCheckAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    return;
-  }
-  
-  if (!isa<CXXRecordDecl>(d)) {
-    S.Diag(Attr.getLoc(),
-           Attr.isCXX0XAttribute() ? diag::err_attribute_wrong_decl_type
-                                   : diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 9 /*class*/;
-    return;
-  }
-  
-  if (d->getAttr<BaseCheckAttr>()) {
-    S.Diag(Attr.getLoc(), diag::err_repeat_attribute) << "base_check";
-    return;
-  }
-  
-  d->addAttr(::new (S.Context) BaseCheckAttr(Attr.getLoc(), S.Context));
+static bool isValidSubjectOfNSAttribute(Sema &S, QualType type) {
+  return type->isObjCObjectPointerType() || S.Context.isObjCNSObjectType(type);
+}
+static bool isValidSubjectOfCFAttribute(Sema &S, QualType type) {
+  return type->isPointerType() || isValidSubjectOfNSAttribute(S, type);
 }
 
-static void HandleHidingAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+static void HandleNSConsumedAttr(Decl *d, const AttributeList &attr, Sema &S) {
+  ParmVarDecl *param = dyn_cast<ParmVarDecl>(d);
+  if (!param) {
+    S.Diag(d->getLocStart(), diag::warn_attribute_wrong_decl_type)
+      << SourceRange(attr.getLoc()) << attr.getName() << 4 /*parameter*/;
     return;
   }
 
-  if (!isa<RecordDecl>(d->getDeclContext())) {
-    // FIXME: It's not the type that's the problem
-    S.Diag(Attr.getLoc(),
-           Attr.isCXX0XAttribute() ? diag::err_attribute_wrong_decl_type
-                                   : diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 11 /*member*/;
-    return;
+  bool typeOK, cf;
+  if (attr.getKind() == AttributeList::AT_ns_consumed) {
+    typeOK = isValidSubjectOfNSAttribute(S, param->getType());
+    cf = false;
+  } else {
+    typeOK = isValidSubjectOfCFAttribute(S, param->getType());
+    cf = true;
   }
 
-  // FIXME: Conform to C++0x redeclaration rules.
-
-  if (d->getAttr<HidingAttr>()) {
-    S.Diag(Attr.getLoc(), diag::err_repeat_attribute) << "hiding";
+  if (!typeOK) {
+    S.Diag(d->getLocStart(), diag::warn_ns_attribute_wrong_parameter_type)
+      << SourceRange(attr.getLoc()) << attr.getName() << cf;
     return;
   }
 
-  d->addAttr(::new (S.Context) HidingAttr(Attr.getLoc(), S.Context));
+  if (cf)
+    param->addAttr(::new (S.Context) CFConsumedAttr(attr.getLoc(), S.Context));
+  else
+    param->addAttr(::new (S.Context) NSConsumedAttr(attr.getLoc(), S.Context));  
 }
 
-static void HandleOverrideAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    return;
-  }
-
-  if (!isa<CXXMethodDecl>(d) || !cast<CXXMethodDecl>(d)->isVirtual()) {
-    // FIXME: It's not the type that's the problem
-    S.Diag(Attr.getLoc(),
-           Attr.isCXX0XAttribute() ? diag::err_attribute_wrong_decl_type
-                                   : diag::warn_attribute_wrong_decl_type)
-      << Attr.getName() << 10 /*virtual method*/;
-    return;
-  }
-
-  // FIXME: Conform to C++0x redeclaration rules.
-
-  if (d->getAttr<OverrideAttr>()) {
-    S.Diag(Attr.getLoc(), diag::err_repeat_attribute) << "override";
+static void HandleNSConsumesSelfAttr(Decl *d, const AttributeList &attr,
+                                     Sema &S) {
+  if (!isa<ObjCMethodDecl>(d)) {
+    S.Diag(d->getLocStart(), diag::warn_attribute_wrong_decl_type)
+      << SourceRange(attr.getLoc()) << attr.getName() << 13 /*method*/;
     return;
   }
 
-  d->addAttr(::new (S.Context) OverrideAttr(Attr.getLoc(), S.Context));
+  d->addAttr(::new (S.Context) NSConsumesSelfAttr(attr.getLoc(), S.Context));
 }
 
-//===----------------------------------------------------------------------===//
-// Checker-specific attribute handlers.
-//===----------------------------------------------------------------------===//
-
-static void HandleNSReturnsRetainedAttr(Decl *d, const AttributeList &Attr,
+static void HandleNSReturnsRetainedAttr(Decl *d, const AttributeList &attr,
                                         Sema &S) {
 
-  QualType RetTy;
+  QualType returnType;
 
   if (ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(d))
-    RetTy = MD->getResultType();
+    returnType = MD->getResultType();
   else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(d))
-    RetTy = FD->getResultType();
+    returnType = FD->getResultType();
   else {
-    SourceLocation L = Attr.getLoc();
     S.Diag(d->getLocStart(), diag::warn_attribute_wrong_decl_type)
-        << SourceRange(L, L) << Attr.getName() << 3 /* function or method */;
+        << SourceRange(attr.getLoc()) << attr.getName()
+        << 3 /* function or method */;
     return;
   }
 
-  if (!(S.Context.isObjCNSObjectType(RetTy) || RetTy->getAs<PointerType>()
-        || RetTy->getAs<ObjCObjectPointerType>())) {
-    SourceLocation L = Attr.getLoc();
+  bool typeOK;
+  bool cf;
+  switch (attr.getKind()) {
+  default: llvm_unreachable("invalid ownership attribute"); return;
+  case AttributeList::AT_ns_returns_autoreleased:
+  case AttributeList::AT_ns_returns_retained:
+  case AttributeList::AT_ns_returns_not_retained:
+    typeOK = isValidSubjectOfNSAttribute(S, returnType);
+    cf = false;
+    break;
+
+  case AttributeList::AT_cf_returns_retained:
+  case AttributeList::AT_cf_returns_not_retained:
+    typeOK = isValidSubjectOfCFAttribute(S, returnType);
+    cf = true;
+    break;
+  }
+
+  if (!typeOK) {
     S.Diag(d->getLocStart(), diag::warn_ns_attribute_wrong_return_type)
-      << SourceRange(L, L) << Attr.getName();
+      << SourceRange(attr.getLoc())
+      << attr.getName() << isa<ObjCMethodDecl>(d) << cf;
     return;
   }
 
-  switch (Attr.getKind()) {
+  switch (attr.getKind()) {
     default:
       assert(0 && "invalid ownership attribute");
       return;
+    case AttributeList::AT_ns_returns_autoreleased:
+      d->addAttr(::new (S.Context) NSReturnsAutoreleasedAttr(attr.getLoc(),
+                                                             S.Context));
+      return;
     case AttributeList::AT_cf_returns_not_retained:
-      d->addAttr(::new (S.Context) CFReturnsNotRetainedAttr(Attr.getLoc(), S.Context));
+      d->addAttr(::new (S.Context) CFReturnsNotRetainedAttr(attr.getLoc(),
+                                                            S.Context));
       return;
     case AttributeList::AT_ns_returns_not_retained:
-      d->addAttr(::new (S.Context) NSReturnsNotRetainedAttr(Attr.getLoc(), S.Context));
+      d->addAttr(::new (S.Context) NSReturnsNotRetainedAttr(attr.getLoc(),
+                                                            S.Context));
       return;
     case AttributeList::AT_cf_returns_retained:
-      d->addAttr(::new (S.Context) CFReturnsRetainedAttr(Attr.getLoc(), S.Context));
+      d->addAttr(::new (S.Context) CFReturnsRetainedAttr(attr.getLoc(),
+                                                         S.Context));
       return;
     case AttributeList::AT_ns_returns_retained:
-      d->addAttr(::new (S.Context) NSReturnsRetainedAttr(Attr.getLoc(), S.Context));
+      d->addAttr(::new (S.Context) NSReturnsRetainedAttr(attr.getLoc(),
+                                                         S.Context));
       return;
   };
 }
 
 static bool isKnownDeclSpecAttr(const AttributeList &Attr) {
   return Attr.getKind() == AttributeList::AT_dllimport ||
-         Attr.getKind() == AttributeList::AT_dllexport;
+         Attr.getKind() == AttributeList::AT_dllexport ||
+         Attr.getKind() == AttributeList::AT_uuid;
+}
+
+//===----------------------------------------------------------------------===//
+// Microsoft specific attribute handlers.
+//===----------------------------------------------------------------------===//
+
+static void HandleUuidAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  if (S.LangOpts.Microsoft || S.LangOpts.Borland) {
+    // check the attribute arguments.
+    if (Attr.getNumArgs() != 1) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+      return;
+    }
+    Expr *Arg = Attr.getArg(0);
+    StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
+    if (Str == 0 || Str->isWide()) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
+        << "uuid" << 1;
+      return;
+    }
+
+    llvm::StringRef StrRef = Str->getString();
+
+    bool IsCurly = StrRef.size() > 1 && StrRef.front() == '{' &&
+                   StrRef.back() == '}';
+    
+    // Validate GUID length.
+    if (IsCurly && StrRef.size() != 38) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_uuid_malformed_guid);
+      return;
+    }
+    if (!IsCurly && StrRef.size() != 36) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_uuid_malformed_guid);
+      return;
+    }
+
+    // GUID format is "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" or 
+    // "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}"
+    llvm::StringRef::iterator I = StrRef.begin();
+    if (IsCurly) // Skip the optional '{'
+       ++I;
+
+    for (int i = 0; i < 36; ++i) {
+      if (i == 8 || i == 13 || i == 18 || i == 23) {
+        if (*I != '-') {
+          S.Diag(Attr.getLoc(), diag::err_attribute_uuid_malformed_guid);
+          return;
+        }
+      } else if (!isxdigit(*I)) {
+        S.Diag(Attr.getLoc(), diag::err_attribute_uuid_malformed_guid);
+        return;
+      }
+      I++;
+    }
+
+    d->addAttr(::new (S.Context) UuidAttr(Attr.getLoc(), S.Context, 
+                                          Str->getString()));
+  } else
+    S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << "uuid";
 }
 
 //===----------------------------------------------------------------------===//
 // Top Level Sema Entry Points
 //===----------------------------------------------------------------------===//
 
-/// ProcessDeclAttribute - Apply the specific attribute to the specified decl if
-/// the attribute applies to decls.  If the attribute is a type attribute, just
-/// silently ignore it if a GNU attribute. FIXME: Applying a C++0x attribute to
-/// the wrong thing is illegal (C++0x [dcl.attr.grammar]/4).
-static void ProcessDeclAttribute(Scope *scope, Decl *D,
-                                 const AttributeList &Attr, Sema &S) {
-  if (Attr.isInvalid())
-    return;
+static void ProcessNonInheritableDeclAttr(Scope *scope, Decl *D,
+                                          const AttributeList &Attr, Sema &S) {
+  switch (Attr.getKind()) {
+  case AttributeList::AT_device:      HandleDeviceAttr      (D, Attr, S); break;
+  case AttributeList::AT_host:        HandleHostAttr        (D, Attr, S); break;
+  case AttributeList::AT_overloadable:HandleOverloadableAttr(D, Attr, S); break;
+  default:
+    break;
+  }
+}
 
-  if (Attr.isDeclspecAttribute() && !isKnownDeclSpecAttr(Attr))
-    // FIXME: Try to deal with other __declspec attributes!
-    return;
+static void ProcessInheritableDeclAttr(Scope *scope, Decl *D,
+                                       const AttributeList &Attr, Sema &S) {
   switch (Attr.getKind()) {
   case AttributeList::AT_IBAction:            HandleIBAction(D, Attr, S); break;
     case AttributeList::AT_IBOutlet:          HandleIBOutlet(D, Attr, S); break;
@@ -2211,9 +2672,17 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_address_space:
   case AttributeList::AT_objc_gc:
   case AttributeList::AT_vector_size:
+  case AttributeList::AT_neon_vector_type:
+  case AttributeList::AT_neon_polyvector_type:
     // Ignore these, these are type attributes, handled by
     // ProcessTypeAttributes.
     break;
+  case AttributeList::AT_device:
+  case AttributeList::AT_host:
+  case AttributeList::AT_overloadable:
+    // Ignore, this is a non-inheritable attribute, handled
+    // by ProcessNonInheritableDeclAttr.
+    break;
   case AttributeList::AT_alias:       HandleAliasAttr       (D, Attr, S); break;
   case AttributeList::AT_aligned:     HandleAlignedAttr     (D, Attr, S); break;
   case AttributeList::AT_always_inline:
@@ -2221,33 +2690,45 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_analyzer_noreturn:
     HandleAnalyzerNoReturnAttr  (D, Attr, S); break;
   case AttributeList::AT_annotate:    HandleAnnotateAttr    (D, Attr, S); break;
-  case AttributeList::AT_base_check:  HandleBaseCheckAttr   (D, Attr, S); break;
   case AttributeList::AT_carries_dependency:
                                       HandleDependencyAttr  (D, Attr, S); break;
+  case AttributeList::AT_common:      HandleCommonAttr      (D, Attr, S); break;
+  case AttributeList::AT_constant:    HandleConstantAttr    (D, Attr, S); break;
   case AttributeList::AT_constructor: HandleConstructorAttr (D, Attr, S); break;
   case AttributeList::AT_deprecated:  HandleDeprecatedAttr  (D, Attr, S); break;
   case AttributeList::AT_destructor:  HandleDestructorAttr  (D, Attr, S); break;
   case AttributeList::AT_ext_vector_type:
     HandleExtVectorTypeAttr(scope, D, Attr, S);
     break;
-  case AttributeList::AT_final:       HandleFinalAttr       (D, Attr, S); break;
   case AttributeList::AT_format:      HandleFormatAttr      (D, Attr, S); break;
   case AttributeList::AT_format_arg:  HandleFormatArgAttr   (D, Attr, S); break;
+  case AttributeList::AT_global:      HandleGlobalAttr      (D, Attr, S); break;
   case AttributeList::AT_gnu_inline:  HandleGNUInlineAttr   (D, Attr, S); break;
-  case AttributeList::AT_hiding:      HandleHidingAttr      (D, Attr, S); break;
+  case AttributeList::AT_launch_bounds:
+    HandleLaunchBoundsAttr(D, Attr, S);
+    break;
   case AttributeList::AT_mode:        HandleModeAttr        (D, Attr, S); break;
   case AttributeList::AT_malloc:      HandleMallocAttr      (D, Attr, S); break;
+  case AttributeList::AT_may_alias:   HandleMayAliasAttr    (D, Attr, S); break;
+  case AttributeList::AT_nocommon:    HandleNoCommonAttr    (D, Attr, S); break;
   case AttributeList::AT_nonnull:     HandleNonNullAttr     (D, Attr, S); break;
   case AttributeList::AT_ownership_returns:
   case AttributeList::AT_ownership_takes:
   case AttributeList::AT_ownership_holds:
       HandleOwnershipAttr     (D, Attr, S); break;
+  case AttributeList::AT_naked:       HandleNakedAttr       (D, Attr, S); break;
   case AttributeList::AT_noreturn:    HandleNoReturnAttr    (D, Attr, S); break;
   case AttributeList::AT_nothrow:     HandleNothrowAttr     (D, Attr, S); break;
-  case AttributeList::AT_override:    HandleOverrideAttr    (D, Attr, S); break;
+  case AttributeList::AT_shared:      HandleSharedAttr      (D, Attr, S); break;
   case AttributeList::AT_vecreturn:   HandleVecReturnAttr   (D, Attr, S); break;
 
   // Checker-specific.
+  case AttributeList::AT_cf_consumed:
+  case AttributeList::AT_ns_consumed: HandleNSConsumedAttr  (D, Attr, S); break;
+  case AttributeList::AT_ns_consumes_self:
+    HandleNSConsumesSelfAttr(D, Attr, S); break;
+
+  case AttributeList::AT_ns_returns_autoreleased:
   case AttributeList::AT_ns_returns_not_retained:
   case AttributeList::AT_cf_returns_not_retained:
   case AttributeList::AT_ns_returns_retained:
@@ -2277,7 +2758,6 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_objc_exception:
     HandleObjCExceptionAttr(D, Attr, S);
     break;
-  case AttributeList::AT_overloadable:HandleOverloadableAttr(D, Attr, S); break;
   case AttributeList::AT_nsobject:    HandleObjCNSObject    (D, Attr, S); break;
   case AttributeList::AT_blocks:      HandleBlocksAttr      (D, Attr, S); break;
   case AttributeList::AT_sentinel:    HandleSentinelAttr    (D, Attr, S); break;
@@ -2300,6 +2780,12 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_pascal:
     HandleCallConvAttr(D, Attr, S);
     break;
+  case AttributeList::AT_opencl_kernel_function:
+    HandleOpenCLKernelAttr(D, Attr, S);
+    break;
+  case AttributeList::AT_uuid:
+    HandleUuidAttr(D, Attr, S);
+    break;
   default:
     // Ask target about the attribute.
     const TargetAttributesSema &TargetAttrs = S.getTargetAttributesSema();
@@ -2310,17 +2796,40 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   }
 }
 
+/// ProcessDeclAttribute - Apply the specific attribute to the specified decl if
+/// the attribute applies to decls.  If the attribute is a type attribute, just
+/// silently ignore it if a GNU attribute. FIXME: Applying a C++0x attribute to
+/// the wrong thing is illegal (C++0x [dcl.attr.grammar]/4).
+static void ProcessDeclAttribute(Scope *scope, Decl *D,
+                                 const AttributeList &Attr, Sema &S,
+                                 bool NonInheritable, bool Inheritable) {
+  if (Attr.isInvalid())
+    return;
+
+  if (Attr.isDeclspecAttribute() && !isKnownDeclSpecAttr(Attr))
+    // FIXME: Try to deal with other __declspec attributes!
+    return;
+
+  if (NonInheritable)
+    ProcessNonInheritableDeclAttr(scope, D, Attr, S);
+
+  if (Inheritable)
+    ProcessInheritableDeclAttr(scope, D, Attr, S);
+}
+
 /// ProcessDeclAttributeList - Apply all the decl attributes in the specified
 /// attribute list to the specified decl, ignoring any type attributes.
-void Sema::ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *AttrList) {
+void Sema::ProcessDeclAttributeList(Scope *S, Decl *D,
+                                    const AttributeList *AttrList,
+                                    bool NonInheritable, bool Inheritable) {
   for (const AttributeList* l = AttrList; l; l = l->getNext()) {
-    ProcessDeclAttribute(S, D, *l, *this);
+    ProcessDeclAttribute(S, D, *l, *this, NonInheritable, Inheritable);
   }
 
   // GCC accepts
   // static int a9 __attribute__((weakref));
   // but that looks really pointless. We reject it.
-  if (D->hasAttr<WeakRefAttr>() && !D->hasAttr<AliasAttr>()) {
+  if (Inheritable && D->hasAttr<WeakRefAttr>() && !D->hasAttr<AliasAttr>()) {
     Diag(AttrList->getLoc(), diag::err_attribute_weakref_without_alias) <<
     dyn_cast<NamedDecl>(D)->getNameAsString();
     return;
@@ -2380,22 +2889,27 @@ void Sema::DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W) {
 /// ProcessDeclAttributes - Given a declarator (PD) with attributes indicated in
 /// it, apply them to D.  This is a bit tricky because PD can have attributes
 /// specified in many different places, and we need to find and apply them all.
-void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD) {
-  // Handle #pragma weak
-  if (NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
-    if (ND->hasLinkage()) {
-      WeakInfo W = WeakUndeclaredIdentifiers.lookup(ND->getIdentifier());
-      if (W != WeakInfo()) {
-        // Identifier referenced by #pragma weak before it was declared
-        DeclApplyPragmaWeak(S, ND, W);
-        WeakUndeclaredIdentifiers[ND->getIdentifier()] = W;
+void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD,
+                                 bool NonInheritable, bool Inheritable) {
+  // It's valid to "forward-declare" #pragma weak, in which case we
+  // have to do this.
+  if (Inheritable && !WeakUndeclaredIdentifiers.empty()) {
+    if (NamedDecl *ND = dyn_cast<NamedDecl>(D)) {
+      if (IdentifierInfo *Id = ND->getIdentifier()) {
+        llvm::DenseMap<IdentifierInfo*,WeakInfo>::iterator I
+          = WeakUndeclaredIdentifiers.find(Id);
+        if (I != WeakUndeclaredIdentifiers.end() && ND->hasLinkage()) {
+          WeakInfo W = I->second;
+          DeclApplyPragmaWeak(S, ND, W);
+          WeakUndeclaredIdentifiers[Id] = W;
+        }
       }
     }
   }
 
   // Apply decl attributes from the DeclSpec if present.
-  if (const AttributeList *Attrs = PD.getDeclSpec().getAttributes())
-    ProcessDeclAttributeList(S, D, Attrs);
+  if (const AttributeList *Attrs = PD.getDeclSpec().getAttributes().getList())
+    ProcessDeclAttributeList(S, D, Attrs, NonInheritable, Inheritable);
 
   // Walk the declarator structure, applying decl attributes that were in a type
   // position to the decl itself.  This handles cases like:
@@ -2403,66 +2917,84 @@ void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD) {
   // when X is a decl attribute.
   for (unsigned i = 0, e = PD.getNumTypeObjects(); i != e; ++i)
     if (const AttributeList *Attrs = PD.getTypeObject(i).getAttrs())
-      ProcessDeclAttributeList(S, D, Attrs);
+      ProcessDeclAttributeList(S, D, Attrs, NonInheritable, Inheritable);
 
   // Finally, apply any attributes on the decl itself.
   if (const AttributeList *Attrs = PD.getAttributes())
-    ProcessDeclAttributeList(S, D, Attrs);
+    ProcessDeclAttributeList(S, D, Attrs, NonInheritable, Inheritable);
 }
 
-/// PushParsingDeclaration - Enter a new "scope" of deprecation
-/// warnings.
-///
-/// The state token we use is the start index of this scope
-/// on the warning stack.
-Sema::ParsingDeclStackState Sema::PushParsingDeclaration() {
-  ParsingDeclDepth++;
-  return (ParsingDeclStackState) DelayedDiagnostics.size();
+// This duplicates a vector push_back but hides the need to know the
+// size of the type.
+void Sema::DelayedDiagnostics::add(const DelayedDiagnostic &diag) {
+  assert(StackSize <= StackCapacity);
+
+  // Grow the stack if necessary.
+  if (StackSize == StackCapacity) {
+    unsigned newCapacity = 2 * StackCapacity + 2;
+    char *newBuffer = new char[newCapacity * sizeof(DelayedDiagnostic)];
+    const char *oldBuffer = (const char*) Stack;
+
+    if (StackCapacity)
+      memcpy(newBuffer, oldBuffer, StackCapacity * sizeof(DelayedDiagnostic));
+    
+    delete[] oldBuffer;
+    Stack = reinterpret_cast<sema::DelayedDiagnostic*>(newBuffer);
+    StackCapacity = newCapacity;
+  }
+
+  assert(StackSize < StackCapacity);
+  new (&Stack[StackSize++]) DelayedDiagnostic(diag);
 }
 
-void Sema::PopParsingDeclaration(ParsingDeclStackState S, Decl *D) {
-  assert(ParsingDeclDepth > 0 && "empty ParsingDeclaration stack");
-  ParsingDeclDepth--;
+void Sema::DelayedDiagnostics::popParsingDecl(Sema &S, ParsingDeclState state,
+                                              Decl *decl) {
+  DelayedDiagnostics &DD = S.DelayedDiagnostics;
 
-  if (DelayedDiagnostics.empty())
-    return;
+  // Check the invariants.
+  assert(DD.StackSize >= state.SavedStackSize);
+  assert(state.SavedStackSize >= DD.ActiveStackBase);
+  assert(DD.ParsingDepth > 0);
 
-  unsigned SavedIndex = (unsigned) S;
-  assert(SavedIndex <= DelayedDiagnostics.size() &&
-         "saved index is out of bounds");
+  // Drop the parsing depth.
+  DD.ParsingDepth--;
 
-  unsigned E = DelayedDiagnostics.size();
+  // If there are no active diagnostics, we're done.
+  if (DD.StackSize == DD.ActiveStackBase)
+    return;
 
   // We only want to actually emit delayed diagnostics when we
   // successfully parsed a decl.
-  if (D) {
-    // We really do want to start with 0 here.  We get one push for a
+  if (decl) {
+    // We emit all the active diagnostics, not just those starting
+    // from the saved state.  The idea is this:  we get one push for a
     // decl spec and another for each declarator;  in a decl group like:
     //   deprecated_typedef foo, *bar, baz();
     // only the declarator pops will be passed decls.  This is correct;
     // we really do need to consider delayed diagnostics from the decl spec
     // for each of the different declarations.
-    for (unsigned I = 0; I != E; ++I) {
-      if (DelayedDiagnostics[I].Triggered)
+    for (unsigned i = DD.ActiveStackBase, e = DD.StackSize; i != e; ++i) {
+      DelayedDiagnostic &diag = DD.Stack[i];
+      if (diag.Triggered)
         continue;
 
-      switch (DelayedDiagnostics[I].Kind) {
+      switch (diag.Kind) {
       case DelayedDiagnostic::Deprecation:
-        HandleDelayedDeprecationCheck(DelayedDiagnostics[I], D);
+        S.HandleDelayedDeprecationCheck(diag, decl);
         break;
 
       case DelayedDiagnostic::Access:
-        HandleDelayedAccessCheck(DelayedDiagnostics[I], D);
+        S.HandleDelayedAccessCheck(diag, decl);
         break;
       }
     }
   }
 
   // Destroy all the delayed diagnostics we're about to pop off.
-  for (unsigned I = SavedIndex; I != E; ++I)
-    DelayedDiagnostics[I].destroy();
+  for (unsigned i = state.SavedStackSize, e = DD.StackSize; i != e; ++i)
+    DD.Stack[i].destroy();
 
-  DelayedDiagnostics.set_size(SavedIndex);
+  DD.StackSize = state.SavedStackSize;
 }
 
 static bool isDeclDeprecated(Decl *D) {
@@ -2479,20 +3011,34 @@ void Sema::HandleDelayedDeprecationCheck(DelayedDiagnostic &DD,
     return;
 
   DD.Triggered = true;
-  Diag(DD.Loc, diag::warn_deprecated)
-    << DD.DeprecationData.Decl->getDeclName();
+  if (!DD.getDeprecationMessage().empty())
+    Diag(DD.Loc, diag::warn_deprecated_message)
+      << DD.getDeprecationDecl()->getDeclName()
+      << DD.getDeprecationMessage();
+  else
+    Diag(DD.Loc, diag::warn_deprecated)
+      << DD.getDeprecationDecl()->getDeclName();
 }
 
-void Sema::EmitDeprecationWarning(NamedDecl *D, SourceLocation Loc) {
+void Sema::EmitDeprecationWarning(NamedDecl *D, llvm::StringRef Message,
+                                  SourceLocation Loc,
+                                  bool UnknownObjCClass) {
   // Delay if we're currently parsing a declaration.
-  if (ParsingDeclDepth) {
-    DelayedDiagnostics.push_back(DelayedDiagnostic::makeDeprecation(Loc, D));
+  if (DelayedDiagnostics.shouldDelayDiagnostics()) {
+    DelayedDiagnostics.add(DelayedDiagnostic::makeDeprecation(Loc, D, Message));
     return;
   }
 
   // Otherwise, don't warn if our current context is deprecated.
   if (isDeclDeprecated(cast<Decl>(CurContext)))
     return;
-
-  Diag(Loc, diag::warn_deprecated) << D->getDeclName();
+  if (!Message.empty())
+    Diag(Loc, diag::warn_deprecated_message) << D->getDeclName() 
+                                             << Message;
+  else {
+    if (!UnknownObjCClass)
+      Diag(Loc, diag::warn_deprecated) << D->getDeclName();
+    else
+      Diag(Loc, diag::warn_deprecated_fwdclass_message) << D->getDeclName();
+  }
 }
diff --git a/lib/Sema/SemaDeclCXX.cpp b/lib/Sema/SemaDeclCXX.cpp
index 63acdb5f1cc6..e8abab847654 100644
--- a/lib/Sema/SemaDeclCXX.cpp
+++ b/lib/Sema/SemaDeclCXX.cpp
@@ -63,8 +63,7 @@ namespace {
   /// VisitExpr - Visit all of the children of this expression.
   bool CheckDefaultArgumentVisitor::VisitExpr(Expr *Node) {
     bool IsInvalid = false;
-    for (Stmt::child_iterator I = Node->child_begin(),
-         E = Node->child_end(); I != E; ++I)
+    for (Stmt::child_range I = Node->children(); I; ++I)
       IsInvalid |= Visit(*I);
     return IsInvalid;
   }
@@ -90,7 +89,7 @@ namespace {
       // C++ [dcl.fct.default]p7
       //   Local variables shall not be used in default argument
       //   expressions.
-      if (VDecl->isBlockVarDecl())
+      if (VDecl->isLocalVarDecl())
         return S->Diag(DRE->getSourceRange().getBegin(),
                        diag::err_param_default_argument_references_local)
           << VDecl->getDeclName() << DefaultArg->getSourceRange();
@@ -125,21 +124,35 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, Expr *Arg,
   //   the same semantic constraints as the initializer expression in
   //   a declaration of a variable of the parameter type, using the
   //   copy-initialization semantics (8.5).
-  InitializedEntity Entity = InitializedEntity::InitializeParameter(Param);
+  InitializedEntity Entity = InitializedEntity::InitializeParameter(Context,
+                                                                    Param);
   InitializationKind Kind = InitializationKind::CreateCopy(Param->getLocation(),
                                                            EqualLoc);
   InitializationSequence InitSeq(*this, Entity, Kind, &Arg, 1);
   ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                            MultiExprArg(*this, &Arg, 1));
+                                      MultiExprArg(*this, &Arg, 1));
   if (Result.isInvalid())
     return true;
   Arg = Result.takeAs<Expr>();
 
-  Arg = MaybeCreateCXXExprWithTemporaries(Arg);
+  CheckImplicitConversions(Arg, EqualLoc);
+  Arg = MaybeCreateExprWithCleanups(Arg);
 
   // Okay: add the default argument to the parameter
   Param->setDefaultArg(Arg);
 
+  // We have already instantiated this parameter; provide each of the 
+  // instantiations with the uninstantiated default argument.
+  UnparsedDefaultArgInstantiationsMap::iterator InstPos
+    = UnparsedDefaultArgInstantiations.find(Param);
+  if (InstPos != UnparsedDefaultArgInstantiations.end()) {
+    for (unsigned I = 0, N = InstPos->second.size(); I != N; ++I)
+      InstPos->second[I]->setUninstantiatedDefaultArg(Arg);
+    
+    // We're done tracking this parameter's instantiations.
+    UnparsedDefaultArgInstantiations.erase(InstPos);
+  }
+  
   return false;
 }
 
@@ -163,6 +176,12 @@ Sema::ActOnParamDefaultArgument(Decl *param, SourceLocation EqualLoc,
     return;
   }
 
+  // Check for unexpanded parameter packs.
+  if (DiagnoseUnexpandedParameterPack(DefaultArg, UPPC_DefaultArgument)) {
+    Param->setInvalidDecl();
+    return;
+  }    
+      
   // Check that the default argument is well-formed
   CheckDefaultArgumentVisitor DefaultArgChecker(DefaultArg, this);
   if (DefaultArgChecker.Visit(DefaultArg)) {
@@ -297,7 +316,7 @@ bool Sema::MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old) {
     } else if (OldParam->hasDefaultArg()) {
       // Merge the old default argument into the new parameter.
       // It's important to use getInit() here;  getDefaultArg()
-      // strips off any top-level CXXExprWithTemporaries.
+      // strips off any top-level ExprWithCleanups.
       NewParam->setHasInheritedDefaultArg();
       if (OldParam->hasUninstantiatedDefaultArg())
         NewParam->setUninstantiatedDefaultArg(
@@ -444,7 +463,8 @@ CXXBaseSpecifier *
 Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
                          SourceRange SpecifierRange,
                          bool Virtual, AccessSpecifier Access,
-                         TypeSourceInfo *TInfo) {
+                         TypeSourceInfo *TInfo,
+                         SourceLocation EllipsisLoc) {
   QualType BaseType = TInfo->getType();
 
   // C++ [class.union]p1:
@@ -455,10 +475,17 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
     return 0;
   }
 
+  if (EllipsisLoc.isValid() && 
+      !TInfo->getType()->containsUnexpandedParameterPack()) {
+    Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+      << TInfo->getTypeLoc().getSourceRange();
+    EllipsisLoc = SourceLocation();
+  }
+  
   if (BaseType->isDependentType())
     return new (Context) CXXBaseSpecifier(SpecifierRange, Virtual,
                                           Class->getTagKind() == TTK_Class,
-                                          Access, TInfo);
+                                          Access, TInfo, EllipsisLoc);
 
   SourceLocation BaseLoc = TInfo->getTypeLoc().getBeginLoc();
 
@@ -493,90 +520,25 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
   CXXRecordDecl * CXXBaseDecl = cast<CXXRecordDecl>(BaseDecl);
   assert(CXXBaseDecl && "Base type is not a C++ type");
 
-  // C++0x CWG Issue #817 indicates that [[final]] classes shouldn't be bases.
+  // C++ [class.derived]p2:
+  //   If a class is marked with the class-virt-specifier final and it appears
+  //   as a base-type-specifier in a base-clause (10 class.derived), the program
+  //   is ill-formed.
   if (CXXBaseDecl->hasAttr<FinalAttr>()) {
-    Diag(BaseLoc, diag::err_final_base) << BaseType.getAsString();
+    Diag(BaseLoc, diag::err_class_marked_final_used_as_base) 
+      << CXXBaseDecl->getDeclName();
     Diag(CXXBaseDecl->getLocation(), diag::note_previous_decl)
-      << BaseType;
+      << CXXBaseDecl->getDeclName();
     return 0;
   }
 
-  SetClassDeclAttributesFromBase(Class, CXXBaseDecl, Virtual);
-
   if (BaseDecl->isInvalidDecl())
     Class->setInvalidDecl();
   
   // Create the base specifier.
   return new (Context) CXXBaseSpecifier(SpecifierRange, Virtual,
                                         Class->getTagKind() == TTK_Class,
-                                        Access, TInfo);
-}
-
-void Sema::SetClassDeclAttributesFromBase(CXXRecordDecl *Class,
-                                          const CXXRecordDecl *BaseClass,
-                                          bool BaseIsVirtual) {
-  // A class with a non-empty base class is not empty.
-  // FIXME: Standard ref?
-  if (!BaseClass->isEmpty())
-    Class->setEmpty(false);
-
-  // C++ [class.virtual]p1:
-  //   A class that [...] inherits a virtual function is called a polymorphic
-  //   class.
-  if (BaseClass->isPolymorphic())
-    Class->setPolymorphic(true);
-
-  // C++ [dcl.init.aggr]p1:
-  //   An aggregate is [...] a class with [...] no base classes [...].
-  Class->setAggregate(false);
-
-  // C++ [class]p4:
-  //   A POD-struct is an aggregate class...
-  Class->setPOD(false);
-
-  if (BaseIsVirtual) {
-    // C++ [class.ctor]p5:
-    //   A constructor is trivial if its class has no virtual base classes.
-    Class->setHasTrivialConstructor(false);
-
-    // C++ [class.copy]p6:
-    //   A copy constructor is trivial if its class has no virtual base classes.
-    Class->setHasTrivialCopyConstructor(false);
-
-    // C++ [class.copy]p11:
-    //   A copy assignment operator is trivial if its class has no virtual
-    //   base classes.
-    Class->setHasTrivialCopyAssignment(false);
-
-    // C++0x [meta.unary.prop] is_empty:
-    //    T is a class type, but not a union type, with ... no virtual base
-    //    classes
-    Class->setEmpty(false);
-  } else {
-    // C++ [class.ctor]p5:
-    //   A constructor is trivial if all the direct base classes of its
-    //   class have trivial constructors.
-    if (!BaseClass->hasTrivialConstructor())
-      Class->setHasTrivialConstructor(false);
-
-    // C++ [class.copy]p6:
-    //   A copy constructor is trivial if all the direct base classes of its
-    //   class have trivial copy constructors.
-    if (!BaseClass->hasTrivialCopyConstructor())
-      Class->setHasTrivialCopyConstructor(false);
-
-    // C++ [class.copy]p11:
-    //   A copy assignment operator is trivial if all the direct base classes
-    //   of its class have trivial copy assignment operators.
-    if (!BaseClass->hasTrivialCopyAssignment())
-      Class->setHasTrivialCopyAssignment(false);
-  }
-
-  // C++ [class.ctor]p3:
-  //   A destructor is trivial if all the direct base classes of its class
-  //   have trivial destructors.
-  if (!BaseClass->hasTrivialDestructor())
-    Class->setHasTrivialDestructor(false);
+                                        Access, TInfo, EllipsisLoc);
 }
 
 /// ActOnBaseSpecifier - Parsed a base specifier. A base specifier is
@@ -587,7 +549,8 @@ void Sema::SetClassDeclAttributesFromBase(CXXRecordDecl *Class,
 BaseResult
 Sema::ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange,
                          bool Virtual, AccessSpecifier Access,
-                         ParsedType basetype, SourceLocation BaseLoc) {
+                         ParsedType basetype, SourceLocation BaseLoc,
+                         SourceLocation EllipsisLoc) {
   if (!classdecl)
     return true;
 
@@ -598,8 +561,15 @@ Sema::ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange,
 
   TypeSourceInfo *TInfo = 0;
   GetTypeFromParser(basetype, &TInfo);
+
+  if (EllipsisLoc.isInvalid() &&
+      DiagnoseUnexpandedParameterPack(SpecifierRange.getBegin(), TInfo, 
+                                      UPPC_BaseType))
+    return true;
+  
   if (CXXBaseSpecifier *BaseSpec = CheckBaseSpecifier(Class, SpecifierRange,
-                                                      Virtual, Access, TInfo))
+                                                      Virtual, Access, TInfo,
+                                                      EllipsisLoc))
     return BaseSpec;
 
   return true;
@@ -885,6 +855,63 @@ Decl *Sema::ActOnAccessSpecifier(AccessSpecifier Access,
   return ASDecl;
 }
 
+/// CheckOverrideControl - Check C++0x override control semantics.
+void Sema::CheckOverrideControl(const Decl *D) {
+  const CXXMethodDecl *MD = llvm::dyn_cast<CXXMethodDecl>(D);
+  if (!MD || !MD->isVirtual())
+    return;
+
+  if (MD->isDependentContext())
+    return;
+
+  // C++0x [class.virtual]p3:
+  //   If a virtual function is marked with the virt-specifier override and does
+  //   not override a member function of a base class, 
+  //   the program is ill-formed.
+  bool HasOverriddenMethods = 
+    MD->begin_overridden_methods() != MD->end_overridden_methods();
+  if (MD->hasAttr<OverrideAttr>() && !HasOverriddenMethods) {
+    Diag(MD->getLocation(), 
+                 diag::err_function_marked_override_not_overriding)
+      << MD->getDeclName();
+    return;
+  }
+
+  // C++0x [class.derived]p8:
+  //   In a class definition marked with the class-virt-specifier explicit,
+  //   if a virtual member function that is neither implicitly-declared nor a 
+  //   destructor overrides a member function of a base class and it is not
+  //   marked with the virt-specifier override, the program is ill-formed.
+  if (MD->getParent()->hasAttr<ExplicitAttr>() && !isa<CXXDestructorDecl>(MD) &&
+      HasOverriddenMethods && !MD->hasAttr<OverrideAttr>()) {
+    llvm::SmallVector<const CXXMethodDecl*, 4> 
+      OverriddenMethods(MD->begin_overridden_methods(), 
+                        MD->end_overridden_methods());
+
+    Diag(MD->getLocation(), diag::err_function_overriding_without_override)
+      << MD->getDeclName() 
+      << (unsigned)OverriddenMethods.size();
+
+    for (unsigned I = 0; I != OverriddenMethods.size(); ++I)
+      Diag(OverriddenMethods[I]->getLocation(),
+           diag::note_overridden_virtual_function);
+  }
+}
+
+/// CheckIfOverriddenFunctionIsMarkedFinal - Checks whether a virtual member 
+/// function overrides a virtual member function marked 'final', according to
+/// C++0x [class.virtual]p3.
+bool Sema::CheckIfOverriddenFunctionIsMarkedFinal(const CXXMethodDecl *New,
+                                                  const CXXMethodDecl *Old) {
+  if (!Old->hasAttr<FinalAttr>())
+    return false;
+
+  Diag(New->getLocation(), diag::err_final_function_overridden)
+    << New->getDeclName();
+  Diag(Old->getLocation(), diag::note_overridden_virtual_function);
+  return true;
+}
+
 /// ActOnCXXMemberDeclarator - This is invoked when a C++ class member
 /// declarator is parsed. 'AS' is the access specifier, 'BW' specifies the
 /// bitfield width if there is one and 'InitExpr' specifies the initializer if
@@ -892,12 +919,18 @@ Decl *Sema::ActOnAccessSpecifier(AccessSpecifier Access,
 Decl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
-                               ExprTy *BW, ExprTy *InitExpr, bool IsDefinition,
+                               ExprTy *BW, const VirtSpecifiers &VS,
+                               ExprTy *InitExpr, bool IsDefinition,
                                bool Deleted) {
   const DeclSpec &DS = D.getDeclSpec();
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
   DeclarationName Name = NameInfo.getName();
   SourceLocation Loc = NameInfo.getLoc();
+
+  // For anonymous bitfields, the location should point to the type.
+  if (Loc.isInvalid())
+    Loc = D.getSourceRange().getBegin();
+
   Expr *BitWidth = static_cast<Expr*>(BW);
   Expr *Init = static_cast<Expr*>(InitExpr);
 
@@ -951,7 +984,29 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
 
   Decl *Member;
   if (isInstField) {
+    CXXScopeSpec &SS = D.getCXXScopeSpec();
+    
+    
+    if (SS.isSet() && !SS.isInvalid()) {
+      // The user provided a superfluous scope specifier inside a class
+      // definition:
+      //
+      // class X {
+      //   int X::member;
+      // };
+      DeclContext *DC = 0;
+      if ((DC = computeDeclContext(SS, false)) && DC->Equals(CurContext))
+        Diag(D.getIdentifierLoc(), diag::warn_member_extra_qualification)
+        << Name << FixItHint::CreateRemoval(SS.getRange());
+      else
+        Diag(D.getIdentifierLoc(), diag::err_member_qualification)
+          << Name << SS.getRange();
+       
+      SS.clear();
+    }
+    
     // FIXME: Check for template parameters!
+    // FIXME: Check that the name is an identifier!
     Member = HandleField(S, cast<CXXRecordDecl>(CurContext), Loc, D, BitWidth,
                          AS);
     assert(Member && "HandleField never returns null");
@@ -994,17 +1049,37 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
       FunTmpl->getTemplatedDecl()->setAccess(AS);
   }
 
+  if (VS.isOverrideSpecified()) {
+    CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Member);
+    if (!MD || !MD->isVirtual()) {
+      Diag(Member->getLocStart(), 
+           diag::override_keyword_only_allowed_on_virtual_member_functions)
+        << "override" << FixItHint::CreateRemoval(VS.getOverrideLoc());
+    } else
+      MD->addAttr(new (Context) OverrideAttr(VS.getOverrideLoc(), Context));
+  }
+  if (VS.isFinalSpecified()) {
+    CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Member);
+    if (!MD || !MD->isVirtual()) {
+      Diag(Member->getLocStart(), 
+           diag::override_keyword_only_allowed_on_virtual_member_functions)
+      << "final" << FixItHint::CreateRemoval(VS.getFinalLoc());
+    } else
+      MD->addAttr(new (Context) FinalAttr(VS.getFinalLoc(), Context));
+  }
+
+  CheckOverrideControl(Member);
+
   assert((Name || isInstField) && "No identifier for non-field ?");
 
   if (Init)
-    AddInitializerToDecl(Member, Init, false);
+    AddInitializerToDecl(Member, Init, false,
+                         DS.getTypeSpecType() == DeclSpec::TST_auto);
   if (Deleted) // FIXME: Source location is not very good.
     SetDeclDeleted(Member, D.getSourceRange().getBegin());
 
-  if (isInstField) {
+  if (isInstField)
     FieldCollector->Add(cast<FieldDecl>(Member));
-    return 0;
-  }
   return Member;
 }
 
@@ -1063,8 +1138,8 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
                           SourceLocation IdLoc,
                           SourceLocation LParenLoc,
                           ExprTy **Args, unsigned NumArgs,
-                          SourceLocation *CommaLocs,
-                          SourceLocation RParenLoc) {
+                          SourceLocation RParenLoc,
+                          SourceLocation EllipsisLoc) {
   if (!ConstructorD)
     return true;
 
@@ -1084,12 +1159,12 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
 
   // C++ [class.base.init]p2:
   //   Names in a mem-initializer-id are looked up in the scope of the
-  //   constructor’s class and, if not found in that scope, are looked
-  //   up in the scope containing the constructor’s
-  //   definition. [Note: if the constructor’s class contains a member
-  //   with the same name as a direct or virtual base class of the
-  //   class, a mem-initializer-id naming the member or base class and
-  //   composed of a single identifier refers to the class member. A
+  //   constructor's class and, if not found in that scope, are looked
+  //   up in the scope containing the constructor's definition.
+  //   [Note: if the constructor's class contains a member with the
+  //   same name as a direct or virtual base class of the class, a
+  //   mem-initializer-id naming the member or base class and composed
+  //   of a single identifier refers to the class member. A
   //   mem-initializer-id for the hidden base class may be specified
   //   using a qualified name. ]
   if (!SS.getScopeRep() && !TemplateTypeTy) {
@@ -1097,14 +1172,30 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
     FieldDecl *Member = 0;
     DeclContext::lookup_result Result
       = ClassDecl->lookup(MemberOrBase);
-    if (Result.first != Result.second)
+    if (Result.first != Result.second) {
       Member = dyn_cast<FieldDecl>(*Result.first);
-
-    // FIXME: Handle members of an anonymous union.
-
-    if (Member)
-      return BuildMemberInitializer(Member, (Expr**)Args, NumArgs, IdLoc,
+    
+      if (Member) {
+        if (EllipsisLoc.isValid())
+          Diag(EllipsisLoc, diag::err_pack_expansion_member_init)
+            << MemberOrBase << SourceRange(IdLoc, RParenLoc);
+        
+        return BuildMemberInitializer(Member, (Expr**)Args, NumArgs, IdLoc,
                                     LParenLoc, RParenLoc);
+      }
+      
+      // Handle anonymous union case.
+      if (IndirectFieldDecl* IndirectField
+            = dyn_cast<IndirectFieldDecl>(*Result.first)) {
+        if (EllipsisLoc.isValid())
+          Diag(EllipsisLoc, diag::err_pack_expansion_member_init)
+            << MemberOrBase << SourceRange(IdLoc, RParenLoc);
+
+         return BuildMemberInitializer(IndirectField, (Expr**)Args,
+                                       NumArgs, IdLoc,
+                                       LParenLoc, RParenLoc);
+      }
+    }
   }
   // It didn't name a member, so see if it names a class.
   QualType BaseType;
@@ -1212,7 +1303,7 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
     TInfo = Context.getTrivialTypeSourceInfo(BaseType, IdLoc);
 
   return BuildBaseInitializer(BaseType, TInfo, (Expr **)Args, NumArgs, 
-                              LParenLoc, RParenLoc, ClassDecl);
+                              LParenLoc, RParenLoc, ClassDecl, EllipsisLoc);
 }
 
 /// Checks an initializer expression for use of uninitialized fields, such as
@@ -1220,8 +1311,10 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
 /// uninitialized field was used an updates the SourceLocation parameter; false
 /// otherwise.
 static bool InitExprContainsUninitializedFields(const Stmt *S,
-                                                const FieldDecl *LhsField,
+                                                const ValueDecl *LhsField,
                                                 SourceLocation *L) {
+  assert(isa<FieldDecl>(LhsField) || isa<IndirectFieldDecl>(LhsField));
+
   if (isa<CallExpr>(S)) {
     // Do not descend into function calls or constructors, as the use
     // of an uninitialized field may be valid. One would have to inspect
@@ -1232,6 +1325,20 @@ static bool InitExprContainsUninitializedFields(const Stmt *S,
   }
   if (const MemberExpr *ME = dyn_cast<MemberExpr>(S)) {
     const NamedDecl *RhsField = ME->getMemberDecl();
+
+    if (const VarDecl *VD = dyn_cast<VarDecl>(RhsField)) {
+      // The member expression points to a static data member.
+      assert(VD->isStaticDataMember() && 
+             "Member points to non-static data member!");
+      (void)VD;
+      return false;
+    }
+    
+    if (isa<EnumConstantDecl>(RhsField)) {
+      // The member expression points to an enum.
+      return false;
+    }
+
     if (RhsField == LhsField) {
       // Initializing a field with itself. Throw a warning.
       // But wait; there are exceptions!
@@ -1247,9 +1354,16 @@ static bool InitExprContainsUninitializedFields(const Stmt *S,
       *L = ME->getMemberLoc();
       return true;
     }
+  } else if (isa<SizeOfAlignOfExpr>(S)) {
+    // sizeof/alignof doesn't reference contents, do not warn.
+    return false;
+  } else if (const UnaryOperator *UOE = dyn_cast<UnaryOperator>(S)) {
+    // address-of doesn't reference contents (the pointer may be dereferenced
+    // in the same expression but it would be rare; and weird).
+    if (UOE->getOpcode() == UO_AddrOf)
+      return false;
   }
-  for (Stmt::const_child_iterator it = S->child_begin(), e = S->child_end();
-       it != e; ++it) {
+  for (Stmt::const_child_range it = S->children(); it; ++it) {
     if (!*it) {
       // An expression such as 'member(arg ?: "")' may trigger this.
       continue;
@@ -1261,10 +1375,18 @@ static bool InitExprContainsUninitializedFields(const Stmt *S,
 }
 
 MemInitResult
-Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
+Sema::BuildMemberInitializer(ValueDecl *Member, Expr **Args,
                              unsigned NumArgs, SourceLocation IdLoc,
                              SourceLocation LParenLoc,
                              SourceLocation RParenLoc) {
+  FieldDecl *DirectMember = dyn_cast<FieldDecl>(Member);
+  IndirectFieldDecl *IndirectMember = dyn_cast<IndirectFieldDecl>(Member);
+  assert((DirectMember || IndirectMember) &&
+         "Member must be a FieldDecl or IndirectFieldDecl");
+
+  if (Member->isInvalidDecl())
+    return true;
+
   // Diagnose value-uses of fields to initialize themselves, e.g.
   //   foo(foo)
   // where foo is not also a parameter to the constructor.
@@ -1286,12 +1408,12 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
   for (unsigned i = 0; i < NumArgs; i++)
     HasDependentArg |= Args[i]->isTypeDependent();
 
+  Expr *Init;
   if (Member->getType()->isDependentType() || HasDependentArg) {
     // Can't check initialization for a member of dependent type or when
     // any of the arguments are type-dependent expressions.
-    Expr *Init
-      = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                    RParenLoc);
+    Init = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
+                                       RParenLoc);
 
     // Erase any temporaries within this evaluation context; we're not
     // going to track them in the AST, since we'll be rebuilding the
@@ -1299,69 +1421,78 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
     ExprTemporaries.erase(
               ExprTemporaries.begin() + ExprEvalContexts.back().NumTemporaries,
                           ExprTemporaries.end());
-    
-    return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
-                                                    LParenLoc, 
-                                                    Init,
-                                                    RParenLoc);
-    
+  } else {
+    // Initialize the member.
+    InitializedEntity MemberEntity =
+      DirectMember ? InitializedEntity::InitializeMember(DirectMember, 0)
+                   : InitializedEntity::InitializeMember(IndirectMember, 0);
+    InitializationKind Kind =
+      InitializationKind::CreateDirect(IdLoc, LParenLoc, RParenLoc);
+
+    InitializationSequence InitSeq(*this, MemberEntity, Kind, Args, NumArgs);
+
+    ExprResult MemberInit =
+      InitSeq.Perform(*this, MemberEntity, Kind,
+                      MultiExprArg(*this, Args, NumArgs), 0);
+    if (MemberInit.isInvalid())
+      return true;
+
+    CheckImplicitConversions(MemberInit.get(), LParenLoc);
+
+    // C++0x [class.base.init]p7:
+    //   The initialization of each base and member constitutes a
+    //   full-expression.
+    MemberInit = MaybeCreateExprWithCleanups(MemberInit);
+    if (MemberInit.isInvalid())
+      return true;
+
+    // If we are in a dependent context, template instantiation will
+    // perform this type-checking again. Just save the arguments that we
+    // received in a ParenListExpr.
+    // FIXME: This isn't quite ideal, since our ASTs don't capture all
+    // of the information that we have about the member
+    // initializer. However, deconstructing the ASTs is a dicey process,
+    // and this approach is far more likely to get the corner cases right.
+    if (CurContext->isDependentContext())
+      Init = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
+                                               RParenLoc);
+    else
+      Init = MemberInit.get();
   }
-  
-  if (Member->isInvalidDecl())
-    return true;
-  
-  // Initialize the member.
-  InitializedEntity MemberEntity =
-    InitializedEntity::InitializeMember(Member, 0);
-  InitializationKind Kind = 
-    InitializationKind::CreateDirect(IdLoc, LParenLoc, RParenLoc);
-  
-  InitializationSequence InitSeq(*this, MemberEntity, Kind, Args, NumArgs);
-  
-  ExprResult MemberInit =
-    InitSeq.Perform(*this, MemberEntity, Kind, 
-                    MultiExprArg(*this, Args, NumArgs), 0);
-  if (MemberInit.isInvalid())
-    return true;
-  
-  // C++0x [class.base.init]p7:
-  //   The initialization of each base and member constitutes a 
-  //   full-expression.
-  MemberInit = MaybeCreateCXXExprWithTemporaries(MemberInit.get());
-  if (MemberInit.isInvalid())
-    return true;
-  
-  // If we are in a dependent context, template instantiation will
-  // perform this type-checking again. Just save the arguments that we
-  // received in a ParenListExpr.
-  // FIXME: This isn't quite ideal, since our ASTs don't capture all
-  // of the information that we have about the member
-  // initializer. However, deconstructing the ASTs is a dicey process,
-  // and this approach is far more likely to get the corner cases right.
-  if (CurContext->isDependentContext()) {
-    // Bump the reference count of all of the arguments.
-    for (unsigned I = 0; I != NumArgs; ++I)
-      Args[I]->Retain();
 
-    Expr *Init = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                             RParenLoc);
-    return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
-                                                    LParenLoc, 
-                                                    Init,
+  if (DirectMember) {
+    return new (Context) CXXCtorInitializer(Context, DirectMember,
+                                                    IdLoc, LParenLoc, Init,
+                                                    RParenLoc);
+  } else {
+    return new (Context) CXXCtorInitializer(Context, IndirectMember,
+                                                    IdLoc, LParenLoc, Init,
                                                     RParenLoc);
   }
+}
 
-  return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
-                                                  LParenLoc, 
-                                                  MemberInit.get(),
-                                                  RParenLoc);
+MemInitResult
+Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo,
+                                 Expr **Args, unsigned NumArgs,
+                                 SourceLocation LParenLoc,
+                                 SourceLocation RParenLoc,
+                                 CXXRecordDecl *ClassDecl,
+                                 SourceLocation EllipsisLoc) {
+  SourceLocation Loc = TInfo->getTypeLoc().getLocalSourceRange().getBegin();
+  if (!LangOpts.CPlusPlus0x)
+    return Diag(Loc, diag::err_delegation_0x_only)
+      << TInfo->getTypeLoc().getLocalSourceRange();
+
+  return Diag(Loc, diag::err_delegation_unimplemented)
+    << TInfo->getTypeLoc().getLocalSourceRange();
 }
 
 MemInitResult
 Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
                            Expr **Args, unsigned NumArgs, 
                            SourceLocation LParenLoc, SourceLocation RParenLoc, 
-                           CXXRecordDecl *ClassDecl) {
+                           CXXRecordDecl *ClassDecl,
+                           SourceLocation EllipsisLoc) {
   bool HasDependentArg = false;
   for (unsigned i = 0; i < NumArgs; i++)
     HasDependentArg |= Args[i]->isTypeDependent();
@@ -1375,16 +1506,40 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
 
   // C++ [class.base.init]p2:
   //   [...] Unless the mem-initializer-id names a nonstatic data
-  //   member of the constructor’s class or a direct or virtual base
+  //   member of the constructor's class or a direct or virtual base
   //   of that class, the mem-initializer is ill-formed. A
   //   mem-initializer-list can initialize a base class using any
   //   name that denotes that base class type.
   bool Dependent = BaseType->isDependentType() || HasDependentArg;
 
+  if (EllipsisLoc.isValid()) {
+    // This is a pack expansion.
+    if (!BaseType->containsUnexpandedParameterPack())  {
+      Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+        << SourceRange(BaseLoc, RParenLoc);
+      
+      EllipsisLoc = SourceLocation();
+    }
+  } else {
+    // Check for any unexpanded parameter packs.
+    if (DiagnoseUnexpandedParameterPack(BaseLoc, BaseTInfo, UPPC_Initializer))
+      return true;
+    
+    for (unsigned I = 0; I != NumArgs; ++I)
+      if (DiagnoseUnexpandedParameterPack(Args[I]))
+        return true;
+  }
+  
   // Check for direct and virtual base classes.
   const CXXBaseSpecifier *DirectBaseSpec = 0;
   const CXXBaseSpecifier *VirtualBaseSpec = 0;
   if (!Dependent) { 
+    if (Context.hasSameUnqualifiedType(QualType(ClassDecl->getTypeForDecl(),0),
+                                       BaseType))
+      return BuildDelegatingInitializer(BaseTInfo, Args, NumArgs,
+                                        LParenLoc, RParenLoc, ClassDecl,
+                                        EllipsisLoc);
+
     FindBaseInitializer(*this, ClassDecl, BaseType, DirectBaseSpec, 
                         VirtualBaseSpec);
 
@@ -1421,11 +1576,12 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
               ExprTemporaries.begin() + ExprEvalContexts.back().NumTemporaries,
                           ExprTemporaries.end());
 
-    return new (Context) CXXBaseOrMemberInitializer(Context, BaseTInfo, 
+    return new (Context) CXXCtorInitializer(Context, BaseTInfo, 
                                                     /*IsVirtual=*/false,
                                                     LParenLoc, 
                                                     BaseInit.takeAs<Expr>(),
-                                                    RParenLoc);
+                                                    RParenLoc,
+                                                    EllipsisLoc);
   }
 
   // C++ [base.class.init]p2:
@@ -1454,11 +1610,13 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
                     MultiExprArg(*this, Args, NumArgs), 0);
   if (BaseInit.isInvalid())
     return true;
+
+  CheckImplicitConversions(BaseInit.get(), LParenLoc);
   
   // C++0x [class.base.init]p7:
   //   The initialization of each base and member constitutes a 
   //   full-expression.
-  BaseInit = MaybeCreateCXXExprWithTemporaries(BaseInit.get());
+  BaseInit = MaybeCreateExprWithCleanups(BaseInit);
   if (BaseInit.isInvalid())
     return true;
 
@@ -1470,25 +1628,23 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   // initializer. However, deconstructing the ASTs is a dicey process,
   // and this approach is far more likely to get the corner cases right.
   if (CurContext->isDependentContext()) {
-    // Bump the reference count of all of the arguments.
-    for (unsigned I = 0; I != NumArgs; ++I)
-      Args[I]->Retain();
-
     ExprResult Init
       = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
                                           RParenLoc));
-    return new (Context) CXXBaseOrMemberInitializer(Context, BaseTInfo,
+    return new (Context) CXXCtorInitializer(Context, BaseTInfo,
                                                     BaseSpec->isVirtual(),
                                                     LParenLoc, 
                                                     Init.takeAs<Expr>(),
-                                                    RParenLoc);
+                                                    RParenLoc,
+                                                    EllipsisLoc);
   }
 
-  return new (Context) CXXBaseOrMemberInitializer(Context, BaseTInfo,
+  return new (Context) CXXCtorInitializer(Context, BaseTInfo,
                                                   BaseSpec->isVirtual(),
                                                   LParenLoc, 
                                                   BaseInit.takeAs<Expr>(),
-                                                  RParenLoc);
+                                                  RParenLoc,
+                                                  EllipsisLoc);
 }
 
 /// ImplicitInitializerKind - How an implicit base or member initializer should
@@ -1504,7 +1660,7 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                              ImplicitInitializerKind ImplicitInitKind,
                              CXXBaseSpecifier *BaseSpec,
                              bool IsInheritedVirtualBase,
-                             CXXBaseOrMemberInitializer *&CXXBaseInit) {
+                             CXXCtorInitializer *&CXXBaseInit) {
   InitializedEntity InitEntity
     = InitializedEntity::InitializeBase(SemaRef.Context, BaseSpec,
                                         IsInheritedVirtualBase);
@@ -1527,7 +1683,8 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     
     Expr *CopyCtorArg = 
       DeclRefExpr::Create(SemaRef.Context, 0, SourceRange(), Param, 
-                          Constructor->getLocation(), ParamType, 0);
+                          Constructor->getLocation(), ParamType,
+                          VK_LValue, 0);
     
     // Cast to the base class to avoid ambiguities.
     QualType ArgTy = 
@@ -1554,20 +1711,18 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     assert(false && "Unhandled initializer kind!");
   }
 
-  if (BaseInit.isInvalid())
-    return true;
-      
-  BaseInit = SemaRef.MaybeCreateCXXExprWithTemporaries(BaseInit.get());
+  BaseInit = SemaRef.MaybeCreateExprWithCleanups(BaseInit);
   if (BaseInit.isInvalid())
     return true;
         
   CXXBaseInit =
-    new (SemaRef.Context) CXXBaseOrMemberInitializer(SemaRef.Context,
+    new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context,
                SemaRef.Context.getTrivialTypeSourceInfo(BaseSpec->getType(), 
                                                         SourceLocation()),
                                              BaseSpec->isVirtual(),
                                              SourceLocation(),
                                              BaseInit.takeAs<Expr>(),
+                                             SourceLocation(),
                                              SourceLocation());
 
   return false;
@@ -1577,7 +1732,7 @@ static bool
 BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                                ImplicitInitializerKind ImplicitInitKind,
                                FieldDecl *Field,
-                               CXXBaseOrMemberInitializer *&CXXMemberInit) {
+                               CXXCtorInitializer *&CXXMemberInit) {
   if (Field->isInvalidDecl())
     return true;
 
@@ -1589,7 +1744,7 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     
     Expr *MemberExprBase = 
       DeclRefExpr::Create(SemaRef.Context, 0, SourceRange(), Param, 
-                          Loc, ParamType, 0);
+                          Loc, ParamType, VK_LValue, 0);
 
     // Build a reference to this field within the parameter.
     CXXScopeSpec SS;
@@ -1634,7 +1789,7 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
       
       // Create a reference to the iteration variable.
       ExprResult IterationVarRef
-        = SemaRef.BuildDeclRefExpr(IterationVar, SizeType, Loc);
+        = SemaRef.BuildDeclRefExpr(IterationVar, SizeType, VK_RValue, Loc);
       assert(!IterationVarRef.isInvalid() &&
              "Reference to invented variable cannot fail!");
       
@@ -1671,12 +1826,12 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     ExprResult MemberInit
       = InitSeq.Perform(SemaRef, Entities.back(), InitKind, 
                         MultiExprArg(&CopyCtorArgE, 1));
-    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(MemberInit.get());
+    MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
     if (MemberInit.isInvalid())
       return true;
 
     CXXMemberInit
-      = CXXBaseOrMemberInitializer::Create(SemaRef.Context, Field, Loc, Loc,
+      = CXXCtorInitializer::Create(SemaRef.Context, Field, Loc, Loc,
                                            MemberInit.takeAs<Expr>(), Loc,
                                            IndexVariables.data(),
                                            IndexVariables.size());
@@ -1696,15 +1851,13 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, 0, 0);
     ExprResult MemberInit = 
       InitSeq.Perform(SemaRef, InitEntity, InitKind, MultiExprArg());
-    if (MemberInit.isInvalid())
-      return true;
 
-    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(MemberInit.get());
+    MemberInit = SemaRef.MaybeCreateExprWithCleanups(MemberInit);
     if (MemberInit.isInvalid())
       return true;
     
     CXXMemberInit =
-      new (SemaRef.Context) CXXBaseOrMemberInitializer(SemaRef.Context,
+      new (SemaRef.Context) CXXCtorInitializer(SemaRef.Context,
                                                        Field, Loc, Loc,
                                                        MemberInit.get(),
                                                        Loc);
@@ -1742,8 +1895,8 @@ struct BaseAndFieldInfo {
   CXXConstructorDecl *Ctor;
   bool AnyErrorsInInits;
   ImplicitInitializerKind IIK;
-  llvm::DenseMap<const void *, CXXBaseOrMemberInitializer*> AllBaseFields;
-  llvm::SmallVector<CXXBaseOrMemberInitializer*, 8> AllToInit;
+  llvm::DenseMap<const void *, CXXCtorInitializer*> AllBaseFields;
+  llvm::SmallVector<CXXCtorInitializer*, 8> AllToInit;
 
   BaseAndFieldInfo(Sema &S, CXXConstructorDecl *Ctor, bool ErrorsInInits)
     : S(S), Ctor(Ctor), AnyErrorsInInits(ErrorsInInits) {
@@ -1756,26 +1909,12 @@ struct BaseAndFieldInfo {
 };
 }
 
-static void RecordFieldInitializer(BaseAndFieldInfo &Info,
-                                   FieldDecl *Top, FieldDecl *Field,
-                                   CXXBaseOrMemberInitializer *Init) {
-  // If the member doesn't need to be initialized, Init will still be null.
-  if (!Init)
-    return;
-
-  Info.AllToInit.push_back(Init);
-  if (Field != Top) {
-    Init->setMember(Top);
-    Init->setAnonUnionMember(Field);
-  }
-}
-
 static bool CollectFieldInitializer(BaseAndFieldInfo &Info,
                                     FieldDecl *Top, FieldDecl *Field) {
 
   // Overwhelmingly common case: we have a direct initializer for this field.
-  if (CXXBaseOrMemberInitializer *Init = Info.AllBaseFields.lookup(Field)) {
-    RecordFieldInitializer(Info, Top, Field, Init);
+  if (CXXCtorInitializer *Init = Info.AllBaseFields.lookup(Field)) {
+    Info.AllToInit.push_back(Init);
     return false;
   }
 
@@ -1794,8 +1933,8 @@ static bool CollectFieldInitializer(BaseAndFieldInfo &Info,
       // First check for an explicit initializer for one field.
       for (RecordDecl::field_iterator FA = FieldClassDecl->field_begin(),
            EA = FieldClassDecl->field_end(); FA != EA; FA++) {
-        if (CXXBaseOrMemberInitializer *Init = Info.AllBaseFields.lookup(*FA)) {
-          RecordFieldInitializer(Info, Top, *FA, Init);
+        if (CXXCtorInitializer *Init = Info.AllBaseFields.lookup(*FA)) {
+          Info.AllToInit.push_back(Init);
 
           // Once we've initialized a field of an anonymous union, the union
           // field in the class is also initialized, so exit immediately.
@@ -1828,29 +1967,31 @@ static bool CollectFieldInitializer(BaseAndFieldInfo &Info,
   if (Info.AnyErrorsInInits)
     return false;
 
-  CXXBaseOrMemberInitializer *Init = 0;
+  CXXCtorInitializer *Init = 0;
   if (BuildImplicitMemberInitializer(Info.S, Info.Ctor, Info.IIK, Field, Init))
     return true;
 
-  RecordFieldInitializer(Info, Top, Field, Init);
+  if (Init)
+    Info.AllToInit.push_back(Init);
+
   return false;
 }
                                
 bool
-Sema::SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
-                                  CXXBaseOrMemberInitializer **Initializers,
+Sema::SetCtorInitializers(CXXConstructorDecl *Constructor,
+                                  CXXCtorInitializer **Initializers,
                                   unsigned NumInitializers,
                                   bool AnyErrors) {
   if (Constructor->getDeclContext()->isDependentContext()) {
     // Just store the initializers as written, they will be checked during
     // instantiation.
     if (NumInitializers > 0) {
-      Constructor->setNumBaseOrMemberInitializers(NumInitializers);
-      CXXBaseOrMemberInitializer **baseOrMemberInitializers =
-        new (Context) CXXBaseOrMemberInitializer*[NumInitializers];
+      Constructor->setNumCtorInitializers(NumInitializers);
+      CXXCtorInitializer **baseOrMemberInitializers =
+        new (Context) CXXCtorInitializer*[NumInitializers];
       memcpy(baseOrMemberInitializers, Initializers,
-             NumInitializers * sizeof(CXXBaseOrMemberInitializer*));
-      Constructor->setBaseOrMemberInitializers(baseOrMemberInitializers);
+             NumInitializers * sizeof(CXXCtorInitializer*));
+      Constructor->setCtorInitializers(baseOrMemberInitializers);
     }
     
     return false;
@@ -1867,12 +2008,12 @@ Sema::SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
   bool HadError = false;
 
   for (unsigned i = 0; i < NumInitializers; i++) {
-    CXXBaseOrMemberInitializer *Member = Initializers[i];
+    CXXCtorInitializer *Member = Initializers[i];
     
     if (Member->isBaseInitializer())
       Info.AllBaseFields[Member->getBaseClass()->getAs<RecordType>()] = Member;
     else
-      Info.AllBaseFields[Member->getMember()] = Member;
+      Info.AllBaseFields[Member->getAnyMember()] = Member;
   }
 
   // Keep track of the direct virtual bases.
@@ -1887,12 +2028,12 @@ Sema::SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
   for (CXXRecordDecl::base_class_iterator VBase = ClassDecl->vbases_begin(),
        E = ClassDecl->vbases_end(); VBase != E; ++VBase) {
 
-    if (CXXBaseOrMemberInitializer *Value
+    if (CXXCtorInitializer *Value
         = Info.AllBaseFields.lookup(VBase->getType()->getAs<RecordType>())) {
       Info.AllToInit.push_back(Value);
     } else if (!AnyErrors) {
       bool IsInheritedVirtualBase = !DirectVBases.count(VBase);
-      CXXBaseOrMemberInitializer *CXXBaseInit;
+      CXXCtorInitializer *CXXBaseInit;
       if (BuildImplicitBaseInitializer(*this, Constructor, Info.IIK,
                                        VBase, IsInheritedVirtualBase, 
                                        CXXBaseInit)) {
@@ -1911,11 +2052,11 @@ Sema::SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
     if (Base->isVirtual())
       continue;
 
-    if (CXXBaseOrMemberInitializer *Value
+    if (CXXCtorInitializer *Value
           = Info.AllBaseFields.lookup(Base->getType()->getAs<RecordType>())) {
       Info.AllToInit.push_back(Value);
     } else if (!AnyErrors) {
-      CXXBaseOrMemberInitializer *CXXBaseInit;
+      CXXCtorInitializer *CXXBaseInit;
       if (BuildImplicitBaseInitializer(*this, Constructor, Info.IIK,
                                        Base, /*IsInheritedVirtualBase=*/false,
                                        CXXBaseInit)) {
@@ -1941,12 +2082,12 @@ Sema::SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
 
   NumInitializers = Info.AllToInit.size();
   if (NumInitializers > 0) {
-    Constructor->setNumBaseOrMemberInitializers(NumInitializers);
-    CXXBaseOrMemberInitializer **baseOrMemberInitializers =
-      new (Context) CXXBaseOrMemberInitializer*[NumInitializers];
+    Constructor->setNumCtorInitializers(NumInitializers);
+    CXXCtorInitializer **baseOrMemberInitializers =
+      new (Context) CXXCtorInitializer*[NumInitializers];
     memcpy(baseOrMemberInitializers, Info.AllToInit.data(),
-           NumInitializers * sizeof(CXXBaseOrMemberInitializer*));
-    Constructor->setBaseOrMemberInitializers(baseOrMemberInitializers);
+           NumInitializers * sizeof(CXXCtorInitializer*));
+    Constructor->setCtorInitializers(baseOrMemberInitializers);
 
     // Constructors implicitly reference the base and member
     // destructors.
@@ -1967,25 +2108,18 @@ static void *GetKeyForTopLevelField(FieldDecl *Field) {
 }
 
 static void *GetKeyForBase(ASTContext &Context, QualType BaseType) {
-  return Context.getCanonicalType(BaseType).getTypePtr();
+  return const_cast<Type*>(Context.getCanonicalType(BaseType).getTypePtr());
 }
 
 static void *GetKeyForMember(ASTContext &Context,
-                             CXXBaseOrMemberInitializer *Member,
-                             bool MemberMaybeAnon = false) {
-  if (!Member->isMemberInitializer())
+                             CXXCtorInitializer *Member) {
+  if (!Member->isAnyMemberInitializer())
     return GetKeyForBase(Context, QualType(Member->getBaseClass(), 0));
     
   // For fields injected into the class via declaration of an anonymous union,
   // use its anonymous union class declaration as the unique key.
-  FieldDecl *Field = Member->getMember();
-
-  // After SetBaseOrMemberInitializers call, Field is the anonymous union
-  // data member of the class. Data member used in the initializer list is
-  // in AnonUnionMember field.
-  if (MemberMaybeAnon && Field->isAnonymousStructOrUnion())
-    Field = Member->getAnonUnionMember();
-  
+  FieldDecl *Field = Member->getAnyMember();
+ 
   // If the field is a member of an anonymous struct or union, our key
   // is the anonymous record decl that's a direct child of the class.
   RecordDecl *RD = Field->getParent();
@@ -2007,13 +2141,24 @@ static void *GetKeyForMember(ASTContext &Context,
 static void
 DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
                                   const CXXConstructorDecl *Constructor,
-                                  CXXBaseOrMemberInitializer **Inits,
+                                  CXXCtorInitializer **Inits,
                                   unsigned NumInits) {
   if (Constructor->getDeclContext()->isDependentContext())
     return;
 
-  if (SemaRef.Diags.getDiagnosticLevel(diag::warn_initializer_out_of_order)
-        == Diagnostic::Ignored)
+  // Don't check initializers order unless the warning is enabled at the
+  // location of at least one initializer. 
+  bool ShouldCheckOrder = false;
+  for (unsigned InitIndex = 0; InitIndex != NumInits; ++InitIndex) {
+    CXXCtorInitializer *Init = Inits[InitIndex];
+    if (SemaRef.Diags.getDiagnosticLevel(diag::warn_initializer_out_of_order,
+                                         Init->getSourceLocation())
+          != Diagnostic::Ignored) {
+      ShouldCheckOrder = true;
+      break;
+    }
+  }
+  if (!ShouldCheckOrder)
     return;
   
   // Build the list of bases and members in the order that they'll
@@ -2045,10 +2190,10 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
   unsigned NumIdealInits = IdealInitKeys.size();
   unsigned IdealIndex = 0;
 
-  CXXBaseOrMemberInitializer *PrevInit = 0;
+  CXXCtorInitializer *PrevInit = 0;
   for (unsigned InitIndex = 0; InitIndex != NumInits; ++InitIndex) {
-    CXXBaseOrMemberInitializer *Init = Inits[InitIndex];
-    void *InitKey = GetKeyForMember(SemaRef.Context, Init, true);
+    CXXCtorInitializer *Init = Inits[InitIndex];
+    void *InitKey = GetKeyForMember(SemaRef.Context, Init);
 
     // Scan forward to try to find this initializer in the idealized
     // initializers list.
@@ -2064,13 +2209,13 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
         SemaRef.Diag(PrevInit->getSourceLocation(),
                      diag::warn_initializer_out_of_order);
 
-      if (PrevInit->isMemberInitializer())
-        D << 0 << PrevInit->getMember()->getDeclName();
+      if (PrevInit->isAnyMemberInitializer())
+        D << 0 << PrevInit->getAnyMember()->getDeclName();
       else
         D << 1 << PrevInit->getBaseClassInfo()->getType();
       
-      if (Init->isMemberInitializer())
-        D << 0 << Init->getMember()->getDeclName();
+      if (Init->isAnyMemberInitializer())
+        D << 0 << Init->getAnyMember()->getDeclName();
       else
         D << 1 << Init->getBaseClassInfo()->getType();
 
@@ -2089,8 +2234,8 @@ DiagnoseBaseOrMemInitializerOrder(Sema &SemaRef,
 
 namespace {
 bool CheckRedundantInit(Sema &S,
-                        CXXBaseOrMemberInitializer *Init,
-                        CXXBaseOrMemberInitializer *&PrevInit) {
+                        CXXCtorInitializer *Init,
+                        CXXCtorInitializer *&PrevInit) {
   if (!PrevInit) {
     PrevInit = Init;
     return false;
@@ -2102,7 +2247,7 @@ bool CheckRedundantInit(Sema &S,
       << Field->getDeclName()
       << Init->getSourceRange();
   else {
-    Type *BaseClass = Init->getBaseClass();
+    const Type *BaseClass = Init->getBaseClass();
     assert(BaseClass && "neither field nor base");
     S.Diag(Init->getSourceLocation(),
            diag::err_multiple_base_initialization)
@@ -2115,13 +2260,13 @@ bool CheckRedundantInit(Sema &S,
   return true;
 }
 
-typedef std::pair<NamedDecl *, CXXBaseOrMemberInitializer *> UnionEntry;
+typedef std::pair<NamedDecl *, CXXCtorInitializer *> UnionEntry;
 typedef llvm::DenseMap<RecordDecl*, UnionEntry> RedundantUnionMap;
 
 bool CheckRedundantUnionInit(Sema &S,
-                             CXXBaseOrMemberInitializer *Init,
+                             CXXCtorInitializer *Init,
                              RedundantUnionMap &Unions) {
-  FieldDecl *Field = Init->getMember();
+  FieldDecl *Field = Init->getAnyMember();
   RecordDecl *Parent = Field->getParent();
   if (!Parent->isAnonymousStructOrUnion())
     return false;
@@ -2170,26 +2315,26 @@ void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
     return;
   }
   
-  CXXBaseOrMemberInitializer **MemInits =
-    reinterpret_cast<CXXBaseOrMemberInitializer **>(meminits);
+  CXXCtorInitializer **MemInits =
+    reinterpret_cast<CXXCtorInitializer **>(meminits);
 
   // Mapping for the duplicate initializers check.
   // For member initializers, this is keyed with a FieldDecl*.
   // For base initializers, this is keyed with a Type*.
-  llvm::DenseMap<void*, CXXBaseOrMemberInitializer *> Members;
+  llvm::DenseMap<void*, CXXCtorInitializer *> Members;
 
   // Mapping for the inconsistent anonymous-union initializers check.
   RedundantUnionMap MemberUnions;
 
   bool HadError = false;
   for (unsigned i = 0; i < NumMemInits; i++) {
-    CXXBaseOrMemberInitializer *Init = MemInits[i];
+    CXXCtorInitializer *Init = MemInits[i];
 
     // Set the source order index.
     Init->setSourceOrder(i);
 
-    if (Init->isMemberInitializer()) {
-      FieldDecl *Field = Init->getMember();
+    if (Init->isAnyMemberInitializer()) {
+      FieldDecl *Field = Init->getAnyMember();
       if (CheckRedundantInit(*this, Init, Members[Field]) ||
           CheckRedundantUnionInit(*this, Init, MemberUnions))
         HadError = true;
@@ -2205,7 +2350,7 @@ void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
 
   DiagnoseBaseOrMemInitializerOrder(*this, Constructor, MemInits, NumMemInits);
 
-  SetBaseOrMemberInitializers(Constructor, MemInits, NumMemInits, AnyErrors);
+  SetCtorInitializers(Constructor, MemInits, NumMemInits, AnyErrors);
 }
 
 void
@@ -2304,7 +2449,7 @@ void Sema::ActOnDefaultCtorInitializers(Decl *CDtorDecl) {
 
   if (CXXConstructorDecl *Constructor
       = dyn_cast<CXXConstructorDecl>(CDtorDecl))
-    SetBaseOrMemberInitializers(Constructor, 0, 0, /*AnyErrors=*/false);
+    SetCtorInitializers(Constructor, 0, 0, /*AnyErrors=*/false);
 }
 
 bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
@@ -2391,7 +2536,7 @@ void Sema::DiagnoseAbstractType(const CXXRecordDecl *RD) {
 
       Diag(SO->second.front().Method->getLocation(), 
            diag::note_pure_virtual_function) 
-        << SO->second.front().Method->getDeclName();
+        << SO->second.front().Method->getDeclName() << RD->getDeclName();
     }
   }
 
@@ -2566,84 +2711,9 @@ static void CheckAbstractClassUsage(AbstractUsageInfo &Info,
 /// completing, introducing implicitly-declared members, checking for
 /// abstract types, etc.
 void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
-  if (!Record || Record->isInvalidDecl())
-    return;
-
-  if (!Record->isDependentType())
-    AddImplicitlyDeclaredMembersToClass(Record);
-  
-  if (Record->isInvalidDecl())
+  if (!Record)
     return;
 
-  // Set access bits correctly on the directly-declared conversions.
-  UnresolvedSetImpl *Convs = Record->getConversionFunctions();
-  for (UnresolvedSetIterator I = Convs->begin(), E = Convs->end(); I != E; ++I)
-    Convs->setAccess(I, (*I)->getAccess());
-
-  // Determine whether we need to check for final overriders. We do
-  // this either when there are virtual base classes (in which case we
-  // may end up finding multiple final overriders for a given virtual
-  // function) or any of the base classes is abstract (in which case
-  // we might detect that this class is abstract).
-  bool CheckFinalOverriders = false;
-  if (Record->isPolymorphic() && !Record->isInvalidDecl() &&
-      !Record->isDependentType()) {
-    if (Record->getNumVBases())
-      CheckFinalOverriders = true;
-    else if (!Record->isAbstract()) {
-      for (CXXRecordDecl::base_class_const_iterator B = Record->bases_begin(),
-                                                 BEnd = Record->bases_end();
-           B != BEnd; ++B) {
-        CXXRecordDecl *BaseDecl 
-          = cast<CXXRecordDecl>(B->getType()->getAs<RecordType>()->getDecl());
-        if (BaseDecl->isAbstract()) {
-          CheckFinalOverriders = true; 
-          break;
-        }
-      }
-    }
-  }
-
-  if (CheckFinalOverriders) {
-    CXXFinalOverriderMap FinalOverriders;
-    Record->getFinalOverriders(FinalOverriders);
-
-    for (CXXFinalOverriderMap::iterator M = FinalOverriders.begin(), 
-                                     MEnd = FinalOverriders.end();
-         M != MEnd; ++M) {
-      for (OverridingMethods::iterator SO = M->second.begin(), 
-                                    SOEnd = M->second.end();
-           SO != SOEnd; ++SO) {
-        assert(SO->second.size() > 0 && 
-               "All virtual functions have overridding virtual functions");
-        if (SO->second.size() == 1) {
-          // C++ [class.abstract]p4:
-          //   A class is abstract if it contains or inherits at least one
-          //   pure virtual function for which the final overrider is pure
-          //   virtual.
-          if (SO->second.front().Method->isPure())
-            Record->setAbstract(true);
-          continue;
-        }
-
-        // C++ [class.virtual]p2:
-        //   In a derived class, if a virtual member function of a base
-        //   class subobject has more than one final overrider the
-        //   program is ill-formed.
-        Diag(Record->getLocation(), diag::err_multiple_final_overriders)
-          << (NamedDecl *)M->first << Record;
-        Diag(M->first->getLocation(), diag::note_overridden_virtual_function);
-        for (OverridingMethods::overriding_iterator OM = SO->second.begin(), 
-                                                 OMEnd = SO->second.end();
-             OM != OMEnd; ++OM)
-          Diag(OM->Method->getLocation(), diag::note_final_overrider)
-            << (NamedDecl *)M->first << OM->Method->getParent();
-        
-        Record->setInvalidDecl();
-      }
-    }
-  }
-
   if (Record->isAbstract() && !Record->isInvalidDecl()) {
     AbstractUsageInfo Info(*this, Record);
     CheckAbstractClassUsage(Info, Record);
@@ -2673,8 +2743,149 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
     }
   }
 
-  if (Record->isDynamicClass())
+  if (Record->isDynamicClass() && !Record->isDependentType())
     DynamicClasses.push_back(Record);
+
+  if (Record->getIdentifier()) {
+    // C++ [class.mem]p13:
+    //   If T is the name of a class, then each of the following shall have a 
+    //   name different from T:
+    //     - every member of every anonymous union that is a member of class T.
+    //
+    // C++ [class.mem]p14:
+    //   In addition, if class T has a user-declared constructor (12.1), every 
+    //   non-static data member of class T shall have a name different from T.
+    for (DeclContext::lookup_result R = Record->lookup(Record->getDeclName());
+         R.first != R.second; ++R.first) {
+      NamedDecl *D = *R.first;
+      if ((isa<FieldDecl>(D) && Record->hasUserDeclaredConstructor()) ||
+          isa<IndirectFieldDecl>(D)) {
+        Diag(D->getLocation(), diag::err_member_name_of_class)
+          << D->getDeclName();
+        break;
+      }
+    }
+  }
+
+  // Warn if the class has virtual methods but non-virtual public destructor.
+  if (Record->isPolymorphic() && !Record->isDependentType()) {
+    CXXDestructorDecl *dtor = Record->getDestructor();
+    if (!dtor || (!dtor->isVirtual() && dtor->getAccess() == AS_public))
+      Diag(dtor ? dtor->getLocation() : Record->getLocation(),
+           diag::warn_non_virtual_dtor) << Context.getRecordType(Record);
+  }
+
+  // See if a method overloads virtual methods in a base
+  /// class without overriding any.
+  if (!Record->isDependentType()) {
+    for (CXXRecordDecl::method_iterator M = Record->method_begin(),
+                                     MEnd = Record->method_end();
+         M != MEnd; ++M) {
+      DiagnoseHiddenVirtualMethods(Record, *M);
+    }
+  }
+
+  // Declare inherited constructors. We do this eagerly here because:
+  // - The standard requires an eager diagnostic for conflicting inherited
+  //   constructors from different classes.
+  // - The lazy declaration of the other implicit constructors is so as to not
+  //   waste space and performance on classes that are not meant to be
+  //   instantiated (e.g. meta-functions). This doesn't apply to classes that
+  //   have inherited constructors.
+  DeclareInheritedConstructors(Record);
+}
+
+/// \brief Data used with FindHiddenVirtualMethod
+struct FindHiddenVirtualMethodData {
+  Sema *S;
+  CXXMethodDecl *Method;
+  llvm::SmallPtrSet<const CXXMethodDecl *, 8> OverridenAndUsingBaseMethods;
+  llvm::SmallVector<CXXMethodDecl *, 8> OverloadedMethods;
+};
+
+/// \brief Member lookup function that determines whether a given C++
+/// method overloads virtual methods in a base class without overriding any,
+/// to be used with CXXRecordDecl::lookupInBases().
+static bool FindHiddenVirtualMethod(const CXXBaseSpecifier *Specifier,
+                                    CXXBasePath &Path,
+                                    void *UserData) {
+  RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
+
+  FindHiddenVirtualMethodData &Data
+    = *static_cast<FindHiddenVirtualMethodData*>(UserData);
+
+  DeclarationName Name = Data.Method->getDeclName();
+  assert(Name.getNameKind() == DeclarationName::Identifier);
+
+  bool foundSameNameMethod = false;
+  llvm::SmallVector<CXXMethodDecl *, 8> overloadedMethods;
+  for (Path.Decls = BaseRecord->lookup(Name);
+       Path.Decls.first != Path.Decls.second;
+       ++Path.Decls.first) {
+    NamedDecl *D = *Path.Decls.first;
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D)) {
+      MD = MD->getCanonicalDecl();
+      foundSameNameMethod = true;
+      // Interested only in hidden virtual methods.
+      if (!MD->isVirtual())
+        continue;
+      // If the method we are checking overrides a method from its base
+      // don't warn about the other overloaded methods.
+      if (!Data.S->IsOverload(Data.Method, MD, false))
+        return true;
+      // Collect the overload only if its hidden.
+      if (!Data.OverridenAndUsingBaseMethods.count(MD))
+        overloadedMethods.push_back(MD);
+    }
+  }
+
+  if (foundSameNameMethod)
+    Data.OverloadedMethods.append(overloadedMethods.begin(),
+                                   overloadedMethods.end());
+  return foundSameNameMethod;
+}
+
+/// \brief See if a method overloads virtual methods in a base class without
+/// overriding any.
+void Sema::DiagnoseHiddenVirtualMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
+  if (Diags.getDiagnosticLevel(diag::warn_overloaded_virtual,
+                               MD->getLocation()) == Diagnostic::Ignored)
+    return;
+  if (MD->getDeclName().getNameKind() != DeclarationName::Identifier)
+    return;
+
+  CXXBasePaths Paths(/*FindAmbiguities=*/true, // true to look in all bases.
+                     /*bool RecordPaths=*/false,
+                     /*bool DetectVirtual=*/false);
+  FindHiddenVirtualMethodData Data;
+  Data.Method = MD;
+  Data.S = this;
+
+  // Keep the base methods that were overriden or introduced in the subclass
+  // by 'using' in a set. A base method not in this set is hidden.
+  for (DeclContext::lookup_result res = DC->lookup(MD->getDeclName());
+       res.first != res.second; ++res.first) {
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(*res.first))
+      for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+                                          E = MD->end_overridden_methods();
+           I != E; ++I)
+        Data.OverridenAndUsingBaseMethods.insert((*I)->getCanonicalDecl());
+    if (UsingShadowDecl *shad = dyn_cast<UsingShadowDecl>(*res.first))
+      if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(shad->getTargetDecl()))
+        Data.OverridenAndUsingBaseMethods.insert(MD->getCanonicalDecl());
+  }
+
+  if (DC->lookupInBases(&FindHiddenVirtualMethod, &Data, Paths) &&
+      !Data.OverloadedMethods.empty()) {
+    Diag(MD->getLocation(), diag::warn_overloaded_virtual)
+      << MD << (Data.OverloadedMethods.size() > 1);
+
+    for (unsigned i = 0, e = Data.OverloadedMethods.size(); i != e; ++i) {
+      CXXMethodDecl *overloadedMD = Data.OverloadedMethods[i];
+      Diag(overloadedMD->getLocation(),
+           diag::note_hidden_overloaded_virtual_declared_here) << overloadedMD;
+    }
+  }
 }
 
 void Sema::ActOnFinishCXXMemberSpecification(Scope* S, SourceLocation RLoc,
@@ -2915,7 +3126,7 @@ QualType Sema::CheckConstructorDeclarator(Declarator &D, QualType R,
     SC = SC_None;
   }
 
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
   if (FTI.TypeQuals != 0) {
     if (FTI.TypeQuals & Qualifiers::Const)
       Diag(D.getIdentifierLoc(), diag::err_invalid_qualified_constructor)
@@ -2926,20 +3137,31 @@ QualType Sema::CheckConstructorDeclarator(Declarator &D, QualType R,
     if (FTI.TypeQuals & Qualifiers::Restrict)
       Diag(D.getIdentifierLoc(), diag::err_invalid_qualified_constructor)
         << "restrict" << SourceRange(D.getIdentifierLoc());
+    D.setInvalidType();
   }
 
+  // C++0x [class.ctor]p4:
+  //   A constructor shall not be declared with a ref-qualifier.
+  if (FTI.hasRefQualifier()) {
+    Diag(FTI.getRefQualifierLoc(), diag::err_ref_qualifier_constructor)
+      << FTI.RefQualifierIsLValueRef 
+      << FixItHint::CreateRemoval(FTI.getRefQualifierLoc());
+    D.setInvalidType();
+  }
+  
   // Rebuild the function type "R" without any type qualifiers (in
   // case any of the errors above fired) and with "void" as the
   // return type, since constructors don't have return types.
   const FunctionProtoType *Proto = R->getAs<FunctionProtoType>();
+  if (Proto->getResultType() == Context.VoidTy && !D.isInvalidType())
+    return R;
+
+  FunctionProtoType::ExtProtoInfo EPI = Proto->getExtProtoInfo();
+  EPI.TypeQuals = 0;
+  EPI.RefQualifier = RQ_None;
+  
   return Context.getFunctionType(Context.VoidTy, Proto->arg_type_begin(),
-                                 Proto->getNumArgs(),
-                                 Proto->isVariadic(), 0,
-                                 Proto->hasExceptionSpec(),
-                                 Proto->hasAnyExceptionSpec(),
-                                 Proto->getNumExceptions(),
-                                 Proto->exception_begin(),
-                                 Proto->getExtInfo());
+                                 Proto->getNumArgs(), EPI);
 }
 
 /// CheckConstructor - Checks a fully-formed constructor for
@@ -2977,13 +3199,6 @@ void Sema::CheckConstructor(CXXConstructorDecl *Constructor) {
       Constructor->setInvalidDecl();
     }
   }
-
-  // Notify the class that we've added a constructor.  In principle we
-  // don't need to do this for out-of-line declarations; in practice
-  // we only instantiate the most recent declaration of a method, so
-  // we have to call this for everything but friends.
-  if (!Constructor->getFriendObjectKind())
-    ClassDecl->addedConstructor(Context, Constructor);
 }
 
 /// CheckDestructor - Checks a fully-formed destructor definition for
@@ -3071,7 +3286,7 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
       << SourceRange(D.getIdentifierLoc());
   }
 
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
   if (FTI.TypeQuals != 0 && !D.isInvalidType()) {
     if (FTI.TypeQuals & Qualifiers::Const)
       Diag(D.getIdentifierLoc(), diag::err_invalid_qualified_destructor)
@@ -3085,6 +3300,15 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
     D.setInvalidType();
   }
 
+  // C++0x [class.dtor]p2:
+  //   A destructor shall not be declared with a ref-qualifier.
+  if (FTI.hasRefQualifier()) {
+    Diag(FTI.getRefQualifierLoc(), diag::err_ref_qualifier_destructor)
+      << FTI.RefQualifierIsLValueRef
+      << FixItHint::CreateRemoval(FTI.getRefQualifierLoc());
+    D.setInvalidType();
+  }
+  
   // Make sure we don't have any parameters.
   if (FTI.NumArgs > 0 && !FTIHasSingleVoidArgument(FTI)) {
     Diag(D.getIdentifierLoc(), diag::err_destructor_with_params);
@@ -3104,16 +3328,15 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
   // parameters (in case any of the errors above fired) and with
   // "void" as the return type, since destructors don't have return
   // types. 
+  if (!D.isInvalidType())
+    return R;
+
   const FunctionProtoType *Proto = R->getAs<FunctionProtoType>();
-  if (!Proto)
-    return QualType();
-  
-  return Context.getFunctionType(Context.VoidTy, 0, 0, false, 0,
-                                 Proto->hasExceptionSpec(),
-                                 Proto->hasAnyExceptionSpec(),
-                                 Proto->getNumExceptions(),
-                                 Proto->exception_begin(),
-                                 Proto->getExtInfo());
+  FunctionProtoType::ExtProtoInfo EPI = Proto->getExtProtoInfo();
+  EPI.Variadic = false;
+  EPI.TypeQuals = 0;
+  EPI.RefQualifier = RQ_None;
+  return Context.getFunctionType(Context.VoidTy, 0, 0, EPI);
 }
 
 /// CheckConversionDeclarator - Called by ActOnDeclarator to check the
@@ -3161,7 +3384,7 @@ void Sema::CheckConversionDeclarator(Declarator &D, QualType &R,
     Diag(D.getIdentifierLoc(), diag::err_conv_function_with_params);
 
     // Delete the parameters.
-    D.getTypeObject(0).Fun.freeArgs();
+    D.getFunctionTypeInfo().freeArgs();
     D.setInvalidType();
   } else if (Proto->isVariadic()) {
     Diag(D.getIdentifierLoc(), diag::err_conv_function_variadic);
@@ -3193,15 +3416,8 @@ void Sema::CheckConversionDeclarator(Declarator &D, QualType &R,
   // Rebuild the function type "R" without any parameters (in case any
   // of the errors above fired) and with the conversion type as the
   // return type.
-  if (D.isInvalidType()) {
-    R = Context.getFunctionType(ConvType, 0, 0, false,
-                                Proto->getTypeQuals(),
-                                Proto->hasExceptionSpec(),
-                                Proto->hasAnyExceptionSpec(),
-                                Proto->getNumExceptions(),
-                                Proto->exception_begin(),
-                                Proto->getExtInfo());
-  }
+  if (D.isInvalidType())
+    R = Context.getFunctionType(ConvType, 0, 0, Proto->getExtProtoInfo());
 
   // C++0x explicit conversion operators.
   if (D.getDeclSpec().isExplicitSpecified() && !getLangOptions().CPlusPlus0x)
@@ -3234,7 +3450,10 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
     = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
   if (const ReferenceType *ConvTypeRef = ConvType->getAs<ReferenceType>())
     ConvType = ConvTypeRef->getPointeeType();
-  if (ConvType->isRecordType()) {
+  if (Conversion->getTemplateSpecializationKind() != TSK_Undeclared &&
+      Conversion->getTemplateSpecializationKind() != TSK_ExplicitSpecialization)
+    /* Suppress diagnostics for instantiations. */;
+  else if (ConvType->isRecordType()) {
     ConvType = Context.getCanonicalType(ConvType).getUnqualifiedType();
     if (ConvType == ClassType)
       Diag(Conversion->getLocation(), diag::warn_conv_to_self_not_used)
@@ -3247,25 +3466,10 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
       << ClassType << ConvType;
   }
 
-  if (Conversion->getPrimaryTemplate()) {
-    // ignore specializations
-  } else if (Conversion->getPreviousDeclaration()) {
-    if (FunctionTemplateDecl *ConversionTemplate
-                                  = Conversion->getDescribedFunctionTemplate()) {
-      if (ClassDecl->replaceConversion(
-                                   ConversionTemplate->getPreviousDeclaration(),
-                                       ConversionTemplate))
-        return ConversionTemplate;
-    } else if (ClassDecl->replaceConversion(Conversion->getPreviousDeclaration(),
-                                            Conversion))
-      return Conversion;
-    assert(Conversion->isInvalidDecl() && "Conversion should not get here.");
-  } else if (FunctionTemplateDecl *ConversionTemplate
-               = Conversion->getDescribedFunctionTemplate())
-    ClassDecl->addConversionFunction(ConversionTemplate);
-  else 
-    ClassDecl->addConversionFunction(Conversion);
-
+  if (FunctionTemplateDecl *ConversionTemplate
+                                = Conversion->getDescribedFunctionTemplate())
+    return ConversionTemplate;
+  
   return Conversion;
 }
 
@@ -3293,20 +3497,22 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
 
   ProcessDeclAttributeList(DeclRegionScope, Namespc, AttrList);
 
-  if (const VisibilityAttr *attr = Namespc->getAttr<VisibilityAttr>())
-    PushVisibilityAttr(attr);
+  if (const VisibilityAttr *Attr = Namespc->getAttr<VisibilityAttr>())
+    PushNamespaceVisibilityAttr(Attr);
 
   if (II) {
     // C++ [namespace.def]p2:
-    // The identifier in an original-namespace-definition shall not have been
-    // previously defined in the declarative region in which the
-    // original-namespace-definition appears. The identifier in an
-    // original-namespace-definition is the name of the namespace. Subsequently
-    // in that declarative region, it is treated as an original-namespace-name.
-
-    NamedDecl *PrevDecl
-      = LookupSingleName(DeclRegionScope, II, IdentLoc, LookupOrdinaryName,
-                         ForRedeclaration);
+    //   The identifier in an original-namespace-definition shall not
+    //   have been previously defined in the declarative region in
+    //   which the original-namespace-definition appears. The
+    //   identifier in an original-namespace-definition is the name of
+    //   the namespace. Subsequently in that declarative region, it is
+    //   treated as an original-namespace-name.
+    //
+    // Since namespace names are unique in their scope, and we don't
+    // look through using directives, just
+    DeclContext::lookup_result R = CurContext->getRedeclContext()->lookup(II);
+    NamedDecl *PrevDecl = R.first == R.second? 0 : *R.first;
 
     if (NamespaceDecl *OrigNS = dyn_cast_or_null<NamespaceDecl>(PrevDecl)) {
       // This is an extended namespace definition.
@@ -3485,6 +3691,10 @@ Decl *Sema::ActOnUsingDirective(Scope *S,
   assert(!SS.isInvalid() && "Invalid CXXScopeSpec.");
   assert(NamespcName && "Invalid NamespcName.");
   assert(IdentLoc.isValid() && "Invalid NamespceName location.");
+
+  // This can only happen along a recovery path.
+  while (S->getFlags() & Scope::TemplateParamScope)
+    S = S->getParent();
   assert(S->getFlags() & Scope::DeclScope && "Invalid Scope.");
 
   UsingDirectiveDecl *UDir = 0;
@@ -3562,7 +3772,6 @@ Decl *Sema::ActOnUsingDirective(Scope *S,
   }
 
   // FIXME: We ignore attributes for now.
-  delete AttrList;
   return UDir;
 }
 
@@ -3580,14 +3789,14 @@ void Sema::PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir) {
 
 
 Decl *Sema::ActOnUsingDeclaration(Scope *S,
-                                            AccessSpecifier AS,
-                                            bool HasUsingKeyword,
-                                            SourceLocation UsingLoc,
-                                            CXXScopeSpec &SS,
-                                            UnqualifiedId &Name,
-                                            AttributeList *AttrList,
-                                            bool IsTypeName,
-                                            SourceLocation TypenameLoc) {
+                                  AccessSpecifier AS,
+                                  bool HasUsingKeyword,
+                                  SourceLocation UsingLoc,
+                                  CXXScopeSpec &SS,
+                                  UnqualifiedId &Name,
+                                  AttributeList *AttrList,
+                                  bool IsTypeName,
+                                  SourceLocation TypenameLoc) {
   assert(S->getFlags() & Scope::DeclScope && "Invalid Scope.");
 
   switch (Name.getKind()) {
@@ -3633,6 +3842,10 @@ Decl *Sema::ActOnUsingDeclaration(Scope *S,
       << FixItHint::CreateInsertion(SS.getRange().getBegin(), "using ");
   }
 
+  if (DiagnoseUnexpandedParameterPack(SS, UPPC_UsingDeclaration) ||
+      DiagnoseUnexpandedParameterPack(TargetNameInfo, UPPC_UsingDeclaration))
+    return 0;
+
   NamedDecl *UD = BuildUsingDeclaration(S, AS, UsingLoc, SS,
                                         TargetNameInfo, AttrList,
                                         /* IsInstantiation */ false,
@@ -3809,20 +4022,16 @@ UsingShadowDecl *Sema::BuildUsingShadowDecl(Scope *S,
     = UsingShadowDecl::Create(Context, CurContext,
                               UD->getLocation(), UD, Target);
   UD->addShadowDecl(Shadow);
-
+  
+  Shadow->setAccess(UD->getAccess());
+  if (Orig->isInvalidDecl() || UD->isInvalidDecl())
+    Shadow->setInvalidDecl();
+  
   if (S)
     PushOnScopeChains(Shadow, S);
   else
     CurContext->addDecl(Shadow);
-  Shadow->setAccess(UD->getAccess());
 
-  // Register it as a conversion if appropriate.
-  if (Shadow->getDeclName().getNameKind()
-        == DeclarationName::CXXConversionFunctionName)
-    cast<CXXRecordDecl>(CurContext)->addConversionFunction(Shadow);
-
-  if (Orig->isInvalidDecl() || UD->isInvalidDecl())
-    Shadow->setInvalidDecl();
 
   return Shadow;
 }
@@ -3893,7 +4102,6 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
   assert(IdentLoc.isValid() && "Invalid TargetName location.");
 
   // FIXME: We ignore attributes for now.
-  delete AttrList;
 
   if (SS.isEmpty()) {
     Diag(IdentLoc, diag::err_using_requires_qualname);
@@ -3921,8 +4129,7 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
     LookupQualifiedName(Previous, CurContext);
   }
 
-  NestedNameSpecifier *NNS =
-    static_cast<NestedNameSpecifier *>(SS.getScopeRep());
+  NestedNameSpecifier *NNS = SS.getScopeRep();
 
   // Check for invalid redeclarations.
   if (CheckUsingDeclRedeclaration(UsingLoc, IsTypeName, SS, IdentLoc, Previous))
@@ -3962,7 +4169,14 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
     return UD;
   }
 
-  // Look up the target name.
+  // Constructor inheriting using decls get special treatment.
+  if (NameInfo.getName().getNameKind() == DeclarationName::CXXConstructorName) {
+    if (CheckInheritedConstructorUsingDecl(UD))
+      UD->setInvalidDecl();
+    return UD;
+  }
+
+  // Otherwise, look up the target name.
 
   LookupResult R(*this, NameInfo, LookupOrdinaryName);
 
@@ -4026,6 +4240,42 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
   return UD;
 }
 
+/// Additional checks for a using declaration referring to a constructor name.
+bool Sema::CheckInheritedConstructorUsingDecl(UsingDecl *UD) {
+  if (UD->isTypeName()) {
+    // FIXME: Cannot specify typename when specifying constructor
+    return true;
+  }
+
+  const Type *SourceType = UD->getTargetNestedNameDecl()->getAsType();
+  assert(SourceType &&
+         "Using decl naming constructor doesn't have type in scope spec.");
+  CXXRecordDecl *TargetClass = cast<CXXRecordDecl>(CurContext);
+
+  // Check whether the named type is a direct base class.
+  CanQualType CanonicalSourceType = SourceType->getCanonicalTypeUnqualified();
+  CXXRecordDecl::base_class_iterator BaseIt, BaseE;
+  for (BaseIt = TargetClass->bases_begin(), BaseE = TargetClass->bases_end();
+       BaseIt != BaseE; ++BaseIt) {
+    CanQualType BaseType = BaseIt->getType()->getCanonicalTypeUnqualified();
+    if (CanonicalSourceType == BaseType)
+      break;
+  }
+
+  if (BaseIt == BaseE) {
+    // Did not find SourceType in the bases.
+    Diag(UD->getUsingLocation(),
+         diag::err_using_decl_constructor_not_in_direct_base)
+      << UD->getNameInfo().getSourceRange()
+      << QualType(SourceType, 0) << TargetClass;
+    return true;
+  }
+
+  BaseIt->setInheritConstructors();
+
+  return false;
+}
+
 /// Checks that the given using declaration is not an invalid
 /// redeclaration.  Note that this is checking only for the using decl
 /// itself, not for any ill-formedness among the UsingShadowDecls.
@@ -4132,6 +4382,10 @@ bool Sema::CheckUsingDeclQualifier(SourceLocation UsingLoc,
     return true;
   }
 
+  if (!NamedContext->isDependentContext() &&
+      RequireCompleteDeclContext(const_cast<CXXScopeSpec&>(SS), NamedContext))
+    return true;
+
   if (getLangOptions().CPlusPlus0x) {
     // C++0x [namespace.udecl]p3:
     //   In a using-declaration used as a member-declaration, the
@@ -4301,13 +4555,12 @@ namespace {
   /// to implicitly define the body of a C++ member function;
   class ImplicitlyDefinedFunctionScope {
     Sema &S;
-    DeclContext *PreviousContext;
+    Sema::ContextRAII SavedContext;
     
   public:
     ImplicitlyDefinedFunctionScope(Sema &S, CXXMethodDecl *Method)
-      : S(S), PreviousContext(S.CurContext) 
+      : S(S), SavedContext(S, Method) 
     {
-      S.CurContext = Method;
       S.PushFunctionScope();
       S.PushExpressionEvaluationContext(Sema::PotentiallyEvaluated);
     }
@@ -4315,11 +4568,32 @@ namespace {
     ~ImplicitlyDefinedFunctionScope() {
       S.PopExpressionEvaluationContext();
       S.PopFunctionOrBlockScope();
-      S.CurContext = PreviousContext;
     }
   };
 }
 
+static CXXConstructorDecl *getDefaultConstructorUnsafe(Sema &Self,
+                                                       CXXRecordDecl *D) {
+  ASTContext &Context = Self.Context;
+  QualType ClassType = Context.getTypeDeclType(D);
+  DeclarationName ConstructorName
+    = Context.DeclarationNames.getCXXConstructorName(
+                      Context.getCanonicalType(ClassType.getUnqualifiedType()));
+
+  DeclContext::lookup_const_iterator Con, ConEnd;
+  for (llvm::tie(Con, ConEnd) = D->lookup(ConstructorName);
+       Con != ConEnd; ++Con) {
+    // FIXME: In C++0x, a constructor template can be a default constructor.
+    if (isa<FunctionTemplateDecl>(*Con))
+      continue;
+
+    CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
+    if (Constructor->isDefaultConstructor())
+      return Constructor;
+  }
+  return 0;
+}
+
 CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
                                                      CXXRecordDecl *ClassDecl) {
   // C++ [class.ctor]p5:
@@ -4347,8 +4621,8 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
       CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
       if (!BaseClassDecl->hasDeclaredDefaultConstructor())
         ExceptSpec.CalledDecl(DeclareImplicitDefaultConstructor(BaseClassDecl));
-      else if (CXXConstructorDecl *Constructor 
-                                       = BaseClassDecl->getDefaultConstructor())
+      else if (CXXConstructorDecl *Constructor
+                            = getDefaultConstructorUnsafe(*this, BaseClassDecl))
         ExceptSpec.CalledDecl(Constructor);
     }
   }
@@ -4362,7 +4636,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
       if (!BaseClassDecl->hasDeclaredDefaultConstructor())
         ExceptSpec.CalledDecl(DeclareImplicitDefaultConstructor(BaseClassDecl));
       else if (CXXConstructorDecl *Constructor
-                                       = BaseClassDecl->getDefaultConstructor())
+                            = getDefaultConstructorUnsafe(*this, BaseClassDecl))
         ExceptSpec.CalledDecl(Constructor);
     }
   }
@@ -4378,11 +4652,16 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
         ExceptSpec.CalledDecl(
                             DeclareImplicitDefaultConstructor(FieldClassDecl));
       else if (CXXConstructorDecl *Constructor
-                                      = FieldClassDecl->getDefaultConstructor())
+                           = getDefaultConstructorUnsafe(*this, FieldClassDecl))
         ExceptSpec.CalledDecl(Constructor);
     }
   }
-  
+
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.HasExceptionSpec = ExceptSpec.hasExceptionSpecification();
+  EPI.HasAnyExceptionSpec = ExceptSpec.hasAnyExceptionSpecification();
+  EPI.NumExceptions = ExceptSpec.size();
+  EPI.Exceptions = ExceptSpec.data();
   
   // Create the actual constructor declaration.
   CanQualType ClassType
@@ -4393,12 +4672,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   CXXConstructorDecl *DefaultCon
     = CXXConstructorDecl::Create(Context, ClassDecl, NameInfo,
                                  Context.getFunctionType(Context.VoidTy,
-                                                         0, 0, false, 0,
-                                       ExceptSpec.hasExceptionSpecification(),
-                                     ExceptSpec.hasAnyExceptionSpecification(),
-                                                         ExceptSpec.size(),
-                                                         ExceptSpec.data(),
-                                                       FunctionType::ExtInfo()),
+                                                         0, 0, EPI),
                                  /*TInfo=*/0,
                                  /*isExplicit=*/false,
                                  /*isInline=*/true,
@@ -4408,7 +4682,6 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   DefaultCon->setTrivial(ClassDecl->hasTrivialConstructor());
   
   // Note that we have declared this constructor.
-  ClassDecl->setDeclaredDefaultConstructor(true);
   ++ASTContext::NumImplicitDefaultConstructorsDeclared;
   
   if (Scope *S = getScopeForContext(ClassDecl))
@@ -4428,15 +4701,193 @@ void Sema::DefineImplicitDefaultConstructor(SourceLocation CurrentLocation,
   assert(ClassDecl && "DefineImplicitDefaultConstructor - invalid constructor");
 
   ImplicitlyDefinedFunctionScope Scope(*this, Constructor);
-  ErrorTrap Trap(*this);
-  if (SetBaseOrMemberInitializers(Constructor, 0, 0, /*AnyErrors=*/false) ||
+  DiagnosticErrorTrap Trap(Diags);
+  if (SetCtorInitializers(Constructor, 0, 0, /*AnyErrors=*/false) ||
       Trap.hasErrorOccurred()) {
     Diag(CurrentLocation, diag::note_member_synthesized_at) 
       << CXXConstructor << Context.getTagDeclType(ClassDecl);
     Constructor->setInvalidDecl();
-  } else {
-    Constructor->setUsed();
-    MarkVTableUsed(CurrentLocation, ClassDecl);
+    return;
+  }
+
+  SourceLocation Loc = Constructor->getLocation();
+  Constructor->setBody(new (Context) CompoundStmt(Context, 0, 0, Loc, Loc));
+
+  Constructor->setUsed();
+  MarkVTableUsed(CurrentLocation, ClassDecl);
+}
+
+void Sema::DeclareInheritedConstructors(CXXRecordDecl *ClassDecl) {
+  // We start with an initial pass over the base classes to collect those that
+  // inherit constructors from. If there are none, we can forgo all further
+  // processing.
+  typedef llvm::SmallVector<const RecordType *, 4> BasesVector;
+  BasesVector BasesToInheritFrom;
+  for (CXXRecordDecl::base_class_iterator BaseIt = ClassDecl->bases_begin(),
+                                          BaseE = ClassDecl->bases_end();
+         BaseIt != BaseE; ++BaseIt) {
+    if (BaseIt->getInheritConstructors()) {
+      QualType Base = BaseIt->getType();
+      if (Base->isDependentType()) {
+        // If we inherit constructors from anything that is dependent, just
+        // abort processing altogether. We'll get another chance for the
+        // instantiations.
+        return;
+      }
+      BasesToInheritFrom.push_back(Base->castAs<RecordType>());
+    }
+  }
+  if (BasesToInheritFrom.empty())
+    return;
+
+  // Now collect the constructors that we already have in the current class.
+  // Those take precedence over inherited constructors.
+  // C++0x [class.inhctor]p3: [...] a constructor is implicitly declared [...]
+  //   unless there is a user-declared constructor with the same signature in
+  //   the class where the using-declaration appears.
+  llvm::SmallSet<const Type *, 8> ExistingConstructors;
+  for (CXXRecordDecl::ctor_iterator CtorIt = ClassDecl->ctor_begin(),
+                                    CtorE = ClassDecl->ctor_end();
+       CtorIt != CtorE; ++CtorIt) {
+    ExistingConstructors.insert(
+        Context.getCanonicalType(CtorIt->getType()).getTypePtr());
+  }
+
+  Scope *S = getScopeForContext(ClassDecl);
+  DeclarationName CreatedCtorName =
+      Context.DeclarationNames.getCXXConstructorName(
+          ClassDecl->getTypeForDecl()->getCanonicalTypeUnqualified());
+
+  // Now comes the true work.
+  // First, we keep a map from constructor types to the base that introduced
+  // them. Needed for finding conflicting constructors. We also keep the
+  // actually inserted declarations in there, for pretty diagnostics.
+  typedef std::pair<CanQualType, CXXConstructorDecl *> ConstructorInfo;
+  typedef llvm::DenseMap<const Type *, ConstructorInfo> ConstructorToSourceMap;
+  ConstructorToSourceMap InheritedConstructors;
+  for (BasesVector::iterator BaseIt = BasesToInheritFrom.begin(),
+                             BaseE = BasesToInheritFrom.end();
+       BaseIt != BaseE; ++BaseIt) {
+    const RecordType *Base = *BaseIt;
+    CanQualType CanonicalBase = Base->getCanonicalTypeUnqualified();
+    CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(Base->getDecl());
+    for (CXXRecordDecl::ctor_iterator CtorIt = BaseDecl->ctor_begin(),
+                                      CtorE = BaseDecl->ctor_end();
+         CtorIt != CtorE; ++CtorIt) {
+      // Find the using declaration for inheriting this base's constructors.
+      DeclarationName Name =
+          Context.DeclarationNames.getCXXConstructorName(CanonicalBase);
+      UsingDecl *UD = dyn_cast_or_null<UsingDecl>(
+          LookupSingleName(S, Name,SourceLocation(), LookupUsingDeclName));
+      SourceLocation UsingLoc = UD ? UD->getLocation() :
+                                     ClassDecl->getLocation();
+
+      // C++0x [class.inhctor]p1: The candidate set of inherited constructors
+      //   from the class X named in the using-declaration consists of actual
+      //   constructors and notional constructors that result from the
+      //   transformation of defaulted parameters as follows:
+      //   - all non-template default constructors of X, and
+      //   - for each non-template constructor of X that has at least one
+      //     parameter with a default argument, the set of constructors that
+      //     results from omitting any ellipsis parameter specification and
+      //     successively omitting parameters with a default argument from the
+      //     end of the parameter-type-list.
+      CXXConstructorDecl *BaseCtor = *CtorIt;
+      bool CanBeCopyOrMove = BaseCtor->isCopyOrMoveConstructor();
+      const FunctionProtoType *BaseCtorType =
+          BaseCtor->getType()->getAs<FunctionProtoType>();
+
+      for (unsigned params = BaseCtor->getMinRequiredArguments(),
+                    maxParams = BaseCtor->getNumParams();
+           params <= maxParams; ++params) {
+        // Skip default constructors. They're never inherited.
+        if (params == 0)
+          continue;
+        // Skip copy and move constructors for the same reason.
+        if (CanBeCopyOrMove && params == 1)
+          continue;
+
+        // Build up a function type for this particular constructor.
+        // FIXME: The working paper does not consider that the exception spec
+        // for the inheriting constructor might be larger than that of the
+        // source. This code doesn't yet, either.
+        const Type *NewCtorType;
+        if (params == maxParams)
+          NewCtorType = BaseCtorType;
+        else {
+          llvm::SmallVector<QualType, 16> Args;
+          for (unsigned i = 0; i < params; ++i) {
+            Args.push_back(BaseCtorType->getArgType(i));
+          }
+          FunctionProtoType::ExtProtoInfo ExtInfo =
+              BaseCtorType->getExtProtoInfo();
+          ExtInfo.Variadic = false;
+          NewCtorType = Context.getFunctionType(BaseCtorType->getResultType(),
+                                                Args.data(), params, ExtInfo)
+                       .getTypePtr();
+        }
+        const Type *CanonicalNewCtorType =
+            Context.getCanonicalType(NewCtorType);
+
+        // Now that we have the type, first check if the class already has a
+        // constructor with this signature.
+        if (ExistingConstructors.count(CanonicalNewCtorType))
+          continue;
+
+        // Then we check if we have already declared an inherited constructor
+        // with this signature.
+        std::pair<ConstructorToSourceMap::iterator, bool> result =
+            InheritedConstructors.insert(std::make_pair(
+                CanonicalNewCtorType,
+                std::make_pair(CanonicalBase, (CXXConstructorDecl*)0)));
+        if (!result.second) {
+          // Already in the map. If it came from a different class, that's an
+          // error. Not if it's from the same.
+          CanQualType PreviousBase = result.first->second.first;
+          if (CanonicalBase != PreviousBase) {
+            const CXXConstructorDecl *PrevCtor = result.first->second.second;
+            const CXXConstructorDecl *PrevBaseCtor =
+                PrevCtor->getInheritedConstructor();
+            assert(PrevBaseCtor && "Conflicting constructor was not inherited");
+
+            Diag(UsingLoc, diag::err_using_decl_constructor_conflict);
+            Diag(BaseCtor->getLocation(),
+                 diag::note_using_decl_constructor_conflict_current_ctor);
+            Diag(PrevBaseCtor->getLocation(),
+                 diag::note_using_decl_constructor_conflict_previous_ctor);
+            Diag(PrevCtor->getLocation(),
+                 diag::note_using_decl_constructor_conflict_previous_using);
+          }
+          continue;
+        }
+
+        // OK, we're there, now add the constructor.
+        // C++0x [class.inhctor]p8: [...] that would be performed by a
+        //   user-writtern inline constructor [...]
+        DeclarationNameInfo DNI(CreatedCtorName, UsingLoc);
+        CXXConstructorDecl *NewCtor = CXXConstructorDecl::Create(
+            Context, ClassDecl, DNI, QualType(NewCtorType, 0), /*TInfo=*/0,
+            BaseCtor->isExplicit(), /*Inline=*/true,
+            /*ImplicitlyDeclared=*/true);
+        NewCtor->setAccess(BaseCtor->getAccess());
+
+        // Build up the parameter decls and add them.
+        llvm::SmallVector<ParmVarDecl *, 16> ParamDecls;
+        for (unsigned i = 0; i < params; ++i) {
+          ParamDecls.push_back(ParmVarDecl::Create(Context, NewCtor, UsingLoc,
+                                                   /*IdentifierInfo=*/0,
+                                                   BaseCtorType->getArgType(i),
+                                                   /*TInfo=*/0, SC_None,
+                                                   SC_None, /*DefaultArg=*/0));
+        }
+        NewCtor->setParams(ParamDecls.data(), ParamDecls.size());
+        NewCtor->setInheritedConstructor(BaseCtor);
+
+        PushOnScopeChains(NewCtor, S, false);
+        ClassDecl->addDecl(NewCtor);
+        result.first->second.second = NewCtor;
+      }
+    }
   }
 }
 
@@ -4483,13 +4934,12 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   }
   
   // Create the actual destructor declaration.
-  QualType Ty = Context.getFunctionType(Context.VoidTy,
-                                        0, 0, false, 0,
-                                        ExceptSpec.hasExceptionSpecification(),
-                                    ExceptSpec.hasAnyExceptionSpecification(),
-                                        ExceptSpec.size(),
-                                        ExceptSpec.data(),
-                                        FunctionType::ExtInfo());
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.HasExceptionSpec = ExceptSpec.hasExceptionSpecification();
+  EPI.HasAnyExceptionSpec = ExceptSpec.hasAnyExceptionSpecification();
+  EPI.NumExceptions = ExceptSpec.size();
+  EPI.Exceptions = ExceptSpec.data();
+  QualType Ty = Context.getFunctionType(Context.VoidTy, 0, 0, EPI);
   
   CanQualType ClassType
     = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
@@ -4497,7 +4947,7 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
     = Context.DeclarationNames.getCXXDestructorName(ClassType);
   DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXDestructorDecl *Destructor
-    = CXXDestructorDecl::Create(Context, ClassDecl, NameInfo, Ty,
+      = CXXDestructorDecl::Create(Context, ClassDecl, NameInfo, Ty, 0,
                                 /*isInline=*/true,
                                 /*isImplicitlyDeclared=*/true);
   Destructor->setAccess(AS_public);
@@ -4505,7 +4955,6 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   Destructor->setTrivial(ClassDecl->hasTrivialDestructor());
   
   // Note that we have declared this destructor.
-  ClassDecl->setDeclaredDestructor(true);
   ++ASTContext::NumImplicitDestructorsDeclared;
   
   // Introduce this destructor into its scope.
@@ -4533,7 +4982,7 @@ void Sema::DefineImplicitDestructor(SourceLocation CurrentLocation,
 
   ImplicitlyDefinedFunctionScope Scope(*this, Destructor);
 
-  ErrorTrap Trap(*this);
+  DiagnosticErrorTrap Trap(Diags);
   MarkBaseAndMemberDestructorsReferenced(Destructor->getLocation(),
                                          Destructor->getParent());
 
@@ -4545,6 +4994,9 @@ void Sema::DefineImplicitDestructor(SourceLocation CurrentLocation,
     return;
   }
 
+  SourceLocation Loc = Destructor->getLocation();
+  Destructor->setBody(new (Context) CompoundStmt(Context, 0, 0, Loc, Loc));
+
   Destructor->setUsed();
   MarkVTableUsed(CurrentLocation, ClassDecl);
 }
@@ -4644,8 +5096,8 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
     // Build the call to the assignment operator.
 
     ExprResult Call = S.BuildCallToMemberFunction(/*Scope=*/0, 
-                                                      OpEqualRef.takeAs<Expr>(),
-                                                        Loc, &From, 1, 0, Loc);
+                                                  OpEqualRef.takeAs<Expr>(),
+                                                  Loc, &From, 1, Loc);
     if (Call.isInvalid())
       return StmtError();
     
@@ -4693,26 +5145,25 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
   // Create a reference to the iteration variable; we'll use this several
   // times throughout.
   Expr *IterationVarRef
-    = S.BuildDeclRefExpr(IterationVar, SizeType, Loc).takeAs<Expr>();
+    = S.BuildDeclRefExpr(IterationVar, SizeType, VK_RValue, Loc).take();
   assert(IterationVarRef && "Reference to invented variable cannot fail!");
   
   // Create the DeclStmt that holds the iteration variable.
   Stmt *InitStmt = new (S.Context) DeclStmt(DeclGroupRef(IterationVar),Loc,Loc);
   
   // Create the comparison against the array bound.
-  llvm::APInt Upper = ArrayTy->getSize();
-  Upper.zextOrTrunc(S.Context.getTypeSize(SizeType));
+  llvm::APInt Upper
+    = ArrayTy->getSize().zextOrTrunc(S.Context.getTypeSize(SizeType));
   Expr *Comparison
-    = new (S.Context) BinaryOperator(IterationVarRef->Retain(),
-                           IntegerLiteral::Create(S.Context,
-                                                  Upper, SizeType, Loc),
-                                                  BO_NE, S.Context.BoolTy, Loc);
+    = new (S.Context) BinaryOperator(IterationVarRef,
+                     IntegerLiteral::Create(S.Context, Upper, SizeType, Loc),
+                                     BO_NE, S.Context.BoolTy,
+                                     VK_RValue, OK_Ordinary, Loc);
   
   // Create the pre-increment of the iteration variable.
   Expr *Increment
-    = new (S.Context) UnaryOperator(IterationVarRef->Retain(),
-                                    UO_PreInc,
-                                    SizeType, Loc);
+    = new (S.Context) UnaryOperator(IterationVarRef, UO_PreInc, SizeType,
+                                    VK_LValue, OK_Ordinary, Loc);
   
   // Subscript the "from" and "to" expressions with the iteration variable.
   From = AssertSuccess(S.CreateBuiltinArraySubscriptExpr(From, Loc,
@@ -4721,10 +5172,9 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
                                                        IterationVarRef, Loc));
   
   // Build the copy for an individual element of the array.
-  StmtResult Copy = BuildSingleCopyAssign(S, Loc, 
-                                                ArrayTy->getElementType(),
-                                                To, From, 
-                                                CopyingBaseSubobject, Depth+1);
+  StmtResult Copy = BuildSingleCopyAssign(S, Loc, ArrayTy->getElementType(),
+                                          To, From, CopyingBaseSubobject,
+                                          Depth + 1);
   if (Copy.isInvalid())
     return StmtError();
   
@@ -4881,24 +5331,22 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   
   //   An implicitly-declared copy assignment operator is an inline public
   //   member of its class.
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.HasExceptionSpec = ExceptSpec.hasExceptionSpecification();
+  EPI.HasAnyExceptionSpec = ExceptSpec.hasAnyExceptionSpecification();
+  EPI.NumExceptions = ExceptSpec.size();
+  EPI.Exceptions = ExceptSpec.data();
   DeclarationName Name = Context.DeclarationNames.getCXXOperatorName(OO_Equal);
   DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXMethodDecl *CopyAssignment
     = CXXMethodDecl::Create(Context, ClassDecl, NameInfo,
-                            Context.getFunctionType(RetType, &ArgType, 1,
-                                                    false, 0,
-                                         ExceptSpec.hasExceptionSpecification(),
-                                      ExceptSpec.hasAnyExceptionSpecification(),
-                                                    ExceptSpec.size(),
-                                                    ExceptSpec.data(),
-                                                    FunctionType::ExtInfo()),
+                            Context.getFunctionType(RetType, &ArgType, 1, EPI),
                             /*TInfo=*/0, /*isStatic=*/false,
                             /*StorageClassAsWritten=*/SC_None,
                             /*isInline=*/true);
   CopyAssignment->setAccess(AS_public);
   CopyAssignment->setImplicit();
   CopyAssignment->setTrivial(ClassDecl->hasTrivialCopyAssignment());
-  CopyAssignment->setCopyAssignment(true);
   
   // Add the parameter to the operator.
   ParmVarDecl *FromParam = ParmVarDecl::Create(Context, CopyAssignment,
@@ -4910,7 +5358,6 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   CopyAssignment->setParams(&FromParam, 1);
   
   // Note that we have added this copy-assignment operator.
-  ClassDecl->setDeclaredCopyAssignment(true);
   ++ASTContext::NumImplicitCopyAssignmentOperatorsDeclared;
   
   if (Scope *S = getScopeForContext(ClassDecl))
@@ -4939,7 +5386,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   CopyAssignOperator->setUsed();
 
   ImplicitlyDefinedFunctionScope Scope(*this, CopyAssignOperator);
-  ErrorTrap Trap(*this);
+  DiagnosticErrorTrap Trap(Diags);
 
   // C++0x [class.copy]p30:
   //   The implicitly-defined or explicitly-defaulted copy assignment operator
@@ -4967,7 +5414,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   
   // Construct a reference to the "other" object. We'll be using this 
   // throughout the generated ASTs.
-  Expr *OtherRef = BuildDeclRefExpr(Other, OtherRefType, Loc).takeAs<Expr>();
+  Expr *OtherRef = BuildDeclRefExpr(Other, OtherRefType, VK_LValue, Loc).take();
   assert(OtherRef && "Reference to parameter cannot fail!");
   
   // Construct the "this" pointer. We'll be using this throughout the generated
@@ -4982,10 +5429,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     // Form the assignment:
     //   static_cast<Base*>(this)->Base::operator=(static_cast<Base&>(other));
     QualType BaseType = Base->getType().getUnqualifiedType();
-    CXXRecordDecl *BaseClassDecl = 0;
-    if (const RecordType *BaseRecordT = BaseType->getAs<RecordType>())
-      BaseClassDecl = cast<CXXRecordDecl>(BaseRecordT->getDecl());
-    else {
+    if (!BaseType->isRecordType()) {
       Invalid = true;
       continue;
     }
@@ -4995,7 +5439,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
 
     // Construct the "from" expression, which is an implicit cast to the
     // appropriately-qualified base type.
-    Expr *From = OtherRef->Retain();
+    Expr *From = OtherRef;
     ImpCastExprToType(From, Context.getQualifiedType(BaseType, OtherQuals),
                       CK_UncheckedDerivedToBase,
                       VK_LValue, &BasePath);
@@ -5073,11 +5517,11 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     MemberLookup.addDecl(*Field);
     MemberLookup.resolveKind();
     ExprResult From = BuildMemberReferenceExpr(OtherRef, OtherRefType,
-                                                     Loc, /*IsArrow=*/false,
-                                                     SS, 0, MemberLookup, 0);
+                                               Loc, /*IsArrow=*/false,
+                                               SS, 0, MemberLookup, 0);
     ExprResult To = BuildMemberReferenceExpr(This, This->getType(),
-                                                   Loc, /*IsArrow=*/true,
-                                                   SS, 0, MemberLookup, 0);
+                                             Loc, /*IsArrow=*/true,
+                                             SS, 0, MemberLookup, 0);
     assert(!From.isInvalid() && "Implicit field reference cannot fail");
     assert(!To.isInvalid() && "Implicit field reference cannot fail");
     
@@ -5097,8 +5541,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
               = Context.getAsConstantArrayType(FieldType);
            Array; 
            Array = Context.getAsConstantArrayType(Array->getElementType())) {
-        llvm::APInt ArraySize = Array->getSize();
-        ArraySize.zextOrTrunc(Size.getBitWidth());
+        llvm::APInt ArraySize
+          = Array->getSize().zextOrTrunc(Size.getBitWidth());
         Size *= ArraySize;
       }
           
@@ -5128,7 +5572,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
         
           CollectableMemCpyRef = BuildDeclRefExpr(CollectableMemCpy, 
                                                   CollectableMemCpy->getType(),
-                                                  Loc, 0).takeAs<Expr>();
+                                                  VK_LValue, Loc, 0).take();
           assert(CollectableMemCpyRef && "Builtin reference cannot fail");
         }
       }
@@ -5148,7 +5592,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
 
         BuiltinMemCpyRef = BuildDeclRefExpr(BuiltinMemCpy, 
                                             BuiltinMemCpy->getType(),
-                                            Loc, 0).takeAs<Expr>();
+                                            VK_LValue, Loc, 0).take();
         assert(BuiltinMemCpyRef && "Builtin reference cannot fail");
       }
           
@@ -5156,20 +5600,17 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
       CallArgs.push_back(To.takeAs<Expr>());
       CallArgs.push_back(From.takeAs<Expr>());
       CallArgs.push_back(IntegerLiteral::Create(Context, Size, SizeType, Loc));
-      llvm::SmallVector<SourceLocation, 4> Commas; // FIXME: Silly
-      Commas.push_back(Loc);
-      Commas.push_back(Loc);
       ExprResult Call = ExprError();
       if (NeedsCollectableMemCpy)
         Call = ActOnCallExpr(/*Scope=*/0,
                              CollectableMemCpyRef,
                              Loc, move_arg(CallArgs), 
-                             Commas.data(), Loc);
+                             Loc);
       else
         Call = ActOnCallExpr(/*Scope=*/0,
                              BuiltinMemCpyRef,
                              Loc, move_arg(CallArgs), 
-                             Commas.data(), Loc);
+                             Loc);
           
       assert(!Call.isInvalid() && "Call to __builtin_memcpy cannot fail!");
       Statements.push_back(Call.takeAs<Expr>());
@@ -5352,6 +5793,11 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   
   //   An implicitly-declared copy constructor is an inline public
   //   member of its class.
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.HasExceptionSpec = ExceptSpec.hasExceptionSpecification();
+  EPI.HasAnyExceptionSpec = ExceptSpec.hasAnyExceptionSpecification();
+  EPI.NumExceptions = ExceptSpec.size();
+  EPI.Exceptions = ExceptSpec.data();
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(
                                            Context.getCanonicalType(ClassType));
@@ -5359,23 +5805,15 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   CXXConstructorDecl *CopyConstructor
     = CXXConstructorDecl::Create(Context, ClassDecl, NameInfo,
                                  Context.getFunctionType(Context.VoidTy,
-                                                         &ArgType, 1,
-                                                         false, 0,
-                                         ExceptSpec.hasExceptionSpecification(),
-                                      ExceptSpec.hasAnyExceptionSpecification(),
-                                                         ExceptSpec.size(),
-                                                         ExceptSpec.data(),
-                                                       FunctionType::ExtInfo()),
+                                                         &ArgType, 1, EPI),
                                  /*TInfo=*/0,
                                  /*isExplicit=*/false,
                                  /*isInline=*/true,
                                  /*isImplicitlyDeclared=*/true);
   CopyConstructor->setAccess(AS_public);
-  CopyConstructor->setImplicit();
   CopyConstructor->setTrivial(ClassDecl->hasTrivialCopyConstructor());
   
   // Note that we have declared this constructor.
-  ClassDecl->setDeclaredCopyConstructor(true);
   ++ASTContext::NumImplicitCopyConstructorsDeclared;
   
   // Add the parameter to the constructor.
@@ -5405,9 +5843,9 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
   assert(ClassDecl && "DefineImplicitCopyConstructor - invalid constructor");
 
   ImplicitlyDefinedFunctionScope Scope(*this, CopyConstructor);
-  ErrorTrap Trap(*this);
+  DiagnosticErrorTrap Trap(Diags);
 
-  if (SetBaseOrMemberInitializers(CopyConstructor, 0, 0, /*AnyErrors=*/false) ||
+  if (SetCtorInitializers(CopyConstructor, 0, 0, /*AnyErrors=*/false) ||
       Trap.hasErrorOccurred()) {
     Diag(CurrentLocation, diag::note_member_synthesized_at) 
       << CXXCopyConstructor << Context.getTagDeclType(ClassDecl);
@@ -5428,7 +5866,8 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor,
                             MultiExprArg ExprArgs,
                             bool RequiresZeroInit,
-                            unsigned ConstructKind) {
+                            unsigned ConstructKind,
+                            SourceRange ParenRange) {
   bool Elidable = false;
 
   // C++0x [class.copy]p34:
@@ -5441,17 +5880,15 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
   //       with the same cv-unqualified type, the copy/move operation
   //       can be omitted by constructing the temporary object
   //       directly into the target of the omitted copy/move
-  if (Constructor->isCopyConstructor() && ExprArgs.size() >= 1) {
+  if (ConstructKind == CXXConstructExpr::CK_Complete &&
+      Constructor->isCopyOrMoveConstructor() && ExprArgs.size() >= 1) {
     Expr *SubExpr = ((Expr **)ExprArgs.get())[0];
-    Elidable = SubExpr->isTemporaryObject() &&
-      ConstructKind == CXXConstructExpr::CK_Complete &&
-      Context.hasSameUnqualifiedType(SubExpr->getType(), 
-                           Context.getTypeDeclType(Constructor->getParent()));
+    Elidable = SubExpr->isTemporaryObject(Context, Constructor->getParent());
   }
 
   return BuildCXXConstructExpr(ConstructLoc, DeclInitType, Constructor,
                                Elidable, move(ExprArgs), RequiresZeroInit,
-                               ConstructKind);
+                               ConstructKind, ParenRange);
 }
 
 /// BuildCXXConstructExpr - Creates a complete call to a constructor,
@@ -5461,7 +5898,8 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor, bool Elidable,
                             MultiExprArg ExprArgs,
                             bool RequiresZeroInit,
-                            unsigned ConstructKind) {
+                            unsigned ConstructKind,
+                            SourceRange ParenRange) {
   unsigned NumExprs = ExprArgs.size();
   Expr **Exprs = (Expr **)ExprArgs.release();
 
@@ -5469,21 +5907,25 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
   return Owned(CXXConstructExpr::Create(Context, DeclInitType, ConstructLoc,
                                         Constructor, Elidable, Exprs, NumExprs, 
                                         RequiresZeroInit,
-              static_cast<CXXConstructExpr::ConstructionKind>(ConstructKind)));
+              static_cast<CXXConstructExpr::ConstructionKind>(ConstructKind),
+                                        ParenRange));
 }
 
 bool Sema::InitializeVarWithConstructor(VarDecl *VD,
                                         CXXConstructorDecl *Constructor,
                                         MultiExprArg Exprs) {
+  // FIXME: Provide the correct paren SourceRange when available.
   ExprResult TempResult =
     BuildCXXConstructExpr(VD->getLocation(), VD->getType(), Constructor,
-                          move(Exprs), false, CXXConstructExpr::CK_Complete);
+                          move(Exprs), false, CXXConstructExpr::CK_Complete,
+                          SourceRange());
   if (TempResult.isInvalid())
     return true;
 
   Expr *Temp = TempResult.takeAs<Expr>();
+  CheckImplicitConversions(Temp, VD->getLocation());
   MarkDeclarationReferenced(VD->getLocation(), Constructor);
-  Temp = MaybeCreateCXXExprWithTemporaries(Temp);
+  Temp = MaybeCreateExprWithCleanups(Temp);
   VD->setInit(Temp);
 
   return false;
@@ -5500,7 +5942,8 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
                             << VD->getDeclName()
                             << VD->getType());
 
-    if (!VD->isInvalidDecl() && VD->hasGlobalStorage())
+    // TODO: this should be re-enabled for static locals by !CXAAtExit
+    if (!VD->isInvalidDecl() && VD->hasGlobalStorage() && !VD->isStaticLocal())
       Diag(VD->getLocation(), diag::warn_global_destructor);
   }
 }
@@ -5511,8 +5954,8 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
 void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
                                          SourceLocation LParenLoc,
                                          MultiExprArg Exprs,
-                                         SourceLocation *CommaLocs,
-                                         SourceLocation RParenLoc) {
+                                         SourceLocation RParenLoc,
+                                         bool TypeMayContainAuto) {
   assert(Exprs.size() != 0 && Exprs.get() && "missing expressions");
 
   // If there is no declaration, there was an error parsing it.  Just ignore
@@ -5527,6 +5970,37 @@ void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
     return;
   }
 
+  // C++0x [decl.spec.auto]p6. Deduce the type which 'auto' stands in for.
+  if (TypeMayContainAuto && VDecl->getType()->getContainedAutoType()) {
+    VDecl->setParsingAutoInit(false);
+
+    // FIXME: n3225 doesn't actually seem to indicate this is ill-formed
+    if (Exprs.size() > 1) {
+      Diag(Exprs.get()[1]->getSourceRange().getBegin(),
+           diag::err_auto_var_init_multiple_expressions)
+        << VDecl->getDeclName() << VDecl->getType()
+        << VDecl->getSourceRange();
+      RealDecl->setInvalidDecl();
+      return;
+    }
+
+    Expr *Init = Exprs.get()[0];
+    QualType DeducedType;
+    if (!DeduceAutoType(VDecl->getType(), Init, DeducedType)) {
+      Diag(VDecl->getLocation(), diag::err_auto_var_deduction_failure)
+        << VDecl->getDeclName() << VDecl->getType() << Init->getType()
+        << Init->getSourceRange();
+      RealDecl->setInvalidDecl();
+      return;
+    }
+    VDecl->setType(DeducedType);
+
+    // If this is a redeclaration, check that the type we just deduced matches
+    // the previously declared type.
+    if (VarDecl *Old = VDecl->getPreviousDeclaration())
+      MergeVarDeclTypes(VDecl, Old);
+  }
+
   // We will represent direct-initialization similarly to copy-initialization:
   //    int x(1);  -as-> int x = 1;
   //    ClassType x(a,b,c); -as-> ClassType x = ClassType(a,b,c);
@@ -5583,11 +6057,21 @@ void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
     return;
   } 
 
+  bool IsDependent = false;
+  for (unsigned I = 0, N = Exprs.size(); I != N; ++I) {
+    if (DiagnoseUnexpandedParameterPack(Exprs.get()[I], UPPC_Expression)) {
+      VDecl->setInvalidDecl();
+      return;
+    }
+
+    if (Exprs.get()[I]->isTypeDependent())
+      IsDependent = true;
+  }
+
   // If either the declaration has a dependent type or if any of the
   // expressions is type-dependent, we represent the initialization
   // via a ParenListExpr for later use during template instantiation.
-  if (VDecl->getType()->isDependentType() ||
-      Expr::hasAnyTypeDependentArguments((Expr **)Exprs.get(), Exprs.size())) {
+  if (VDecl->getType()->isDependentType() || IsDependent) {
     // Let clients know that initialization was done with a direct initializer.
     VDecl->setCXXDirectInitializer(true);
 
@@ -5615,21 +6099,14 @@ void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
     VDecl->setInvalidDecl();
     return;
   }
+
+  CheckImplicitConversions(Result.get(), LParenLoc);
   
-  Result = MaybeCreateCXXExprWithTemporaries(Result.get());
+  Result = MaybeCreateExprWithCleanups(Result);
   VDecl->setInit(Result.takeAs<Expr>());
   VDecl->setCXXDirectInitializer(true);
 
-    if (!VDecl->isInvalidDecl() &&
-        !VDecl->getDeclContext()->isDependentContext() &&
-        VDecl->hasGlobalStorage() &&
-        !VDecl->getInit()->isConstantInitializer(Context,
-                                        VDecl->getType()->isReferenceType()))
-      Diag(VDecl->getLocation(), diag::warn_global_constructor)
-        << VDecl->getInit()->getSourceRange();
-
-  if (const RecordType *Record = VDecl->getType()->getAs<RecordType>())
-    FinalizeVarWithDestructor(VDecl, Record);
+  CheckCompleteVariableDeclaration(VDecl);
 }
 
 /// \brief Given a constructor and the set of arguments provided for the
@@ -5926,15 +6403,6 @@ bool Sema::CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl) {
         << LastParam->getType() << (Op == OO_MinusMinus);
   }
 
-  // Notify the class if it got an assignment operator.
-  if (Op == OO_Equal) {
-    // Would have returned earlier otherwise.
-    assert(isa<CXXMethodDecl>(FnDecl) &&
-      "Overloaded = not member, but not filtered.");
-    CXXMethodDecl *Method = cast<CXXMethodDecl>(FnDecl);
-    Method->getParent()->addedAssignmentOperator(Context, Method);
-  }
-
   return false;
 }
 
@@ -5964,8 +6432,6 @@ bool Sema::CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl) {
           cast<NonTypeTemplateParmDecl>(Params->getParam(0));
 
         // The template parameter must be a char parameter pack.
-        // FIXME: This test will always fail because non-type parameter packs
-        //   have not been implemented.
         if (PmDecl && PmDecl->isTemplateParameterPack() &&
             Context.hasSameType(PmDecl->getType(), Context.CharTy))
           Valid = true;
@@ -6042,11 +6508,10 @@ FinishedParams:
 /// by Lang/StrSize. LBraceLoc, if valid, provides the location of
 /// the '{' brace. Otherwise, this linkage specification does not
 /// have any braces.
-Decl *Sema::ActOnStartLinkageSpecification(Scope *S,
-                                                     SourceLocation ExternLoc,
-                                                     SourceLocation LangLoc,
-                                                     llvm::StringRef Lang,
-                                                     SourceLocation LBraceLoc) {
+Decl *Sema::ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc,
+                                           SourceLocation LangLoc,
+                                           llvm::StringRef Lang,
+                                           SourceLocation LBraceLoc) {
   LinkageSpecDecl::LanguageIDs Language;
   if (Lang == "\"C\"")
     Language = LinkageSpecDecl::lang_c;
@@ -6082,13 +6547,13 @@ Decl *Sema::ActOnFinishLinkageSpecification(Scope *S,
 /// \brief Perform semantic analysis for the variable declaration that
 /// occurs within a C++ catch clause, returning the newly-created
 /// variable.
-VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
+VarDecl *Sema::BuildExceptionDeclaration(Scope *S, 
                                          TypeSourceInfo *TInfo,
                                          IdentifierInfo *Name,
-                                         SourceLocation Loc,
-                                         SourceRange Range) {
+                                         SourceLocation Loc) {
   bool Invalid = false;
-
+  QualType ExDeclType = TInfo->getType();
+  
   // Arrays and functions decay.
   if (ExDeclType->isArrayType())
     ExDeclType = Context.getArrayDecayedType(ExDeclType);
@@ -6100,7 +6565,7 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
   // incomplete type, other than [cv] void*.
   // N2844 forbids rvalue references.
   if (!ExDeclType->isDependentType() && ExDeclType->isRValueReferenceType()) {
-    Diag(Loc, diag::err_catch_rvalue_ref) << Range;
+    Diag(Loc, diag::err_catch_rvalue_ref);
     Invalid = true;
   }
 
@@ -6161,7 +6626,7 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
   ExDecl->setExceptionVariable(true);
   
   if (!Invalid) {
-    if (const RecordType *RecordTy = ExDeclType->getAs<RecordType>()) {
+    if (const RecordType *recordType = ExDeclType->getAs<RecordType>()) {
       // C++ [except.handle]p16:
       //   The object declared in an exception-declaration or, if the 
       //   exception-declaration does not specify a name, a temporary (12.2) is 
@@ -6171,18 +6636,32 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
       //
       // We just pretend to initialize the object with itself, then make sure 
       // it can be destroyed later.
-      InitializedEntity Entity = InitializedEntity::InitializeVariable(ExDecl);
-      Expr *ExDeclRef = DeclRefExpr::Create(Context, 0, SourceRange(), ExDecl, 
-                                            Loc, ExDeclType, 0);
-      InitializationKind Kind = InitializationKind::CreateCopy(Loc, 
-                                                               SourceLocation());
-      InitializationSequence InitSeq(*this, Entity, Kind, &ExDeclRef, 1);
-      ExprResult Result = InitSeq.Perform(*this, Entity, Kind, 
-                                         MultiExprArg(*this, &ExDeclRef, 1));
-      if (Result.isInvalid())
+      QualType initType = ExDeclType;
+
+      InitializedEntity entity =
+        InitializedEntity::InitializeVariable(ExDecl);
+      InitializationKind initKind =
+        InitializationKind::CreateCopy(Loc, SourceLocation());
+
+      Expr *opaqueValue =
+        new (Context) OpaqueValueExpr(Loc, initType, VK_LValue, OK_Ordinary);
+      InitializationSequence sequence(*this, entity, initKind, &opaqueValue, 1);
+      ExprResult result = sequence.Perform(*this, entity, initKind,
+                                           MultiExprArg(&opaqueValue, 1));
+      if (result.isInvalid())
         Invalid = true;
-      else 
-        FinalizeVarWithDestructor(ExDecl, RecordTy);
+      else {
+        // If the constructor used was non-trivial, set this as the
+        // "initializer".
+        CXXConstructExpr *construct = cast<CXXConstructExpr>(result.take());
+        if (!construct->getConstructor()->isTrivial()) {
+          Expr *init = MaybeCreateExprWithCleanups(construct);
+          ExDecl->setInit(init);
+        }
+        
+        // And make sure it's destructable.
+        FinalizeVarWithDestructor(ExDecl, recordType);
+      }
     }
   }
   
@@ -6196,9 +6675,16 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
 /// handler.
 Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
-  QualType ExDeclType = TInfo->getType();
-
   bool Invalid = D.isInvalidType();
+
+  // Check for unexpanded parameter packs.
+  if (TInfo && DiagnoseUnexpandedParameterPack(D.getIdentifierLoc(), TInfo,
+                                               UPPC_ExceptionType)) {
+    TInfo = Context.getTrivialTypeSourceInfo(Context.IntTy, 
+                                             D.getIdentifierLoc());
+    Invalid = true;
+  }
+
   IdentifierInfo *II = D.getIdentifier();
   if (NamedDecl *PrevDecl = LookupSingleName(S, II, D.getIdentifierLoc(),
                                              LookupOrdinaryName,
@@ -6218,10 +6704,9 @@ Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
     Invalid = true;
   }
 
-  VarDecl *ExDecl = BuildExceptionDeclaration(S, ExDeclType, TInfo,
+  VarDecl *ExDecl = BuildExceptionDeclaration(S, TInfo,
                                               D.getIdentifier(),
-                                              D.getIdentifierLoc(),
-                                            D.getDeclSpec().getSourceRange());
+                                              D.getIdentifierLoc());
 
   if (Invalid)
     ExDecl->setInvalidDecl();
@@ -6255,6 +6740,9 @@ Decl *Sema::ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
     }
   }
 
+  if (DiagnoseUnexpandedParameterPack(AssertExpr, UPPC_StaticAssertExpression))
+    return 0;
+
   Decl *Decl = StaticAssertDecl::Create(Context, CurContext, AssertLoc,
                                         AssertExpr, AssertMessage);
 
@@ -6318,6 +6806,110 @@ FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation FriendLoc,
   return FriendDecl::Create(Context, CurContext, FriendLoc, TSInfo, FriendLoc);
 }
 
+/// Handle a friend tag declaration where the scope specifier was
+/// templated.
+Decl *Sema::ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc,
+                                    unsigned TagSpec, SourceLocation TagLoc,
+                                    CXXScopeSpec &SS,
+                                    IdentifierInfo *Name, SourceLocation NameLoc,
+                                    AttributeList *Attr,
+                                    MultiTemplateParamsArg TempParamLists) {
+  TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForTypeSpec(TagSpec);
+
+  bool isExplicitSpecialization = false;
+  unsigned NumMatchedTemplateParamLists = TempParamLists.size();
+  bool Invalid = false;
+
+  if (TemplateParameterList *TemplateParams
+        = MatchTemplateParametersToScopeSpecifier(TagLoc, SS,
+                                                  TempParamLists.get(),
+                                                  TempParamLists.size(),
+                                                  /*friend*/ true,
+                                                  isExplicitSpecialization,
+                                                  Invalid)) {
+    --NumMatchedTemplateParamLists;
+
+    if (TemplateParams->size() > 0) {
+      // This is a declaration of a class template.
+      if (Invalid)
+        return 0;
+        
+      return CheckClassTemplate(S, TagSpec, TUK_Friend, TagLoc,
+                                SS, Name, NameLoc, Attr,
+                                TemplateParams, AS_public).take();
+    } else {
+      // The "template<>" header is extraneous.
+      Diag(TemplateParams->getTemplateLoc(), diag::err_template_tag_noparams)
+        << TypeWithKeyword::getTagTypeKindName(Kind) << Name;
+      isExplicitSpecialization = true;
+    }
+  }
+
+  if (Invalid) return 0;
+
+  assert(SS.isNotEmpty() && "valid templated tag with no SS and no direct?");
+
+  bool isAllExplicitSpecializations = true;
+  for (unsigned I = 0; I != NumMatchedTemplateParamLists; ++I) {
+    if (TempParamLists.get()[I]->size()) {
+      isAllExplicitSpecializations = false;
+      break;
+    }
+  }
+
+  // FIXME: don't ignore attributes.
+
+  // If it's explicit specializations all the way down, just forget
+  // about the template header and build an appropriate non-templated
+  // friend.  TODO: for source fidelity, remember the headers.
+  if (isAllExplicitSpecializations) {
+    ElaboratedTypeKeyword Keyword
+      = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
+    QualType T = CheckTypenameType(Keyword, SS.getScopeRep(), *Name,
+                                   TagLoc, SS.getRange(), NameLoc);
+    if (T.isNull())
+      return 0;
+
+    TypeSourceInfo *TSI = Context.CreateTypeSourceInfo(T);
+    if (isa<DependentNameType>(T)) {
+      DependentNameTypeLoc TL = cast<DependentNameTypeLoc>(TSI->getTypeLoc());
+      TL.setKeywordLoc(TagLoc);
+      TL.setQualifierRange(SS.getRange());
+      TL.setNameLoc(NameLoc);
+    } else {
+      ElaboratedTypeLoc TL = cast<ElaboratedTypeLoc>(TSI->getTypeLoc());
+      TL.setKeywordLoc(TagLoc);
+      TL.setQualifierRange(SS.getRange());
+      cast<TypeSpecTypeLoc>(TL.getNamedTypeLoc()).setNameLoc(NameLoc);
+    }
+
+    FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
+                                            TSI, FriendLoc);
+    Friend->setAccess(AS_public);
+    CurContext->addDecl(Friend);
+    return Friend;
+  }
+
+  // Handle the case of a templated-scope friend class.  e.g.
+  //   template <class T> class A<T>::B;
+  // FIXME: we don't support these right now.
+  ElaboratedTypeKeyword ETK = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
+  QualType T = Context.getDependentNameType(ETK, SS.getScopeRep(), Name);
+  TypeSourceInfo *TSI = Context.CreateTypeSourceInfo(T);
+  DependentNameTypeLoc TL = cast<DependentNameTypeLoc>(TSI->getTypeLoc());
+  TL.setKeywordLoc(TagLoc);
+  TL.setQualifierRange(SS.getRange());
+  TL.setNameLoc(NameLoc);
+
+  FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
+                                          TSI, FriendLoc);
+  Friend->setAccess(AS_public);
+  Friend->setUnsupportedFriend(true);
+  CurContext->addDecl(Friend);
+  return Friend;
+}
+
+
 /// Handle a friend type declaration.  This works in tandem with
 /// ActOnTag.
 ///
@@ -6336,7 +6928,7 @@ FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation FriendLoc,
 /// parameters present at all, require proper matching, i.e.
 ///   template <> template <class T> friend class A<int>::B;
 Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                                          MultiTemplateParamsArg TempParams) {
+                                MultiTemplateParamsArg TempParams) {
   SourceLocation Loc = DS.getSourceRange().getBegin();
 
   assert(DS.isFriendSpecified());
@@ -6351,6 +6943,9 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (TheDeclarator.isInvalidType())
     return 0;
 
+  if (DiagnoseUnexpandedParameterPack(Loc, TSI, UPPC_FriendDeclaration))
+    return 0;
+
   // This is definitely an error in C++98.  It's probably meant to
   // be forbidden in C++0x, too, but the specification is just
   // poorly written.
@@ -6386,7 +6981,7 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (unsigned NumTempParamLists = TempParams.size())
     D = FriendTemplateDecl::Create(Context, CurContext, Loc,
                                    NumTempParamLists,
-                                 (TemplateParameterList**) TempParams.release(),
+                                   TempParams.release(),
                                    TSI,
                                    DS.getFriendSpecLoc());
   else
@@ -6401,10 +6996,8 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   return D;
 }
 
-Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
-                                         Declarator &D,
-                                         bool IsDefinition,
-                              MultiTemplateParamsArg TemplateParams) {
+Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
+                                    MultiTemplateParamsArg TemplateParams) {
   const DeclSpec &DS = D.getDeclSpec();
 
   assert(DS.isFriendSpecified());
@@ -6447,26 +7040,94 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
   //    declared as a friend, scopes outside the innermost enclosing
   //    namespace scope are not considered.
 
-  CXXScopeSpec &ScopeQual = D.getCXXScopeSpec();
+  CXXScopeSpec &SS = D.getCXXScopeSpec();
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
   DeclarationName Name = NameInfo.getName();
   assert(Name);
 
+  // Check for unexpanded parameter packs.
+  if (DiagnoseUnexpandedParameterPack(Loc, TInfo, UPPC_FriendDeclaration) ||
+      DiagnoseUnexpandedParameterPack(NameInfo, UPPC_FriendDeclaration) ||
+      DiagnoseUnexpandedParameterPack(SS, UPPC_FriendDeclaration))
+    return 0;
+
   // The context we found the declaration in, or in which we should
   // create the declaration.
   DeclContext *DC;
+  Scope *DCScope = S;
+  LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
+                        ForRedeclaration);
 
-  // FIXME: handle local classes
+  // FIXME: there are different rules in local classes
 
+  // There are four cases here.
+  //   - There's no scope specifier, in which case we just go to the
+  //     appropriate scope and look for a function or function template
+  //     there as appropriate.
   // Recover from invalid scope qualifiers as if they just weren't there.
-  LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
-                        ForRedeclaration);
-  if (!ScopeQual.isInvalid() && ScopeQual.isSet()) {
-    DC = computeDeclContext(ScopeQual);
+  if (SS.isInvalid() || !SS.isSet()) {
+    // C++0x [namespace.memdef]p3:
+    //   If the name in a friend declaration is neither qualified nor
+    //   a template-id and the declaration is a function or an
+    //   elaborated-type-specifier, the lookup to determine whether
+    //   the entity has been previously declared shall not consider
+    //   any scopes outside the innermost enclosing namespace.
+    // C++0x [class.friend]p11:
+    //   If a friend declaration appears in a local class and the name
+    //   specified is an unqualified name, a prior declaration is
+    //   looked up without considering scopes that are outside the
+    //   innermost enclosing non-class scope. For a friend function
+    //   declaration, if there is no prior declaration, the program is
+    //   ill-formed.
+    bool isLocal = cast<CXXRecordDecl>(CurContext)->isLocalClass();
+    bool isTemplateId = D.getName().getKind() == UnqualifiedId::IK_TemplateId;
+
+    // Find the appropriate context according to the above.
+    DC = CurContext;
+    while (true) {
+      // Skip class contexts.  If someone can cite chapter and verse
+      // for this behavior, that would be nice --- it's what GCC and
+      // EDG do, and it seems like a reasonable intent, but the spec
+      // really only says that checks for unqualified existing
+      // declarations should stop at the nearest enclosing namespace,
+      // not that they should only consider the nearest enclosing
+      // namespace.
+      while (DC->isRecord()) 
+        DC = DC->getParent();
+
+      LookupQualifiedName(Previous, DC);
 
-    // FIXME: handle dependent contexts
+      // TODO: decide what we think about using declarations.
+      if (isLocal || !Previous.empty())
+        break;
+
+      if (isTemplateId) {
+        if (isa<TranslationUnitDecl>(DC)) break;
+      } else {
+        if (DC->isFileContext()) break;
+      }
+      DC = DC->getParent();
+    }
+
+    // C++ [class.friend]p1: A friend of a class is a function or
+    //   class that is not a member of the class . . .
+    // C++0x changes this for both friend types and functions.
+    // Most C++ 98 compilers do seem to give an error here, so
+    // we do, too.
+    if (!Previous.empty() && DC->Equals(CurContext)
+        && !getLangOptions().CPlusPlus0x)
+      Diag(DS.getFriendSpecLoc(), diag::err_friend_is_member);
+
+    DCScope = getScopeForDeclContext(S, DC);
+
+  //   - There's a non-dependent scope specifier, in which case we
+  //     compute it and do a previous lookup there for a function
+  //     or function template.
+  } else if (!SS.getScopeRep()->isDependent()) {
+    DC = computeDeclContext(SS);
     if (!DC) return 0;
-    if (RequireCompleteDeclContext(ScopeQual, DC)) return 0;
+
+    if (RequireCompleteDeclContext(SS, DC)) return 0;
 
     LookupQualifiedName(Previous, DC);
 
@@ -6493,43 +7154,17 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
     if (DC->Equals(CurContext))
       Diag(DS.getFriendSpecLoc(), diag::err_friend_is_member);
 
-  // Otherwise walk out to the nearest namespace scope looking for matches.
+  //   - There's a scope specifier that does not match any template
+  //     parameter lists, in which case we use some arbitrary context,
+  //     create a method or method template, and wait for instantiation.
+  //   - There's a scope specifier that does match some template
+  //     parameter lists, which we don't handle right now.
   } else {
-    // TODO: handle local class contexts.
-
     DC = CurContext;
-    while (true) {
-      // Skip class contexts.  If someone can cite chapter and verse
-      // for this behavior, that would be nice --- it's what GCC and
-      // EDG do, and it seems like a reasonable intent, but the spec
-      // really only says that checks for unqualified existing
-      // declarations should stop at the nearest enclosing namespace,
-      // not that they should only consider the nearest enclosing
-      // namespace.
-      while (DC->isRecord()) 
-        DC = DC->getParent();
-
-      LookupQualifiedName(Previous, DC);
-
-      // TODO: decide what we think about using declarations.
-      if (!Previous.empty())
-        break;
-      
-      if (DC->isFileContext()) break;
-      DC = DC->getParent();
-    }
-
-    // C++ [class.friend]p1: A friend of a class is a function or
-    //   class that is not a member of the class . . .
-    // C++0x changes this for both friend types and functions.
-    // Most C++ 98 compilers do seem to give an error here, so
-    // we do, too.
-    if (!Previous.empty() && DC->Equals(CurContext)
-        && !getLangOptions().CPlusPlus0x)
-      Diag(DS.getFriendSpecLoc(), diag::err_friend_is_member);
+    assert(isa<CXXRecordDecl>(DC) && "friend declaration not in class?");
   }
 
-  if (DC->isFileContext()) {
+  if (!DC->isRecord()) {
     // This implies that it has to be an operator or function.
     if (D.getName().getKind() == UnqualifiedId::IK_ConstructorName ||
         D.getName().getKind() == UnqualifiedId::IK_DestructorName ||
@@ -6542,7 +7177,7 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
   }
 
   bool Redeclaration = false;
-  NamedDecl *ND = ActOnFunctionDeclarator(S, D, DC, T, TInfo, Previous,
+  NamedDecl *ND = ActOnFunctionDeclarator(DCScope, D, DC, T, TInfo, Previous,
                                           move(TemplateParams),
                                           IsDefinition,
                                           Redeclaration);
@@ -6570,6 +7205,20 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
   FrD->setAccess(AS_public);
   CurContext->addDecl(FrD);
 
+  if (ND->isInvalidDecl())
+    FrD->setInvalidDecl();
+  else {
+    FunctionDecl *FD;
+    if (FunctionTemplateDecl *FTD = dyn_cast<FunctionTemplateDecl>(ND))
+      FD = FTD->getTemplatedDecl();
+    else
+      FD = cast<FunctionDecl>(ND);
+
+    // Mark templated-scope function declarations as unsupported.
+    if (FD->getNumTemplateParameterLists())
+      FrD->setUnsupportedFriend(true);
+  }
+
   return ND;
 }
 
@@ -6591,8 +7240,7 @@ void Sema::SetDeclDeleted(Decl *Dcl, SourceLocation DelLoc) {
 }
 
 static void SearchForReturnInStmt(Sema &Self, Stmt *S) {
-  for (Stmt::child_iterator CI = S->child_begin(), E = S->child_end(); CI != E;
-       ++CI) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI) {
     Stmt *SubStmt = *CI;
     if (!SubStmt)
       continue;
@@ -6676,6 +7324,10 @@ bool Sema::CheckOverridingFunctionReturnType(const CXXMethodDecl *New,
                     diag::err_covariant_return_ambiguous_derived_to_base_conv,
                     // FIXME: Should this point to the return type?
                     New->getLocation(), SourceRange(), New->getDeclName(), 0)) {
+      // FIXME: this note won't trigger for delayed access control
+      // diagnostics, and it's impossible to get an undelayed error
+      // here from access control during the original parse because
+      // the ParsingDeclSpec/ParsingDeclarator are still in scope.
       Diag(Old->getLocation(), diag::note_overridden_virtual_function);
       return true;
     }
@@ -6703,19 +7355,6 @@ bool Sema::CheckOverridingFunctionReturnType(const CXXMethodDecl *New,
   return false;
 }
 
-bool Sema::CheckOverridingFunctionAttributes(const CXXMethodDecl *New,
-                                             const CXXMethodDecl *Old)
-{
-  if (Old->hasAttr<FinalAttr>()) {
-    Diag(New->getLocation(), diag::err_final_function_overridden)
-      << New->getDeclName();
-    Diag(Old->getLocation(), diag::note_overridden_virtual_function);
-    return true;
-  }
-
-  return false;
-}
-
 /// \brief Mark the given method pure.
 ///
 /// \param Method the method to be marked pure.
@@ -6724,9 +7363,6 @@ bool Sema::CheckOverridingFunctionAttributes(const CXXMethodDecl *New,
 bool Sema::CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange) {
   if (Method->isVirtual() || Method->getParent()->isDependentContext()) {
     Method->setPure();
-    
-    // A class is abstract if at least one function is pure virtual.
-    Method->getParent()->setAbstract(true);
     return false;
   } 
 
@@ -6831,21 +7467,9 @@ void Sema::MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
 }
 
 bool Sema::DefineUsedVTables() {
-  // If any dynamic classes have their key function defined within
-  // this translation unit, then those vtables are considered "used" and must
-  // be emitted.
-  for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I) {
-    if (const CXXMethodDecl *KeyFunction
-                             = Context.getKeyFunction(DynamicClasses[I])) {
-      const FunctionDecl *Definition = 0;
-      if (KeyFunction->hasBody(Definition))
-        MarkVTableUsed(Definition->getLocation(), DynamicClasses[I], true);
-    }
-  }
-
   if (VTableUses.empty())
     return false;
-  
+
   // Note: The VTableUses vector could grow as a result of marking
   // the members of a class as "used", so we check the size each
   // time through the loop and prefer indices (with are stable) to
@@ -6954,13 +7578,13 @@ void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
     CollectIvarsToConstructOrDestruct(OID, ivars);
     if (ivars.empty())
       return;
-    llvm::SmallVector<CXXBaseOrMemberInitializer*, 32> AllToInit;
+    llvm::SmallVector<CXXCtorInitializer*, 32> AllToInit;
     for (unsigned i = 0; i < ivars.size(); i++) {
       FieldDecl *Field = ivars[i];
       if (Field->isInvalidDecl())
         continue;
       
-      CXXBaseOrMemberInitializer *Member;
+      CXXCtorInitializer *Member;
       InitializedEntity InitEntity = InitializedEntity::InitializeMember(Field);
       InitializationKind InitKind = 
         InitializationKind::CreateDefault(ObjCImplementation->getLocation());
@@ -6968,18 +7592,17 @@ void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
       InitializationSequence InitSeq(*this, InitEntity, InitKind, 0, 0);
       ExprResult MemberInit = 
         InitSeq.Perform(*this, InitEntity, InitKind, MultiExprArg());
-      MemberInit = MaybeCreateCXXExprWithTemporaries(MemberInit.get());
+      MemberInit = MaybeCreateExprWithCleanups(MemberInit);
       // Note, MemberInit could actually come back empty if no initialization 
       // is required (e.g., because it would call a trivial default constructor)
       if (!MemberInit.get() || MemberInit.isInvalid())
         continue;
-      
+
       Member =
-        new (Context) CXXBaseOrMemberInitializer(Context,
-                                                 Field, SourceLocation(),
-                                                 SourceLocation(),
-                                                 MemberInit.takeAs<Expr>(),
-                                                 SourceLocation());
+        new (Context) CXXCtorInitializer(Context, Field, SourceLocation(),
+                                         SourceLocation(),
+                                         MemberInit.takeAs<Expr>(),
+                                         SourceLocation());
       AllToInit.push_back(Member);
       
       // Be sure that the destructor is accessible and is marked as referenced.
diff --git a/lib/Sema/SemaDeclObjC.cpp b/lib/Sema/SemaDeclObjC.cpp
index a6902a3e391c..66f6f2b960d1 100644
--- a/lib/Sema/SemaDeclObjC.cpp
+++ b/lib/Sema/SemaDeclObjC.cpp
@@ -24,6 +24,19 @@
 
 using namespace clang;
 
+static void DiagnoseObjCImplementedDeprecations(Sema &S,
+                                                NamedDecl *ND,
+                                                SourceLocation ImplLoc,
+                                                int select) {
+  if (ND && ND->getAttr<DeprecatedAttr>()) {
+    S.Diag(ImplLoc, diag::warn_deprecated_def) << select;
+    if (select == 0)
+      S.Diag(ND->getLocation(), diag::note_method_declared_at);
+    else
+      S.Diag(ND->getLocation(), diag::note_previous_decl) << "class";
+  }
+}
+
 /// ActOnStartOfObjCMethodDef - This routine sets up parameters; invisible
 /// and user declared, in the method definition's AST.
 void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
@@ -55,9 +68,23 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
 
   // Introduce all of the other parameters into this scope.
   for (ObjCMethodDecl::param_iterator PI = MDecl->param_begin(),
-       E = MDecl->param_end(); PI != E; ++PI)
+       E = MDecl->param_end(); PI != E; ++PI) {
+    ParmVarDecl *Param = (*PI);
+    if (!Param->isInvalidDecl() &&
+        RequireCompleteType(Param->getLocation(), Param->getType(),
+                            diag::err_typecheck_decl_incomplete_type))
+          Param->setInvalidDecl();
     if ((*PI)->getIdentifier())
       PushOnScopeChains(*PI, FnBodyScope);
+  }
+  // Warn on implementating deprecated methods under 
+  // -Wdeprecated-implementations flag.
+  if (ObjCInterfaceDecl *IC = MDecl->getClassInterface())
+    if (ObjCMethodDecl *IMD = 
+          IC->lookupMethod(MDecl->getSelector(), MDecl->isInstanceMethod()))
+      DiagnoseObjCImplementedDeprecations(*this, 
+                                          dyn_cast<NamedDecl>(IMD), 
+                                          MDecl->getLocation(), 0);
 }
 
 Decl *Sema::
@@ -531,8 +558,14 @@ Decl *Sema::ActOnStartCategoryImplementation(
         << CatName;
       Diag(CatIDecl->getImplementation()->getLocation(),
            diag::note_previous_definition);
-    } else
+    } else {
       CatIDecl->setImplementation(CDecl);
+      // Warn on implementating category of deprecated class under 
+      // -Wdeprecated-implementations flag.
+      DiagnoseObjCImplementedDeprecations(*this, 
+                                          dyn_cast<NamedDecl>(IDecl), 
+                                          CDecl->getLocation(), 2);
+    }
   }
 
   CheckObjCDeclScope(CDecl);
@@ -641,6 +674,11 @@ Decl *Sema::ActOnStartClassImplementation(
   } else { // add it to the list.
     IDecl->setImplementation(IMPDecl);
     PushOnScopeChains(IMPDecl, TUScope);
+    // Warn on implementating deprecated class under 
+    // -Wdeprecated-implementations flag.
+    DiagnoseObjCImplementedDeprecations(*this, 
+                                        dyn_cast<NamedDecl>(IDecl), 
+                                        IMPDecl->getLocation(), 1);
   }
   return IMPDecl;
 }
@@ -739,36 +777,164 @@ void Sema::WarnUndefinedMethod(SourceLocation ImpLoc, ObjCMethodDecl *method,
     Diag(ImpLoc, diag::warn_incomplete_impl);
     IncompleteImpl = true;
   }
-  Diag(method->getLocation(), DiagID) 
-    << method->getDeclName();
+  if (DiagID == diag::warn_unimplemented_protocol_method)
+    Diag(ImpLoc, DiagID) << method->getDeclName();
+  else
+    Diag(method->getLocation(), DiagID) << method->getDeclName();
 }
 
+/// Determines if type B can be substituted for type A.  Returns true if we can
+/// guarantee that anything that the user will do to an object of type A can 
+/// also be done to an object of type B.  This is trivially true if the two 
+/// types are the same, or if B is a subclass of A.  It becomes more complex
+/// in cases where protocols are involved.
+///
+/// Object types in Objective-C describe the minimum requirements for an
+/// object, rather than providing a complete description of a type.  For
+/// example, if A is a subclass of B, then B* may refer to an instance of A.
+/// The principle of substitutability means that we may use an instance of A
+/// anywhere that we may use an instance of B - it will implement all of the
+/// ivars of B and all of the methods of B.  
+///
+/// This substitutability is important when type checking methods, because 
+/// the implementation may have stricter type definitions than the interface.
+/// The interface specifies minimum requirements, but the implementation may
+/// have more accurate ones.  For example, a method may privately accept 
+/// instances of B, but only publish that it accepts instances of A.  Any
+/// object passed to it will be type checked against B, and so will implicitly
+/// by a valid A*.  Similarly, a method may return a subclass of the class that
+/// it is declared as returning.
+///
+/// This is most important when considering subclassing.  A method in a
+/// subclass must accept any object as an argument that its superclass's
+/// implementation accepts.  It may, however, accept a more general type
+/// without breaking substitutability (i.e. you can still use the subclass
+/// anywhere that you can use the superclass, but not vice versa).  The
+/// converse requirement applies to return types: the return type for a
+/// subclass method must be a valid object of the kind that the superclass
+/// advertises, but it may be specified more accurately.  This avoids the need
+/// for explicit down-casting by callers.
+///
+/// Note: This is a stricter requirement than for assignment.  
+static bool isObjCTypeSubstitutable(ASTContext &Context,
+                                    const ObjCObjectPointerType *A,
+                                    const ObjCObjectPointerType *B,
+                                    bool rejectId) {
+  // Reject a protocol-unqualified id.
+  if (rejectId && B->isObjCIdType()) return false;
+
+  // If B is a qualified id, then A must also be a qualified id and it must
+  // implement all of the protocols in B.  It may not be a qualified class.
+  // For example, MyClass<A> can be assigned to id<A>, but MyClass<A> is a
+  // stricter definition so it is not substitutable for id<A>.
+  if (B->isObjCQualifiedIdType()) {
+    return A->isObjCQualifiedIdType() &&
+           Context.ObjCQualifiedIdTypesAreCompatible(QualType(A, 0),
+                                                     QualType(B,0),
+                                                     false);
+  }
+
+  /*
+  // id is a special type that bypasses type checking completely.  We want a
+  // warning when it is used in one place but not another.
+  if (C.isObjCIdType(A) || C.isObjCIdType(B)) return false;
+
+
+  // If B is a qualified id, then A must also be a qualified id (which it isn't
+  // if we've got this far)
+  if (B->isObjCQualifiedIdType()) return false;
+  */
+
+  // Now we know that A and B are (potentially-qualified) class types.  The
+  // normal rules for assignment apply.
+  return Context.canAssignObjCInterfaces(A, B);
+}
+
+static SourceRange getTypeRange(TypeSourceInfo *TSI) {
+  return (TSI ? TSI->getTypeLoc().getSourceRange() : SourceRange());
+}
+
+static void CheckMethodOverrideReturn(Sema &S,
+                                      ObjCMethodDecl *MethodImpl,
+                                      ObjCMethodDecl *MethodIface) {
+  if (S.Context.hasSameUnqualifiedType(MethodImpl->getResultType(),
+                                       MethodIface->getResultType()))
+    return;
+
+  unsigned DiagID = diag::warn_conflicting_ret_types;
+
+  // Mismatches between ObjC pointers go into a different warning
+  // category, and sometimes they're even completely whitelisted.
+  if (const ObjCObjectPointerType *ImplPtrTy =
+        MethodImpl->getResultType()->getAs<ObjCObjectPointerType>()) {
+    if (const ObjCObjectPointerType *IfacePtrTy =
+          MethodIface->getResultType()->getAs<ObjCObjectPointerType>()) {
+      // Allow non-matching return types as long as they don't violate
+      // the principle of substitutability.  Specifically, we permit
+      // return types that are subclasses of the declared return type,
+      // or that are more-qualified versions of the declared type.
+      if (isObjCTypeSubstitutable(S.Context, IfacePtrTy, ImplPtrTy, false))
+        return;
+
+      DiagID = diag::warn_non_covariant_ret_types;
+    }
+  }
+
+  S.Diag(MethodImpl->getLocation(), DiagID)
+    << MethodImpl->getDeclName()
+    << MethodIface->getResultType()
+    << MethodImpl->getResultType()
+    << getTypeRange(MethodImpl->getResultTypeSourceInfo());
+  S.Diag(MethodIface->getLocation(), diag::note_previous_definition)
+    << getTypeRange(MethodIface->getResultTypeSourceInfo());
+}
+
+static void CheckMethodOverrideParam(Sema &S,
+                                     ObjCMethodDecl *MethodImpl,
+                                     ObjCMethodDecl *MethodIface,
+                                     ParmVarDecl *ImplVar,
+                                     ParmVarDecl *IfaceVar) {
+  QualType ImplTy = ImplVar->getType();
+  QualType IfaceTy = IfaceVar->getType();
+  if (S.Context.hasSameUnqualifiedType(ImplTy, IfaceTy))
+    return;
+
+  unsigned DiagID = diag::warn_conflicting_param_types;
+
+  // Mismatches between ObjC pointers go into a different warning
+  // category, and sometimes they're even completely whitelisted.
+  if (const ObjCObjectPointerType *ImplPtrTy =
+        ImplTy->getAs<ObjCObjectPointerType>()) {
+    if (const ObjCObjectPointerType *IfacePtrTy =
+          IfaceTy->getAs<ObjCObjectPointerType>()) {
+      // Allow non-matching argument types as long as they don't
+      // violate the principle of substitutability.  Specifically, the
+      // implementation must accept any objects that the superclass
+      // accepts, however it may also accept others.
+      if (isObjCTypeSubstitutable(S.Context, ImplPtrTy, IfacePtrTy, true))
+        return;
+
+      DiagID = diag::warn_non_contravariant_param_types;
+    }
+  }
+
+  S.Diag(ImplVar->getLocation(), DiagID)
+    << getTypeRange(ImplVar->getTypeSourceInfo())
+    << MethodImpl->getDeclName() << IfaceTy << ImplTy;
+  S.Diag(IfaceVar->getLocation(), diag::note_previous_definition)
+    << getTypeRange(IfaceVar->getTypeSourceInfo());
+}
+                                     
+
 void Sema::WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethodDecl,
                                        ObjCMethodDecl *IntfMethodDecl) {
-  if (!Context.typesAreCompatible(IntfMethodDecl->getResultType(),
-                                  ImpMethodDecl->getResultType()) &&
-      !Context.QualifiedIdConformsQualifiedId(IntfMethodDecl->getResultType(),
-                                              ImpMethodDecl->getResultType())) {
-    Diag(ImpMethodDecl->getLocation(), diag::warn_conflicting_ret_types)
-      << ImpMethodDecl->getDeclName() << IntfMethodDecl->getResultType()
-      << ImpMethodDecl->getResultType();
-    Diag(IntfMethodDecl->getLocation(), diag::note_previous_definition);
-  }
+  CheckMethodOverrideReturn(*this, ImpMethodDecl, IntfMethodDecl);
 
   for (ObjCMethodDecl::param_iterator IM = ImpMethodDecl->param_begin(),
        IF = IntfMethodDecl->param_begin(), EM = ImpMethodDecl->param_end();
-       IM != EM; ++IM, ++IF) {
-    QualType ParmDeclTy = (*IF)->getType().getUnqualifiedType();
-    QualType ParmImpTy = (*IM)->getType().getUnqualifiedType();
-    if (Context.typesAreCompatible(ParmDeclTy, ParmImpTy) ||
-        Context.QualifiedIdConformsQualifiedId(ParmDeclTy, ParmImpTy))
-      continue;
+       IM != EM; ++IM, ++IF)
+    CheckMethodOverrideParam(*this, ImpMethodDecl, IntfMethodDecl, *IM, *IF);
 
-    Diag((*IM)->getLocation(), diag::warn_conflicting_param_types)
-      << ImpMethodDecl->getDeclName() << (*IF)->getType()
-      << (*IM)->getType();
-    Diag((*IF)->getLocation(), diag::note_previous_definition);
-  }
   if (ImpMethodDecl->isVariadic() != IntfMethodDecl->isVariadic()) {
     Diag(ImpMethodDecl->getLocation(), diag::warn_conflicting_variadic);
     Diag(IntfMethodDecl->getLocation(), diag::note_previous_declaration);
@@ -835,8 +1001,10 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
             IDecl->lookupInstanceMethod(method->getSelector());
             if (!MethodInClass || !MethodInClass->isSynthesized()) {
               unsigned DIAG = diag::warn_unimplemented_protocol_method;
-              if (Diags.getDiagnosticLevel(DIAG) != Diagnostic::Ignored) {
+              if (Diags.getDiagnosticLevel(DIAG, ImpLoc)
+                      != Diagnostic::Ignored) {
                 WarnUndefinedMethod(ImpLoc, method, IncompleteImpl, DIAG);
+                Diag(method->getLocation(), diag::note_method_declared_at);
                 Diag(CDecl->getLocation(), diag::note_required_for_protocol_at)
                   << PDecl->getDeclName();
               }
@@ -852,8 +1020,9 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
         !ClsMap.count(method->getSelector()) &&
         (!Super || !Super->lookupClassMethod(method->getSelector()))) {
       unsigned DIAG = diag::warn_unimplemented_protocol_method;
-      if (Diags.getDiagnosticLevel(DIAG) != Diagnostic::Ignored) {
+      if (Diags.getDiagnosticLevel(DIAG, ImpLoc) != Diagnostic::Ignored) {
         WarnUndefinedMethod(ImpLoc, method, IncompleteImpl, DIAG);
+        Diag(method->getLocation(), diag::note_method_declared_at);
         Diag(IDecl->getLocation(), diag::note_required_for_protocol_at) <<
           PDecl->getDeclName();
       }
@@ -921,7 +1090,16 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
       WarnConflictingTypedMethods(ImpMethodDecl, IntfMethodDecl);
     }
   }
+  
   if (ObjCInterfaceDecl *I = dyn_cast<ObjCInterfaceDecl> (CDecl)) {
+    // Also methods in class extensions need be looked at next.
+    for (const ObjCCategoryDecl *ClsExtDecl = I->getFirstClassExtension(); 
+         ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension())
+      MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
+                                 IMPDecl,
+                                 const_cast<ObjCCategoryDecl *>(ClsExtDecl), 
+                                 IncompleteImpl, false);
+    
     // Check for any implementation of a methods declared in protocol.
     for (ObjCInterfaceDecl::all_protocol_iterator
           PI = I->all_referenced_protocol_begin(),
@@ -949,7 +1127,8 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
   // Check and see if properties declared in the interface have either 1)
   // an implementation or 2) there is a @synthesize/@dynamic implementation
   // of the property in the @implementation.
-  if (isa<ObjCInterfaceDecl>(CDecl) && !LangOpts.ObjCNonFragileABI2)
+  if (isa<ObjCInterfaceDecl>(CDecl) &&
+        !(LangOpts.ObjCDefaultSynthProperties && LangOpts.ObjCNonFragileABI2))
     DiagnoseUnimplementedProperties(S, IMPDecl, CDecl, InsMap);
       
   llvm::DenseSet<Selector> ClsMap;
@@ -1156,7 +1335,21 @@ void Sema::AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl,
   // signature.
   for (ObjCMethodList *List = &Entry; List; List = List->Next)
     if (MatchTwoMethodDeclarations(Method, List->Method)) {
-      List->Method->setDefined(impl);
+      ObjCMethodDecl *PrevObjCMethod = List->Method;
+      PrevObjCMethod->setDefined(impl);
+      // If a method is deprecated, push it in the global pool.
+      // This is used for better diagnostics.
+      if (Method->getAttr<DeprecatedAttr>()) {
+        if (!PrevObjCMethod->getAttr<DeprecatedAttr>())
+          List->Method = Method;
+      }
+      // If new method is unavailable, push it into global pool
+      // unless previous one is deprecated.
+      if (Method->getAttr<UnavailableAttr>()) {
+        if (!PrevObjCMethod->getAttr<UnavailableAttr>() &&
+            !PrevObjCMethod->getAttr<DeprecatedAttr>())
+          List->Method = Method;
+      }
       return;
     }
 
@@ -1180,7 +1373,8 @@ ObjCMethodDecl *Sema::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
   ObjCMethodList &MethList = instance ? Pos->second.first : Pos->second.second;
 
   bool strictSelectorMatch = receiverIdOrClass && warn &&
-    (Diags.getDiagnosticLevel(diag::warn_strict_multiple_method_decl) != 
+    (Diags.getDiagnosticLevel(diag::warn_strict_multiple_method_decl,
+                              R.getBegin()) != 
       Diagnostic::Ignored);
   if (warn && MethList.Method && MethList.Next) {
     bool issueWarning = false;
@@ -1314,8 +1508,6 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
     Diag(L, diag::warn_missing_atend);
   }
   
-  DeclContext *DC = dyn_cast<DeclContext>(ClassDecl);
-
   // FIXME: Remove these and use the ObjCContainerDecl/DeclContext.
   llvm::DenseMap<Selector, const ObjCMethodDecl*> InsMap;
   llvm::DenseMap<Selector, const ObjCMethodDecl*> ClsMap;
@@ -1335,8 +1527,8 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
           Diag(Method->getLocation(), diag::err_duplicate_method_decl)
             << Method->getDeclName();
           Diag(PrevMethod->getLocation(), diag::note_previous_declaration);
+        Method->setInvalidDecl();
       } else {
-        DC->addDecl(Method);
         InsMap[Method->getSelector()] = Method;
         /// The following allows us to typecheck messages to "id".
         AddInstanceMethodToGlobalPool(Method);
@@ -1354,8 +1546,8 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
         Diag(Method->getLocation(), diag::err_duplicate_method_decl)
           << Method->getDeclName();
         Diag(PrevMethod->getLocation(), diag::note_previous_declaration);
+        Method->setInvalidDecl();
       } else {
-        DC->addDecl(Method);
         ClsMap[Method->getSelector()] = Method;
         /// The following allows us to typecheck messages to "Class".
         AddFactoryMethodToGlobalPool(Method);
@@ -1377,8 +1569,10 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
 
     // Compare protocol properties with those in category
     CompareProperties(C, C);
-    if (C->IsClassExtension())
-      DiagnoseClassExtensionDupMethods(C, C->getClassInterface());
+    if (C->IsClassExtension()) {
+      ObjCInterfaceDecl *CCPrimary = C->getClassInterface();
+      DiagnoseClassExtensionDupMethods(C, CCPrimary);
+    }
   }
   if (ObjCContainerDecl *CDecl = dyn_cast<ObjCContainerDecl>(ClassDecl)) {
     if (CDecl->getIdentifier())
@@ -1394,7 +1588,40 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
   if (ObjCImplementationDecl *IC=dyn_cast<ObjCImplementationDecl>(ClassDecl)) {
     IC->setAtEndRange(AtEnd);
     if (ObjCInterfaceDecl* IDecl = IC->getClassInterface()) {
-      if (LangOpts.ObjCNonFragileABI2)
+      // Any property declared in a class extension might have user
+      // declared setter or getter in current class extension or one
+      // of the other class extensions. Mark them as synthesized as
+      // property will be synthesized when property with same name is
+      // seen in the @implementation.
+      for (const ObjCCategoryDecl *ClsExtDecl =
+           IDecl->getFirstClassExtension();
+           ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension()) {
+        for (ObjCContainerDecl::prop_iterator I = ClsExtDecl->prop_begin(),
+             E = ClsExtDecl->prop_end(); I != E; ++I) {
+          ObjCPropertyDecl *Property = (*I);
+          // Skip over properties declared @dynamic
+          if (const ObjCPropertyImplDecl *PIDecl
+              = IC->FindPropertyImplDecl(Property->getIdentifier()))
+            if (PIDecl->getPropertyImplementation() 
+                  == ObjCPropertyImplDecl::Dynamic)
+              continue;
+          
+          for (const ObjCCategoryDecl *CExtDecl =
+               IDecl->getFirstClassExtension();
+               CExtDecl; CExtDecl = CExtDecl->getNextClassExtension()) {
+            if (ObjCMethodDecl *GetterMethod =
+                CExtDecl->getInstanceMethod(Property->getGetterName()))
+              GetterMethod->setSynthesized(true);
+            if (!Property->isReadOnly())
+              if (ObjCMethodDecl *SetterMethod =
+                  CExtDecl->getInstanceMethod(Property->getSetterName()))
+                SetterMethod->setSynthesized(true);
+          }        
+        }
+      }
+      
+      if (LangOpts.ObjCDefaultSynthProperties &&
+          LangOpts.ObjCNonFragileABI2)
         DefaultSynthesizeProperties(S, IC, IDecl);
       ImplMethodsVsClassMethods(S, IC, IDecl);
       AtomicPropertySetterGetterRules(IC, IDecl);
@@ -1469,6 +1696,7 @@ bool containsInvalidMethodImplAttribute(const AttrVec &A) {
 }
 
 Decl *Sema::ActOnMethodDeclaration(
+    Scope *S,
     SourceLocation MethodLoc, SourceLocation EndLoc,
     tok::TokenKind MethodType, Decl *ClassDecl,
     ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
@@ -1482,7 +1710,6 @@ Decl *Sema::ActOnMethodDeclaration(
   // Make sure we can establish a context for the method.
   if (!ClassDecl) {
     Diag(MethodLoc, diag::error_missing_method_context);
-    getCurFunction()->LabelMap.clear();
     return 0;
   }
   QualType resultDeclType;
@@ -1526,6 +1753,20 @@ Decl *Sema::ActOnMethodDeclaration(
       ArgType = adjustParameterType(ArgType);
     }
 
+    LookupResult R(*this, ArgInfo[i].Name, ArgInfo[i].NameLoc, 
+                   LookupOrdinaryName, ForRedeclaration);
+    LookupName(R, S);
+    if (R.isSingleResult()) {
+      NamedDecl *PrevDecl = R.getFoundDecl();
+      if (S->isDeclScope(PrevDecl)) {
+        // FIXME. This should be an error; but will break projects.
+        Diag(ArgInfo[i].NameLoc, diag::warn_method_param_redefinition) 
+          << ArgInfo[i].Name;
+        Diag(PrevDecl->getLocation(), 
+             diag::note_previous_declaration);
+      }
+    }
+
     ParmVarDecl* Param
       = ParmVarDecl::Create(Context, ObjCMethod, ArgInfo[i].NameLoc,
                             ArgInfo[i].Name, ArgType, DI,
@@ -1544,9 +1785,12 @@ Decl *Sema::ActOnMethodDeclaration(
     // Apply the attributes to the parameter.
     ProcessDeclAttributeList(TUScope, Param, ArgInfo[i].ArgAttrs);
 
+    S->AddDecl(Param);
+    IdResolver.AddDecl(Param);
+
     Params.push_back(Param);
   }
-
+  
   for (unsigned i = 0, e = CNumArgs; i != e; ++i) {
     ParmVarDecl *Param = cast<ParmVarDecl>(CParamInfo[i].Param);
     QualType ArgType = Param->getType();
@@ -1562,8 +1806,7 @@ Decl *Sema::ActOnMethodDeclaration(
       Param->setInvalidDecl();
     }
     Param->setDeclContext(ObjCMethod);
-    if (Param->getDeclName())
-      IdResolver.RemoveDecl(Param);
+    
     Params.push_back(Param);
   }
   
@@ -1578,10 +1821,7 @@ Decl *Sema::ActOnMethodDeclaration(
 
   const ObjCMethodDecl *InterfaceMD = 0;
 
-  // For implementations (which can be very "coarse grain"), we add the
-  // method now. This allows the AST to implement lookup methods that work
-  // incrementally (without waiting until we parse the @end). It also allows
-  // us to flag multiple declaration errors as they occur.
+  // Add the method now.
   if (ObjCImplementationDecl *ImpDecl =
         dyn_cast<ObjCImplementationDecl>(ClassDecl)) {
     if (MethodType == tok::minus) {
@@ -1608,6 +1848,8 @@ Decl *Sema::ActOnMethodDeclaration(
     if (ObjCMethod->hasAttrs() &&
         containsInvalidMethodImplAttribute(ObjCMethod->getAttrs()))
       Diag(EndLoc, diag::warn_attribute_method_def);
+  } else {
+    cast<DeclContext>(ClassDecl)->addDecl(ObjCMethod);
   }
   if (PrevMethod) {
     // You can never have two method definitions with the same name.
@@ -1619,9 +1861,13 @@ Decl *Sema::ActOnMethodDeclaration(
   // If the interface declared this method, and it was deprecated there,
   // mark it deprecated here.
   if (InterfaceMD)
-   if (Attr *DA = InterfaceMD->getAttr<DeprecatedAttr>())
-    ObjCMethod->addAttr(::new (Context) DeprecatedAttr(DA->getLocation(),
-                                                       Context));
+   if (Attr *DA = InterfaceMD->getAttr<DeprecatedAttr>()) {
+    StringLiteral *SE = StringLiteral::CreateEmpty(Context, 1);
+    ObjCMethod->addAttr(::new (Context) 
+                        DeprecatedAttr(DA->getLocation(),
+                                       Context, 
+                                       SE->getString()));
+   }
 
   return ObjCMethod;
 }
@@ -1783,20 +2029,24 @@ void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
 }
 
 void ObjCImplementationDecl::setIvarInitializers(ASTContext &C,
-                                    CXXBaseOrMemberInitializer ** initializers,
+                                             CXXCtorInitializer ** initializers,
                                                  unsigned numInitializers) {
   if (numInitializers > 0) {
     NumIvarInitializers = numInitializers;
-    CXXBaseOrMemberInitializer **ivarInitializers =
-    new (C) CXXBaseOrMemberInitializer*[NumIvarInitializers];
+    CXXCtorInitializer **ivarInitializers =
+    new (C) CXXCtorInitializer*[NumIvarInitializers];
     memcpy(ivarInitializers, initializers,
-           numInitializers * sizeof(CXXBaseOrMemberInitializer*));
+           numInitializers * sizeof(CXXCtorInitializer*));
     IvarInitializers = ivarInitializers;
   }
 }
 
 void Sema::DiagnoseUseOfUnimplementedSelectors() {
-  if (ReferencedSelectors.empty())
+  // Warning will be issued only when selector table is
+  // generated (which means there is at lease one implementation
+  // in the TU). This is to match gcc's behavior.
+  if (ReferencedSelectors.empty() || 
+      !Context.AnyObjCImplementation())
     return;
   for (llvm::DenseMap<Selector, SourceLocation>::iterator S = 
         ReferencedSelectors.begin(),
diff --git a/lib/Sema/SemaExceptionSpec.cpp b/lib/Sema/SemaExceptionSpec.cpp
index c902e7787096..5d7993b1afba 100644
--- a/lib/Sema/SemaExceptionSpec.cpp
+++ b/lib/Sema/SemaExceptionSpec.cpp
@@ -113,6 +113,9 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   if (!MissingExceptionSpecification && !MissingEmptyExceptionSpecification)
     return true;
 
+  const FunctionProtoType *NewProto 
+    = New->getType()->getAs<FunctionProtoType>();
+
   // The new function declaration is only missing an empty exception
   // specification "throw()". If the throw() specification came from a
   // function in a system header that has C linkage, just add an empty
@@ -121,42 +124,38 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   // to many libc functions as an optimization. Unfortunately, that
   // optimization isn't permitted by the C++ standard, so we're forced
   // to work around it here.
-  if (MissingEmptyExceptionSpecification &&
-      isa<FunctionProtoType>(New->getType()) &&
+  if (MissingEmptyExceptionSpecification && NewProto &&
       (Old->getLocation().isInvalid() ||
        Context.getSourceManager().isInSystemHeader(Old->getLocation())) &&
       Old->isExternC()) {
-    const FunctionProtoType *NewProto 
-      = cast<FunctionProtoType>(New->getType());
+    FunctionProtoType::ExtProtoInfo EPI = NewProto->getExtProtoInfo();
+    EPI.HasExceptionSpec = true;
+    EPI.HasAnyExceptionSpec = false;
+    EPI.NumExceptions = 0;
     QualType NewType = Context.getFunctionType(NewProto->getResultType(),
                                                NewProto->arg_type_begin(),
                                                NewProto->getNumArgs(),
-                                               NewProto->isVariadic(),
-                                               NewProto->getTypeQuals(),
-                                               true, false, 0, 0,
-                                               NewProto->getExtInfo());
+                                               EPI);
     New->setType(NewType);
     return false;
   }
 
-  if (MissingExceptionSpecification && isa<FunctionProtoType>(New->getType())) {
-    const FunctionProtoType *NewProto 
-      = cast<FunctionProtoType>(New->getType());
+  if (MissingExceptionSpecification && NewProto) {
     const FunctionProtoType *OldProto
       = Old->getType()->getAs<FunctionProtoType>();
 
+    FunctionProtoType::ExtProtoInfo EPI = NewProto->getExtProtoInfo();
+    EPI.HasExceptionSpec = OldProto->hasExceptionSpec();
+    EPI.HasAnyExceptionSpec = OldProto->hasAnyExceptionSpec();
+    EPI.NumExceptions = OldProto->getNumExceptions();
+    EPI.Exceptions = OldProto->exception_begin();
+
     // Update the type of the function with the appropriate exception
     // specification.
     QualType NewType = Context.getFunctionType(NewProto->getResultType(),
                                                NewProto->arg_type_begin(),
                                                NewProto->getNumArgs(),
-                                               NewProto->isVariadic(),
-                                               NewProto->getTypeQuals(),
-                                               OldProto->hasExceptionSpec(),
-                                               OldProto->hasAnyExceptionSpec(),
-                                               OldProto->getNumExceptions(),
-                                               OldProto->exception_begin(),
-                                               NewProto->getExtInfo());
+                                               EPI);
     New->setType(NewType);
 
     // If exceptions are disabled, suppress the warning about missing
@@ -197,7 +196,7 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
 
     SourceLocation AfterParenLoc;
     if (TypeSourceInfo *TSInfo = New->getTypeSourceInfo()) {
-      TypeLoc TL = TSInfo->getTypeLoc();
+      TypeLoc TL = TSInfo->getTypeLoc().IgnoreParens();
       if (const FunctionTypeLoc *FTLoc = dyn_cast<FunctionTypeLoc>(&TL))
         AfterParenLoc = PP.getLocForEndOfToken(FTLoc->getRParenLoc());
     }
diff --git a/lib/Sema/SemaExpr.cpp b/lib/Sema/SemaExpr.cpp
index 80b465230e14..65b57c30cd7d 100644
--- a/lib/Sema/SemaExpr.cpp
+++ b/lib/Sema/SemaExpr.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/AST/EvaluatedExprVisitor.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
@@ -48,21 +49,57 @@ using namespace sema;
 /// used, or produce an error (and return true) if a C++0x deleted
 /// function is being used.
 ///
-/// If IgnoreDeprecated is set to true, this should not want about deprecated
+/// If IgnoreDeprecated is set to true, this should not warn about deprecated
 /// decls.
 ///
 /// \returns true if there was an error (this declaration cannot be
 /// referenced), false otherwise.
 ///
-bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc) {
-  // See if the decl is deprecated.
-  if (D->getAttr<DeprecatedAttr>()) {
-    EmitDeprecationWarning(D, Loc);
+bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc,
+                             bool UnknownObjCClass) {
+  if (getLangOptions().CPlusPlus && isa<FunctionDecl>(D)) {
+    // If there were any diagnostics suppressed by template argument deduction,
+    // emit them now.
+    llvm::DenseMap<Decl *, llvm::SmallVector<PartialDiagnosticAt, 1> >::iterator
+      Pos = SuppressedDiagnostics.find(D->getCanonicalDecl());
+    if (Pos != SuppressedDiagnostics.end()) {
+      llvm::SmallVectorImpl<PartialDiagnosticAt> &Suppressed = Pos->second;
+      for (unsigned I = 0, N = Suppressed.size(); I != N; ++I)
+        Diag(Suppressed[I].first, Suppressed[I].second);
+      
+      // Clear out the list of suppressed diagnostics, so that we don't emit
+      // them again for this specialization. However, we don't remove this
+      // entry from the table, because we want to avoid ever emitting these
+      // diagnostics again.
+      Suppressed.clear();
+    }
+  }
+
+  // See if this is an auto-typed variable whose initializer we are parsing.
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    if (VD->isParsingAutoInit()) {
+      Diag(Loc, diag::err_auto_variable_cannot_appear_in_own_initializer)
+        << D->getDeclName();
+      return true;
+    }
   }
 
+  // See if the decl is deprecated.
+  if (const DeprecatedAttr *DA = D->getAttr<DeprecatedAttr>())
+    EmitDeprecationWarning(D, DA->getMessage(), Loc, UnknownObjCClass);
+
   // See if the decl is unavailable
-  if (D->getAttr<UnavailableAttr>()) {
-    Diag(Loc, diag::err_unavailable) << D->getDeclName();
+  if (const UnavailableAttr *UA = D->getAttr<UnavailableAttr>()) {
+    if (UA->getMessage().empty()) {
+      if (!UnknownObjCClass)
+        Diag(Loc, diag::err_unavailable) << D->getDeclName();
+      else
+        Diag(Loc, diag::warn_unavailable_fwdclass_message) 
+             << D->getDeclName();
+    }
+    else 
+      Diag(Loc, diag::err_unavailable_message) 
+        << D->getDeclName() << UA->getMessage();
     Diag(D->getLocation(), diag::note_unavailable_here) << 0;
   }
 
@@ -75,6 +112,10 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc) {
     }
   }
 
+  // Warn if this is used but marked unused.
+  if (D->hasAttr<UnusedAttr>())
+    Diag(Loc, diag::warn_used_but_marked_unused) << D->getDeclName();
+
   return false;
 }
 
@@ -167,6 +208,10 @@ void Sema::DiagnoseSentinelCalls(NamedDecl *D, SourceLocation Loc,
   if (!sentinelExpr) return;
   if (sentinelExpr->isTypeDependent()) return;
   if (sentinelExpr->isValueDependent()) return;
+
+  // nullptr_t is always treated as null.
+  if (sentinelExpr->getType()->isNullPtrType()) return;
+
   if (sentinelExpr->getType()->isAnyPointerType() &&
       sentinelExpr->IgnoreParenCasts()->isNullPointerConstant(Context,
                                             Expr::NPC_ValueDependentIsNull))
@@ -208,32 +253,66 @@ void Sema::DefaultFunctionArrayConversion(Expr *&E) {
     // An lvalue or rvalue of type "array of N T" or "array of unknown bound of
     // T" can be converted to an rvalue of type "pointer to T".
     //
-    if (getLangOptions().C99 || getLangOptions().CPlusPlus ||
-        E->isLvalue(Context) == Expr::LV_Valid)
+    if (getLangOptions().C99 || getLangOptions().CPlusPlus || E->isLValue())
       ImpCastExprToType(E, Context.getArrayDecayedType(Ty),
                         CK_ArrayToPointerDecay);
   }
 }
 
-void Sema::DefaultFunctionArrayLvalueConversion(Expr *&E) {
-  DefaultFunctionArrayConversion(E);
+void Sema::DefaultLvalueConversion(Expr *&E) {
+  // C++ [conv.lval]p1:
+  //   A glvalue of a non-function, non-array type T can be
+  //   converted to a prvalue.
+  if (!E->isGLValue()) return;
 
-  QualType Ty = E->getType();
-  assert(!Ty.isNull() && "DefaultFunctionArrayLvalueConversion - missing type");
-  if (!Ty->isDependentType() && Ty.hasQualifiers() &&
-      (!getLangOptions().CPlusPlus || !Ty->isRecordType()) &&
-      E->isLvalue(Context) == Expr::LV_Valid) {
-    // C++ [conv.lval]p1:
-    //   [...] If T is a non-class type, the type of the rvalue is the
-    //   cv-unqualified version of T. Otherwise, the type of the
-    //   rvalue is T
-    //
-    // C99 6.3.2.1p2:
-    //   If the lvalue has qualified type, the value has the unqualified
-    //   version of the type of the lvalue; otherwise, the value has the
-    //   type of the lvalue.
-    ImpCastExprToType(E, Ty.getUnqualifiedType(), CK_NoOp);
+  QualType T = E->getType();
+  assert(!T.isNull() && "r-value conversion on typeless expression?");
+
+  // Create a load out of an ObjCProperty l-value, if necessary.
+  if (E->getObjectKind() == OK_ObjCProperty) {
+    ConvertPropertyForRValue(E);
+    if (!E->isGLValue())
+      return;
   }
+
+  // We don't want to throw lvalue-to-rvalue casts on top of
+  // expressions of certain types in C++.
+  if (getLangOptions().CPlusPlus &&
+      (E->getType() == Context.OverloadTy ||
+       T->isDependentType() ||
+       T->isRecordType()))
+    return;
+
+  // The C standard is actually really unclear on this point, and
+  // DR106 tells us what the result should be but not why.  It's
+  // generally best to say that void types just doesn't undergo
+  // lvalue-to-rvalue at all.  Note that expressions of unqualified
+  // 'void' type are never l-values, but qualified void can be.
+  if (T->isVoidType())
+    return;
+
+  // C++ [conv.lval]p1:
+  //   [...] If T is a non-class type, the type of the prvalue is the
+  //   cv-unqualified version of T. Otherwise, the type of the
+  //   rvalue is T.
+  //
+  // C99 6.3.2.1p2:
+  //   If the lvalue has qualified type, the value has the unqualified
+  //   version of the type of the lvalue; otherwise, the value has the
+  //   type of the lvalue.    
+  if (T.hasQualifiers())
+    T = T.getUnqualifiedType();
+
+  if (const ArraySubscriptExpr *ae = dyn_cast<ArraySubscriptExpr>(E))
+    CheckArrayAccess(ae);
+  
+  E = ImplicitCastExpr::Create(Context, T, CK_LValueToRValue,
+                               E, 0, VK_RValue);
+}
+
+void Sema::DefaultFunctionArrayLvalueConversion(Expr *&E) {
+  DefaultFunctionArrayConversion(E);
+  DefaultLvalueConversion(E);
 }
 
 
@@ -242,36 +321,43 @@ void Sema::DefaultFunctionArrayLvalueConversion(Expr *&E) {
 /// sometimes surpressed. For example, the array->pointer conversion doesn't
 /// apply if the array is an argument to the sizeof or address (&) operators.
 /// In these instances, this routine should *not* be called.
-Expr *Sema::UsualUnaryConversions(Expr *&Expr) {
-  QualType Ty = Expr->getType();
+Expr *Sema::UsualUnaryConversions(Expr *&E) {
+  // First, convert to an r-value.
+  DefaultFunctionArrayLvalueConversion(E);
+  
+  QualType Ty = E->getType();
   assert(!Ty.isNull() && "UsualUnaryConversions - missing type");
-
-  // C99 6.3.1.1p2:
-  //
-  //   The following may be used in an expression wherever an int or
-  //   unsigned int may be used:
-  //     - an object or expression with an integer type whose integer
-  //       conversion rank is less than or equal to the rank of int
-  //       and unsigned int.
-  //     - A bit-field of type _Bool, int, signed int, or unsigned int.
-  //
-  //   If an int can represent all values of the original type, the
-  //   value is converted to an int; otherwise, it is converted to an
-  //   unsigned int. These are called the integer promotions. All
-  //   other types are unchanged by the integer promotions.
-  QualType PTy = Context.isPromotableBitField(Expr);
-  if (!PTy.isNull()) {
-    ImpCastExprToType(Expr, PTy, CK_IntegralCast);
-    return Expr;
-  }
-  if (Ty->isPromotableIntegerType()) {
-    QualType PT = Context.getPromotedIntegerType(Ty);
-    ImpCastExprToType(Expr, PT, CK_IntegralCast);
-    return Expr;
+  
+  // Try to perform integral promotions if the object has a theoretically
+  // promotable type.
+  if (Ty->isIntegralOrUnscopedEnumerationType()) {
+    // C99 6.3.1.1p2:
+    //
+    //   The following may be used in an expression wherever an int or
+    //   unsigned int may be used:
+    //     - an object or expression with an integer type whose integer
+    //       conversion rank is less than or equal to the rank of int
+    //       and unsigned int.
+    //     - A bit-field of type _Bool, int, signed int, or unsigned int.
+    //
+    //   If an int can represent all values of the original type, the
+    //   value is converted to an int; otherwise, it is converted to an
+    //   unsigned int. These are called the integer promotions. All
+    //   other types are unchanged by the integer promotions.
+  
+    QualType PTy = Context.isPromotableBitField(E);
+    if (!PTy.isNull()) {
+      ImpCastExprToType(E, PTy, CK_IntegralCast);
+      return E;
+    }
+    if (Ty->isPromotableIntegerType()) {
+      QualType PT = Context.getPromotedIntegerType(Ty);
+      ImpCastExprToType(E, PT, CK_IntegralCast);
+      return E;
+    }
   }
 
-  DefaultFunctionArrayLvalueConversion(Expr);
-  return Expr;
+  return E;
 }
 
 /// DefaultArgumentPromotion (C99 6.5.2.2p6). Used for function calls that
@@ -281,12 +367,11 @@ void Sema::DefaultArgumentPromotion(Expr *&Expr) {
   QualType Ty = Expr->getType();
   assert(!Ty.isNull() && "DefaultArgumentPromotion - missing type");
 
+  UsualUnaryConversions(Expr);
+
   // If this is a 'float' (CVR qualified or typedef) promote to double.
   if (Ty->isSpecificBuiltinType(BuiltinType::Float))
-    return ImpCastExprToType(Expr, Context.DoubleTy,
-                             CK_FloatingCast);
-
-  UsualUnaryConversions(Expr);
+    return ImpCastExprToType(Expr, Context.DoubleTy, CK_FloatingCast);
 }
 
 /// DefaultVariadicArgumentPromotion - Like DefaultArgumentPromotion, but
@@ -318,7 +403,6 @@ bool Sema::DefaultVariadicArgumentPromotion(Expr *&Expr, VariadicCallType CT,
   return false;
 }
 
-
 /// UsualArithmeticConversions - Performs various conversions that are common to
 /// binary operators (C99 6.3.1.8). If both operands aren't arithmetic, this
 /// routine returns the first non-arithmetic type found. The client is
@@ -348,19 +432,271 @@ QualType Sema::UsualArithmeticConversions(Expr *&lhsExpr, Expr *&rhsExpr,
   if (!lhs->isArithmeticType() || !rhs->isArithmeticType())
     return lhs;
 
-  // Perform bitfield promotions.
+  // Apply unary and bitfield promotions to the LHS's type.
+  QualType lhs_unpromoted = lhs;
+  if (lhs->isPromotableIntegerType())
+    lhs = Context.getPromotedIntegerType(lhs);
   QualType LHSBitfieldPromoteTy = Context.isPromotableBitField(lhsExpr);
   if (!LHSBitfieldPromoteTy.isNull())
     lhs = LHSBitfieldPromoteTy;
-  QualType RHSBitfieldPromoteTy = Context.isPromotableBitField(rhsExpr);
-  if (!RHSBitfieldPromoteTy.isNull())
-    rhs = RHSBitfieldPromoteTy;
+  if (lhs != lhs_unpromoted && !isCompAssign)
+    ImpCastExprToType(lhsExpr, lhs, CK_IntegralCast);
 
-  QualType destType = Context.UsualArithmeticConversionsType(lhs, rhs);
-  if (!isCompAssign)
-    ImpCastExprToType(lhsExpr, destType, CK_Unknown);
-  ImpCastExprToType(rhsExpr, destType, CK_Unknown);
-  return destType;
+  // If both types are identical, no conversion is needed.
+  if (lhs == rhs)
+    return lhs;
+
+  // At this point, we have two different arithmetic types.
+
+  // Handle complex types first (C99 6.3.1.8p1).
+  bool LHSComplexFloat = lhs->isComplexType();
+  bool RHSComplexFloat = rhs->isComplexType();
+  if (LHSComplexFloat || RHSComplexFloat) {
+    // if we have an integer operand, the result is the complex type.
+
+    if (!RHSComplexFloat && !rhs->isRealFloatingType()) {
+      if (rhs->isIntegerType()) {
+        QualType fp = cast<ComplexType>(lhs)->getElementType();
+        ImpCastExprToType(rhsExpr, fp, CK_IntegralToFloating);
+        ImpCastExprToType(rhsExpr, lhs, CK_FloatingRealToComplex);
+      } else {
+        assert(rhs->isComplexIntegerType());
+        ImpCastExprToType(rhsExpr, lhs, CK_IntegralComplexToFloatingComplex);
+      }
+      return lhs;
+    }
+
+    if (!LHSComplexFloat && !lhs->isRealFloatingType()) {
+      if (!isCompAssign) {
+        // int -> float -> _Complex float
+        if (lhs->isIntegerType()) {
+          QualType fp = cast<ComplexType>(rhs)->getElementType();
+          ImpCastExprToType(lhsExpr, fp, CK_IntegralToFloating);
+          ImpCastExprToType(lhsExpr, rhs, CK_FloatingRealToComplex);
+        } else {
+          assert(lhs->isComplexIntegerType());
+          ImpCastExprToType(lhsExpr, rhs, CK_IntegralComplexToFloatingComplex);
+        }
+      }
+      return rhs;
+    }
+
+    // This handles complex/complex, complex/float, or float/complex.
+    // When both operands are complex, the shorter operand is converted to the
+    // type of the longer, and that is the type of the result. This corresponds
+    // to what is done when combining two real floating-point operands.
+    // The fun begins when size promotion occur across type domains.
+    // From H&S 6.3.4: When one operand is complex and the other is a real
+    // floating-point type, the less precise type is converted, within it's
+    // real or complex domain, to the precision of the other type. For example,
+    // when combining a "long double" with a "double _Complex", the
+    // "double _Complex" is promoted to "long double _Complex".
+    int order = Context.getFloatingTypeOrder(lhs, rhs);
+
+    // If both are complex, just cast to the more precise type.
+    if (LHSComplexFloat && RHSComplexFloat) {
+      if (order > 0) {
+        // _Complex float -> _Complex double
+        ImpCastExprToType(rhsExpr, lhs, CK_FloatingComplexCast);
+        return lhs;
+
+      } else if (order < 0) {
+        // _Complex float -> _Complex double
+        if (!isCompAssign)
+          ImpCastExprToType(lhsExpr, rhs, CK_FloatingComplexCast);
+        return rhs;
+      }
+      return lhs;
+    }
+
+    // If just the LHS is complex, the RHS needs to be converted,
+    // and the LHS might need to be promoted.
+    if (LHSComplexFloat) {
+      if (order > 0) { // LHS is wider
+        // float -> _Complex double
+        QualType fp = cast<ComplexType>(lhs)->getElementType();
+        ImpCastExprToType(rhsExpr, fp, CK_FloatingCast);
+        ImpCastExprToType(rhsExpr, lhs, CK_FloatingRealToComplex);
+        return lhs;        
+      }
+
+      // RHS is at least as wide.  Find its corresponding complex type.
+      QualType result = (order == 0 ? lhs : Context.getComplexType(rhs));
+
+      // double -> _Complex double
+      ImpCastExprToType(rhsExpr, result, CK_FloatingRealToComplex);
+
+      // _Complex float -> _Complex double
+      if (!isCompAssign && order < 0)
+        ImpCastExprToType(lhsExpr, result, CK_FloatingComplexCast);
+
+      return result;
+    }
+
+    // Just the RHS is complex, so the LHS needs to be converted
+    // and the RHS might need to be promoted.
+    assert(RHSComplexFloat);
+
+    if (order < 0) { // RHS is wider
+      // float -> _Complex double
+      if (!isCompAssign) {
+        QualType fp = cast<ComplexType>(rhs)->getElementType();
+        ImpCastExprToType(lhsExpr, fp, CK_FloatingCast);
+        ImpCastExprToType(lhsExpr, rhs, CK_FloatingRealToComplex);
+      }
+      return rhs;
+    }
+
+    // LHS is at least as wide.  Find its corresponding complex type.
+    QualType result = (order == 0 ? rhs : Context.getComplexType(lhs));
+
+    // double -> _Complex double
+    if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, result, CK_FloatingRealToComplex);
+
+    // _Complex float -> _Complex double
+    if (order > 0)
+      ImpCastExprToType(rhsExpr, result, CK_FloatingComplexCast);
+
+    return result;
+  }
+
+  // Now handle "real" floating types (i.e. float, double, long double).
+  bool LHSFloat = lhs->isRealFloatingType();
+  bool RHSFloat = rhs->isRealFloatingType();
+  if (LHSFloat || RHSFloat) {
+    // If we have two real floating types, convert the smaller operand
+    // to the bigger result.
+    if (LHSFloat && RHSFloat) {
+      int order = Context.getFloatingTypeOrder(lhs, rhs);
+      if (order > 0) {
+        ImpCastExprToType(rhsExpr, lhs, CK_FloatingCast);
+        return lhs;
+      }
+
+      assert(order < 0 && "illegal float comparison");
+      if (!isCompAssign)
+        ImpCastExprToType(lhsExpr, rhs, CK_FloatingCast);
+      return rhs;
+    }
+
+    // If we have an integer operand, the result is the real floating type.
+    if (LHSFloat) {
+      if (rhs->isIntegerType()) {
+        // Convert rhs to the lhs floating point type.
+        ImpCastExprToType(rhsExpr, lhs, CK_IntegralToFloating);
+        return lhs;
+      }
+
+      // Convert both sides to the appropriate complex float.
+      assert(rhs->isComplexIntegerType());
+      QualType result = Context.getComplexType(lhs);
+
+      // _Complex int -> _Complex float
+      ImpCastExprToType(rhsExpr, result, CK_IntegralComplexToFloatingComplex);
+
+      // float -> _Complex float
+      if (!isCompAssign)
+        ImpCastExprToType(lhsExpr, result, CK_FloatingRealToComplex);
+
+      return result;
+    }
+
+    assert(RHSFloat);
+    if (lhs->isIntegerType()) {
+      // Convert lhs to the rhs floating point type.
+      if (!isCompAssign)
+        ImpCastExprToType(lhsExpr, rhs, CK_IntegralToFloating);
+      return rhs;
+    }
+
+    // Convert both sides to the appropriate complex float.
+    assert(lhs->isComplexIntegerType());
+    QualType result = Context.getComplexType(rhs);
+
+    // _Complex int -> _Complex float
+    if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, result, CK_IntegralComplexToFloatingComplex);
+
+    // float -> _Complex float
+    ImpCastExprToType(rhsExpr, result, CK_FloatingRealToComplex);
+
+    return result;
+  }
+
+  // Handle GCC complex int extension.
+  // FIXME: if the operands are (int, _Complex long), we currently
+  // don't promote the complex.  Also, signedness?
+  const ComplexType *lhsComplexInt = lhs->getAsComplexIntegerType();
+  const ComplexType *rhsComplexInt = rhs->getAsComplexIntegerType();
+  if (lhsComplexInt && rhsComplexInt) {
+    int order = Context.getIntegerTypeOrder(lhsComplexInt->getElementType(),
+                                            rhsComplexInt->getElementType());
+    assert(order && "inequal types with equal element ordering");
+    if (order > 0) {
+      // _Complex int -> _Complex long
+      ImpCastExprToType(rhsExpr, lhs, CK_IntegralComplexCast);
+      return lhs;
+    }
+
+    if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, rhs, CK_IntegralComplexCast);
+    return rhs;
+  } else if (lhsComplexInt) {
+    // int -> _Complex int
+    ImpCastExprToType(rhsExpr, lhs, CK_IntegralRealToComplex);
+    return lhs;
+  } else if (rhsComplexInt) {
+    // int -> _Complex int
+    if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, rhs, CK_IntegralRealToComplex);
+    return rhs;
+  }
+
+  // Finally, we have two differing integer types.
+  // The rules for this case are in C99 6.3.1.8
+  int compare = Context.getIntegerTypeOrder(lhs, rhs);
+  bool lhsSigned = lhs->hasSignedIntegerRepresentation(),
+       rhsSigned = rhs->hasSignedIntegerRepresentation();
+  if (lhsSigned == rhsSigned) {
+    // Same signedness; use the higher-ranked type
+    if (compare >= 0) {
+      ImpCastExprToType(rhsExpr, lhs, CK_IntegralCast);
+      return lhs;
+    } else if (!isCompAssign) 
+      ImpCastExprToType(lhsExpr, rhs, CK_IntegralCast);
+    return rhs;
+  } else if (compare != (lhsSigned ? 1 : -1)) {
+    // The unsigned type has greater than or equal rank to the
+    // signed type, so use the unsigned type
+    if (rhsSigned) {
+      ImpCastExprToType(rhsExpr, lhs, CK_IntegralCast);
+      return lhs;
+    } else if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, rhs, CK_IntegralCast);
+    return rhs;
+  } else if (Context.getIntWidth(lhs) != Context.getIntWidth(rhs)) {
+    // The two types are different widths; if we are here, that
+    // means the signed type is larger than the unsigned type, so
+    // use the signed type.
+    if (lhsSigned) {
+      ImpCastExprToType(rhsExpr, lhs, CK_IntegralCast);
+      return lhs;
+    } else if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, rhs, CK_IntegralCast);
+    return rhs;
+  } else {
+    // The signed type is higher-ranked than the unsigned type,
+    // but isn't actually any bigger (like unsigned int and long
+    // on most 32-bit systems).  Use the unsigned type corresponding
+    // to the signed type.
+    QualType result =
+      Context.getCorrespondingUnsignedType(lhsSigned ? lhs : rhs);
+    ImpCastExprToType(rhsExpr, result, CK_IntegralCast);
+    if (!isCompAssign)
+      ImpCastExprToType(lhsExpr, result, CK_IntegralCast);
+    return result;
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -409,245 +745,366 @@ Sema::ActOnStringLiteral(const Token *StringToks, unsigned NumStringToks) {
                                      StringTokLocs.size()));
 }
 
-/// ShouldSnapshotBlockValueReference - Return true if a reference inside of
-/// CurBlock to VD should cause it to be snapshotted (as we do for auto
-/// variables defined outside the block) or false if this is not needed (e.g.
-/// for values inside the block or for globals).
+enum CaptureResult {
+  /// No capture is required.
+  CR_NoCapture,
+
+  /// A capture is required.
+  CR_Capture,
+
+  /// A by-ref capture is required.
+  CR_CaptureByRef,
+
+  /// An error occurred when trying to capture the given variable.
+  CR_Error
+};
+
+/// Diagnose an uncapturable value reference.
 ///
-/// This also keeps the 'hasBlockDeclRefExprs' in the BlockScopeInfo records
-/// up-to-date.
+/// \param var - the variable referenced
+/// \param DC - the context which we couldn't capture through
+static CaptureResult
+diagnoseUncapturableValueReference(Sema &S, SourceLocation loc,
+                                   VarDecl *var, DeclContext *DC) {
+  switch (S.ExprEvalContexts.back().Context) {
+  case Sema::Unevaluated:
+    // The argument will never be evaluated, so don't complain.
+    return CR_NoCapture;
+
+  case Sema::PotentiallyEvaluated:
+  case Sema::PotentiallyEvaluatedIfUsed:
+    break;
+
+  case Sema::PotentiallyPotentiallyEvaluated:
+    // FIXME: delay these!
+    break;
+  }
+
+  // Don't diagnose about capture if we're not actually in code right
+  // now; in general, there are more appropriate places that will
+  // diagnose this.
+  if (!S.CurContext->isFunctionOrMethod()) return CR_NoCapture;
+
+  // This particular madness can happen in ill-formed default
+  // arguments; claim it's okay and let downstream code handle it.
+  if (isa<ParmVarDecl>(var) &&
+      S.CurContext == var->getDeclContext()->getParent())
+    return CR_NoCapture;
+
+  DeclarationName functionName;
+  if (FunctionDecl *fn = dyn_cast<FunctionDecl>(var->getDeclContext()))
+    functionName = fn->getDeclName();
+  // FIXME: variable from enclosing block that we couldn't capture from!
+
+  S.Diag(loc, diag::err_reference_to_local_var_in_enclosing_function)
+    << var->getIdentifier() << functionName;
+  S.Diag(var->getLocation(), diag::note_local_variable_declared_here)
+    << var->getIdentifier();
+
+  return CR_Error;
+}
+
+/// There is a well-formed capture at a particular scope level;
+/// propagate it through all the nested blocks.
+static CaptureResult propagateCapture(Sema &S, unsigned validScopeIndex,
+                                      const BlockDecl::Capture &capture) {
+  VarDecl *var = capture.getVariable();
+
+  // Update all the inner blocks with the capture information.
+  for (unsigned i = validScopeIndex + 1, e = S.FunctionScopes.size();
+         i != e; ++i) {
+    BlockScopeInfo *innerBlock = cast<BlockScopeInfo>(S.FunctionScopes[i]);
+    innerBlock->Captures.push_back(
+      BlockDecl::Capture(capture.getVariable(), capture.isByRef(),
+                         /*nested*/ true, capture.getCopyExpr()));
+    innerBlock->CaptureMap[var] = innerBlock->Captures.size(); // +1
+  }
+
+  return capture.isByRef() ? CR_CaptureByRef : CR_Capture;
+}
+
+/// shouldCaptureValueReference - Determine if a reference to the
+/// given value in the current context requires a variable capture.
 ///
-static bool ShouldSnapshotBlockValueReference(Sema &S, BlockScopeInfo *CurBlock,
-                                              ValueDecl *VD) {
-  // If the value is defined inside the block, we couldn't snapshot it even if
-  // we wanted to.
-  if (CurBlock->TheDecl == VD->getDeclContext())
-    return false;
+/// This also keeps the captures set in the BlockScopeInfo records
+/// up-to-date.
+static CaptureResult shouldCaptureValueReference(Sema &S, SourceLocation loc,
+                                                 ValueDecl *value) {
+  // Only variables ever require capture.
+  VarDecl *var = dyn_cast<VarDecl>(value);
+  if (!var) return CR_NoCapture;
 
-  // If this is an enum constant or function, it is constant, don't snapshot.
-  if (isa<EnumConstantDecl>(VD) || isa<FunctionDecl>(VD))
-    return false;
+  // Fast path: variables from the current context never require capture.
+  DeclContext *DC = S.CurContext;
+  if (var->getDeclContext() == DC) return CR_NoCapture;
 
-  // If this is a reference to an extern, static, or global variable, no need to
-  // snapshot it.
+  // Only variables with local storage require capture.
   // FIXME: What about 'const' variables in C++?
-  if (const VarDecl *Var = dyn_cast<VarDecl>(VD))
-    if (!Var->hasLocalStorage())
-      return false;
+  if (!var->hasLocalStorage()) return CR_NoCapture;
 
-  // Blocks that have these can't be constant.
-  CurBlock->hasBlockDeclRefExprs = true;
+  // Otherwise, we need to capture.
 
-  // If we have nested blocks, the decl may be declared in an outer block (in
-  // which case that outer block doesn't get "hasBlockDeclRefExprs") or it may
-  // be defined outside all of the current blocks (in which case the blocks do
-  // all get the bit).  Walk the nesting chain.
-  for (unsigned I = S.FunctionScopes.size() - 1; I; --I) {
-    BlockScopeInfo *NextBlock = dyn_cast<BlockScopeInfo>(S.FunctionScopes[I]);
+  unsigned functionScopesIndex = S.FunctionScopes.size() - 1;
+  do {
+    // Only blocks (and eventually C++0x closures) can capture; other
+    // scopes don't work.
+    if (!isa<BlockDecl>(DC))
+      return diagnoseUncapturableValueReference(S, loc, var, DC);
 
-    if (!NextBlock)
-      continue;
+    BlockScopeInfo *blockScope =
+      cast<BlockScopeInfo>(S.FunctionScopes[functionScopesIndex]);
+    assert(blockScope->TheDecl == static_cast<BlockDecl*>(DC));
 
-    // If we found the defining block for the variable, don't mark the block as
-    // having a reference outside it.
-    if (NextBlock->TheDecl == VD->getDeclContext())
-      break;
+    // Check whether we've already captured it in this block.  If so,
+    // we're done.
+    if (unsigned indexPlus1 = blockScope->CaptureMap[var])
+      return propagateCapture(S, functionScopesIndex,
+                              blockScope->Captures[indexPlus1 - 1]);
+
+    functionScopesIndex--;
+    DC = cast<BlockDecl>(DC)->getDeclContext();
+  } while (var->getDeclContext() != DC);
 
-    // Otherwise, the DeclRef from the inner block causes the outer one to need
-    // a snapshot as well.
-    NextBlock->hasBlockDeclRefExprs = true;
+  // Okay, we descended all the way to the block that defines the variable.
+  // Actually try to capture it.
+  QualType type = var->getType();
+
+  // Prohibit variably-modified types.
+  if (type->isVariablyModifiedType()) {
+    S.Diag(loc, diag::err_ref_vm_type);
+    S.Diag(var->getLocation(), diag::note_declared_at);
+    return CR_Error;
   }
 
-  return true;
-}
+  // Prohibit arrays, even in __block variables, but not references to
+  // them.
+  if (type->isArrayType()) {
+    S.Diag(loc, diag::err_ref_array_type);
+    S.Diag(var->getLocation(), diag::note_declared_at);
+    return CR_Error;
+  }
+
+  S.MarkDeclarationReferenced(loc, var);
+
+  // The BlocksAttr indicates the variable is bound by-reference.
+  bool byRef = var->hasAttr<BlocksAttr>();
+
+  // Build a copy expression.
+  Expr *copyExpr = 0;
+  if (!byRef && S.getLangOptions().CPlusPlus &&
+      !type->isDependentType() && type->isStructureOrClassType()) {
+    // According to the blocks spec, the capture of a variable from
+    // the stack requires a const copy constructor.  This is not true
+    // of the copy/move done to move a __block variable to the heap.
+    type.addConst();
+
+    Expr *declRef = new (S.Context) DeclRefExpr(var, type, VK_LValue, loc);
+    ExprResult result =
+      S.PerformCopyInitialization(
+                      InitializedEntity::InitializeBlock(var->getLocation(),
+                                                         type, false),
+                                  loc, S.Owned(declRef));
+
+    // Build a full-expression copy expression if initialization
+    // succeeded and used a non-trivial constructor.  Recover from
+    // errors by pretending that the copy isn't necessary.
+    if (!result.isInvalid() &&
+        !cast<CXXConstructExpr>(result.get())->getConstructor()->isTrivial()) {
+      result = S.MaybeCreateExprWithCleanups(result);
+      copyExpr = result.take();
+    }
+  }
+
+  // We're currently at the declarer; go back to the closure.
+  functionScopesIndex++;
+  BlockScopeInfo *blockScope =
+    cast<BlockScopeInfo>(S.FunctionScopes[functionScopesIndex]);
+
+  // Build a valid capture in this scope.
+  blockScope->Captures.push_back(
+                 BlockDecl::Capture(var, byRef, /*nested*/ false, copyExpr));
+  blockScope->CaptureMap[var] = blockScope->Captures.size(); // +1
+
+  // Propagate that to inner captures if necessary.
+  return propagateCapture(S, functionScopesIndex,
+                          blockScope->Captures.back());
+}
+
+static ExprResult BuildBlockDeclRefExpr(Sema &S, ValueDecl *vd,
+                                        const DeclarationNameInfo &NameInfo,
+                                        bool byRef) {
+  assert(isa<VarDecl>(vd) && "capturing non-variable");
+
+  VarDecl *var = cast<VarDecl>(vd);
+  assert(var->hasLocalStorage() && "capturing non-local");
+  assert(byRef == var->hasAttr<BlocksAttr>() && "byref set wrong");
+
+  QualType exprType = var->getType().getNonReferenceType();
+
+  BlockDeclRefExpr *BDRE;
+  if (!byRef) {
+    // The variable will be bound by copy; make it const within the
+    // closure, but record that this was done in the expression.
+    bool constAdded = !exprType.isConstQualified();
+    exprType.addConst();
+
+    BDRE = new (S.Context) BlockDeclRefExpr(var, exprType, VK_LValue,
+                                            NameInfo.getLoc(), false,
+                                            constAdded);
+  } else {
+    BDRE = new (S.Context) BlockDeclRefExpr(var, exprType, VK_LValue,
+                                            NameInfo.getLoc(), true);
+  }
 
+  return S.Owned(BDRE);
+}
 
 ExprResult
-Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
+Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
+                       SourceLocation Loc,
                        const CXXScopeSpec *SS) {
   DeclarationNameInfo NameInfo(D->getDeclName(), Loc);
-  return BuildDeclRefExpr(D, Ty, NameInfo, SS);
+  return BuildDeclRefExpr(D, Ty, VK, NameInfo, SS);
 }
 
-/// BuildDeclRefExpr - Build a DeclRefExpr.
+/// BuildDeclRefExpr - Build an expression that references a
+/// declaration that does not require a closure capture.
 ExprResult
-Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty,
+Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
                        const DeclarationNameInfo &NameInfo,
                        const CXXScopeSpec *SS) {
-  if (Context.getCanonicalType(Ty) == Context.UndeducedAutoTy) {
-    Diag(NameInfo.getLoc(),
-         diag::err_auto_variable_cannot_appear_in_own_initializer)
-      << D->getDeclName();
-    return ExprError();
-  }
-
-  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
-    if (isa<NonTypeTemplateParmDecl>(VD)) {
-      // Non-type template parameters can be referenced anywhere they are
-      // visible.
-      Ty = Ty.getNonLValueExprType(Context);
-    } else if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(CurContext)) {
-      if (const FunctionDecl *FD = MD->getParent()->isLocalClass()) {
-        if (VD->hasLocalStorage() && VD->getDeclContext() != CurContext) {
-          Diag(NameInfo.getLoc(),
-               diag::err_reference_to_local_var_in_enclosing_function)
-            << D->getIdentifier() << FD->getDeclName();
-          Diag(D->getLocation(), diag::note_local_variable_declared_here)
-            << D->getIdentifier();
-          return ExprError();
-        }
-      }
-    }
-  }
-
   MarkDeclarationReferenced(NameInfo.getLoc(), D);
 
-  return Owned(DeclRefExpr::Create(Context,
+  Expr *E = DeclRefExpr::Create(Context,
                               SS? (NestedNameSpecifier *)SS->getScopeRep() : 0,
-                                   SS? SS->getRange() : SourceRange(),
-                                   D, NameInfo, Ty));
-}
+                                SS? SS->getRange() : SourceRange(),
+                                D, NameInfo, Ty, VK);
 
-/// \brief Given a field that represents a member of an anonymous
-/// struct/union, build the path from that field's context to the
-/// actual member.
-///
-/// Construct the sequence of field member references we'll have to
-/// perform to get to the field in the anonymous union/struct. The
-/// list of members is built from the field outward, so traverse it
-/// backwards to go from an object in the current context to the field
-/// we found.
-///
-/// \returns The variable from which the field access should begin,
-/// for an anonymous struct/union that is not a member of another
-/// class. Otherwise, returns NULL.
-VarDecl *Sema::BuildAnonymousStructUnionMemberPath(FieldDecl *Field,
-                                   llvm::SmallVectorImpl<FieldDecl *> &Path) {
-  assert(Field->getDeclContext()->isRecord() &&
-         cast<RecordDecl>(Field->getDeclContext())->isAnonymousStructOrUnion()
-         && "Field must be stored inside an anonymous struct or union");
-
-  Path.push_back(Field);
-  VarDecl *BaseObject = 0;
-  DeclContext *Ctx = Field->getDeclContext();
-  do {
-    RecordDecl *Record = cast<RecordDecl>(Ctx);
-    ValueDecl *AnonObject = Record->getAnonymousStructOrUnionObject();
-    if (FieldDecl *AnonField = dyn_cast<FieldDecl>(AnonObject))
-      Path.push_back(AnonField);
-    else {
-      BaseObject = cast<VarDecl>(AnonObject);
-      break;
-    }
-    Ctx = Ctx->getParent();
-  } while (Ctx->isRecord() &&
-           cast<RecordDecl>(Ctx)->isAnonymousStructOrUnion());
+  // Just in case we're building an illegal pointer-to-member.
+  if (isa<FieldDecl>(D) && cast<FieldDecl>(D)->getBitWidth())
+    E->setObjectKind(OK_BitField);
 
-  return BaseObject;
+  return Owned(E);
 }
 
+static ExprResult
+BuildFieldReferenceExpr(Sema &S, Expr *BaseExpr, bool IsArrow,
+                        const CXXScopeSpec &SS, FieldDecl *Field,
+                        DeclAccessPair FoundDecl,
+                        const DeclarationNameInfo &MemberNameInfo);
+
 ExprResult
-Sema::BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
-                                               FieldDecl *Field,
-                                               Expr *BaseObjectExpr,
-                                               SourceLocation OpLoc) {
-  llvm::SmallVector<FieldDecl *, 4> AnonFields;
-  VarDecl *BaseObject = BuildAnonymousStructUnionMemberPath(Field,
-                                                            AnonFields);
-
-  // Build the expression that refers to the base object, from
-  // which we will build a sequence of member references to each
-  // of the anonymous union objects and, eventually, the field we
-  // found via name lookup.
-  bool BaseObjectIsPointer = false;
-  Qualifiers BaseQuals;
-  if (BaseObject) {
-    // BaseObject is an anonymous struct/union variable (and is,
-    // therefore, not part of another non-anonymous record).
-    MarkDeclarationReferenced(Loc, BaseObject);
-    BaseObjectExpr = new (Context) DeclRefExpr(BaseObject,BaseObject->getType(),
-                                               SourceLocation());
-    BaseQuals
-      = Context.getCanonicalType(BaseObject->getType()).getQualifiers();
-  } else if (BaseObjectExpr) {
+Sema::BuildAnonymousStructUnionMemberReference(const CXXScopeSpec &SS,
+                                               SourceLocation loc,
+                                               IndirectFieldDecl *indirectField,
+                                               Expr *baseObjectExpr,
+                                               SourceLocation opLoc) {
+  // First, build the expression that refers to the base object.
+
+  bool baseObjectIsPointer = false;
+  Qualifiers baseQuals;
+
+  // Case 1:  the base of the indirect field is not a field.
+  VarDecl *baseVariable = indirectField->getVarDecl();
+  CXXScopeSpec EmptySS;
+  if (baseVariable) {
+    assert(baseVariable->getType()->isRecordType());
+
+    // In principle we could have a member access expression that
+    // accesses an anonymous struct/union that's a static member of
+    // the base object's class.  However, under the current standard,
+    // static data members cannot be anonymous structs or unions.
+    // Supporting this is as easy as building a MemberExpr here.
+    assert(!baseObjectExpr && "anonymous struct/union is static data member?");
+
+    DeclarationNameInfo baseNameInfo(DeclarationName(), loc);
+
+    ExprResult result =
+      BuildDeclarationNameExpr(EmptySS, baseNameInfo, baseVariable);
+    if (result.isInvalid()) return ExprError();
+
+    baseObjectExpr = result.take();    
+    baseObjectIsPointer = false;
+    baseQuals = baseObjectExpr->getType().getQualifiers();
+
+  // Case 2: the base of the indirect field is a field and the user
+  // wrote a member expression.
+  } else if (baseObjectExpr) {
     // The caller provided the base object expression. Determine
     // whether its a pointer and whether it adds any qualifiers to the
     // anonymous struct/union fields we're looking into.
-    QualType ObjectType = BaseObjectExpr->getType();
-    if (const PointerType *ObjectPtr = ObjectType->getAs<PointerType>()) {
-      BaseObjectIsPointer = true;
-      ObjectType = ObjectPtr->getPointeeType();
+    QualType objectType = baseObjectExpr->getType();
+    
+    if (const PointerType *ptr = objectType->getAs<PointerType>()) {
+      baseObjectIsPointer = true;
+      objectType = ptr->getPointeeType();
+    } else {
+      baseObjectIsPointer = false;
     }
-    BaseQuals
-      = Context.getCanonicalType(ObjectType).getQualifiers();
+    baseQuals = objectType.getQualifiers();
+
+  // Case 3: the base of the indirect field is a field and we should
+  // build an implicit member access.
   } else {
     // We've found a member of an anonymous struct/union that is
     // inside a non-anonymous struct/union, so in a well-formed
     // program our base object expression is "this".
-    DeclContext *DC = getFunctionLevelDeclContext();
-    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(DC)) {
-      if (!MD->isStatic()) {
-        QualType AnonFieldType
-          = Context.getTagDeclType(
-                     cast<RecordDecl>(AnonFields.back()->getDeclContext()));
-        QualType ThisType = Context.getTagDeclType(MD->getParent());
-        if ((Context.getCanonicalType(AnonFieldType)
-               == Context.getCanonicalType(ThisType)) ||
-            IsDerivedFrom(ThisType, AnonFieldType)) {
-          // Our base object expression is "this".
-          BaseObjectExpr = new (Context) CXXThisExpr(Loc,
-                                                     MD->getThisType(Context),
-                                                     /*isImplicit=*/true);
-          BaseObjectIsPointer = true;
-        }
-      } else {
-        return ExprError(Diag(Loc,diag::err_invalid_member_use_in_static_method)
-          << Field->getDeclName());
-      }
-      BaseQuals = Qualifiers::fromCVRMask(MD->getTypeQualifiers());
+    CXXMethodDecl *method = tryCaptureCXXThis();
+    if (!method) {
+      Diag(loc, diag::err_invalid_member_use_in_static_method)
+        << indirectField->getDeclName();
+      return ExprError();
     }
 
-    if (!BaseObjectExpr)
-      return ExprError(Diag(Loc, diag::err_invalid_non_static_member_use)
-        << Field->getDeclName());
+    // Our base object expression is "this".
+    baseObjectExpr =
+      new (Context) CXXThisExpr(loc, method->getThisType(Context),
+                                /*isImplicit=*/ true);
+    baseObjectIsPointer = true;
+    baseQuals = Qualifiers::fromCVRMask(method->getTypeQualifiers());
   }
 
   // Build the implicit member references to the field of the
   // anonymous struct/union.
-  Expr *Result = BaseObjectExpr;
-  Qualifiers ResultQuals = BaseQuals;
-  for (llvm::SmallVector<FieldDecl *, 4>::reverse_iterator
-         FI = AnonFields.rbegin(), FIEnd = AnonFields.rend();
-       FI != FIEnd; ++FI) {
-    QualType MemberType = (*FI)->getType();
-    Qualifiers MemberTypeQuals =
-      Context.getCanonicalType(MemberType).getQualifiers();
+  Expr *result = baseObjectExpr;
+  IndirectFieldDecl::chain_iterator
+    FI = indirectField->chain_begin(), FEnd = indirectField->chain_end();
 
-    // CVR attributes from the base are picked up by members,
-    // except that 'mutable' members don't pick up 'const'.
-    if ((*FI)->isMutable())
-      ResultQuals.removeConst();
+  // Build the first member access in the chain with full information.
+  if (!baseVariable) {
+    FieldDecl *field = cast<FieldDecl>(*FI);
 
-    // GC attributes are never picked up by members.
-    ResultQuals.removeObjCGCAttr();
+    // FIXME: use the real found-decl info!
+    DeclAccessPair foundDecl = DeclAccessPair::make(field, field->getAccess());
 
-    // TR 18037 does not allow fields to be declared with address spaces.
-    assert(!MemberTypeQuals.hasAddressSpace());
+    // Make a nameInfo that properly uses the anonymous name.
+    DeclarationNameInfo memberNameInfo(field->getDeclName(), loc);
 
-    Qualifiers NewQuals = ResultQuals + MemberTypeQuals;
-    if (NewQuals != MemberTypeQuals)
-      MemberType = Context.getQualifiedType(MemberType, NewQuals);
+    result = BuildFieldReferenceExpr(*this, result, baseObjectIsPointer,
+                                     EmptySS, field, foundDecl,
+                                     memberNameInfo).take();
+    baseObjectIsPointer = false;
 
-    MarkDeclarationReferenced(Loc, *FI);
-    PerformObjectMemberConversion(Result, /*FIXME:Qualifier=*/0, *FI, *FI);
-    // FIXME: Might this end up being a qualified name?
-    Result = new (Context) MemberExpr(Result, BaseObjectIsPointer, *FI,
-                                      OpLoc, MemberType);
-    BaseObjectIsPointer = false;
-    ResultQuals = NewQuals;
+    // FIXME: check qualified member access
   }
 
-  return Owned(Result);
+  // In all cases, we should now skip the first declaration in the chain.
+  ++FI;
+
+  while (FI != FEnd) {
+    FieldDecl *field = cast<FieldDecl>(*FI++);
+
+    // FIXME: these are somewhat meaningless
+    DeclarationNameInfo memberNameInfo(field->getDeclName(), loc);
+    DeclAccessPair foundDecl = DeclAccessPair::make(field, field->getAccess());
+
+    result = BuildFieldReferenceExpr(*this, result, /*isarrow*/ false,
+                                     (FI == FEnd? SS : EmptySS), field, 
+                                     foundDecl, memberNameInfo)
+      .take();
+  }
+
+  return Owned(result);
 }
 
 /// Decomposes the given name into a DeclarationNameInfo, its location, and
@@ -684,28 +1141,6 @@ static void DecomposeUnqualifiedId(Sema &SemaRef,
   }
 }
 
-/// Determines whether the given record is "fully-formed" at the given
-/// location, i.e. whether a qualified lookup into it is assured of
-/// getting consistent results already.
-static bool IsFullyFormedScope(Sema &SemaRef, CXXRecordDecl *Record) {
-  if (!Record->hasDefinition())
-    return false;
-
-  for (CXXRecordDecl::base_class_iterator I = Record->bases_begin(),
-         E = Record->bases_end(); I != E; ++I) {
-    CanQualType BaseT = SemaRef.Context.getCanonicalType((*I).getType());
-    CanQual<RecordType> BaseRT = BaseT->getAs<RecordType>();
-    if (!BaseRT) return false;
-
-    CXXRecordDecl *BaseRecord = cast<CXXRecordDecl>(BaseRT->getDecl());
-    if (!BaseRecord->hasDefinition() ||
-        !IsFullyFormedScope(SemaRef, BaseRecord))
-      return false;
-  }
-
-  return true;
-}
-
 /// Determines if the given class is provably not derived from all of
 /// the prospective base classes.
 static bool IsProvablyNotDerivedFrom(Sema &SemaRef,
@@ -757,10 +1192,6 @@ enum IMAKind {
   /// context is not an instance method.
   IMA_Unresolved_StaticContext,
 
-  /// The reference is to a member of an anonymous structure in a
-  /// non-class context.
-  IMA_AnonymousMember,
-
   /// All possible referrents are instance members and the current
   /// context is not an instance method.
   IMA_Error_StaticContext,
@@ -790,19 +1221,16 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
 
   // Collect all the declaring classes of instance members we find.
   bool hasNonInstance = false;
+  bool hasField = false;
   llvm::SmallPtrSet<CXXRecordDecl*, 4> Classes;
   for (LookupResult::iterator I = R.begin(), E = R.end(); I != E; ++I) {
     NamedDecl *D = *I;
+
     if (D->isCXXInstanceMember()) {
-      CXXRecordDecl *R = cast<CXXRecordDecl>(D->getDeclContext());
+      if (dyn_cast<FieldDecl>(D))
+        hasField = true;
 
-      // If this is a member of an anonymous record, move out to the
-      // innermost non-anonymous struct or union.  If there isn't one,
-      // that's a special case.
-      while (R->isAnonymousStructOrUnion()) {
-        R = dyn_cast<CXXRecordDecl>(R->getParent());
-        if (!R) return IMA_AnonymousMember;
-      }
+      CXXRecordDecl *R = cast<CXXRecordDecl>(D->getDeclContext());
       Classes.insert(R->getCanonicalDecl());
     }
     else
@@ -816,8 +1244,24 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
 
   // If the current context is not an instance method, it can't be
   // an implicit member reference.
-  if (isStaticContext)
-    return (hasNonInstance ? IMA_Mixed_StaticContext : IMA_Error_StaticContext);
+  if (isStaticContext) {
+    if (hasNonInstance)
+        return IMA_Mixed_StaticContext;
+        
+    if (SemaRef.getLangOptions().CPlusPlus0x && hasField) {
+      // C++0x [expr.prim.general]p10:
+      //   An id-expression that denotes a non-static data member or non-static
+      //   member function of a class can only be used:
+      //   (...)
+      //   - if that id-expression denotes a non-static data member and it appears in an unevaluated operand.
+      const Sema::ExpressionEvaluationContextRecord& record = SemaRef.ExprEvalContexts.back();
+      bool isUnevaluatedExpression = record.Context == Sema::Unevaluated;
+      if (isUnevaluatedExpression)
+        return IMA_Mixed_StaticContext;
+    }
+    
+    return IMA_Error_StaticContext;
+  }
 
   // If we can prove that the current context is unrelated to all the
   // declaring classes, it can't be an implicit member reference (in
@@ -833,23 +1277,24 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
 /// Diagnose a reference to a field with no object available.
 static void DiagnoseInstanceReference(Sema &SemaRef,
                                       const CXXScopeSpec &SS,
-                                      const LookupResult &R) {
-  SourceLocation Loc = R.getNameLoc();
+                                      NamedDecl *rep,
+                                      const DeclarationNameInfo &nameInfo) {
+  SourceLocation Loc = nameInfo.getLoc();
   SourceRange Range(Loc);
   if (SS.isSet()) Range.setBegin(SS.getRange().getBegin());
 
-  if (R.getAsSingle<FieldDecl>()) {
+  if (isa<FieldDecl>(rep) || isa<IndirectFieldDecl>(rep)) {
     if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(SemaRef.CurContext)) {
       if (MD->isStatic()) {
         // "invalid use of member 'x' in static member function"
         SemaRef.Diag(Loc, diag::err_invalid_member_use_in_static_method)
-          << Range << R.getLookupName();
+          << Range << nameInfo.getName();
         return;
       }
     }
 
     SemaRef.Diag(Loc, diag::err_invalid_non_static_member_use)
-      << R.getLookupName() << Range;
+      << nameInfo.getName() << Range;
     return;
   }
 
@@ -1001,25 +1446,41 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   return true;
 }
 
-static ObjCPropertyDecl *OkToSynthesizeProvisionalIvar(Sema &SemaRef,
-                                                       IdentifierInfo *II,
-                                                       SourceLocation NameLoc) {
-  ObjCMethodDecl *CurMeth = SemaRef.getCurMethodDecl();
+ObjCPropertyDecl *Sema::canSynthesizeProvisionalIvar(IdentifierInfo *II) {
+  ObjCMethodDecl *CurMeth = getCurMethodDecl();
   ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
   if (!IDecl)
     return 0;
   ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
   if (!ClassImpDecl)
     return 0;
-  ObjCPropertyDecl *property = SemaRef.LookupPropertyDecl(IDecl, II);
+  ObjCPropertyDecl *property = LookupPropertyDecl(IDecl, II);
   if (!property)
     return 0;
   if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II))
-    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic ||
+        PIDecl->getPropertyIvarDecl())
       return 0;
   return property;
 }
 
+bool Sema::canSynthesizeProvisionalIvar(ObjCPropertyDecl *Property) {
+  ObjCMethodDecl *CurMeth = getCurMethodDecl();
+  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
+  if (!IDecl)
+    return false;
+  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
+  if (!ClassImpDecl)
+    return false;
+  if (ObjCPropertyImplDecl *PIDecl
+                = ClassImpDecl->FindPropertyImplDecl(Property->getIdentifier()))
+    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic ||
+        PIDecl->getPropertyIvarDecl())
+      return false;
+  
+  return true;
+}
+
 static ObjCIvarDecl *SynthesizeProvisionalIvar(Sema &SemaRef,
                                                LookupResult &Lookup,
                                                IdentifierInfo *II,
@@ -1032,7 +1493,8 @@ static ObjCIvarDecl *SynthesizeProvisionalIvar(Sema &SemaRef,
     LookForIvars = false;
   else
     LookForIvars = (Lookup.isSingleResult() &&
-                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod());
+                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod() &&
+                    (Lookup.getAsSingle<VarDecl>() != 0));
   if (!LookForIvars)
     return 0;
   
@@ -1046,15 +1508,20 @@ static ObjCIvarDecl *SynthesizeProvisionalIvar(Sema &SemaRef,
   ObjCPropertyDecl *property = SemaRef.LookupPropertyDecl(IDecl, II);
   if (!property)
     return 0;
-  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II))
+  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II)) {
     DynamicImplSeen = 
       (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic);
+    // property implementation has a designated ivar. No need to assume a new
+    // one.
+    if (!DynamicImplSeen && PIDecl->getPropertyIvarDecl())
+      return 0;
+  }
   if (!DynamicImplSeen) {
     QualType PropType = SemaRef.Context.getCanonicalType(property->getType());
     ObjCIvarDecl *Ivar = ObjCIvarDecl::Create(SemaRef.Context, ClassImpDecl, 
                                               NameLoc,
                                               II, PropType, /*Dinfo=*/0,
-                                              ObjCIvarDecl::Protected,
+                                              ObjCIvarDecl::Private,
                                               (Expr *)0, true);
     ClassImpDecl->addDecl(Ivar);
     IDecl->makeDeclVisibleInContext(Ivar, false);
@@ -1102,22 +1569,18 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
       Name.getCXXNameType()->isDependentType()) {
     DependentID = true;
   } else if (SS.isSet()) {
-    DeclContext *DC = computeDeclContext(SS, false);
-    if (DC) {
+    if (DeclContext *DC = computeDeclContext(SS, false)) {
       if (RequireCompleteDeclContext(SS, DC))
         return ExprError();
-      // FIXME: We should be checking whether DC is the current instantiation.
-      if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC))
-        DependentID = !IsFullyFormedScope(*this, RD);
     } else {
       DependentID = true;
     }
   }
 
-  if (DependentID) {
+  if (DependentID)
     return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
                                       TemplateArgs);
-  }
+
   bool IvarLookupFollowUp = false;
   // Perform the required lookup.
   LookupResult R(*this, NameInfo, LookupOrdinaryName);
@@ -1130,10 +1593,21 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
     bool MemberOfUnknownSpecialization;
     LookupTemplateName(R, S, SS, QualType(), /*EnteringContext=*/false,
                        MemberOfUnknownSpecialization);
+    
+    if (MemberOfUnknownSpecialization ||
+        (R.getResultKind() == LookupResult::NotFoundInCurrentInstantiation))
+      return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+                                        TemplateArgs);
   } else {
     IvarLookupFollowUp = (!SS.isSet() && II && getCurMethodDecl());
     LookupParsedName(R, S, &SS, !IvarLookupFollowUp);
 
+    // If the result might be in a dependent base class, this is a dependent 
+    // id-expression.
+    if (R.getResultKind() == LookupResult::NotFoundInCurrentInstantiation)
+      return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+                                        TemplateArgs);
+      
     // If this reference is in an Objective-C method, then we need to do
     // some special Objective-C lookup, too.
     if (IvarLookupFollowUp) {
@@ -1141,13 +1615,21 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
       if (E.isInvalid())
         return ExprError();
 
-      Expr *Ex = E.takeAs<Expr>();
-      if (Ex) return Owned(Ex);
-      // Synthesize ivars lazily
-      if (getLangOptions().ObjCNonFragileABI2) {
-        if (SynthesizeProvisionalIvar(*this, R, II, NameLoc))
+      if (Expr *Ex = E.takeAs<Expr>())
+        return Owned(Ex);
+      
+      // Synthesize ivars lazily.
+      if (getLangOptions().ObjCDefaultSynthProperties &&
+          getLangOptions().ObjCNonFragileABI2) {
+        if (SynthesizeProvisionalIvar(*this, R, II, NameLoc)) {
+          if (const ObjCPropertyDecl *Property = 
+                canSynthesizeProvisionalIvar(II)) {
+            Diag(NameLoc, diag::warn_synthesized_ivar_access) << II;
+            Diag(Property->getLocation(), diag::note_property_declare);
+          }
           return ActOnIdExpression(S, SS, Id, HasTrailingLParen,
                                    isAddressOfOperand);
+        }
       }
       // for further use, this must be set to false if in class method.
       IvarLookupFollowUp = getCurMethodDecl()->isInstanceMethod();
@@ -1195,33 +1677,16 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
 
   if (VarDecl *Var = R.getAsSingle<VarDecl>()) {
     if (getLangOptions().ObjCNonFragileABI && IvarLookupFollowUp &&
-        !getLangOptions().ObjCNonFragileABI2 &&
+        !(getLangOptions().ObjCDefaultSynthProperties && 
+          getLangOptions().ObjCNonFragileABI2) &&
         Var->isFileVarDecl()) {
-      ObjCPropertyDecl *Property = 
-        OkToSynthesizeProvisionalIvar(*this, II, NameLoc);
+      ObjCPropertyDecl *Property = canSynthesizeProvisionalIvar(II);
       if (Property) {
         Diag(NameLoc, diag::warn_ivar_variable_conflict) << Var->getDeclName();
         Diag(Property->getLocation(), diag::note_property_declare);
         Diag(Var->getLocation(), diag::note_global_declared_at);
       }
     }
-  } else if (FunctionDecl *Func = R.getAsSingle<FunctionDecl>()) {
-    if (!getLangOptions().CPlusPlus && !Func->hasPrototype()) {
-      // C99 DR 316 says that, if a function type comes from a
-      // function definition (without a prototype), that type is only
-      // used for checking compatibility. Therefore, when referencing
-      // the function, we pretend that we don't have the full function
-      // type.
-      if (DiagnoseUseOfDecl(Func, NameLoc))
-        return ExprError();
-
-      QualType T = Func->getType();
-      QualType NoProtoType = T;
-      if (const FunctionProtoType *Proto = T->getAs<FunctionProtoType>())
-        NoProtoType = Context.getFunctionNoProtoType(Proto->getResultType(),
-                                                     Proto->getExtInfo());
-      return BuildDeclRefExpr(Func, NoProtoType, NameLoc, &SS);
-    }
   }
 
   // Check whether this might be a C++ implicit instance member access.
@@ -1259,7 +1724,8 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
     else if (R.isUnresolvableResult())
       MightBeImplicitMember = true;
     else
-      MightBeImplicitMember = isa<FieldDecl>(R.getFoundDecl());
+      MightBeImplicitMember = isa<FieldDecl>(R.getFoundDecl()) ||
+                              isa<IndirectFieldDecl>(R.getFoundDecl());
 
     if (MightBeImplicitMember)
       return BuildPossibleImplicitMemberExpr(SS, R, TemplateArgs);
@@ -1280,11 +1746,6 @@ Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
   case IMA_Instance:
     return BuildImplicitMemberExpr(SS, R, TemplateArgs, true);
 
-  case IMA_AnonymousMember:
-    assert(R.isSingleResult());
-    return BuildAnonymousStructUnionMemberReference(R.getNameLoc(),
-                                                    R.getAsSingle<FieldDecl>());
-
   case IMA_Mixed:
   case IMA_Mixed_Unrelated:
   case IMA_Unresolved:
@@ -1299,7 +1760,8 @@ Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
 
   case IMA_Error_StaticContext:
   case IMA_Error_Unrelated:
-    DiagnoseInstanceReference(*this, SS, R);
+    DiagnoseInstanceReference(*this, SS, R.getRepresentativeDecl(),
+                              R.getLookupNameInfo());
     return ExprError();
   }
 
@@ -1400,11 +1862,17 @@ Sema::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
       SelfName.setIdentifier(&II, SourceLocation());
       CXXScopeSpec SelfScopeSpec;
       ExprResult SelfExpr = ActOnIdExpression(S, SelfScopeSpec,
-                                                    SelfName, false, false);
+                                              SelfName, false, false);
+      if (SelfExpr.isInvalid())
+        return ExprError();
+
+      Expr *SelfE = SelfExpr.take();
+      DefaultLvalueConversion(SelfE);
+
       MarkDeclarationReferenced(Loc, IV);
       return Owned(new (Context)
                    ObjCIvarRefExpr(IV, IV->getType(), Loc,
-                                   SelfExpr.takeAs<Expr>(), true, true));
+                                   SelfE, true, true));
     }
   } else if (CurMethod->isInstanceMethod()) {
     // We should warn if a local variable hides an ivar.
@@ -1607,6 +2075,7 @@ static MemberExpr *BuildMemberExpr(ASTContext &C, Expr *Base, bool isArrow,
                                    DeclAccessPair FoundDecl,
                                    const DeclarationNameInfo &MemberNameInfo,
                                    QualType Ty,
+                                   ExprValueKind VK, ExprObjectKind OK,
                           const TemplateArgumentListInfo *TemplateArgs = 0) {
   NestedNameSpecifier *Qualifier = 0;
   SourceRange QualifierRange;
@@ -1617,7 +2086,65 @@ static MemberExpr *BuildMemberExpr(ASTContext &C, Expr *Base, bool isArrow,
 
   return MemberExpr::Create(C, Base, isArrow, Qualifier, QualifierRange,
                             Member, FoundDecl, MemberNameInfo,
-                            TemplateArgs, Ty);
+                            TemplateArgs, Ty, VK, OK);
+}
+
+static ExprResult
+BuildFieldReferenceExpr(Sema &S, Expr *BaseExpr, bool IsArrow,
+                        const CXXScopeSpec &SS, FieldDecl *Field,
+                        DeclAccessPair FoundDecl,
+                        const DeclarationNameInfo &MemberNameInfo) {
+  // x.a is an l-value if 'a' has a reference type. Otherwise:
+  // x.a is an l-value/x-value/pr-value if the base is (and note
+  //   that *x is always an l-value), except that if the base isn't
+  //   an ordinary object then we must have an rvalue.
+  ExprValueKind VK = VK_LValue;
+  ExprObjectKind OK = OK_Ordinary;
+  if (!IsArrow) {
+    if (BaseExpr->getObjectKind() == OK_Ordinary)
+      VK = BaseExpr->getValueKind();
+    else
+      VK = VK_RValue;
+  }
+  if (VK != VK_RValue && Field->isBitField())
+    OK = OK_BitField;
+
+  // Figure out the type of the member; see C99 6.5.2.3p3, C++ [expr.ref]
+  QualType MemberType = Field->getType();
+  if (const ReferenceType *Ref = MemberType->getAs<ReferenceType>()) {
+    MemberType = Ref->getPointeeType();
+    VK = VK_LValue;
+  } else {
+    QualType BaseType = BaseExpr->getType();
+    if (IsArrow) BaseType = BaseType->getAs<PointerType>()->getPointeeType();
+
+    Qualifiers BaseQuals = BaseType.getQualifiers();
+
+    // GC attributes are never picked up by members.
+    BaseQuals.removeObjCGCAttr();
+
+    // CVR attributes from the base are picked up by members,
+    // except that 'mutable' members don't pick up 'const'.
+    if (Field->isMutable()) BaseQuals.removeConst();
+
+    Qualifiers MemberQuals
+      = S.Context.getCanonicalType(MemberType).getQualifiers();
+
+    // TR 18037 does not allow fields to be declared with address spaces.
+    assert(!MemberQuals.hasAddressSpace());
+
+    Qualifiers Combined = BaseQuals + MemberQuals;
+    if (Combined != MemberQuals)
+      MemberType = S.Context.getQualifiedType(MemberType, Combined);
+  }
+
+  S.MarkDeclarationReferenced(MemberNameInfo.getLoc(), Field);
+  if (S.PerformObjectMemberConversion(BaseExpr, SS.getScopeRep(),
+                                      FoundDecl, Field))
+    return ExprError();
+  return S.Owned(BuildMemberExpr(S.Context, BaseExpr, IsArrow, SS,
+                                 Field, FoundDecl, MemberNameInfo,
+                                 MemberType, VK, OK));
 }
 
 /// Builds an implicit member access expression.  The current context
@@ -1631,29 +2158,30 @@ Sema::BuildImplicitMemberExpr(const CXXScopeSpec &SS,
                               bool IsKnownInstance) {
   assert(!R.empty() && !R.isAmbiguous());
 
-  SourceLocation Loc = R.getNameLoc();
+  SourceLocation loc = R.getNameLoc();
 
   // We may have found a field within an anonymous union or struct
   // (C++ [class.union]).
-  // FIXME: This needs to happen post-isImplicitMemberReference?
   // FIXME: template-ids inside anonymous structs?
-  if (FieldDecl *FD = R.getAsSingle<FieldDecl>())
-    if (cast<RecordDecl>(FD->getDeclContext())->isAnonymousStructOrUnion())
-      return BuildAnonymousStructUnionMemberReference(Loc, FD);
+  if (IndirectFieldDecl *FD = R.getAsSingle<IndirectFieldDecl>())
+    return BuildAnonymousStructUnionMemberReference(SS, R.getNameLoc(), FD);
 
-  // If this is known to be an instance access, go ahead and build a
+  // If this is known to be an instance access, go ahead and build an
+  // implicit 'this' expression now.
   // 'this' expression now.
-  DeclContext *DC = getFunctionLevelDeclContext();
-  QualType ThisType = cast<CXXMethodDecl>(DC)->getThisType(Context);
-  Expr *This = 0; // null signifies implicit access
+  CXXMethodDecl *method = tryCaptureCXXThis();
+  assert(method && "didn't correctly pre-flight capture of 'this'");
+
+  QualType thisType = method->getThisType(Context);
+  Expr *baseExpr = 0; // null signifies implicit access
   if (IsKnownInstance) {
     SourceLocation Loc = R.getNameLoc();
     if (SS.getRange().isValid())
       Loc = SS.getRange().getBegin();
-    This = new (Context) CXXThisExpr(Loc, ThisType, /*isImplicit=*/true);
+    baseExpr = new (Context) CXXThisExpr(loc, thisType, /*isImplicit=*/true);
   }
 
-  return BuildMemberReferenceExpr(This, ThisType,
+  return BuildMemberReferenceExpr(baseExpr, thisType,
                                   /*OpLoc*/ SourceLocation(),
                                   /*IsArrow*/ true,
                                   SS,
@@ -1762,10 +2290,8 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   // we've picked a target.
   R.suppressDiagnostics();
 
-  bool Dependent
-    = UnresolvedLookupExpr::ComputeDependence(R.begin(), R.end(), 0);
   UnresolvedLookupExpr *ULE
-    = UnresolvedLookupExpr::Create(Context, Dependent, R.getNamingClass(),
+    = UnresolvedLookupExpr::Create(Context, R.getNamingClass(),
                                    (NestedNameSpecifier*) SS.getScopeRep(),
                                    SS.getRange(), R.getLookupNameInfo(),
                                    NeedsADL, R.isOverloadedResult(),
@@ -1774,7 +2300,6 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   return Owned(ULE);
 }
 
-
 /// \brief Complete semantic analysis for a reference to the given declaration.
 ExprResult
 Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
@@ -1817,6 +2342,14 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   if (VD->isInvalidDecl())
     return ExprError();
 
+  // Handle members of anonymous structs and unions.  If we got here,
+  // and the reference is to a class member indirect field, then this
+  // must be the subject of a pointer-to-member expression.
+  if (IndirectFieldDecl *indirectField = dyn_cast<IndirectFieldDecl>(VD))
+    if (!indirectField->isCXXClassMember())
+      return BuildAnonymousStructUnionMemberReference(SS, NameInfo.getLoc(),
+                                                      indirectField);
+
   // If the identifier reference is inside a block, and it refers to a value
   // that is outside the block, create a BlockDeclRefExpr instead of a
   // DeclRefExpr.  This ensures the value is treated as a copy-in snapshot when
@@ -1825,59 +2358,140 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   // We do not do this for things like enum constants, global variables, etc,
   // as they do not get snapshotted.
   //
-  if (getCurBlock() &&
-      ShouldSnapshotBlockValueReference(*this, getCurBlock(), VD)) {
-    if (VD->getType().getTypePtr()->isVariablyModifiedType()) {
-      Diag(Loc, diag::err_ref_vm_type);
-      Diag(D->getLocation(), diag::note_declared_at);
+  switch (shouldCaptureValueReference(*this, NameInfo.getLoc(), VD)) {
+  case CR_Error:
+    return ExprError();
+
+  case CR_Capture:
+    assert(!SS.isSet() && "referenced local variable with scope specifier?");
+    return BuildBlockDeclRefExpr(*this, VD, NameInfo, /*byref*/ false);
+
+  case CR_CaptureByRef:
+    assert(!SS.isSet() && "referenced local variable with scope specifier?");
+    return BuildBlockDeclRefExpr(*this, VD, NameInfo, /*byref*/ true);
+
+  case CR_NoCapture: {
+    // If this reference is not in a block or if the referenced
+    // variable is within the block, create a normal DeclRefExpr.
+
+    QualType type = VD->getType();
+    ExprValueKind valueKind = VK_RValue;
+
+    switch (D->getKind()) {
+    // Ignore all the non-ValueDecl kinds.
+#define ABSTRACT_DECL(kind)
+#define VALUE(type, base)
+#define DECL(type, base) \
+    case Decl::type:
+#include "clang/AST/DeclNodes.inc"
+      llvm_unreachable("invalid value decl kind");
       return ExprError();
-    }
 
-    if (VD->getType()->isArrayType()) {
-      Diag(Loc, diag::err_ref_array_type);
-      Diag(D->getLocation(), diag::note_declared_at);
+    // These shouldn't make it here.
+    case Decl::ObjCAtDefsField:
+    case Decl::ObjCIvar:
+      llvm_unreachable("forming non-member reference to ivar?");
       return ExprError();
+
+    // Enum constants are always r-values and never references.
+    // Unresolved using declarations are dependent.
+    case Decl::EnumConstant:
+    case Decl::UnresolvedUsingValue:
+      valueKind = VK_RValue;
+      break;
+
+    // Fields and indirect fields that got here must be for
+    // pointer-to-member expressions; we just call them l-values for
+    // internal consistency, because this subexpression doesn't really
+    // exist in the high-level semantics.
+    case Decl::Field:
+    case Decl::IndirectField:
+      assert(getLangOptions().CPlusPlus &&
+             "building reference to field in C?");
+
+      // These can't have reference type in well-formed programs, but
+      // for internal consistency we do this anyway.
+      type = type.getNonReferenceType();
+      valueKind = VK_LValue;
+      break;
+
+    // Non-type template parameters are either l-values or r-values
+    // depending on the type.
+    case Decl::NonTypeTemplateParm: {
+      if (const ReferenceType *reftype = type->getAs<ReferenceType>()) {
+        type = reftype->getPointeeType();
+        valueKind = VK_LValue; // even if the parameter is an r-value reference
+        break;
+      }
+
+      // For non-references, we need to strip qualifiers just in case
+      // the template parameter was declared as 'const int' or whatever.
+      valueKind = VK_RValue;
+      type = type.getUnqualifiedType();
+      break;
     }
 
-    MarkDeclarationReferenced(Loc, VD);
-    QualType ExprTy = VD->getType().getNonReferenceType();
-    // The BlocksAttr indicates the variable is bound by-reference.
-    if (VD->getAttr<BlocksAttr>())
-      return Owned(new (Context) BlockDeclRefExpr(VD, ExprTy, Loc, true));
-    // This is to record that a 'const' was actually synthesize and added.
-    bool constAdded = !ExprTy.isConstQualified();
-    // Variable will be bound by-copy, make it const within the closure.
-
-    ExprTy.addConst();
-    QualType T = VD->getType();
-    BlockDeclRefExpr *BDRE = new (Context) BlockDeclRefExpr(VD, 
-                                                            ExprTy, Loc, false,
-                                                            constAdded);
-    if (getLangOptions().CPlusPlus) {
-      if (!T->isDependentType() && !T->isReferenceType()) {
-        Expr *E = new (Context) 
-                    DeclRefExpr(const_cast<ValueDecl*>(BDRE->getDecl()), T,
-                                          SourceLocation());
+    case Decl::Var:
+      // In C, "extern void blah;" is valid and is an r-value.
+      if (!getLangOptions().CPlusPlus &&
+          !type.hasQualifiers() &&
+          type->isVoidType()) {
+        valueKind = VK_RValue;
+        break;
+      }
+      // fallthrough
+
+    case Decl::ImplicitParam:
+    case Decl::ParmVar:
+      // These are always l-values.
+      valueKind = VK_LValue;
+      type = type.getNonReferenceType();
+      break;
+
+    case Decl::Function: {
+      // Functions are l-values in C++.
+      if (getLangOptions().CPlusPlus) {
+        valueKind = VK_LValue;
+        break;
+      }
       
-        ExprResult Res = PerformCopyInitialization(
-                          InitializedEntity::InitializeBlock(VD->getLocation(), 
-                                                         T, false),
-                                                         SourceLocation(),
-                                                         Owned(E));
-        if (!Res.isInvalid()) {
-          Res = MaybeCreateCXXExprWithTemporaries(Res.get());
-          Expr *Init = Res.takeAs<Expr>();
-          BDRE->setCopyConstructorExpr(Init);
-        }
+      // C99 DR 316 says that, if a function type comes from a
+      // function definition (without a prototype), that type is only
+      // used for checking compatibility. Therefore, when referencing
+      // the function, we pretend that we don't have the full function
+      // type.
+      if (!cast<FunctionDecl>(VD)->hasPrototype())
+        if (const FunctionProtoType *proto = type->getAs<FunctionProtoType>())
+          type = Context.getFunctionNoProtoType(proto->getResultType(),
+                                                proto->getExtInfo());
+
+      // Functions are r-values in C.
+      valueKind = VK_RValue;
+      break;
+    }
+
+    case Decl::CXXMethod:
+      // C++ methods are l-values if static, r-values if non-static.
+      if (cast<CXXMethodDecl>(VD)->isStatic()) {
+        valueKind = VK_LValue;
+        break;
       }
+      // fallthrough
+
+    case Decl::CXXConversion:
+    case Decl::CXXDestructor:
+    case Decl::CXXConstructor:
+      valueKind = VK_RValue;
+      break;
     }
-    return Owned(BDRE);
+
+    return BuildDeclRefExpr(VD, type, valueKind, NameInfo, &SS);
   }
-  // If this reference is not in a block or if the referenced variable is
-  // within the block, create a normal DeclRefExpr.
 
-  return BuildDeclRefExpr(VD, VD->getType().getNonReferenceType(),
-                          NameInfo, &SS);
+  }
+
+  llvm_unreachable("unknown capture result");
+  return ExprError();
 }
 
 ExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc,
@@ -2008,6 +2622,9 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
     bool isExact = (result == APFloat::opOK);
     Res = FloatingLiteral::Create(Context, Val, isExact, Ty, Tok.getLocation());
 
+    if (getLangOptions().SinglePrecisionConstants && Ty == Context.DoubleTy)
+      ImpCastExprToType(Res, Context.FloatTy, CK_FloatingCast);
+
   } else if (!Literal.isIntegerLiteral()) {
     return ExprError();
   } else {
@@ -2074,7 +2691,10 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
         // Does it fit in a unsigned long long?
         if (ResultVal.isIntN(LongLongSize)) {
           // Does it fit in a signed long long?
-          if (!Literal.isUnsigned && ResultVal[LongLongSize-1] == 0)
+          // To be compatible with MSVC, hex integer literals ending with the
+          // LL or i64 suffix are always signed in Microsoft mode.
+          if (!Literal.isUnsigned && (ResultVal[LongLongSize-1] == 0 ||
+              (getLangOptions().Microsoft && Literal.isLongLong)))
             Ty = Context.LongLongTy;
           else if (AllowUnsigned)
             Ty = Context.UnsignedLongLongTy;
@@ -2091,7 +2711,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
       }
 
       if (ResultVal.getBitWidth() != Width)
-        ResultVal.trunc(Width);
+        ResultVal = ResultVal.trunc(Width);
     }
     Res = IntegerLiteral::Create(Context, ResultVal, Ty, Tok.getLocation());
   }
@@ -2114,7 +2734,7 @@ ExprResult Sema::ActOnParenExpr(SourceLocation L,
 /// See C99 6.3.2.1p[2-4] for more details.
 bool Sema::CheckSizeOfAlignOfOperand(QualType exprType,
                                      SourceLocation OpLoc,
-                                     const SourceRange &ExprRange,
+                                     SourceRange ExprRange,
                                      bool isSizeof) {
   if (exprType->isDependentType())
     return false;
@@ -2153,17 +2773,11 @@ bool Sema::CheckSizeOfAlignOfOperand(QualType exprType,
     return true;
   }
 
-  if (Context.hasSameUnqualifiedType(exprType, Context.OverloadTy)) {
-    Diag(OpLoc, diag::err_sizeof_alignof_overloaded_function_type)
-      << !isSizeof << ExprRange;
-    return true;
-  }
-  
   return false;
 }
 
-bool Sema::CheckAlignOfExpr(Expr *E, SourceLocation OpLoc,
-                            const SourceRange &ExprRange) {
+static bool CheckAlignOfExpr(Sema &S, Expr *E, SourceLocation OpLoc,
+                             SourceRange ExprRange) {
   E = E->IgnoreParens();
 
   // alignof decl is always ok.
@@ -2175,7 +2789,7 @@ bool Sema::CheckAlignOfExpr(Expr *E, SourceLocation OpLoc,
     return false;
 
   if (E->getBitField()) {
-    Diag(OpLoc, diag::err_sizeof_alignof_bitfield) << 1 << ExprRange;
+   S. Diag(OpLoc, diag::err_sizeof_alignof_bitfield) << 1 << ExprRange;
     return true;
   }
 
@@ -2185,7 +2799,7 @@ bool Sema::CheckAlignOfExpr(Expr *E, SourceLocation OpLoc,
     if (isa<FieldDecl>(ME->getMemberDecl()))
       return false;
 
-  return CheckSizeOfAlignOfOperand(E->getType(), OpLoc, ExprRange, false);
+  return S.CheckSizeOfAlignOfOperand(E->getType(), OpLoc, ExprRange, false);
 }
 
 /// \brief Build a sizeof or alignof expression given a type operand.
@@ -2218,10 +2832,14 @@ Sema::CreateSizeOfAlignOfExpr(Expr *E, SourceLocation OpLoc,
   if (E->isTypeDependent()) {
     // Delay type-checking for type-dependent expressions.
   } else if (!isSizeOf) {
-    isInvalid = CheckAlignOfExpr(E, OpLoc, R);
+    isInvalid = CheckAlignOfExpr(*this, E, OpLoc, R);
   } else if (E->getBitField()) {  // C99 6.5.3.4p1.
     Diag(OpLoc, diag::err_sizeof_alignof_bitfield) << 0;
     isInvalid = true;
+  } else if (E->getType()->isPlaceholderType()) {
+    ExprResult PE = CheckPlaceholderExpr(E, OpLoc);
+    if (PE.isInvalid()) return ExprError();
+    return CreateSizeOfAlignOfExpr(PE.take(), OpLoc, isSizeOf, R);
   } else {
     isInvalid = CheckSizeOfAlignOfOperand(E->getType(), OpLoc, R, true);
   }
@@ -2257,9 +2875,14 @@ Sema::ActOnSizeOfAlignOfExpr(SourceLocation OpLoc, bool isSizeof, bool isType,
   return move(Result);
 }
 
-QualType Sema::CheckRealImagOperand(Expr *&V, SourceLocation Loc, bool isReal) {
+static QualType CheckRealImagOperand(Sema &S, Expr *&V, SourceLocation Loc,
+                                     bool isReal) {
   if (V->isTypeDependent())
-    return Context.DependentTy;
+    return S.Context.DependentTy;
+
+  // _Real and _Imag are only l-values for normal l-values.
+  if (V->getObjectKind() != OK_Ordinary)
+    S.DefaultLvalueConversion(V);
 
   // These operators return the element type of a complex type.
   if (const ComplexType *CT = V->getType()->getAs<ComplexType>())
@@ -2269,8 +2892,16 @@ QualType Sema::CheckRealImagOperand(Expr *&V, SourceLocation Loc, bool isReal) {
   if (V->getType()->isArithmeticType())
     return V->getType();
 
+  // Test for placeholders.
+  ExprResult PR = S.CheckPlaceholderExpr(V, Loc);
+  if (PR.isInvalid()) return QualType();
+  if (PR.take() != V) {
+    V = PR.take();
+    return CheckRealImagOperand(S, V, Loc, isReal);
+  }
+
   // Reject anything else.
-  Diag(Loc, diag::err_realimag_invalid_type) << V->getType()
+  S.Diag(Loc, diag::err_realimag_invalid_type) << V->getType()
     << (isReal ? "__real" : "__imag");
   return QualType();
 }
@@ -2290,6 +2921,19 @@ Sema::ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc,
   return BuildUnaryOp(S, OpLoc, Opc, Input);
 }
 
+/// Expressions of certain arbitrary types are forbidden by C from
+/// having l-value type.  These are:
+///   - 'void', but not qualified void
+///   - function types
+///
+/// The exact rule here is C99 6.3.2.1:
+///   An lvalue is an expression with an object type or an incomplete
+///   type other than void.
+static bool IsCForbiddenLValueType(ASTContext &C, QualType T) {
+  return ((T->isVoidType() && !T.hasQualifiers()) ||
+          T->isFunctionType());
+}
+
 ExprResult
 Sema::ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc,
                               Expr *Idx, SourceLocation RLoc) {
@@ -2303,7 +2947,9 @@ Sema::ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc,
   if (getLangOptions().CPlusPlus &&
       (LHSExp->isTypeDependent() || RHSExp->isTypeDependent())) {
     return Owned(new (Context) ArraySubscriptExpr(LHSExp, RHSExp,
-                                                  Context.DependentTy, RLoc));
+                                                  Context.DependentTy,
+                                                  VK_LValue, OK_Ordinary,
+                                                  RLoc));
   }
 
   if (getLangOptions().CPlusPlus &&
@@ -2330,6 +2976,8 @@ Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
   DefaultFunctionArrayLvalueConversion(RHSExp);
 
   QualType LHSTy = LHSExp->getType(), RHSTy = RHSExp->getType();
+  ExprValueKind VK = VK_LValue;
+  ExprObjectKind OK = OK_Ordinary;
 
   // C99 6.5.2.1p2: the expression e1[e2] is by definition precisely equivalent
   // to the expression *((e1)+(e2)). This means the array "Base" may actually be
@@ -2364,6 +3012,9 @@ Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
   } else if (const VectorType *VTy = LHSTy->getAs<VectorType>()) {
     BaseExpr = LHSExp;    // vectors: V[123]
     IndexExpr = RHSExp;
+    VK = LHSExp->getValueKind();
+    if (VK != VK_RValue)
+      OK = OK_VectorComponent;
 
     // FIXME: need to deal with const...
     ResultType = VTy->getElementType();
@@ -2417,7 +3068,15 @@ Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
     return ExprError();
   }
 
-  if (!ResultType->isDependentType() &&
+  if (ResultType->isVoidType() && !getLangOptions().CPlusPlus) {
+    // GNU extension: subscripting on pointer to void
+    Diag(LLoc, diag::ext_gnu_void_ptr)
+      << BaseExpr->getSourceRange();
+
+    // C forbids expressions of unqualified void type from being l-values.
+    // See IsCForbiddenLValueType.
+    if (!ResultType.hasQualifiers()) VK = VK_RValue;
+  } else if (!ResultType->isDependentType() &&
       RequireCompleteType(LLoc, ResultType,
                           PDiag(diag::err_subscript_incomplete_type)
                             << BaseExpr->getSourceRange()))
@@ -2430,13 +3089,20 @@ Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
     return ExprError();
   }
 
+  assert(VK == VK_RValue || LangOpts.CPlusPlus ||
+         !IsCForbiddenLValueType(Context, ResultType));
+
   return Owned(new (Context) ArraySubscriptExpr(LHSExp, RHSExp,
-                                                ResultType, RLoc));
+                                                ResultType, VK, OK, RLoc));
 }
 
-QualType Sema::
-CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
-                        const IdentifierInfo *CompName,
+/// Check an ext-vector component access expression.
+///
+/// VK should be set in advance to the value kind of the base
+/// expression.
+static QualType
+CheckExtVectorComponent(Sema &S, QualType baseType, ExprValueKind &VK,
+                        SourceLocation OpLoc, const IdentifierInfo *CompName,
                         SourceLocation CompLoc) {
   // FIXME: Share logic with ExtVectorElementExpr::containsDuplicateElements,
   // see FIXME there.
@@ -2457,25 +3123,36 @@ CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
   // indicating that it is a string of hex values to be used as vector indices.
   bool HexSwizzle = *compStr == 's' || *compStr == 'S';
 
+  bool HasRepeated = false;
+  bool HasIndex[16] = {};
+
+  int Idx;
+
   // Check that we've found one of the special components, or that the component
   // names must come from the same set.
   if (!strcmp(compStr, "hi") || !strcmp(compStr, "lo") ||
       !strcmp(compStr, "even") || !strcmp(compStr, "odd")) {
     HalvingSwizzle = true;
-  } else if (vecType->getPointAccessorIdx(*compStr) != -1) {
-    do
+  } else if (!HexSwizzle &&
+             (Idx = vecType->getPointAccessorIdx(*compStr)) != -1) {
+    do {
+      if (HasIndex[Idx]) HasRepeated = true;
+      HasIndex[Idx] = true;
       compStr++;
-    while (*compStr && vecType->getPointAccessorIdx(*compStr) != -1);
-  } else if (HexSwizzle || vecType->getNumericAccessorIdx(*compStr) != -1) {
-    do
+    } while (*compStr && (Idx = vecType->getPointAccessorIdx(*compStr)) != -1);
+  } else {
+    if (HexSwizzle) compStr++;
+    while ((Idx = vecType->getNumericAccessorIdx(*compStr)) != -1) {
+      if (HasIndex[Idx]) HasRepeated = true;
+      HasIndex[Idx] = true;
       compStr++;
-    while (*compStr && vecType->getNumericAccessorIdx(*compStr) != -1);
+    }
   }
 
   if (!HalvingSwizzle && *compStr) {
     // We didn't get to the end of the string. This means the component names
     // didn't come from the same set *or* we encountered an illegal name.
-    Diag(OpLoc, diag::err_ext_vector_component_name_illegal)
+    S.Diag(OpLoc, diag::err_ext_vector_component_name_illegal)
       << llvm::StringRef(compStr, 1) << SourceRange(CompLoc);
     return QualType();
   }
@@ -2490,7 +3167,7 @@ CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
 
     while (*compStr) {
       if (!vecType->isAccessorWithinNumElements(*compStr++)) {
-        Diag(OpLoc, diag::err_ext_vector_component_exceeds_length)
+        S.Diag(OpLoc, diag::err_ext_vector_component_exceeds_length)
           << baseType << SourceRange(CompLoc);
         return QualType();
       }
@@ -2510,48 +3187,51 @@ CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
   if (CompSize == 1)
     return vecType->getElementType();
 
-  QualType VT = Context.getExtVectorType(vecType->getElementType(), CompSize);
+  if (HasRepeated) VK = VK_RValue;
+
+  QualType VT = S.Context.getExtVectorType(vecType->getElementType(), CompSize);
   // Now look up the TypeDefDecl from the vector type. Without this,
   // diagostics look bad. We want extended vector types to appear built-in.
-  for (unsigned i = 0, E = ExtVectorDecls.size(); i != E; ++i) {
-    if (ExtVectorDecls[i]->getUnderlyingType() == VT)
-      return Context.getTypedefType(ExtVectorDecls[i]);
+  for (unsigned i = 0, E = S.ExtVectorDecls.size(); i != E; ++i) {
+    if (S.ExtVectorDecls[i]->getUnderlyingType() == VT)
+      return S.Context.getTypedefType(S.ExtVectorDecls[i]);
   }
   return VT; // should never get here (a typedef type should always be found).
 }
 
-static Decl *FindGetterNameDeclFromProtocolList(const ObjCProtocolDecl*PDecl,
+static Decl *FindGetterSetterNameDeclFromProtocolList(const ObjCProtocolDecl*PDecl,
                                                 IdentifierInfo *Member,
                                                 const Selector &Sel,
                                                 ASTContext &Context) {
-
-  if (ObjCPropertyDecl *PD = PDecl->FindPropertyDeclaration(Member))
-    return PD;
+  if (Member)
+    if (ObjCPropertyDecl *PD = PDecl->FindPropertyDeclaration(Member))
+      return PD;
   if (ObjCMethodDecl *OMD = PDecl->getInstanceMethod(Sel))
     return OMD;
 
   for (ObjCProtocolDecl::protocol_iterator I = PDecl->protocol_begin(),
        E = PDecl->protocol_end(); I != E; ++I) {
-    if (Decl *D = FindGetterNameDeclFromProtocolList(*I, Member, Sel,
-                                                     Context))
+    if (Decl *D = FindGetterSetterNameDeclFromProtocolList(*I, Member, Sel,
+                                                           Context))
       return D;
   }
   return 0;
 }
 
-static Decl *FindGetterNameDecl(const ObjCObjectPointerType *QIdTy,
-                                IdentifierInfo *Member,
-                                const Selector &Sel,
-                                ASTContext &Context) {
+static Decl *FindGetterSetterNameDecl(const ObjCObjectPointerType *QIdTy,
+                                      IdentifierInfo *Member,
+                                      const Selector &Sel,
+                                      ASTContext &Context) {
   // Check protocols on qualified interfaces.
   Decl *GDecl = 0;
   for (ObjCObjectPointerType::qual_iterator I = QIdTy->qual_begin(),
        E = QIdTy->qual_end(); I != E; ++I) {
-    if (ObjCPropertyDecl *PD = (*I)->FindPropertyDeclaration(Member)) {
-      GDecl = PD;
-      break;
-    }
-    // Also must look for a getter name which uses property syntax.
+    if (Member)
+      if (ObjCPropertyDecl *PD = (*I)->FindPropertyDeclaration(Member)) {
+        GDecl = PD;
+        break;
+      }
+    // Also must look for a getter or setter name which uses property syntax.
     if (ObjCMethodDecl *OMD = (*I)->getInstanceMethod(Sel)) {
       GDecl = OMD;
       break;
@@ -2561,7 +3241,8 @@ static Decl *FindGetterNameDecl(const ObjCObjectPointerType *QIdTy,
     for (ObjCObjectPointerType::qual_iterator I = QIdTy->qual_begin(),
          E = QIdTy->qual_end(); I != E; ++I) {
       // Search in the protocol-qualifier list of current protocol.
-      GDecl = FindGetterNameDeclFromProtocolList(*I, Member, Sel, Context);
+      GDecl = FindGetterSetterNameDeclFromProtocolList(*I, Member, Sel, 
+                                                       Context);
       if (GDecl)
         return GDecl;
     }
@@ -2617,14 +3298,15 @@ static void DiagnoseQualifiedMemberReference(Sema &SemaRef,
                                              Expr *BaseExpr,
                                              QualType BaseType,
                                              const CXXScopeSpec &SS,
-                                             const LookupResult &R) {
+                                             NamedDecl *rep,
+                                       const DeclarationNameInfo &nameInfo) {
   // If this is an implicit member access, use a different set of
   // diagnostics.
   if (!BaseExpr)
-    return DiagnoseInstanceReference(SemaRef, SS, R);
+    return DiagnoseInstanceReference(SemaRef, SS, rep, nameInfo);
 
-  SemaRef.Diag(R.getNameLoc(), diag::err_qualified_member_of_unrelated)
-    << SS.getRange() << R.getRepresentativeDecl() << BaseType;
+  SemaRef.Diag(nameInfo.getLoc(), diag::err_qualified_member_of_unrelated)
+    << SS.getRange() << rep << BaseType;
 }
 
 // Check whether the declarations we found through a nested-name
@@ -2673,7 +3355,9 @@ bool Sema::CheckQualifiedMemberReference(Expr *BaseExpr,
       return false;
   }
 
-  DiagnoseQualifiedMemberReference(*this, BaseExpr, BaseType, SS, R);
+  DiagnoseQualifiedMemberReference(*this, BaseExpr, BaseType, SS,
+                                   R.getRepresentativeDecl(),
+                                   R.getLookupNameInfo());
   return true;
 }
 
@@ -2846,18 +3530,12 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
   // Construct an unresolved result if we in fact got an unresolved
   // result.
   if (R.isOverloadedResult() || R.isUnresolvableResult()) {
-    bool Dependent =
-      BaseExprType->isDependentType() ||
-      R.isUnresolvableResult() ||
-      OverloadExpr::ComputeDependence(R.begin(), R.end(), TemplateArgs);
-
     // Suppress any lookup-related diagnostics; we'll do these when we
     // pick a member.
     R.suppressDiagnostics();
 
     UnresolvedMemberExpr *MemExpr
-      = UnresolvedMemberExpr::Create(Context, Dependent,
-                                     R.isUnresolvableResult(),
+      = UnresolvedMemberExpr::Create(Context, R.isUnresolvableResult(),
                                      BaseExpr, BaseExprType,
                                      IsArrow, OpLoc,
                                      Qualifier, SS.getRange(),
@@ -2905,58 +3583,44 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
     return ExprError();
   }
 
-  if (FieldDecl *FD = dyn_cast<FieldDecl>(MemberDecl)) {
-    // We may have found a field within an anonymous union or struct
-    // (C++ [class.union]).
-    if (cast<RecordDecl>(FD->getDeclContext())->isAnonymousStructOrUnion() &&
-        !BaseType->getAs<RecordType>()->getDecl()->isAnonymousStructOrUnion())
-      return BuildAnonymousStructUnionMemberReference(MemberLoc, FD,
-                                                      BaseExpr, OpLoc);
-
-    // Figure out the type of the member; see C99 6.5.2.3p3, C++ [expr.ref]
-    QualType MemberType = FD->getType();
-    if (const ReferenceType *Ref = MemberType->getAs<ReferenceType>())
-      MemberType = Ref->getPointeeType();
-    else {
-      Qualifiers BaseQuals = BaseType.getQualifiers();
-      BaseQuals.removeObjCGCAttr();
-      if (FD->isMutable()) BaseQuals.removeConst();
+  // Perform a property load on the base regardless of whether we
+  // actually need it for the declaration.
+  if (BaseExpr->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(BaseExpr);
 
-      Qualifiers MemberQuals
-        = Context.getCanonicalType(MemberType).getQualifiers();
+  if (FieldDecl *FD = dyn_cast<FieldDecl>(MemberDecl))
+    return BuildFieldReferenceExpr(*this, BaseExpr, IsArrow,
+                                   SS, FD, FoundDecl, MemberNameInfo);
 
-      Qualifiers Combined = BaseQuals + MemberQuals;
-      if (Combined != MemberQuals)
-        MemberType = Context.getQualifiedType(MemberType, Combined);
-    }
-
-    MarkDeclarationReferenced(MemberLoc, FD);
-    if (PerformObjectMemberConversion(BaseExpr, Qualifier, FoundDecl, FD))
-      return ExprError();
-    return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
-                                 FD, FoundDecl, MemberNameInfo,
-                                 MemberType));
-  }
+  if (IndirectFieldDecl *FD = dyn_cast<IndirectFieldDecl>(MemberDecl))
+    // We may have found a field within an anonymous union or struct
+    // (C++ [class.union]).
+    return BuildAnonymousStructUnionMemberReference(SS, MemberLoc, FD,
+                                                    BaseExpr, OpLoc);
 
   if (VarDecl *Var = dyn_cast<VarDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, Var);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
                                  Var, FoundDecl, MemberNameInfo,
-                                 Var->getType().getNonReferenceType()));
+                                 Var->getType().getNonReferenceType(),
+                                 VK_LValue, OK_Ordinary));
   }
 
-  if (FunctionDecl *MemberFn = dyn_cast<FunctionDecl>(MemberDecl)) {
+  if (CXXMethodDecl *MemberFn = dyn_cast<CXXMethodDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, MemberDecl);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
                                  MemberFn, FoundDecl, MemberNameInfo,
-                                 MemberFn->getType()));
+                                 MemberFn->getType(),
+                                 MemberFn->isInstance() ? VK_RValue : VK_LValue,
+                                 OK_Ordinary));
   }
+  assert(!isa<FunctionDecl>(MemberDecl) && "member function not C++ method?");
 
   if (EnumConstantDecl *Enum = dyn_cast<EnumConstantDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, MemberDecl);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
                                  Enum, FoundDecl, MemberNameInfo,
-                                 Enum->getType()));
+                                 Enum->getType(), VK_RValue, OK_Ordinary));
   }
 
   Owned(BaseExpr);
@@ -2975,6 +3639,38 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
   return ExprError();
 }
 
+/// Given that normal member access failed on the given expression,
+/// and given that the expression's type involves builtin-id or
+/// builtin-Class, decide whether substituting in the redefinition
+/// types would be profitable.  The redefinition type is whatever
+/// this translation unit tried to typedef to id/Class;  we store
+/// it to the side and then re-use it in places like this.
+static bool ShouldTryAgainWithRedefinitionType(Sema &S, Expr *&base) {
+  const ObjCObjectPointerType *opty
+    = base->getType()->getAs<ObjCObjectPointerType>();
+  if (!opty) return false;
+
+  const ObjCObjectType *ty = opty->getObjectType();
+
+  QualType redef;
+  if (ty->isObjCId()) {
+    redef = S.Context.ObjCIdRedefinitionType;
+  } else if (ty->isObjCClass()) {
+    redef = S.Context.ObjCClassRedefinitionType;
+  } else {
+    return false;
+  }
+
+  // Do the substitution as long as the redefinition type isn't just a
+  // possibly-qualified pointer to builtin-id or builtin-Class again.
+  opty = redef->getAs<ObjCObjectPointerType>();
+  if (opty && !opty->getObjectType()->getInterface() != 0)
+    return false;
+
+  S.ImpCastExprToType(base, redef, CK_BitCast);
+  return true;
+}
+
 /// Look up the given member of the given non-type-dependent
 /// expression.  This can return in one of two ways:
 ///  * If it returns a sentinel null-but-valid result, the caller will
@@ -2994,6 +3690,7 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
 
   // Perform default conversions.
   DefaultFunctionArrayConversion(BaseExpr);
+  if (IsArrow) DefaultLvalueConversion(BaseExpr);
 
   QualType BaseType = BaseExpr->getType();
   assert(!BaseType->isDependentType());
@@ -3001,96 +3698,235 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
   DeclarationName MemberName = R.getLookupName();
   SourceLocation MemberLoc = R.getNameLoc();
 
-  // If the user is trying to apply -> or . to a function pointer
-  // type, it's probably because they forgot parentheses to call that
-  // function. Suggest the addition of those parentheses, build the
-  // call, and continue on.
-  if (const PointerType *Ptr = BaseType->getAs<PointerType>()) {
-    if (const FunctionProtoType *Fun
-          = Ptr->getPointeeType()->getAs<FunctionProtoType>()) {
-      QualType ResultTy = Fun->getResultType();
-      if (Fun->getNumArgs() == 0 &&
-          ((!IsArrow && ResultTy->isRecordType()) ||
-           (IsArrow && ResultTy->isPointerType() &&
-            ResultTy->getAs<PointerType>()->getPointeeType()
-                                                          ->isRecordType()))) {
-        SourceLocation Loc = PP.getLocForEndOfToken(BaseExpr->getLocEnd());
-        Diag(Loc, diag::err_member_reference_needs_call)
-          << QualType(Fun, 0)
-          << FixItHint::CreateInsertion(Loc, "()");
+  // For later type-checking purposes, turn arrow accesses into dot
+  // accesses.  The only access type we support that doesn't follow
+  // the C equivalence "a->b === (*a).b" is ObjC property accesses,
+  // and those never use arrows, so this is unaffected.
+  if (IsArrow) {
+    if (const PointerType *Ptr = BaseType->getAs<PointerType>())
+      BaseType = Ptr->getPointeeType();
+    else if (const ObjCObjectPointerType *Ptr
+               = BaseType->getAs<ObjCObjectPointerType>())
+      BaseType = Ptr->getPointeeType();
+    else if (BaseType->isRecordType()) {
+      // Recover from arrow accesses to records, e.g.:
+      //   struct MyRecord foo;
+      //   foo->bar
+      // This is actually well-formed in C++ if MyRecord has an
+      // overloaded operator->, but that should have been dealt with
+      // by now.
+      Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
+        << BaseType << int(IsArrow) << BaseExpr->getSourceRange()
+        << FixItHint::CreateReplacement(OpLoc, ".");
+      IsArrow = false;
+    } else {
+      Diag(MemberLoc, diag::err_typecheck_member_reference_arrow)
+        << BaseType << BaseExpr->getSourceRange();
+      return ExprError();
+    }
+  }
 
-        ExprResult NewBase
-          = ActOnCallExpr(0, BaseExpr, Loc,
-                          MultiExprArg(*this, 0, 0), 0, Loc);
-        BaseExpr = 0;
-        if (NewBase.isInvalid())
-          return ExprError();
+  // Handle field access to simple records.
+  if (const RecordType *RTy = BaseType->getAs<RecordType>()) {
+    if (LookupMemberExprInRecord(*this, R, BaseExpr->getSourceRange(),
+                                 RTy, OpLoc, SS, HasTemplateArgs))
+      return ExprError();
 
-        BaseExpr = NewBase.takeAs<Expr>();
-        DefaultFunctionArrayConversion(BaseExpr);
-        BaseType = BaseExpr->getType();
-      }
-    }
+    // Returning valid-but-null is how we indicate to the caller that
+    // the lookup result was filled in.
+    return Owned((Expr*) 0);
   }
 
-  // If this is an Objective-C pseudo-builtin and a definition is provided then
-  // use that.
-  if (BaseType->isObjCIdType()) {
-    if (IsArrow) {
-      // Handle the following exceptional case PObj->isa.
-      if (const ObjCObjectPointerType *OPT =
-          BaseType->getAs<ObjCObjectPointerType>()) {
-        if (OPT->getObjectType()->isObjCId() &&
-            MemberName.getAsIdentifierInfo()->isStr("isa"))
-          return Owned(new (Context) ObjCIsaExpr(BaseExpr, true, MemberLoc,
-                                                 Context.getObjCClassType()));
-      }
+  // Handle ivar access to Objective-C objects.
+  if (const ObjCObjectType *OTy = BaseType->getAs<ObjCObjectType>()) {
+    IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
+
+    // There are three cases for the base type:
+    //   - builtin id (qualified or unqualified)
+    //   - builtin Class (qualified or unqualified)
+    //   - an interface
+    ObjCInterfaceDecl *IDecl = OTy->getInterface();
+    if (!IDecl) {
+      // There's an implicit 'isa' ivar on all objects.
+      // But we only actually find it this way on objects of type 'id',
+      // apparently.
+      if (OTy->isObjCId() && Member->isStr("isa"))
+        return Owned(new (Context) ObjCIsaExpr(BaseExpr, IsArrow, MemberLoc,
+                                               Context.getObjCClassType()));
+
+      if (ShouldTryAgainWithRedefinitionType(*this, BaseExpr))
+        return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                                ObjCImpDecl, HasTemplateArgs);
+      goto fail;
     }
-    // We have an 'id' type. Rather than fall through, we check if this
-    // is a reference to 'isa'.
-    if (BaseType != Context.ObjCIdRedefinitionType) {
-      BaseType = Context.ObjCIdRedefinitionType;
-      ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
+
+    ObjCInterfaceDecl *ClassDeclared;
+    ObjCIvarDecl *IV = IDecl->lookupInstanceVariable(Member, ClassDeclared);
+
+    if (!IV) {
+      // Attempt to correct for typos in ivar names.
+      LookupResult Res(*this, R.getLookupName(), R.getNameLoc(),
+                       LookupMemberName);
+      if (CorrectTypo(Res, 0, 0, IDecl, false, 
+                      IsArrow ? CTC_ObjCIvarLookup
+                              : CTC_ObjCPropertyLookup) &&
+          (IV = Res.getAsSingle<ObjCIvarDecl>())) {
+        Diag(R.getNameLoc(),
+             diag::err_typecheck_member_reference_ivar_suggest)
+          << IDecl->getDeclName() << MemberName << IV->getDeclName()
+          << FixItHint::CreateReplacement(R.getNameLoc(),
+                                          IV->getNameAsString());
+        Diag(IV->getLocation(), diag::note_previous_decl)
+          << IV->getDeclName();
+      } else {
+        Res.clear();
+        Res.setLookupName(Member);
+
+        Diag(MemberLoc, diag::err_typecheck_member_reference_ivar)
+          << IDecl->getDeclName() << MemberName
+          << BaseExpr->getSourceRange();
+        return ExprError();
+      }
     }
-  }
 
-  // If this is an Objective-C pseudo-builtin and a definition is provided then
-  // use that.
-  if (Context.isObjCSelType(BaseType)) {
-    // We have an 'SEL' type. Rather than fall through, we check if this
-    // is a reference to 'sel_id'.
-    if (BaseType != Context.ObjCSelRedefinitionType) {
-      BaseType = Context.ObjCSelRedefinitionType;
-      ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
+    // If the decl being referenced had an error, return an error for this
+    // sub-expr without emitting another error, in order to avoid cascading
+    // error cases.
+    if (IV->isInvalidDecl())
+      return ExprError();
+
+    // Check whether we can reference this field.
+    if (DiagnoseUseOfDecl(IV, MemberLoc))
+      return ExprError();
+    if (IV->getAccessControl() != ObjCIvarDecl::Public &&
+        IV->getAccessControl() != ObjCIvarDecl::Package) {
+      ObjCInterfaceDecl *ClassOfMethodDecl = 0;
+      if (ObjCMethodDecl *MD = getCurMethodDecl())
+        ClassOfMethodDecl =  MD->getClassInterface();
+      else if (ObjCImpDecl && getCurFunctionDecl()) {
+        // Case of a c-function declared inside an objc implementation.
+        // FIXME: For a c-style function nested inside an objc implementation
+        // class, there is no implementation context available, so we pass
+        // down the context as argument to this routine. Ideally, this context
+        // need be passed down in the AST node and somehow calculated from the
+        // AST for a function decl.
+        if (ObjCImplementationDecl *IMPD =
+              dyn_cast<ObjCImplementationDecl>(ObjCImpDecl))
+          ClassOfMethodDecl = IMPD->getClassInterface();
+        else if (ObjCCategoryImplDecl* CatImplClass =
+                   dyn_cast<ObjCCategoryImplDecl>(ObjCImpDecl))
+          ClassOfMethodDecl = CatImplClass->getClassInterface();
+      }
+
+      if (IV->getAccessControl() == ObjCIvarDecl::Private) {
+        if (ClassDeclared != IDecl ||
+            ClassOfMethodDecl != ClassDeclared)
+          Diag(MemberLoc, diag::error_private_ivar_access)
+            << IV->getDeclName();
+      } else if (!IDecl->isSuperClassOf(ClassOfMethodDecl))
+        // @protected
+        Diag(MemberLoc, diag::error_protected_ivar_access)
+          << IV->getDeclName();
     }
+
+    return Owned(new (Context) ObjCIvarRefExpr(IV, IV->getType(),
+                                               MemberLoc, BaseExpr,
+                                               IsArrow));
   }
 
-  assert(!BaseType.isNull() && "no type for member expression");
+  // Objective-C property access.
+  const ObjCObjectPointerType *OPT;
+  if (!IsArrow && (OPT = BaseType->getAs<ObjCObjectPointerType>())) {
+    // This actually uses the base as an r-value.
+    DefaultLvalueConversion(BaseExpr);
+    assert(Context.hasSameUnqualifiedType(BaseType, BaseExpr->getType()));
 
-  // Handle properties on ObjC 'Class' types.
-  if (!IsArrow && BaseType->isObjCClassType()) {
-    // Also must look for a getter name which uses property syntax.
     IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
-    Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
-    if (ObjCMethodDecl *MD = getCurMethodDecl()) {
+
+    const ObjCObjectType *OT = OPT->getObjectType();
+
+    // id, with and without qualifiers.
+    if (OT->isObjCId()) {
+      // Check protocols on qualified interfaces.
+      Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
+      if (Decl *PMDecl = FindGetterSetterNameDecl(OPT, Member, Sel, Context)) {
+        if (ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(PMDecl)) {
+          // Check the use of this declaration
+          if (DiagnoseUseOfDecl(PD, MemberLoc))
+            return ExprError();
+
+          return Owned(new (Context) ObjCPropertyRefExpr(PD, PD->getType(),
+                                                         VK_LValue,
+                                                         OK_ObjCProperty,
+                                                         MemberLoc, 
+                                                         BaseExpr));
+        }
+
+        if (ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(PMDecl)) {
+          // Check the use of this method.
+          if (DiagnoseUseOfDecl(OMD, MemberLoc))
+            return ExprError();
+          Selector SetterSel =
+            SelectorTable::constructSetterName(PP.getIdentifierTable(),
+                                               PP.getSelectorTable(), Member);
+          ObjCMethodDecl *SMD = 0;
+          if (Decl *SDecl = FindGetterSetterNameDecl(OPT, /*Property id*/0, 
+                                                     SetterSel, Context))
+            SMD = dyn_cast<ObjCMethodDecl>(SDecl);
+          QualType PType = OMD->getSendResultType();
+          
+          ExprValueKind VK = VK_LValue;
+          if (!getLangOptions().CPlusPlus &&
+              IsCForbiddenLValueType(Context, PType))
+            VK = VK_RValue;
+          ExprObjectKind OK = (VK == VK_RValue ? OK_Ordinary : OK_ObjCProperty);
+
+          return Owned(new (Context) ObjCPropertyRefExpr(OMD, SMD, PType,
+                                                         VK, OK,
+                                                         MemberLoc, BaseExpr));
+        }
+      }
+
+      if (ShouldTryAgainWithRedefinitionType(*this, BaseExpr))
+        return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                                ObjCImpDecl, HasTemplateArgs);
+
+      return ExprError(Diag(MemberLoc, diag::err_property_not_found)
+                         << MemberName << BaseType);
+    }
+
+    // 'Class', unqualified only.
+    if (OT->isObjCClass()) {
+      // Only works in a method declaration (??!).
+      ObjCMethodDecl *MD = getCurMethodDecl();
+      if (!MD) {
+        if (ShouldTryAgainWithRedefinitionType(*this, BaseExpr))
+          return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                                  ObjCImpDecl, HasTemplateArgs);
+
+        goto fail;
+      }
+
+      // Also must look for a getter name which uses property syntax.
+      Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
       ObjCInterfaceDecl *IFace = MD->getClassInterface();
       ObjCMethodDecl *Getter;
-      // FIXME: need to also look locally in the implementation.
       if ((Getter = IFace->lookupClassMethod(Sel))) {
         // Check the use of this method.
         if (DiagnoseUseOfDecl(Getter, MemberLoc))
           return ExprError();
-      }
+      } else
+        Getter = IFace->lookupPrivateMethod(Sel, false);
       // If we found a getter then this may be a valid dot-reference, we
       // will look for the matching setter, in case it is needed.
       Selector SetterSel =
-      SelectorTable::constructSetterName(PP.getIdentifierTable(),
-                                         PP.getSelectorTable(), Member);
+        SelectorTable::constructSetterName(PP.getIdentifierTable(),
+                                           PP.getSelectorTable(), Member);
       ObjCMethodDecl *Setter = IFace->lookupClassMethod(SetterSel);
       if (!Setter) {
         // If this reference is in an @implementation, also check for 'private'
         // methods.
-        Setter = IFace->lookupPrivateInstanceMethod(SetterSel);
+        Setter = IFace->lookupPrivateMethod(SetterSel, false);
       }
       // Look through local category implementations associated with the class.
       if (!Setter)
@@ -3102,49 +3938,73 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
       if (Getter || Setter) {
         QualType PType;
 
-        if (Getter)
+        ExprValueKind VK = VK_LValue;
+        if (Getter) {
           PType = Getter->getSendResultType();
-        else
+          if (!getLangOptions().CPlusPlus &&
+              IsCForbiddenLValueType(Context, PType))
+            VK = VK_RValue;
+        } else {
           // Get the expression type from Setter's incoming parameter.
           PType = (*(Setter->param_end() -1))->getType();
+        }
+        ExprObjectKind OK = (VK == VK_RValue ? OK_Ordinary : OK_ObjCProperty);
+
         // FIXME: we must check that the setter has property type.
-        return Owned(new (Context) ObjCImplicitSetterGetterRefExpr(Getter,
-                                                  PType,
-                                                  Setter, MemberLoc, BaseExpr));
+        return Owned(new (Context) ObjCPropertyRefExpr(Getter, Setter,
+                                                       PType, VK, OK,
+                                                       MemberLoc, BaseExpr));
       }
+
+      if (ShouldTryAgainWithRedefinitionType(*this, BaseExpr))
+        return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                                ObjCImpDecl, HasTemplateArgs);
+
       return ExprError(Diag(MemberLoc, diag::err_property_not_found)
-                       << MemberName << BaseType);
+                         << MemberName << BaseType);
     }
-  }
 
-  if (BaseType->isObjCClassType() &&
-      BaseType != Context.ObjCClassRedefinitionType) {
-    BaseType = Context.ObjCClassRedefinitionType;
-    ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
+    // Normal property access.
+    return HandleExprPropertyRefExpr(OPT, BaseExpr, MemberName, MemberLoc,
+                                     SourceLocation(), QualType(), false);
   }
 
-  if (IsArrow) {
-    if (const PointerType *PT = BaseType->getAs<PointerType>())
-      BaseType = PT->getPointeeType();
-    else if (BaseType->isObjCObjectPointerType())
-      ;
-    else if (BaseType->isRecordType()) {
-      // Recover from arrow accesses to records, e.g.:
-      //   struct MyRecord foo;
-      //   foo->bar
-      // This is actually well-formed in C++ if MyRecord has an
-      // overloaded operator->, but that should have been dealt with
-      // by now.
-      Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
-        << BaseType << int(IsArrow) << BaseExpr->getSourceRange()
-        << FixItHint::CreateReplacement(OpLoc, ".");
-      IsArrow = false;
-    } else {
-      Diag(MemberLoc, diag::err_typecheck_member_reference_arrow)
-        << BaseType << BaseExpr->getSourceRange();
+  // Handle 'field access' to vectors, such as 'V.xx'.
+  if (BaseType->isExtVectorType()) {
+    // FIXME: this expr should store IsArrow.
+    IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
+    ExprValueKind VK = (IsArrow ? VK_LValue : BaseExpr->getValueKind());
+    QualType ret = CheckExtVectorComponent(*this, BaseType, VK, OpLoc,
+                                           Member, MemberLoc);
+    if (ret.isNull())
       return ExprError();
-    }
-  } else {
+
+    return Owned(new (Context) ExtVectorElementExpr(ret, VK, BaseExpr,
+                                                    *Member, MemberLoc));
+  }
+
+  // Adjust builtin-sel to the appropriate redefinition type if that's
+  // not just a pointer to builtin-sel again.
+  if (IsArrow &&
+      BaseType->isSpecificBuiltinType(BuiltinType::ObjCSel) &&
+      !Context.ObjCSelRedefinitionType->isObjCSelType()) {
+    ImpCastExprToType(BaseExpr, Context.ObjCSelRedefinitionType, CK_BitCast);
+    return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                            ObjCImpDecl, HasTemplateArgs);
+  }
+
+  // Failure cases.
+ fail:
+
+  // There's a possible road to recovery for function types.
+  const FunctionType *Fun = 0;
+  SourceLocation ParenInsertionLoc =
+      PP.getLocForEndOfToken(BaseExpr->getLocEnd());
+
+  if (const PointerType *Ptr = BaseType->getAs<PointerType>()) {
+    if ((Fun = Ptr->getPointeeType()->getAs<FunctionType>())) {
+      // fall out, handled below.
+
     // Recover from dot accesses to pointers, e.g.:
     //   type *foo;
     //   foo.bar
@@ -3152,165 +4012,113 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
     //   - 'type' is an Objective C type
     //   - 'bar' is a pseudo-destructor name which happens to refer to
     //     the appropriate pointer type
-    if (MemberName.getNameKind() != DeclarationName::CXXDestructorName) {
-      const PointerType *PT = BaseType->getAs<PointerType>();
-      if (PT && PT->getPointeeType()->isRecordType()) {
-        Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
-          << BaseType << int(IsArrow) << BaseExpr->getSourceRange()
-          << FixItHint::CreateReplacement(OpLoc, "->");
-        BaseType = PT->getPointeeType();
-        IsArrow = true;
-      }
-    }
-  }
+    } else if (!IsArrow && Ptr->getPointeeType()->isRecordType() &&
+               MemberName.getNameKind() != DeclarationName::CXXDestructorName) {
+      Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
+        << BaseType << int(IsArrow) << BaseExpr->getSourceRange()
+        << FixItHint::CreateReplacement(OpLoc, "->");
 
-  // Handle field access to simple records.
-  if (const RecordType *RTy = BaseType->getAs<RecordType>()) {
-    if (LookupMemberExprInRecord(*this, R, BaseExpr->getSourceRange(),
-                                 RTy, OpLoc, SS, HasTemplateArgs))
-      return ExprError();
-    return Owned((Expr*) 0);
+      // Recurse as an -> access.
+      IsArrow = true;
+      return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                              ObjCImpDecl, HasTemplateArgs);
+    }
+  } else {
+    Fun = BaseType->getAs<FunctionType>();
   }
 
-  // Handle access to Objective-C instance variables, such as "Obj->ivar" and
-  // (*Obj).ivar.
-  if ((IsArrow && BaseType->isObjCObjectPointerType()) ||
-      (!IsArrow && BaseType->isObjCObjectType())) {
-    const ObjCObjectPointerType *OPT = BaseType->getAs<ObjCObjectPointerType>();
-    ObjCInterfaceDecl *IDecl =
-      OPT ? OPT->getInterfaceDecl()
-          : BaseType->getAs<ObjCObjectType>()->getInterface();
-    if (IDecl) {
-      IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
-
-      ObjCInterfaceDecl *ClassDeclared;
-      ObjCIvarDecl *IV = IDecl->lookupInstanceVariable(Member, ClassDeclared);
-
-      if (!IV) {
-        // Attempt to correct for typos in ivar names.
-        LookupResult Res(*this, R.getLookupName(), R.getNameLoc(),
-                         LookupMemberName);
-        if (CorrectTypo(Res, 0, 0, IDecl, false, CTC_MemberLookup) &&
-            (IV = Res.getAsSingle<ObjCIvarDecl>())) {
-          Diag(R.getNameLoc(),
-               diag::err_typecheck_member_reference_ivar_suggest)
-            << IDecl->getDeclName() << MemberName << IV->getDeclName()
-            << FixItHint::CreateReplacement(R.getNameLoc(),
-                                            IV->getNameAsString());
-          Diag(IV->getLocation(), diag::note_previous_decl)
-            << IV->getDeclName();
-        } else {
-          Res.clear();
-          Res.setLookupName(Member);
+  // If the user is trying to apply -> or . to a function pointer
+  // type, it's probably because they forgot parentheses to call that
+  // function. Suggest the addition of those parentheses, build the
+  // call, and continue on.
+  if (Fun || BaseType == Context.OverloadTy) {
+    bool TryCall;
+    if (BaseType == Context.OverloadTy) {
+      // Plunder the overload set for something that would make the member
+      // expression valid.
+      const OverloadExpr *Overloads = cast<OverloadExpr>(BaseExpr);
+      UnresolvedSet<4> CandidateOverloads;
+      bool HasZeroArgCandidateOverload = false;
+      for (OverloadExpr::decls_iterator it = Overloads->decls_begin(),
+           DeclsEnd = Overloads->decls_end(); it != DeclsEnd; ++it) {
+        const FunctionDecl *OverloadDecl = cast<FunctionDecl>(*it);
+        QualType ResultTy = OverloadDecl->getResultType();
+        if ((!IsArrow && ResultTy->isRecordType()) ||
+            (IsArrow && ResultTy->isPointerType() &&
+             ResultTy->getPointeeType()->isRecordType())) {
+          CandidateOverloads.addDecl(*it);
+          if (OverloadDecl->getNumParams() == 0) {
+            HasZeroArgCandidateOverload = true;
+          }
         }
       }
-
-      if (IV) {
-        // If the decl being referenced had an error, return an error for this
-        // sub-expr without emitting another error, in order to avoid cascading
-        // error cases.
-        if (IV->isInvalidDecl())
-          return ExprError();
-
-        // Check whether we can reference this field.
-        if (DiagnoseUseOfDecl(IV, MemberLoc))
-          return ExprError();
-        if (IV->getAccessControl() != ObjCIvarDecl::Public &&
-            IV->getAccessControl() != ObjCIvarDecl::Package) {
-          ObjCInterfaceDecl *ClassOfMethodDecl = 0;
-          if (ObjCMethodDecl *MD = getCurMethodDecl())
-            ClassOfMethodDecl =  MD->getClassInterface();
-          else if (ObjCImpDecl && getCurFunctionDecl()) {
-            // Case of a c-function declared inside an objc implementation.
-            // FIXME: For a c-style function nested inside an objc implementation
-            // class, there is no implementation context available, so we pass
-            // down the context as argument to this routine. Ideally, this context
-            // need be passed down in the AST node and somehow calculated from the
-            // AST for a function decl.
-            if (ObjCImplementationDecl *IMPD =
-                dyn_cast<ObjCImplementationDecl>(ObjCImpDecl))
-              ClassOfMethodDecl = IMPD->getClassInterface();
-            else if (ObjCCategoryImplDecl* CatImplClass =
-                        dyn_cast<ObjCCategoryImplDecl>(ObjCImpDecl))
-              ClassOfMethodDecl = CatImplClass->getClassInterface();
+      if (HasZeroArgCandidateOverload && CandidateOverloads.size() == 1) {
+        // We have one reasonable overload, and there's only one way to call it,
+        // so emit a fixit and try to recover
+        Diag(ParenInsertionLoc, diag::err_member_reference_needs_call)
+            << 1
+            << BaseExpr->getSourceRange()
+            << FixItHint::CreateInsertion(ParenInsertionLoc, "()");
+        TryCall = true;
+      } else {
+        Diag(BaseExpr->getExprLoc(), diag::err_member_reference_needs_call)
+            << 0
+            << BaseExpr->getSourceRange();
+        int CandidateOverloadCount = CandidateOverloads.size();
+        int I;
+        for (I = 0; I < CandidateOverloadCount; ++I) {
+          // FIXME: Magic number for max shown overloads stolen from
+          // OverloadCandidateSet::NoteCandidates.
+          if (I >= 4 && Diags.getShowOverloads() == Diagnostic::Ovl_Best) {
+            break;
           }
-
-          if (IV->getAccessControl() == ObjCIvarDecl::Private) {
-            if (ClassDeclared != IDecl ||
-                ClassOfMethodDecl != ClassDeclared)
-              Diag(MemberLoc, diag::error_private_ivar_access)
-                << IV->getDeclName();
-          } else if (!IDecl->isSuperClassOf(ClassOfMethodDecl))
-            // @protected
-            Diag(MemberLoc, diag::error_protected_ivar_access)
-              << IV->getDeclName();
+          Diag(CandidateOverloads[I].getDecl()->getSourceRange().getBegin(),
+               diag::note_member_ref_possible_intended_overload);
+        }
+        if (I != CandidateOverloadCount) {
+          Diag(BaseExpr->getExprLoc(), diag::note_ovl_too_many_candidates)
+              << int(CandidateOverloadCount - I);
         }
+        return ExprError();
+      }
+    } else {
+      if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(Fun)) {
+        TryCall = (FPT->getNumArgs() == 0);
+      } else {
+        TryCall = true;
+      }
 
-        return Owned(new (Context) ObjCIvarRefExpr(IV, IV->getType(),
-                                                   MemberLoc, BaseExpr,
-                                                   IsArrow));
+      if (TryCall) {
+        QualType ResultTy = Fun->getResultType();
+        TryCall = (!IsArrow && ResultTy->isRecordType()) ||
+                  (IsArrow && ResultTy->isPointerType() &&
+                   ResultTy->getAs<PointerType>()->getPointeeType()->isRecordType());
       }
-      return ExprError(Diag(MemberLoc, diag::err_typecheck_member_reference_ivar)
-                         << IDecl->getDeclName() << MemberName
-                         << BaseExpr->getSourceRange());
     }
-  }
-  // Handle properties on 'id' and qualified "id".
-  if (!IsArrow && (BaseType->isObjCIdType() ||
-                   BaseType->isObjCQualifiedIdType())) {
-    const ObjCObjectPointerType *QIdTy = BaseType->getAs<ObjCObjectPointerType>();
-    IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
-
-    // Check protocols on qualified interfaces.
-    Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
-    if (Decl *PMDecl = FindGetterNameDecl(QIdTy, Member, Sel, Context)) {
-      if (ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(PMDecl)) {
-        // Check the use of this declaration
-        if (DiagnoseUseOfDecl(PD, MemberLoc))
-          return ExprError();
 
-        return Owned(new (Context) ObjCPropertyRefExpr(PD, PD->getType(),
-                                                       MemberLoc, BaseExpr));
-      }
-      if (ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(PMDecl)) {
-        // Check the use of this method.
-        if (DiagnoseUseOfDecl(OMD, MemberLoc))
-          return ExprError();
 
-        return Owned(ObjCMessageExpr::Create(Context,
-                                             OMD->getSendResultType(),
-                                             OpLoc, BaseExpr, Sel,
-                                             OMD, NULL, 0, MemberLoc));
+    if (TryCall) {
+      if (Fun) {
+        Diag(BaseExpr->getExprLoc(),
+             diag::err_member_reference_needs_call_zero_arg)
+          << QualType(Fun, 0)
+          << FixItHint::CreateInsertion(ParenInsertionLoc, "()");
       }
-    }
 
-    return ExprError(Diag(MemberLoc, diag::err_property_not_found)
-                       << MemberName << BaseType);
-  }
+      ExprResult NewBase
+        = ActOnCallExpr(0, BaseExpr, ParenInsertionLoc,
+                        MultiExprArg(*this, 0, 0), ParenInsertionLoc);
+      if (NewBase.isInvalid())
+        return ExprError();
+      BaseExpr = NewBase.takeAs<Expr>();
 
-  // Handle Objective-C property access, which is "Obj.property" where Obj is a
-  // pointer to a (potentially qualified) interface type.
-  if (!IsArrow)
-    if (const ObjCObjectPointerType *OPT =
-          BaseType->getAsObjCInterfacePointerType())
-      return HandleExprPropertyRefExpr(OPT, BaseExpr, MemberName, MemberLoc);
 
-  // Handle the following exceptional case (*Obj).isa.
-  if (!IsArrow &&
-      BaseType->isObjCObjectType() &&
-      BaseType->getAs<ObjCObjectType>()->isObjCId() &&
-      MemberName.getAsIdentifierInfo()->isStr("isa"))
-    return Owned(new (Context) ObjCIsaExpr(BaseExpr, false, MemberLoc,
-                                           Context.getObjCClassType()));
+      DefaultFunctionArrayConversion(BaseExpr);
+      BaseType = BaseExpr->getType();
 
-  // Handle 'field access' to vectors, such as 'V.xx'.
-  if (BaseType->isExtVectorType()) {
-    IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
-    QualType ret = CheckExtVectorComponent(BaseType, OpLoc, Member, MemberLoc);
-    if (ret.isNull())
-      return ExprError();
-    return Owned(new (Context) ExtVectorElementExpr(ret, BaseExpr, *Member,
-                                                    MemberLoc));
+      return LookupMemberExpr(R, BaseExpr, IsArrow, OpLoc, SS,
+                              ObjCImpDecl, HasTemplateArgs);
+    }
   }
 
   Diag(MemberLoc, diag::err_typecheck_member_reference_struct_union)
@@ -3333,15 +4141,21 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
 ///   this is an ugly hack around the fact that ObjC @implementations
 ///   aren't properly put in the context chain
 ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
-                                                   SourceLocation OpLoc,
-                                                   tok::TokenKind OpKind,
-                                                   CXXScopeSpec &SS,
-                                                   UnqualifiedId &Id,
-                                                   Decl *ObjCImpDecl,
-                                                   bool HasTrailingLParen) {
+                                       SourceLocation OpLoc,
+                                       tok::TokenKind OpKind,
+                                       CXXScopeSpec &SS,
+                                       UnqualifiedId &Id,
+                                       Decl *ObjCImpDecl,
+                                       bool HasTrailingLParen) {
   if (SS.isSet() && SS.isInvalid())
     return ExprError();
 
+  // Warn about the explicit constructor calls Microsoft extension.
+  if (getLangOptions().Microsoft &&
+      Id.getKind() == UnqualifiedId::IK_ConstructorName)
+    Diag(Id.getSourceRange().getBegin(),
+         diag::ext_ms_explicit_constructor_call);
+
   TemplateArgumentListInfo TemplateArgsBuffer;
 
   // Decompose the name into its component parts.
@@ -3399,60 +4213,76 @@ ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
 }
 
 ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
-                                                    FunctionDecl *FD,
-                                                    ParmVarDecl *Param) {
+                                        FunctionDecl *FD,
+                                        ParmVarDecl *Param) {
   if (Param->hasUnparsedDefaultArg()) {
-    Diag (CallLoc,
-          diag::err_use_of_default_argument_to_function_declared_later) <<
+    Diag(CallLoc,
+         diag::err_use_of_default_argument_to_function_declared_later) <<
       FD << cast<CXXRecordDecl>(FD->getDeclContext())->getDeclName();
     Diag(UnparsedDefaultArgLocs[Param],
-          diag::note_default_argument_declared_here);
-  } else {
-    if (Param->hasUninstantiatedDefaultArg()) {
-      Expr *UninstExpr = Param->getUninstantiatedDefaultArg();
-
-      // Instantiate the expression.
-      MultiLevelTemplateArgumentList ArgList
-        = getTemplateInstantiationArgs(FD, 0, /*RelativeToPrimary=*/true);
-
-      std::pair<const TemplateArgument *, unsigned> Innermost 
-        = ArgList.getInnermost();
-      InstantiatingTemplate Inst(*this, CallLoc, Param, Innermost.first,
-                                 Innermost.second);
-
-      ExprResult Result = SubstExpr(UninstExpr, ArgList);
-      if (Result.isInvalid())
-        return ExprError();
+         diag::note_default_argument_declared_here);
+    return ExprError();
+  }
+  
+  if (Param->hasUninstantiatedDefaultArg()) {
+    Expr *UninstExpr = Param->getUninstantiatedDefaultArg();
+
+    // Instantiate the expression.
+    MultiLevelTemplateArgumentList ArgList
+      = getTemplateInstantiationArgs(FD, 0, /*RelativeToPrimary=*/true);
+
+    std::pair<const TemplateArgument *, unsigned> Innermost
+      = ArgList.getInnermost();
+    InstantiatingTemplate Inst(*this, CallLoc, Param, Innermost.first,
+                               Innermost.second);
+
+    ExprResult Result;
+    {
+      // C++ [dcl.fct.default]p5:
+      //   The names in the [default argument] expression are bound, and
+      //   the semantic constraints are checked, at the point where the
+      //   default argument expression appears.
+      ContextRAII SavedContext(*this, FD);
+      Result = SubstExpr(UninstExpr, ArgList);
+    }
+    if (Result.isInvalid())
+      return ExprError();
 
-      // Check the expression as an initializer for the parameter.
-      InitializedEntity Entity
-        = InitializedEntity::InitializeParameter(Param);
-      InitializationKind Kind
-        = InitializationKind::CreateCopy(Param->getLocation(),
-               /*FIXME:EqualLoc*/UninstExpr->getSourceRange().getBegin());
-      Expr *ResultE = Result.takeAs<Expr>();
-
-      InitializationSequence InitSeq(*this, Entity, Kind, &ResultE, 1);
-      Result = InitSeq.Perform(*this, Entity, Kind,
-                               MultiExprArg(*this, &ResultE, 1));
-      if (Result.isInvalid())
-        return ExprError();
+    // Check the expression as an initializer for the parameter.
+    InitializedEntity Entity
+      = InitializedEntity::InitializeParameter(Context, Param);
+    InitializationKind Kind
+      = InitializationKind::CreateCopy(Param->getLocation(),
+             /*FIXME:EqualLoc*/UninstExpr->getSourceRange().getBegin());
+    Expr *ResultE = Result.takeAs<Expr>();
+
+    InitializationSequence InitSeq(*this, Entity, Kind, &ResultE, 1);
+    Result = InitSeq.Perform(*this, Entity, Kind,
+                             MultiExprArg(*this, &ResultE, 1));
+    if (Result.isInvalid())
+      return ExprError();
 
-      // Build the default argument expression.
-      return Owned(CXXDefaultArgExpr::Create(Context, CallLoc, Param,
-                                             Result.takeAs<Expr>()));
-    }
+    // Build the default argument expression.
+    return Owned(CXXDefaultArgExpr::Create(Context, CallLoc, Param,
+                                           Result.takeAs<Expr>()));
+  }
 
-    // If the default expression creates temporaries, we need to
-    // push them to the current stack of expression temporaries so they'll
-    // be properly destroyed.
-    // FIXME: We should really be rebuilding the default argument with new
-    // bound temporaries; see the comment in PR5810.
-    for (unsigned i = 0, e = Param->getNumDefaultArgTemporaries(); i != e; ++i)
-      ExprTemporaries.push_back(Param->getDefaultArgTemporary(i));
+  // If the default expression creates temporaries, we need to
+  // push them to the current stack of expression temporaries so they'll
+  // be properly destroyed.
+  // FIXME: We should really be rebuilding the default argument with new
+  // bound temporaries; see the comment in PR5810.
+  for (unsigned i = 0, e = Param->getNumDefaultArgTemporaries(); i != e; ++i) {
+    CXXTemporary *Temporary = Param->getDefaultArgTemporary(i);
+    MarkDeclarationReferenced(Param->getDefaultArg()->getLocStart(), 
+                    const_cast<CXXDestructorDecl*>(Temporary->getDestructor()));
+    ExprTemporaries.push_back(Temporary);
   }
 
-  // We already type-checked the argument, so we know it works.
+  // We already type-checked the argument, so we know it works. 
+  // Just mark all of the declarations in this potentially-evaluated expression
+  // as being "referenced".
+  MarkDeclarationsReferencedInExpr(Param->getDefaultArg());
   return Owned(CXXDefaultArgExpr::Create(Context, CallLoc, Param));
 }
 
@@ -3549,13 +4379,12 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
       if (FDecl && i < FDecl->getNumParams())
         Param = FDecl->getParamDecl(i);
 
-
       InitializedEntity Entity =
-        Param? InitializedEntity::InitializeParameter(Param)
-             : InitializedEntity::InitializeParameter(ProtoArgType);
+        Param? InitializedEntity::InitializeParameter(Context, Param)
+             : InitializedEntity::InitializeParameter(Context, ProtoArgType);
       ExprResult ArgE = PerformCopyInitialization(Entity,
-                                                        SourceLocation(),
-                                                        Owned(Arg));
+                                                  SourceLocation(),
+                                                  Owned(Arg));
       if (ArgE.isInvalid())
         return true;
 
@@ -3590,8 +4419,8 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
 /// locations.
 ExprResult
 Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
-                    MultiExprArg args,
-                    SourceLocation *CommaLocs, SourceLocation RParenLoc) {
+                    MultiExprArg args, SourceLocation RParenLoc,
+                    Expr *ExecConfig) {
   unsigned NumArgs = args.size();
 
   // Since this might be a postfix expression, get rid of ParenListExprs.
@@ -3615,7 +4444,7 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
       }
 
       return Owned(new (Context) CallExpr(Context, Fn, 0, 0, Context.VoidTy,
-                                          RParenLoc));
+                                          VK_RValue, RParenLoc));
     }
 
     // Determine whether this is a dependent call inside a C++ template,
@@ -3628,14 +4457,22 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
     else if (Expr::hasAnyTypeDependentArguments(Args, NumArgs))
       Dependent = true;
 
-    if (Dependent)
-      return Owned(new (Context) CallExpr(Context, Fn, Args, NumArgs,
-                                          Context.DependentTy, RParenLoc));
+    if (Dependent) {
+      if (ExecConfig) {
+        return Owned(new (Context) CUDAKernelCallExpr(
+            Context, Fn, cast<CallExpr>(ExecConfig), Args, NumArgs,
+            Context.DependentTy, VK_RValue, RParenLoc));
+      } else {
+        return Owned(new (Context) CallExpr(Context, Fn, Args, NumArgs,
+                                            Context.DependentTy, VK_RValue,
+                                            RParenLoc));
+      }
+    }
 
     // Determine whether this is a call to an object (C++ [over.call.object]).
     if (Fn->getType()->isRecordType())
       return Owned(BuildCallToObjectOfClassType(S, Fn, LParenLoc, Args, NumArgs,
-                                                CommaLocs, RParenLoc));
+                                                RParenLoc));
 
     Expr *NakedFn = Fn->IgnoreParens();
 
@@ -3652,7 +4489,7 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
       (void)MemE;
 
       return BuildCallToMemberFunction(S, Fn, LParenLoc, Args, NumArgs,
-                                       CommaLocs, RParenLoc);
+                                       RParenLoc);
     }
 
     // Determine whether this is a call to a member function.
@@ -3660,7 +4497,7 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
       NamedDecl *MemDecl = MemExpr->getMemberDecl();
       if (isa<CXXMethodDecl>(MemDecl))
         return BuildCallToMemberFunction(S, Fn, LParenLoc, Args, NumArgs,
-                                         CommaLocs, RParenLoc);
+                                         RParenLoc);
     }
 
     // Determine whether this is a call to a pointer-to-member function.
@@ -3670,10 +4507,31 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
         if (const FunctionProtoType *FPT
                                 = BO->getType()->getAs<FunctionProtoType>()) {
           QualType ResultTy = FPT->getCallResultType(Context);
-
+          ExprValueKind VK = Expr::getValueKindForType(FPT->getResultType());
+
+          // Check that the object type isn't more qualified than the
+          // member function we're calling.
+          Qualifiers FuncQuals = Qualifiers::fromCVRMask(FPT->getTypeQuals());
+          Qualifiers ObjectQuals 
+            = BO->getOpcode() == BO_PtrMemD
+                ? BO->getLHS()->getType().getQualifiers()
+                : BO->getLHS()->getType()->getAs<PointerType>()
+                                            ->getPointeeType().getQualifiers();
+
+          Qualifiers Difference = ObjectQuals - FuncQuals;
+          Difference.removeObjCGCAttr();
+          Difference.removeAddressSpace();
+          if (Difference) {
+            std::string QualsString = Difference.getAsString();
+            Diag(LParenLoc, diag::err_pointer_to_member_call_drops_quals)
+              << BO->getType().getUnqualifiedType()
+              << QualsString
+              << (QualsString.find(' ') == std::string::npos? 1 : 2);
+          }
+              
           CXXMemberCallExpr *TheCall
-            = new (Context) CXXMemberCallExpr(Context, BO, Args,
-                                              NumArgs, ResultTy,
+            = new (Context) CXXMemberCallExpr(Context, Fn, Args,
+                                              NumArgs, ResultTy, VK,
                                               RParenLoc);
 
           if (CheckCallReturnType(FPT->getResultType(),
@@ -3702,14 +4560,34 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
   if (isa<UnresolvedLookupExpr>(NakedFn)) {
     UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(NakedFn);
     return BuildOverloadedCallExpr(S, Fn, ULE, LParenLoc, Args, NumArgs,
-                                   CommaLocs, RParenLoc);
+                                   RParenLoc, ExecConfig);
   }
 
   NamedDecl *NDecl = 0;
+  if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(NakedFn))
+    if (UnOp->getOpcode() == UO_AddrOf)
+      NakedFn = UnOp->getSubExpr()->IgnoreParens();
+  
   if (isa<DeclRefExpr>(NakedFn))
     NDecl = cast<DeclRefExpr>(NakedFn)->getDecl();
 
-  return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, Args, NumArgs, RParenLoc);
+  return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, Args, NumArgs, RParenLoc,
+                               ExecConfig);
+}
+
+ExprResult
+Sema::ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
+                              MultiExprArg execConfig, SourceLocation GGGLoc) {
+  FunctionDecl *ConfigDecl = Context.getcudaConfigureCallDecl();
+  if (!ConfigDecl)
+    return ExprError(Diag(LLLLoc, diag::err_undeclared_var_use)
+                          << "cudaConfigureCall");
+  QualType ConfigQTy = ConfigDecl->getType();
+
+  DeclRefExpr *ConfigDR = new (Context) DeclRefExpr(
+      ConfigDecl, ConfigQTy, VK_LValue, LLLLoc);
+
+  return ActOnCallExpr(S, ConfigDR, LLLLoc, execConfig, GGGLoc, 0);
 }
 
 /// BuildResolvedCallExpr - Build a call to a resolved expression,
@@ -3722,7 +4600,8 @@ ExprResult
 Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                             SourceLocation LParenLoc,
                             Expr **Args, unsigned NumArgs,
-                            SourceLocation RParenLoc) {
+                            SourceLocation RParenLoc,
+                            Expr *Config) {
   FunctionDecl *FDecl = dyn_cast_or_null<FunctionDecl>(NDecl);
 
   // Promote the function operand.
@@ -3730,10 +4609,21 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   // Make the call expr early, before semantic checks.  This guarantees cleanup
   // of arguments and function on error.
-  CallExpr *TheCall = new (Context) CallExpr(Context, Fn,
-                                             Args, NumArgs,
-                                             Context.BoolTy,
-                                             RParenLoc);
+  CallExpr *TheCall;
+  if (Config) {
+    TheCall = new (Context) CUDAKernelCallExpr(Context, Fn,
+                                               cast<CallExpr>(Config),
+                                               Args, NumArgs,
+                                               Context.BoolTy,
+                                               VK_RValue,
+                                               RParenLoc);
+  } else {
+    TheCall = new (Context) CallExpr(Context, Fn,
+                                     Args, NumArgs,
+                                     Context.BoolTy,
+                                     VK_RValue,
+                                     RParenLoc);
+  }
 
   const FunctionType *FuncT;
   if (!Fn->getType()->isBlockPointerType()) {
@@ -3760,6 +4650,7 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   // We know the result type of the call, set it.
   TheCall->setType(FuncT->getCallResultType(Context));
+  TheCall->setValueKind(Expr::getValueKindForType(FuncT->getResultType()));
 
   if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FuncT)) {
     if (ConvertArgumentsForCall(TheCall, Fn, FDecl, Proto, Args, NumArgs,
@@ -3773,24 +4664,45 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
       // on our knowledge of the function definition.
       const FunctionDecl *Def = 0;
       if (FDecl->hasBody(Def) && NumArgs != Def->param_size()) {
-        const FunctionProtoType *Proto =
-            Def->getType()->getAs<FunctionProtoType>();
-        if (!Proto || !(Proto->isVariadic() && NumArgs >= Def->param_size())) {
+        const FunctionProtoType *Proto 
+          = Def->getType()->getAs<FunctionProtoType>();
+        if (!Proto || !(Proto->isVariadic() && NumArgs >= Def->param_size()))
           Diag(RParenLoc, diag::warn_call_wrong_number_of_arguments)
             << (NumArgs > Def->param_size()) << FDecl << Fn->getSourceRange();
-        }
       }
+      
+      // If the function we're calling isn't a function prototype, but we have
+      // a function prototype from a prior declaratiom, use that prototype.
+      if (!FDecl->hasPrototype())
+        Proto = FDecl->getType()->getAs<FunctionProtoType>();
     }
 
     // Promote the arguments (C99 6.5.2.2p6).
     for (unsigned i = 0; i != NumArgs; i++) {
       Expr *Arg = Args[i];
-      DefaultArgumentPromotion(Arg);
+
+      if (Proto && i < Proto->getNumArgs()) {
+        InitializedEntity Entity
+          = InitializedEntity::InitializeParameter(Context, 
+                                                   Proto->getArgType(i));
+        ExprResult ArgE = PerformCopyInitialization(Entity,
+                                                    SourceLocation(),
+                                                    Owned(Arg));
+        if (ArgE.isInvalid())
+          return true;
+        
+        Arg = ArgE.takeAs<Expr>();
+
+      } else {
+        DefaultArgumentPromotion(Arg);
+      }
+      
       if (RequireCompleteType(Arg->getSourceRange().getBegin(),
                               Arg->getType(),
                               PDiag(diag::err_call_incomplete_argument)
                                 << Arg->getSourceRange()))
         return ExprError();
+
       TheCall->setArg(i, Arg);
     }
   }
@@ -3840,6 +4752,11 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
   QualType literalType = TInfo->getType();
 
   if (literalType->isArrayType()) {
+    if (RequireCompleteType(LParenLoc, Context.getBaseElementType(literalType),
+             PDiag(diag::err_illegal_decl_array_incomplete_type)
+               << SourceRange(LParenLoc,
+                              literalExpr->getSourceRange().getEnd())))
+      return ExprError();
     if (literalType->isVariableArrayType())
       return ExprError(Diag(LParenLoc, diag::err_variable_object_no_init)
         << SourceRange(LParenLoc, literalExpr->getSourceRange().getEnd()));
@@ -3869,8 +4786,11 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
       return ExprError();
   }
 
+  // In C, compound literals are l-values for some reason.
+  ExprValueKind VK = getLangOptions().CPlusPlus ? VK_RValue : VK_LValue;
+
   return Owned(new (Context) CompoundLiteralExpr(LParenLoc, TInfo, literalType,
-                                                 literalExpr, isFileScope));
+                                                 VK, literalExpr, isFileScope));
 }
 
 ExprResult
@@ -3888,60 +4808,171 @@ Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg initlist,
   return Owned(E);
 }
 
-static CastKind getScalarCastKind(ASTContext &Context,
-                                            QualType SrcTy, QualType DestTy) {
-  if (Context.hasSameUnqualifiedType(SrcTy, DestTy))
+/// Prepares for a scalar cast, performing all the necessary stages
+/// except the final cast and returning the kind required.
+static CastKind PrepareScalarCast(Sema &S, Expr *&Src, QualType DestTy) {
+  // Both Src and Dest are scalar types, i.e. arithmetic or pointer.
+  // Also, callers should have filtered out the invalid cases with
+  // pointers.  Everything else should be possible.
+
+  QualType SrcTy = Src->getType();
+  if (S.Context.hasSameUnqualifiedType(SrcTy, DestTy))
     return CK_NoOp;
 
-  if (SrcTy->hasPointerRepresentation()) {
-    if (DestTy->hasPointerRepresentation())
+  switch (SrcTy->getScalarTypeKind()) {
+  case Type::STK_MemberPointer:
+    llvm_unreachable("member pointer type in C");
+
+  case Type::STK_Pointer:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_Pointer:
       return DestTy->isObjCObjectPointerType() ?
                 CK_AnyPointerToObjCPointerCast :
                 CK_BitCast;
-    if (DestTy->isIntegerType())
+    case Type::STK_Bool:
+      return CK_PointerToBoolean;
+    case Type::STK_Integral:
       return CK_PointerToIntegral;
-  }
+    case Type::STK_Floating:
+    case Type::STK_FloatingComplex:
+    case Type::STK_IntegralComplex:
+    case Type::STK_MemberPointer:
+      llvm_unreachable("illegal cast from pointer");
+    }
+    break;
 
-  if (SrcTy->isIntegerType()) {
-    if (DestTy->isIntegerType())
-      return CK_IntegralCast;
-    if (DestTy->hasPointerRepresentation())
+  case Type::STK_Bool: // casting from bool is like casting from an integer
+  case Type::STK_Integral:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_Pointer:
+      if (Src->isNullPointerConstant(S.Context, Expr::NPC_ValueDependentIsNull))
+        return CK_NullToPointer;
       return CK_IntegralToPointer;
-    if (DestTy->isRealFloatingType())
+    case Type::STK_Bool:
+      return CK_IntegralToBoolean;
+    case Type::STK_Integral:
+      return CK_IntegralCast;
+    case Type::STK_Floating:
       return CK_IntegralToFloating;
-  }
+    case Type::STK_IntegralComplex:
+      S.ImpCastExprToType(Src, DestTy->getAs<ComplexType>()->getElementType(),
+                          CK_IntegralCast);
+      return CK_IntegralRealToComplex;
+    case Type::STK_FloatingComplex:
+      S.ImpCastExprToType(Src, DestTy->getAs<ComplexType>()->getElementType(),
+                          CK_IntegralToFloating);
+      return CK_FloatingRealToComplex;
+    case Type::STK_MemberPointer:
+      llvm_unreachable("member pointer type in C");
+    }
+    break;
 
-  if (SrcTy->isRealFloatingType()) {
-    if (DestTy->isRealFloatingType())
+  case Type::STK_Floating:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_Floating:
       return CK_FloatingCast;
-    if (DestTy->isIntegerType())
+    case Type::STK_Bool:
+      return CK_FloatingToBoolean;
+    case Type::STK_Integral:
       return CK_FloatingToIntegral;
+    case Type::STK_FloatingComplex:
+      S.ImpCastExprToType(Src, DestTy->getAs<ComplexType>()->getElementType(),
+                          CK_FloatingCast);
+      return CK_FloatingRealToComplex;
+    case Type::STK_IntegralComplex:
+      S.ImpCastExprToType(Src, DestTy->getAs<ComplexType>()->getElementType(),
+                          CK_FloatingToIntegral);
+      return CK_IntegralRealToComplex;
+    case Type::STK_Pointer:
+      llvm_unreachable("valid float->pointer cast?");
+    case Type::STK_MemberPointer:
+      llvm_unreachable("member pointer type in C");
+    }
+    break;
+
+  case Type::STK_FloatingComplex:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_FloatingComplex:
+      return CK_FloatingComplexCast;
+    case Type::STK_IntegralComplex:
+      return CK_FloatingComplexToIntegralComplex;
+    case Type::STK_Floating: {
+      QualType ET = SrcTy->getAs<ComplexType>()->getElementType();
+      if (S.Context.hasSameType(ET, DestTy))
+        return CK_FloatingComplexToReal;
+      S.ImpCastExprToType(Src, ET, CK_FloatingComplexToReal);
+      return CK_FloatingCast;
+    }
+    case Type::STK_Bool:
+      return CK_FloatingComplexToBoolean;
+    case Type::STK_Integral:
+      S.ImpCastExprToType(Src, SrcTy->getAs<ComplexType>()->getElementType(),
+                          CK_FloatingComplexToReal);
+      return CK_FloatingToIntegral;
+    case Type::STK_Pointer:
+      llvm_unreachable("valid complex float->pointer cast?");
+    case Type::STK_MemberPointer:
+      llvm_unreachable("member pointer type in C");
+    }
+    break;
+
+  case Type::STK_IntegralComplex:
+    switch (DestTy->getScalarTypeKind()) {
+    case Type::STK_FloatingComplex:
+      return CK_IntegralComplexToFloatingComplex;
+    case Type::STK_IntegralComplex:
+      return CK_IntegralComplexCast;
+    case Type::STK_Integral: {
+      QualType ET = SrcTy->getAs<ComplexType>()->getElementType();
+      if (S.Context.hasSameType(ET, DestTy))
+        return CK_IntegralComplexToReal;
+      S.ImpCastExprToType(Src, ET, CK_IntegralComplexToReal);
+      return CK_IntegralCast;
+    }
+    case Type::STK_Bool:
+      return CK_IntegralComplexToBoolean;
+    case Type::STK_Floating:
+      S.ImpCastExprToType(Src, SrcTy->getAs<ComplexType>()->getElementType(),
+                          CK_IntegralComplexToReal);
+      return CK_IntegralToFloating;
+    case Type::STK_Pointer:
+      llvm_unreachable("valid complex int->pointer cast?");
+    case Type::STK_MemberPointer:
+      llvm_unreachable("member pointer type in C");
+    }
+    break;
   }
 
-  // FIXME: Assert here.
-  // assert(false && "Unhandled cast combination!");
-  return CK_Unknown;
+  llvm_unreachable("Unhandled scalar cast");
+  return CK_BitCast;
 }
 
 /// CheckCastTypes - Check type constraints for casting between types.
-bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
-                          CastKind& Kind,
-                          CXXCastPath &BasePath,
-                          bool FunctionalStyle) {
+bool Sema::CheckCastTypes(SourceRange TyR, QualType castType,
+                          Expr *&castExpr, CastKind& Kind, ExprValueKind &VK,
+                          CXXCastPath &BasePath, bool FunctionalStyle) {
   if (getLangOptions().CPlusPlus)
-    return CXXCheckCStyleCast(TyR, castType, castExpr, Kind, BasePath,
+    return CXXCheckCStyleCast(SourceRange(TyR.getBegin(),
+                                          castExpr->getLocEnd()), 
+                              castType, VK, castExpr, Kind, BasePath,
                               FunctionalStyle);
 
-  DefaultFunctionArrayLvalueConversion(castExpr);
+  // We only support r-value casts in C.
+  VK = VK_RValue;
 
   // C99 6.5.4p2: the cast type needs to be void or scalar and the expression
   // type needs to be scalar.
   if (castType->isVoidType()) {
+    // We don't necessarily do lvalue-to-rvalue conversions on this.
+    IgnoredValueConversions(castExpr);
+
     // Cast to void allows any expr type.
     Kind = CK_ToVoid;
     return false;
   }
 
+  DefaultFunctionArrayLvalueConversion(castExpr);
+
   if (RequireCompleteType(TyR.getBegin(), castType,
                           diag::err_typecheck_cast_to_incomplete))
     return true;
@@ -3964,7 +4995,8 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
       for (Field = RD->field_begin(), FieldEnd = RD->field_end();
            Field != FieldEnd; ++Field) {
         if (Context.hasSameUnqualifiedType(Field->getType(),
-                                           castExpr->getType())) {
+                                           castExpr->getType()) &&
+            !Field->isUnnamedBitfield()) {
           Diag(TyR.getBegin(), diag::ext_typecheck_cast_to_union)
             << castExpr->getSourceRange();
           break;
@@ -3982,6 +5014,9 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
       << castType << castExpr->getSourceRange();
   }
 
+  // The type we're casting to is known to be a scalar or vector.
+
+  // Require the operand to be a scalar or vector.
   if (!castExpr->getType()->isScalarType() &&
       !castExpr->getType()->isVectorType()) {
     return Diag(castExpr->getLocStart(),
@@ -3997,9 +5032,16 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
   if (castExpr->getType()->isVectorType())
     return CheckVectorCast(TyR, castExpr->getType(), castType, Kind);
 
+  // The source and target types are both scalars, i.e.
+  //   - arithmetic types (fundamental, enum, and complex)
+  //   - all kinds of pointers
+  // Note that member pointers were filtered out with C++, above.
+
   if (isa<ObjCSelectorExpr>(castExpr))
     return Diag(castExpr->getLocStart(), diag::err_cast_selector_expr);
 
+  // If either type is a pointer, the other type has to be either an
+  // integer or a pointer.
   if (!castType->isArithmeticType()) {
     QualType castExprType = castExpr->getType();
     if (!castExprType->isIntegralType(Context) && 
@@ -4014,9 +5056,9 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
         << castType << castExpr->getSourceRange();
   }
 
-  Kind = getScalarCastKind(Context, castExpr->getType(), castType);
+  Kind = PrepareScalarCast(*this, castExpr, castType);
 
-  if (Kind == CK_Unknown || Kind == CK_BitCast)
+  if (Kind == CK_BitCast)
     CheckCastAlign(castExpr, castType, TyR);
 
   return false;
@@ -4068,7 +5110,7 @@ bool Sema::CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *&CastExpr,
 
   QualType DestElemTy = DestTy->getAs<ExtVectorType>()->getElementType();
   ImpCastExprToType(CastExpr, DestElemTy,
-                    getScalarCastKind(Context, SrcTy, DestElemTy));
+                    PrepareScalarCast(*this, CastExpr, DestElemTy));
 
   Kind = CK_VectorSplat;
   return false;
@@ -4096,15 +5138,16 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc, ParsedType Ty,
 ExprResult
 Sema::BuildCStyleCastExpr(SourceLocation LParenLoc, TypeSourceInfo *Ty,
                           SourceLocation RParenLoc, Expr *castExpr) {
-  CastKind Kind = CK_Unknown;
+  CastKind Kind = CK_Invalid;
+  ExprValueKind VK = VK_RValue;
   CXXCastPath BasePath;
   if (CheckCastTypes(SourceRange(LParenLoc, RParenLoc), Ty->getType(), castExpr,
-                     Kind, BasePath))
+                     Kind, VK, BasePath))
     return ExprError();
 
   return Owned(CStyleCastExpr::Create(Context,
                                     Ty->getType().getNonLValueExprType(Context),
-                                      Kind, castExpr, &BasePath, Ty,
+                                      VK, Kind, castExpr, &BasePath, Ty,
                                       LParenLoc, RParenLoc));
 }
 
@@ -4188,14 +5231,73 @@ ExprResult Sema::ActOnParenOrParenListExpr(SourceLocation L,
   return Owned(expr);
 }
 
+/// \brief Emit a specialized diagnostic when one expression is a null pointer
+/// constant and the other is not a pointer.
+bool Sema::DiagnoseConditionalForNull(Expr *LHS, Expr *RHS,
+                                      SourceLocation QuestionLoc) {
+  Expr *NullExpr = LHS;
+  Expr *NonPointerExpr = RHS;
+  Expr::NullPointerConstantKind NullKind =
+      NullExpr->isNullPointerConstant(Context,
+                                      Expr::NPC_ValueDependentIsNotNull);
+
+  if (NullKind == Expr::NPCK_NotNull) {
+    NullExpr = RHS;
+    NonPointerExpr = LHS;
+    NullKind =
+        NullExpr->isNullPointerConstant(Context,
+                                        Expr::NPC_ValueDependentIsNotNull);
+  }
+
+  if (NullKind == Expr::NPCK_NotNull)
+    return false;
+
+  if (NullKind == Expr::NPCK_ZeroInteger) {
+    // In this case, check to make sure that we got here from a "NULL"
+    // string in the source code.
+    NullExpr = NullExpr->IgnoreParenImpCasts();
+    SourceManager& SM = Context.getSourceManager();
+    SourceLocation Loc = SM.getInstantiationLoc(NullExpr->getExprLoc());
+    unsigned Len =
+        Lexer::MeasureTokenLength(Loc, SM, Context.getLangOptions());
+    if (Len != 4 || memcmp(SM.getCharacterData(Loc), "NULL", 4))
+      return false;
+  }
+
+  int DiagType = (NullKind == Expr::NPCK_CXX0X_nullptr);
+  Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands_null)
+      << NonPointerExpr->getType() << DiagType
+      << NonPointerExpr->getSourceRange();
+  return true;
+}
+
 /// Note that lhs is not null here, even if this is the gnu "x ?: y" extension.
 /// In that case, lhs = cond.
 /// C99 6.5.15
 QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
+                                        ExprValueKind &VK, ExprObjectKind &OK,
                                         SourceLocation QuestionLoc) {
+  // If both LHS and RHS are overloaded functions, try to resolve them.
+  if (Context.hasSameType(LHS->getType(), RHS->getType()) && 
+      LHS->getType()->isSpecificBuiltinType(BuiltinType::Overload)) {
+    ExprResult LHSResult = CheckPlaceholderExpr(LHS, QuestionLoc);
+    if (LHSResult.isInvalid())
+      return QualType();
+
+    ExprResult RHSResult = CheckPlaceholderExpr(RHS, QuestionLoc);
+    if (RHSResult.isInvalid())
+      return QualType();
+
+    LHS = LHSResult.take();
+    RHS = RHSResult.take();
+  }
+
   // C++ is sufficiently different to merit its own checker.
   if (getLangOptions().CPlusPlus)
-    return CXXCheckConditionalOperands(Cond, LHS, RHS, QuestionLoc);
+    return CXXCheckConditionalOperands(Cond, LHS, RHS, VK, OK, QuestionLoc);
+
+  VK = VK_RValue;
+  OK = OK_Ordinary;
 
   UsualUnaryConversions(Cond);
   UsualUnaryConversions(LHS);
@@ -4206,15 +5308,47 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
 
   // first, check the condition.
   if (!CondTy->isScalarType()) { // C99 6.5.15p2
-    Diag(Cond->getLocStart(), diag::err_typecheck_cond_expect_scalar)
-      << CondTy;
-    return QualType();
+    // OpenCL: Sec 6.3.i says the condition is allowed to be a vector or scalar.
+    // Throw an error if its not either.
+    if (getLangOptions().OpenCL) {
+      if (!CondTy->isVectorType()) {
+        Diag(Cond->getLocStart(), 
+             diag::err_typecheck_cond_expect_scalar_or_vector)
+          << CondTy;
+        return QualType();
+      }
+    }
+    else {
+      Diag(Cond->getLocStart(), diag::err_typecheck_cond_expect_scalar)
+        << CondTy;
+      return QualType();
+    }
   }
 
   // Now check the two expressions.
   if (LHSTy->isVectorType() || RHSTy->isVectorType())
     return CheckVectorOperands(QuestionLoc, LHS, RHS);
 
+  // OpenCL: If the condition is a vector, and both operands are scalar,
+  // attempt to implicity convert them to the vector type to act like the
+  // built in select.
+  if (getLangOptions().OpenCL && CondTy->isVectorType()) {
+    // Both operands should be of scalar type.
+    if (!LHSTy->isScalarType()) {
+      Diag(LHS->getLocStart(), diag::err_typecheck_cond_expect_scalar)
+        << CondTy;
+      return QualType();
+    }
+    if (!RHSTy->isScalarType()) {
+      Diag(RHS->getLocStart(), diag::err_typecheck_cond_expect_scalar)
+        << CondTy;
+      return QualType();
+    }
+    // Implicity convert these scalars to the type of the condition.
+    ImpCastExprToType(LHS, CondTy, CK_IntegralCast);
+    ImpCastExprToType(RHS, CondTy, CK_IntegralCast);
+  }
+  
   // If both operands have arithmetic type, do the usual arithmetic conversions
   // to find a common type: C99 6.5.15p3,5.
   if (LHSTy->isArithmeticType() && RHSTy->isArithmeticType()) {
@@ -4251,12 +5385,12 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   if ((LHSTy->isAnyPointerType() || LHSTy->isBlockPointerType()) &&
       RHS->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
     // promote the null to a pointer.
-    ImpCastExprToType(RHS, LHSTy, CK_Unknown);
+    ImpCastExprToType(RHS, LHSTy, CK_NullToPointer);
     return LHSTy;
   }
   if ((RHSTy->isAnyPointerType() || RHSTy->isBlockPointerType()) &&
       LHS->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
-    ImpCastExprToType(LHS, RHSTy, CK_Unknown);
+    ImpCastExprToType(LHS, RHSTy, CK_NullToPointer);
     return RHSTy;
   }
 
@@ -4364,7 +5498,8 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     return LHSTy;
   }
 
-  // GCC compatibility: soften pointer/integer mismatch.
+  // GCC compatibility: soften pointer/integer mismatch.  Note that
+  // null pointers have been filtered out by this point.
   if (RHSTy->isPointerType() && LHSTy->isIntegerType()) {
     Diag(QuestionLoc, diag::warn_typecheck_cond_pointer_integer_mismatch)
       << LHSTy << RHSTy << LHS->getSourceRange() << RHS->getSourceRange();
@@ -4378,6 +5513,12 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     return LHSTy;
   }
 
+  // Emit a better diagnostic if one of the expressions is a null pointer
+  // constant and the other is not a pointer type. In this case, the user most
+  // likely forgot to take the address of the other expression.
+  if (DiagnoseConditionalForNull(LHS, RHS, QuestionLoc))
+    return QualType();
+
   // Otherwise, the operands are not compatible.
   Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
     << LHSTy << RHSTy << LHS->getSourceRange() << RHS->getSourceRange();
@@ -4395,34 +5536,34 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
   // to the pseudo-builtin, because that will be implicitly cast back to the
   // redefinition type if an attempt is made to access its fields.
   if (LHSTy->isObjCClassType() &&
-      (RHSTy.getDesugaredType() == Context.ObjCClassRedefinitionType)) {
+      (Context.hasSameType(RHSTy, Context.ObjCClassRedefinitionType))) {
     ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (RHSTy->isObjCClassType() &&
-      (LHSTy.getDesugaredType() == Context.ObjCClassRedefinitionType)) {
+      (Context.hasSameType(LHSTy, Context.ObjCClassRedefinitionType))) {
     ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
   // And the same for struct objc_object* / id
   if (LHSTy->isObjCIdType() &&
-      (RHSTy.getDesugaredType() == Context.ObjCIdRedefinitionType)) {
+      (Context.hasSameType(RHSTy, Context.ObjCIdRedefinitionType))) {
     ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (RHSTy->isObjCIdType() &&
-      (LHSTy.getDesugaredType() == Context.ObjCIdRedefinitionType)) {
+      (Context.hasSameType(LHSTy, Context.ObjCIdRedefinitionType))) {
     ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
   // And the same for struct objc_selector* / SEL
   if (Context.isObjCSelType(LHSTy) &&
-      (RHSTy.getDesugaredType() == Context.ObjCSelRedefinitionType)) {
+      (Context.hasSameType(RHSTy, Context.ObjCSelRedefinitionType))) {
     ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (Context.isObjCSelType(RHSTy) &&
-      (LHSTy.getDesugaredType() == Context.ObjCSelRedefinitionType)) {
+      (Context.hasSameType(LHSTy, Context.ObjCSelRedefinitionType))) {
     ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
@@ -4512,60 +5653,84 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
 /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
 /// in the case of a the GNU conditional expr extension.
 ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
-                                                  SourceLocation ColonLoc,
-                                                  Expr *CondExpr, Expr *LHSExpr,
-                                                  Expr *RHSExpr) {
+                                    SourceLocation ColonLoc,
+                                    Expr *CondExpr, Expr *LHSExpr,
+                                    Expr *RHSExpr) {
   // If this is the gnu "x ?: y" extension, analyze the types as though the LHS
   // was the condition.
-  bool isLHSNull = LHSExpr == 0;
-  Expr *SAVEExpr = 0;
-  if (isLHSNull) {
-    LHSExpr = SAVEExpr = CondExpr;
-  }
-
-  QualType result = CheckConditionalOperands(CondExpr, LHSExpr,
-                                             RHSExpr, QuestionLoc);
+  OpaqueValueExpr *opaqueValue = 0;
+  Expr *commonExpr = 0;
+  if (LHSExpr == 0) {
+    commonExpr = CondExpr;
+
+    // We usually want to apply unary conversions *before* saving, except
+    // in the special case of a C++ l-value conditional.
+    if (!(getLangOptions().CPlusPlus
+          && !commonExpr->isTypeDependent()
+          && commonExpr->getValueKind() == RHSExpr->getValueKind()
+          && commonExpr->isGLValue()
+          && commonExpr->isOrdinaryOrBitFieldObject()
+          && RHSExpr->isOrdinaryOrBitFieldObject()
+          && Context.hasSameType(commonExpr->getType(), RHSExpr->getType()))) {
+      UsualUnaryConversions(commonExpr);
+    }
+
+    opaqueValue = new (Context) OpaqueValueExpr(commonExpr->getExprLoc(),
+                                                commonExpr->getType(),
+                                                commonExpr->getValueKind(),
+                                                commonExpr->getObjectKind());
+    LHSExpr = CondExpr = opaqueValue;
+  }
+
+  ExprValueKind VK = VK_RValue;
+  ExprObjectKind OK = OK_Ordinary;
+  QualType result = CheckConditionalOperands(CondExpr, LHSExpr, RHSExpr, 
+                                             VK, OK, QuestionLoc);
   if (result.isNull())
     return ExprError();
 
-  return Owned(new (Context) ConditionalOperator(CondExpr, QuestionLoc,
-                                                 LHSExpr, ColonLoc, 
-                                                 RHSExpr, SAVEExpr,
-                                                 result));
+  if (!commonExpr)
+    return Owned(new (Context) ConditionalOperator(CondExpr, QuestionLoc,
+                                                   LHSExpr, ColonLoc, 
+                                                   RHSExpr, result, VK, OK));
+
+  return Owned(new (Context)
+    BinaryConditionalOperator(commonExpr, opaqueValue, CondExpr, LHSExpr,
+                              RHSExpr, QuestionLoc, ColonLoc, result, VK, OK));
 }
 
-// CheckPointerTypesForAssignment - This is a very tricky routine (despite
+// checkPointerTypesForAssignment - This is a very tricky routine (despite
 // being closely modeled after the C99 spec:-). The odd characteristic of this
 // routine is it effectively iqnores the qualifiers on the top level pointee.
 // This circumvents the usual type rules specified in 6.2.7p1 & 6.7.5.[1-3].
 // FIXME: add a couple examples in this comment.
-Sema::AssignConvertType
-Sema::CheckPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
-  QualType lhptee, rhptee;
-
-  if ((lhsType->isObjCClassType() &&
-       (rhsType.getDesugaredType() == Context.ObjCClassRedefinitionType)) ||
-     (rhsType->isObjCClassType() &&
-       (lhsType.getDesugaredType() == Context.ObjCClassRedefinitionType))) {
-      return Compatible;
-  }
+static Sema::AssignConvertType
+checkPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
+  assert(lhsType.isCanonical() && "LHS not canonicalized!");
+  assert(rhsType.isCanonical() && "RHS not canonicalized!");
 
   // get the "pointed to" type (ignoring qualifiers at the top level)
-  lhptee = lhsType->getAs<PointerType>()->getPointeeType();
-  rhptee = rhsType->getAs<PointerType>()->getPointeeType();
+  const Type *lhptee, *rhptee;
+  Qualifiers lhq, rhq;
+  llvm::tie(lhptee, lhq) = cast<PointerType>(lhsType)->getPointeeType().split();
+  llvm::tie(rhptee, rhq) = cast<PointerType>(rhsType)->getPointeeType().split();
 
-  // make sure we operate on the canonical type
-  lhptee = Context.getCanonicalType(lhptee);
-  rhptee = Context.getCanonicalType(rhptee);
-
-  AssignConvertType ConvTy = Compatible;
+  Sema::AssignConvertType ConvTy = Sema::Compatible;
 
   // C99 6.5.16.1p1: This following citation is common to constraints
   // 3 & 4 (below). ...and the type *pointed to* by the left has all the
   // qualifiers of the type *pointed to* by the right;
-  // FIXME: Handle ExtQualType
-  if (!lhptee.isAtLeastAsQualifiedAs(rhptee))
-    ConvTy = CompatiblePointerDiscardsQualifiers;
+  Qualifiers lq;
+
+  if (!lhq.compatiblyIncludes(rhq)) {
+    // Treat address-space mismatches as fatal.  TODO: address subspaces
+    if (lhq.getAddressSpace() != rhq.getAddressSpace())
+      ConvTy = Sema::IncompatiblePointerDiscardsQualifiers;
+
+    // For GCC compatibility, other qualifier mismatches are treated
+    // as still compatible in C.
+    else ConvTy = Sema::CompatiblePointerDiscardsQualifiers;
+  }
 
   // C99 6.5.16.1p1 (constraint 4): If one operand is a pointer to an object or
   // incomplete type and the other is a pointer to a qualified or unqualified
@@ -4576,7 +5741,7 @@ Sema::CheckPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
 
     // As an extension, we allow cast to/from void* to function pointer.
     assert(rhptee->isFunctionType());
-    return FunctionVoidPointer;
+    return Sema::FunctionVoidPointer;
   }
 
   if (rhptee->isVoidType()) {
@@ -4585,123 +5750,136 @@ Sema::CheckPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
 
     // As an extension, we allow cast to/from void* to function pointer.
     assert(lhptee->isFunctionType());
-    return FunctionVoidPointer;
+    return Sema::FunctionVoidPointer;
   }
+
   // C99 6.5.16.1p1 (constraint 3): both operands are pointers to qualified or
   // unqualified versions of compatible types, ...
-  lhptee = lhptee.getUnqualifiedType();
-  rhptee = rhptee.getUnqualifiedType();
-  if (!Context.typesAreCompatible(lhptee, rhptee)) {
+  QualType ltrans = QualType(lhptee, 0), rtrans = QualType(rhptee, 0);
+  if (!S.Context.typesAreCompatible(ltrans, rtrans)) {
     // Check if the pointee types are compatible ignoring the sign.
     // We explicitly check for char so that we catch "char" vs
     // "unsigned char" on systems where "char" is unsigned.
     if (lhptee->isCharType())
-      lhptee = Context.UnsignedCharTy;
+      ltrans = S.Context.UnsignedCharTy;
     else if (lhptee->hasSignedIntegerRepresentation())
-      lhptee = Context.getCorrespondingUnsignedType(lhptee);
+      ltrans = S.Context.getCorrespondingUnsignedType(ltrans);
 
     if (rhptee->isCharType())
-      rhptee = Context.UnsignedCharTy;
+      rtrans = S.Context.UnsignedCharTy;
     else if (rhptee->hasSignedIntegerRepresentation())
-      rhptee = Context.getCorrespondingUnsignedType(rhptee);
+      rtrans = S.Context.getCorrespondingUnsignedType(rtrans);
 
-    if (lhptee == rhptee) {
+    if (ltrans == rtrans) {
       // Types are compatible ignoring the sign. Qualifier incompatibility
       // takes priority over sign incompatibility because the sign
       // warning can be disabled.
-      if (ConvTy != Compatible)
+      if (ConvTy != Sema::Compatible)
         return ConvTy;
-      return IncompatiblePointerSign;
+
+      return Sema::IncompatiblePointerSign;
     }
 
     // If we are a multi-level pointer, it's possible that our issue is simply
     // one of qualification - e.g. char ** -> const char ** is not allowed. If
     // the eventual target type is the same and the pointers have the same
     // level of indirection, this must be the issue.
-    if (lhptee->isPointerType() && rhptee->isPointerType()) {
+    if (isa<PointerType>(lhptee) && isa<PointerType>(rhptee)) {
       do {
-        lhptee = lhptee->getAs<PointerType>()->getPointeeType();
-        rhptee = rhptee->getAs<PointerType>()->getPointeeType();
-
-        lhptee = Context.getCanonicalType(lhptee);
-        rhptee = Context.getCanonicalType(rhptee);
-      } while (lhptee->isPointerType() && rhptee->isPointerType());
+        lhptee = cast<PointerType>(lhptee)->getPointeeType().getTypePtr();
+        rhptee = cast<PointerType>(rhptee)->getPointeeType().getTypePtr();
+      } while (isa<PointerType>(lhptee) && isa<PointerType>(rhptee));
 
-      if (Context.hasSameUnqualifiedType(lhptee, rhptee))
-        return IncompatibleNestedPointerQualifiers;
+      if (lhptee == rhptee)
+        return Sema::IncompatibleNestedPointerQualifiers;
     }
 
     // General pointer incompatibility takes priority over qualifiers.
-    return IncompatiblePointer;
+    return Sema::IncompatiblePointer;
   }
   return ConvTy;
 }
 
-/// CheckBlockPointerTypesForAssignment - This routine determines whether two
+/// checkBlockPointerTypesForAssignment - This routine determines whether two
 /// block pointer types are compatible or whether a block and normal pointer
 /// are compatible. It is more restrict than comparing two function pointer
 // types.
-Sema::AssignConvertType
-Sema::CheckBlockPointerTypesForAssignment(QualType lhsType,
-                                          QualType rhsType) {
+static Sema::AssignConvertType
+checkBlockPointerTypesForAssignment(Sema &S, QualType lhsType,
+                                    QualType rhsType) {
+  assert(lhsType.isCanonical() && "LHS not canonicalized!");
+  assert(rhsType.isCanonical() && "RHS not canonicalized!");
+
   QualType lhptee, rhptee;
 
   // get the "pointed to" type (ignoring qualifiers at the top level)
-  lhptee = lhsType->getAs<BlockPointerType>()->getPointeeType();
-  rhptee = rhsType->getAs<BlockPointerType>()->getPointeeType();
+  lhptee = cast<BlockPointerType>(lhsType)->getPointeeType();
+  rhptee = cast<BlockPointerType>(rhsType)->getPointeeType();
 
-  // make sure we operate on the canonical type
-  lhptee = Context.getCanonicalType(lhptee);
-  rhptee = Context.getCanonicalType(rhptee);
+  // In C++, the types have to match exactly.
+  if (S.getLangOptions().CPlusPlus)
+    return Sema::IncompatibleBlockPointer;
 
-  AssignConvertType ConvTy = Compatible;
+  Sema::AssignConvertType ConvTy = Sema::Compatible;
 
   // For blocks we enforce that qualifiers are identical.
-  if (lhptee.getLocalCVRQualifiers() != rhptee.getLocalCVRQualifiers())
-    ConvTy = CompatiblePointerDiscardsQualifiers;
+  if (lhptee.getLocalQualifiers() != rhptee.getLocalQualifiers())
+    ConvTy = Sema::CompatiblePointerDiscardsQualifiers;
+
+  if (!S.Context.typesAreBlockPointerCompatible(lhsType, rhsType))
+    return Sema::IncompatibleBlockPointer;
 
-  if (!getLangOptions().CPlusPlus) {
-    if (!Context.typesAreBlockPointerCompatible(lhsType, rhsType))
-      return IncompatibleBlockPointer;
-  }
-  else if (!Context.typesAreCompatible(lhptee, rhptee))
-    return IncompatibleBlockPointer;
   return ConvTy;
 }
 
-/// CheckObjCPointerTypesForAssignment - Compares two objective-c pointer types
+/// checkObjCPointerTypesForAssignment - Compares two objective-c pointer types
 /// for assignment compatibility.
-Sema::AssignConvertType
-Sema::CheckObjCPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
+static Sema::AssignConvertType
+checkObjCPointerTypesForAssignment(Sema &S, QualType lhsType, QualType rhsType) {
+  assert(lhsType.isCanonical() && "LHS was not canonicalized!");
+  assert(rhsType.isCanonical() && "RHS was not canonicalized!");
+
   if (lhsType->isObjCBuiltinType()) {
     // Class is not compatible with ObjC object pointers.
     if (lhsType->isObjCClassType() && !rhsType->isObjCBuiltinType() &&
         !rhsType->isObjCQualifiedClassType())
-      return IncompatiblePointer;
-    return Compatible;
+      return Sema::IncompatiblePointer;
+    return Sema::Compatible;
   }
   if (rhsType->isObjCBuiltinType()) {
     // Class is not compatible with ObjC object pointers.
     if (rhsType->isObjCClassType() && !lhsType->isObjCBuiltinType() &&
         !lhsType->isObjCQualifiedClassType())
-      return IncompatiblePointer;
-    return Compatible;
+      return Sema::IncompatiblePointer;
+    return Sema::Compatible;
   }
   QualType lhptee =
   lhsType->getAs<ObjCObjectPointerType>()->getPointeeType();
   QualType rhptee =
   rhsType->getAs<ObjCObjectPointerType>()->getPointeeType();
-  // make sure we operate on the canonical type
-  lhptee = Context.getCanonicalType(lhptee);
-  rhptee = Context.getCanonicalType(rhptee);
+
   if (!lhptee.isAtLeastAsQualifiedAs(rhptee))
-    return CompatiblePointerDiscardsQualifiers;
+    return Sema::CompatiblePointerDiscardsQualifiers;
 
-  if (Context.typesAreCompatible(lhsType, rhsType))
-    return Compatible;
+  if (S.Context.typesAreCompatible(lhsType, rhsType))
+    return Sema::Compatible;
   if (lhsType->isObjCQualifiedIdType() || rhsType->isObjCQualifiedIdType())
-    return IncompatibleObjCQualifiedId;
-  return IncompatiblePointer;
+    return Sema::IncompatibleObjCQualifiedId;
+  return Sema::IncompatiblePointer;
+}
+
+Sema::AssignConvertType
+Sema::CheckAssignmentConstraints(SourceLocation Loc,
+                                 QualType lhsType, QualType rhsType) {
+  // Fake up an opaque expression.  We don't actually care about what
+  // cast operations are required, so if CheckAssignmentConstraints
+  // adds casts to this they'll be wasted, but fortunately that doesn't
+  // usually happen on valid code.
+  OpaqueValueExpr rhs(Loc, rhsType, VK_RValue);
+  Expr *rhsPtr = &rhs;
+  CastKind K = CK_Invalid;
+
+  return CheckAssignmentConstraints(lhsType, rhsPtr, K);
 }
 
 /// CheckAssignmentConstraints (C99 6.5.16) - This routine currently
@@ -4720,21 +5898,21 @@ Sema::CheckObjCPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
 /// As a result, the code for dealing with pointers is more complex than the
 /// C99 spec dictates.
 ///
+/// Sets 'Kind' for any result kind except Incompatible.
 Sema::AssignConvertType
-Sema::CheckAssignmentConstraints(QualType lhsType, QualType rhsType) {
+Sema::CheckAssignmentConstraints(QualType lhsType, Expr *&rhs,
+                                 CastKind &Kind) {
+  QualType rhsType = rhs->getType();
+
   // Get canonical types.  We're not formatting these types, just comparing
   // them.
   lhsType = Context.getCanonicalType(lhsType).getUnqualifiedType();
   rhsType = Context.getCanonicalType(rhsType).getUnqualifiedType();
 
-  if (lhsType == rhsType)
-    return Compatible; // Common case: fast path an exact match.
-
-  if ((lhsType->isObjCClassType() &&
-       (rhsType.getDesugaredType() == Context.ObjCClassRedefinitionType)) ||
-     (rhsType->isObjCClassType() &&
-       (lhsType.getDesugaredType() == Context.ObjCClassRedefinitionType))) {
-      return Compatible;
+  // Common case: no conversion required.
+  if (lhsType == rhsType) {
+    Kind = CK_NoOp;
+    return Compatible;
   }
 
   // If the left-hand side is a reference type, then we are in a
@@ -4745,144 +5923,220 @@ Sema::CheckAssignmentConstraints(QualType lhsType, QualType rhsType) {
   // lhsType so that the resulting expression does not have reference
   // type.
   if (const ReferenceType *lhsTypeRef = lhsType->getAs<ReferenceType>()) {
-    if (Context.typesAreCompatible(lhsTypeRef->getPointeeType(), rhsType))
+    if (Context.typesAreCompatible(lhsTypeRef->getPointeeType(), rhsType)) {
+      Kind = CK_LValueBitCast;
       return Compatible;
+    }
     return Incompatible;
   }
+
   // Allow scalar to ExtVector assignments, and assignments of an ExtVector type
   // to the same ExtVector type.
   if (lhsType->isExtVectorType()) {
     if (rhsType->isExtVectorType())
-      return lhsType == rhsType ? Compatible : Incompatible;
-    if (rhsType->isArithmeticType())
+      return Incompatible;
+    if (rhsType->isArithmeticType()) {
+      // CK_VectorSplat does T -> vector T, so first cast to the
+      // element type.
+      QualType elType = cast<ExtVectorType>(lhsType)->getElementType();
+      if (elType != rhsType) {
+        Kind = PrepareScalarCast(*this, rhs, elType);
+        ImpCastExprToType(rhs, elType, Kind);
+      }
+      Kind = CK_VectorSplat;
       return Compatible;
+    }
   }
 
+  // Conversions to or from vector type.
   if (lhsType->isVectorType() || rhsType->isVectorType()) {
     if (lhsType->isVectorType() && rhsType->isVectorType()) {
+      // Allow assignments of an AltiVec vector type to an equivalent GCC
+      // vector type and vice versa
+      if (Context.areCompatibleVectorTypes(lhsType, rhsType)) {
+        Kind = CK_BitCast;
+        return Compatible;
+      }
+
       // If we are allowing lax vector conversions, and LHS and RHS are both
       // vectors, the total size only needs to be the same. This is a bitcast;
       // no bits are changed but the result type is different.
       if (getLangOptions().LaxVectorConversions &&
-         (Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType)))
+          (Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType))) {
+        Kind = CK_BitCast;
         return IncompatibleVectors;
-
-      // Allow assignments of an AltiVec vector type to an equivalent GCC
-      // vector type and vice versa
-      if (Context.areCompatibleVectorTypes(lhsType, rhsType))
-        return Compatible;
+      }
     }
     return Incompatible;
   }
 
+  // Arithmetic conversions.
   if (lhsType->isArithmeticType() && rhsType->isArithmeticType() &&
-      !(getLangOptions().CPlusPlus && lhsType->isEnumeralType()))
+      !(getLangOptions().CPlusPlus && lhsType->isEnumeralType())) {
+    Kind = PrepareScalarCast(*this, rhs, lhsType);
     return Compatible;
+  }
 
-  if (isa<PointerType>(lhsType)) {
-    if (rhsType->isIntegerType())
-      return IntToPointer;
+  // Conversions to normal pointers.
+  if (const PointerType *lhsPointer = dyn_cast<PointerType>(lhsType)) {
+    // U* -> T*
+    if (isa<PointerType>(rhsType)) {
+      Kind = CK_BitCast;
+      return checkPointerTypesForAssignment(*this, lhsType, rhsType);
+    }
 
-    if (isa<PointerType>(rhsType))
-      return CheckPointerTypesForAssignment(lhsType, rhsType);
+    // int -> T*
+    if (rhsType->isIntegerType()) {
+      Kind = CK_IntegralToPointer; // FIXME: null?
+      return IntToPointer;
+    }
 
-    // In general, C pointers are not compatible with ObjC object pointers.
+    // C pointers are not compatible with ObjC object pointers,
+    // with two exceptions:
     if (isa<ObjCObjectPointerType>(rhsType)) {
-      if (lhsType->isVoidPointerType()) // an exception to the rule.
+      //  - conversions to void*
+      if (lhsPointer->getPointeeType()->isVoidType()) {
+        Kind = CK_AnyPointerToObjCPointerCast;
         return Compatible;
+      }
+
+      //  - conversions from 'Class' to the redefinition type
+      if (rhsType->isObjCClassType() &&
+          Context.hasSameType(lhsType, Context.ObjCClassRedefinitionType)) {
+        Kind = CK_BitCast;
+        return Compatible;
+      }
+
+      Kind = CK_BitCast;
       return IncompatiblePointer;
     }
-    if (rhsType->getAs<BlockPointerType>()) {
-      if (lhsType->getAs<PointerType>()->getPointeeType()->isVoidType())
-        return Compatible;
 
-      // Treat block pointers as objects.
-      if (getLangOptions().ObjC1 && lhsType->isObjCIdType())
+    // U^ -> void*
+    if (rhsType->getAs<BlockPointerType>()) {
+      if (lhsPointer->getPointeeType()->isVoidType()) {
+        Kind = CK_BitCast;
         return Compatible;
+      }
     }
+
     return Incompatible;
   }
 
+  // Conversions to block pointers.
   if (isa<BlockPointerType>(lhsType)) {
-    if (rhsType->isIntegerType())
+    // U^ -> T^
+    if (rhsType->isBlockPointerType()) {
+      Kind = CK_AnyPointerToBlockPointerCast;
+      return checkBlockPointerTypesForAssignment(*this, lhsType, rhsType);
+    }
+
+    // int or null -> T^
+    if (rhsType->isIntegerType()) {
+      Kind = CK_IntegralToPointer; // FIXME: null
       return IntToBlockPointer;
+    }
 
-    // Treat block pointers as objects.
-    if (getLangOptions().ObjC1 && rhsType->isObjCIdType())
+    // id -> T^
+    if (getLangOptions().ObjC1 && rhsType->isObjCIdType()) {
+      Kind = CK_AnyPointerToBlockPointerCast;
       return Compatible;
+    }
 
-    if (rhsType->isBlockPointerType())
-      return CheckBlockPointerTypesForAssignment(lhsType, rhsType);
-
-    if (const PointerType *RHSPT = rhsType->getAs<PointerType>()) {
-      if (RHSPT->getPointeeType()->isVoidType())
+    // void* -> T^
+    if (const PointerType *RHSPT = rhsType->getAs<PointerType>())
+      if (RHSPT->getPointeeType()->isVoidType()) {
+        Kind = CK_AnyPointerToBlockPointerCast;
         return Compatible;
-    }
+      }
+
     return Incompatible;
   }
 
+  // Conversions to Objective-C pointers.
   if (isa<ObjCObjectPointerType>(lhsType)) {
-    if (rhsType->isIntegerType())
+    // A* -> B*
+    if (rhsType->isObjCObjectPointerType()) {
+      Kind = CK_BitCast;
+      return checkObjCPointerTypesForAssignment(*this, lhsType, rhsType);
+    }
+
+    // int or null -> A*
+    if (rhsType->isIntegerType()) {
+      Kind = CK_IntegralToPointer; // FIXME: null
       return IntToPointer;
+    }
 
-    // In general, C pointers are not compatible with ObjC object pointers.
+    // In general, C pointers are not compatible with ObjC object pointers,
+    // with two exceptions:
     if (isa<PointerType>(rhsType)) {
-      if (rhsType->isVoidPointerType()) // an exception to the rule.
+      //  - conversions from 'void*'
+      if (rhsType->isVoidPointerType()) {
+        Kind = CK_AnyPointerToObjCPointerCast;
         return Compatible;
-      return IncompatiblePointer;
-    }
-    if (rhsType->isObjCObjectPointerType()) {
-      return CheckObjCPointerTypesForAssignment(lhsType, rhsType);
-    }
-    if (const PointerType *RHSPT = rhsType->getAs<PointerType>()) {
-      if (RHSPT->getPointeeType()->isVoidType())
+      }
+
+      //  - conversions to 'Class' from its redefinition type
+      if (lhsType->isObjCClassType() &&
+          Context.hasSameType(rhsType, Context.ObjCClassRedefinitionType)) {
+        Kind = CK_BitCast;
         return Compatible;
+      }
+
+      Kind = CK_AnyPointerToObjCPointerCast;
+      return IncompatiblePointer;
     }
-    // Treat block pointers as objects.
-    if (rhsType->isBlockPointerType())
+
+    // T^ -> A*
+    if (rhsType->isBlockPointerType()) {
+      Kind = CK_AnyPointerToObjCPointerCast;
       return Compatible;
+    }
+
     return Incompatible;
   }
+
+  // Conversions from pointers that are not covered by the above.
   if (isa<PointerType>(rhsType)) {
-    // C99 6.5.16.1p1: the left operand is _Bool and the right is a pointer.
-    if (lhsType == Context.BoolTy)
+    // T* -> _Bool
+    if (lhsType == Context.BoolTy) {
+      Kind = CK_PointerToBoolean;
       return Compatible;
+    }
 
-    if (lhsType->isIntegerType())
+    // T* -> int
+    if (lhsType->isIntegerType()) {
+      Kind = CK_PointerToIntegral;
       return PointerToInt;
+    }
 
-    if (isa<PointerType>(lhsType))
-      return CheckPointerTypesForAssignment(lhsType, rhsType);
-
-    if (isa<BlockPointerType>(lhsType) &&
-        rhsType->getAs<PointerType>()->getPointeeType()->isVoidType())
-      return Compatible;
     return Incompatible;
   }
+
+  // Conversions from Objective-C pointers that are not covered by the above.
   if (isa<ObjCObjectPointerType>(rhsType)) {
-    // C99 6.5.16.1p1: the left operand is _Bool and the right is a pointer.
-    if (lhsType == Context.BoolTy)
+    // T* -> _Bool
+    if (lhsType == Context.BoolTy) {
+      Kind = CK_PointerToBoolean;
       return Compatible;
+    }
 
-    if (lhsType->isIntegerType())
+    // T* -> int
+    if (lhsType->isIntegerType()) {
+      Kind = CK_PointerToIntegral;
       return PointerToInt;
-
-    // In general, C pointers are not compatible with ObjC object pointers.
-    if (isa<PointerType>(lhsType)) {
-      if (lhsType->isVoidPointerType()) // an exception to the rule.
-        return Compatible;
-      return IncompatiblePointer;
     }
-    if (isa<BlockPointerType>(lhsType) &&
-        rhsType->getAs<PointerType>()->getPointeeType()->isVoidType())
-      return Compatible;
+
     return Incompatible;
   }
 
+  // struct A -> struct B
   if (isa<TagType>(lhsType) && isa<TagType>(rhsType)) {
-    if (Context.typesAreCompatible(lhsType, rhsType))
+    if (Context.typesAreCompatible(lhsType, rhsType)) {
+      Kind = CK_NoOp;
       return Compatible;
+    }
   }
+
   return Incompatible;
 }
 
@@ -4902,7 +6156,7 @@ static void ConstructTransparentUnion(ASTContext &C, Expr *&E,
   // union type from this initializer list.
   TypeSourceInfo *unionTInfo = C.getTrivialTypeSourceInfo(UnionType);
   E = new (C) CompoundLiteralExpr(SourceLocation(), unionTInfo, UnionType,
-                                  Initializer, false);
+                                  VK_RValue, Initializer, false);
 }
 
 Sema::AssignConvertType
@@ -4935,14 +6189,18 @@ Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType, Expr *&rExpr) {
 
       if (rExpr->isNullPointerConstant(Context,
                                        Expr::NPC_ValueDependentIsNull)) {
-        ImpCastExprToType(rExpr, it->getType(), CK_IntegralToPointer);
+        ImpCastExprToType(rExpr, it->getType(), CK_NullToPointer);
         InitField = *it;
         break;
       }
     }
 
-    if (CheckAssignmentConstraints(it->getType(), rExpr->getType())
+    Expr *rhs = rExpr;
+    CastKind Kind = CK_Invalid;
+    if (CheckAssignmentConstraints(it->getType(), rhs, Kind)
           == Compatible) {
+      ImpCastExprToType(rhs, it->getType(), Kind);
+      rExpr = rhs;
       InitField = *it;
       break;
     }
@@ -4970,7 +6228,7 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
 
     // FIXME: Currently, we fall through and treat C++ classes like C
     // structures.
-  }
+  }  
 
   // C99 6.5.16.1p1: the left operand is a pointer and the right is
   // a null pointer constant.
@@ -4979,7 +6237,7 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
        lhsType->isBlockPointerType())
       && rExpr->isNullPointerConstant(Context,
                                       Expr::NPC_ValueDependentIsNull)) {
-    ImpCastExprToType(rExpr, lhsType, CK_Unknown);
+    ImpCastExprToType(rExpr, lhsType, CK_NullToPointer);
     return Compatible;
   }
 
@@ -4992,8 +6250,9 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
   if (!lhsType->isReferenceType())
     DefaultFunctionArrayLvalueConversion(rExpr);
 
+  CastKind Kind = CK_Invalid;
   Sema::AssignConvertType result =
-    CheckAssignmentConstraints(lhsType, rExpr->getType());
+    CheckAssignmentConstraints(lhsType, rExpr, Kind);
 
   // C99 6.5.16.1p2: The value of the right operand is converted to the
   // type of the assignment expression.
@@ -5002,8 +6261,7 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
   // The getNonReferenceType() call makes sure that the resulting expression
   // does not have reference type.
   if (result != Incompatible && rExpr->getType() != lhsType)
-    ImpCastExprToType(rExpr, lhsType.getNonLValueExprType(Context),
-                      CK_Unknown);
+    ImpCastExprToType(rExpr, lhsType.getNonLValueExprType(Context), Kind);
   return result;
 }
 
@@ -5071,16 +6329,22 @@ QualType Sema::CheckVectorOperands(SourceLocation Loc, Expr *&lex, Expr *&rex) {
   if (const ExtVectorType *LV = lhsType->getAs<ExtVectorType>()) {
     QualType EltTy = LV->getElementType();
     if (EltTy->isIntegralType(Context) && rhsType->isIntegralType(Context)) {
-      if (Context.getIntegerTypeOrder(EltTy, rhsType) >= 0) {
-        ImpCastExprToType(rex, lhsType, CK_IntegralCast);
+      int order = Context.getIntegerTypeOrder(EltTy, rhsType);
+      if (order > 0)
+        ImpCastExprToType(rex, EltTy, CK_IntegralCast);
+      if (order >= 0) {
+        ImpCastExprToType(rex, lhsType, CK_VectorSplat);
         if (swapped) std::swap(rex, lex);
         return lhsType;
       }
     }
     if (EltTy->isRealFloatingType() && rhsType->isScalarType() &&
         rhsType->isRealFloatingType()) {
-      if (Context.getFloatingTypeOrder(EltTy, rhsType) >= 0) {
-        ImpCastExprToType(rex, lhsType, CK_FloatingCast);
+      int order = Context.getFloatingTypeOrder(EltTy, rhsType);
+      if (order > 0)
+        ImpCastExprToType(rex, EltTy, CK_FloatingCast);
+      if (order >= 0) {
+        ImpCastExprToType(rex, lhsType, CK_VectorSplat);
         if (swapped) std::swap(rex, lex);
         return lhsType;
       }
@@ -5359,6 +6623,12 @@ QualType Sema::CheckSubtractionOperands(Expr *&lex, Expr *&rex,
   return InvalidOperands(Loc, lex, rex);
 }
 
+static bool isScopedEnumerationType(QualType T) {
+  if (const EnumType *ET = dyn_cast<EnumType>(T))
+    return ET->getDecl()->isScoped();
+  return false;
+}
+
 // C99 6.5.7
 QualType Sema::CheckShiftOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
                                   bool isCompAssign) {
@@ -5367,21 +6637,28 @@ QualType Sema::CheckShiftOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
       !rex->getType()->hasIntegerRepresentation())
     return InvalidOperands(Loc, lex, rex);
 
+  // C++0x: Don't allow scoped enums. FIXME: Use something better than
+  // hasIntegerRepresentation() above instead of this.
+  if (isScopedEnumerationType(lex->getType()) ||
+      isScopedEnumerationType(rex->getType())) {
+    return InvalidOperands(Loc, lex, rex);
+  }
+
   // Vector shifts promote their scalar inputs to vector type.
   if (lex->getType()->isVectorType() || rex->getType()->isVectorType())
     return CheckVectorOperands(Loc, lex, rex);
 
   // Shifts don't perform usual arithmetic conversions, they just do integer
   // promotions on each operand. C99 6.5.7p3
-  QualType LHSTy = Context.isPromotableBitField(lex);
-  if (LHSTy.isNull()) {
-    LHSTy = lex->getType();
-    if (LHSTy->isPromotableIntegerType())
-      LHSTy = Context.getPromotedIntegerType(LHSTy);
-  }
-  if (!isCompAssign)
-    ImpCastExprToType(lex, LHSTy, CK_IntegralCast);
 
+  // For the LHS, do usual unary conversions, but then reset them away
+  // if this is a compound assignment.
+  Expr *old_lex = lex;
+  UsualUnaryConversions(lex);
+  QualType LHSTy = lex->getType();
+  if (isCompAssign) lex = old_lex;
+
+  // The RHS is simpler.
   UsualUnaryConversions(rex);
 
   // Sanity-check shift operands
@@ -5425,8 +6702,28 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
   QualType lType = lex->getType();
   QualType rType = rex->getType();
 
+  Expr *LHSStripped = lex->IgnoreParenImpCasts();
+  Expr *RHSStripped = rex->IgnoreParenImpCasts();
+  QualType LHSStrippedType = LHSStripped->getType();
+  QualType RHSStrippedType = RHSStripped->getType();
+
+  // Two different enums will raise a warning when compared.
+  if (const EnumType *LHSEnumType = LHSStrippedType->getAs<EnumType>()) {
+    if (const EnumType *RHSEnumType = RHSStrippedType->getAs<EnumType>()) {
+      if (LHSEnumType->getDecl()->getIdentifier() &&
+          RHSEnumType->getDecl()->getIdentifier() &&
+          !Context.hasSameUnqualifiedType(LHSStrippedType, RHSStrippedType)) {
+        Diag(Loc, diag::warn_comparison_of_mixed_enum_types)
+          << LHSStrippedType << RHSStrippedType
+          << lex->getSourceRange() << rex->getSourceRange();
+      }
+    }
+  }
+
   if (!lType->hasFloatingRepresentation() &&
-      !(lType->isBlockPointerType() && isRelational)) {
+      !(lType->isBlockPointerType() && isRelational) &&
+      !lex->getLocStart().isMacroID() &&
+      !rex->getLocStart().isMacroID()) {
     // For non-floating point types, check for self-comparisons of the form
     // x == x, x != x, x < x, etc.  These always evaluate to a constant, and
     // often indicate logic errors in the program.
@@ -5437,11 +6734,9 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
     // obvious cases in the definition of the template anyways. The idea is to
     // warn when the typed comparison operator will always evaluate to the same
     // result.
-    Expr *LHSStripped = lex->IgnoreParens();
-    Expr *RHSStripped = rex->IgnoreParens();
     if (DeclRefExpr* DRL = dyn_cast<DeclRefExpr>(LHSStripped)) {
       if (DeclRefExpr* DRR = dyn_cast<DeclRefExpr>(RHSStripped)) {
-        if (DRL->getDecl() == DRR->getDecl() && !Loc.isMacroID() &&
+        if (DRL->getDecl() == DRR->getDecl() &&
             !IsWithinTemplateSpecialization(DRL->getDecl())) {
           DiagRuntimeBehavior(Loc, PDiag(diag::warn_comparison_always)
                               << 0 // self-
@@ -5525,7 +6820,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
   rType = rex->getType();
 
   // The result of comparisons is 'bool' in C++, 'int' in C.
-  QualType ResultTy = getLangOptions().CPlusPlus ? Context.BoolTy:Context.IntTy;
+  QualType ResultTy = Context.getLogicalOperationType();
 
   if (isRelational) {
     if (lType->isRealType() && rType->isRealType())
@@ -5576,6 +6871,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
           return ResultTy;
         }
       }
+
       // C++ [expr.rel]p2:
       //   [...] Pointer conversions (4.10) and qualification
       //   conversions (4.4) are performed on pointer operands (or on
@@ -5629,24 +6925,28 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
   }
 
   if (getLangOptions().CPlusPlus) {
+    // Comparison of nullptr_t with itself.
+    if (lType->isNullPtrType() && rType->isNullPtrType())
+      return ResultTy;
+    
     // Comparison of pointers with null pointer constants and equality
     // comparisons of member pointers to null pointer constants.
     if (RHSIsNull &&
-        (lType->isPointerType() ||
+        ((lType->isPointerType() || lType->isNullPtrType()) ||
          (!isRelational && lType->isMemberPointerType()))) {
       ImpCastExprToType(rex, lType, 
                         lType->isMemberPointerType()
                           ? CK_NullToMemberPointer
-                          : CK_IntegralToPointer);
+                          : CK_NullToPointer);
       return ResultTy;
     }
     if (LHSIsNull &&
-        (rType->isPointerType() ||
+        ((rType->isPointerType() || rType->isNullPtrType()) ||
          (!isRelational && rType->isMemberPointerType()))) {
       ImpCastExprToType(lex, rType, 
                         rType->isMemberPointerType()
                           ? CK_NullToMemberPointer
-                          : CK_IntegralToPointer);
+                          : CK_NullToPointer);
       return ResultTy;
     }
 
@@ -5681,10 +6981,6 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
       ImpCastExprToType(rex, T, CK_BitCast);
       return ResultTy;
     }
-
-    // Comparison of nullptr_t with itself.
-    if (lType->isNullPtrType() && rType->isNullPtrType())
-      return ResultTy;
   }
 
   // Handle block pointer types.
@@ -5765,21 +7061,23 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
     }
     
     if (lType->isIntegerType())
-      ImpCastExprToType(lex, rType, CK_IntegralToPointer);
+      ImpCastExprToType(lex, rType,
+                        LHSIsNull ? CK_NullToPointer : CK_IntegralToPointer);
     else
-      ImpCastExprToType(rex, lType, CK_IntegralToPointer);
+      ImpCastExprToType(rex, lType,
+                        RHSIsNull ? CK_NullToPointer : CK_IntegralToPointer);
     return ResultTy;
   }
   
   // Handle block pointers.
   if (!isRelational && RHSIsNull
       && lType->isBlockPointerType() && rType->isIntegerType()) {
-    ImpCastExprToType(rex, lType, CK_IntegralToPointer);
+    ImpCastExprToType(rex, lType, CK_NullToPointer);
     return ResultTy;
   }
   if (!isRelational && LHSIsNull
       && lType->isIntegerType() && rType->isBlockPointerType()) {
-    ImpCastExprToType(lex, rType, CK_IntegralToPointer);
+    ImpCastExprToType(lex, rType, CK_NullToPointer);
     return ResultTy;
   }
   return InvalidOperands(Loc, lex, rex);
@@ -5798,6 +7096,11 @@ QualType Sema::CheckVectorCompareOperands(Expr *&lex, Expr *&rex,
   if (vType.isNull())
     return vType;
 
+  // If AltiVec, the comparison results in a numeric type, i.e.
+  // bool for C++, int for C
+  if (getLangOptions().AltiVec)
+    return Context.getLogicalOperationType();
+
   QualType lType = lex->getType();
   QualType rType = rex->getType();
 
@@ -5851,7 +7154,8 @@ inline QualType Sema::CheckBitwiseOperands(
 
   QualType compType = UsualArithmeticConversions(lex, rex, isCompAssign);
 
-  if (lex->getType()->isIntegerType() && rex->getType()->isIntegerType())
+  if (lex->getType()->isIntegralOrUnscopedEnumerationType() &&
+      rex->getType()->isIntegralOrUnscopedEnumerationType())
     return compType;
   return InvalidOperands(Loc, lex, rex);
 }
@@ -5912,14 +7216,18 @@ inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
 static bool IsReadonlyProperty(Expr *E, Sema &S) {
   if (E->getStmtClass() == Expr::ObjCPropertyRefExprClass) {
     const ObjCPropertyRefExpr* PropExpr = cast<ObjCPropertyRefExpr>(E);
-    if (ObjCPropertyDecl *PDecl = PropExpr->getProperty()) {
-      QualType BaseType = PropExpr->getBase()->getType();
-      if (const ObjCObjectPointerType *OPT =
-            BaseType->getAsObjCInterfacePointerType())
-        if (ObjCInterfaceDecl *IFace = OPT->getInterfaceDecl())
-          if (S.isPropertyReadonly(PDecl, IFace))
-            return true;
-    }
+    if (PropExpr->isImplicitProperty()) return false;
+
+    ObjCPropertyDecl *PDecl = PropExpr->getExplicitProperty();
+    QualType BaseType = PropExpr->isSuperReceiver() ? 
+                            PropExpr->getSuperReceiverType() :  
+                            PropExpr->getBase()->getType();
+      
+    if (const ObjCObjectPointerType *OPT =
+          BaseType->getAsObjCInterfacePointerType())
+      if (ObjCInterfaceDecl *IFace = OPT->getInterfaceDecl())
+        if (S.isPropertyReadonly(PDecl, IFace))
+          return true;
   }
   return false;
 }
@@ -6005,17 +7313,8 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
   if (CompoundType.isNull()) {
     QualType LHSTy(LHSType);
     // Simple assignment "x = y".
-    if (const ObjCImplicitSetterGetterRefExpr *OISGE = 
-        dyn_cast<ObjCImplicitSetterGetterRefExpr>(LHS)) {
-      // If using property-dot syntax notation for assignment, and there is a
-      // setter, RHS expression is being passed to the setter argument. So,
-      // type conversion (and comparison) is RHS to setter's argument type.
-      if (const ObjCMethodDecl *SetterMD = OISGE->getSetterMethod()) {
-        ObjCMethodDecl::param_iterator P = SetterMD->param_begin();
-        LHSTy = (*P)->getType();
-      }
-    }
-    
+    if (LHS->getObjectKind() == OK_ObjCProperty)
+      ConvertPropertyForLValue(LHS, RHS, LHSTy);
     ConvTy = CheckSingleAssignmentConstraints(LHSTy, RHS);
     // Special case of NSObject attributes on c-style pointer types.
     if (ConvTy == IncompatiblePointer &&
@@ -6025,6 +7324,12 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
           LHSType->isObjCObjectPointerType())))
       ConvTy = Compatible;
 
+    if (ConvTy == Compatible &&
+        getLangOptions().ObjCNonFragileABI &&
+        LHSType->isObjCObjectType())
+      Diag(Loc, diag::err_assignment_requires_nonfragile_object)
+        << LHSType;
+
     // If the RHS is a unary plus or minus, check to see if they = and + are
     // right next to each other.  If so, the user may have typo'd "x =+ 4"
     // instead of "x += 4".
@@ -6048,7 +7353,7 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
     }
   } else {
     // Compound assignment "x += y"
-    ConvTy = CheckAssignmentConstraints(LHSType, RHSType);
+    ConvTy = CheckAssignmentConstraints(Loc, LHSType, RHSType);
   }
 
   if (DiagnoseAssignmentResult(ConvTy, Loc, LHSType, RHSType,
@@ -6072,6 +7377,11 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
     Diag(UO->getOperatorLoc(), diag::note_indirection_through_null);
   }
   
+  // Check for trivial buffer overflows.
+  if (const ArraySubscriptExpr *ae
+      = dyn_cast<ArraySubscriptExpr>(LHS->IgnoreParenCasts()))
+    CheckArrayAccess(ae);
+  
   // C99 6.5.16p3: The type of an assignment expression is the type of the
   // left operand unless the left operand has qualified type, in which case
   // it is the unqualified version of the type of the left operand.
@@ -6079,42 +7389,61 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
   // is converted to the type of the assignment expression (above).
   // C++ 5.17p1: the type of the assignment expression is that of its left
   // operand.
-  return LHSType.getUnqualifiedType();
+  return (getLangOptions().CPlusPlus
+          ? LHSType : LHSType.getUnqualifiedType());
 }
 
 // C99 6.5.17
-QualType Sema::CheckCommaOperands(Expr *LHS, Expr *&RHS, SourceLocation Loc) {
-  DiagnoseUnusedExprResult(LHS);
+static QualType CheckCommaOperands(Sema &S, Expr *&LHS, Expr *&RHS,
+                                   SourceLocation Loc) {
+  S.DiagnoseUnusedExprResult(LHS);
 
-  // Comma performs lvalue conversion (C99 6.3.2.1), but not unary conversions.
-  // C++ does not perform this conversion (C++ [expr.comma]p1).
-  if (!getLangOptions().CPlusPlus)
-    DefaultFunctionArrayLvalueConversion(RHS);
+  ExprResult LHSResult = S.CheckPlaceholderExpr(LHS, Loc);
+  if (LHSResult.isInvalid()) 
+    return QualType();
+
+  ExprResult RHSResult = S.CheckPlaceholderExpr(RHS, Loc);
+  if (RHSResult.isInvalid())
+    return QualType();
+  RHS = RHSResult.take();
 
-  // FIXME: Check that RHS type is complete in C mode (it's legal for it to be
-  // incomplete in C++).
+  // C's comma performs lvalue conversion (C99 6.3.2.1) on both its
+  // operands, but not unary promotions.
+  // C++'s comma does not do any conversions at all (C++ [expr.comma]p1).
+
+  // So we treat the LHS as a ignored value, and in C++ we allow the
+  // containing site to determine what should be done with the RHS.
+  S.IgnoredValueConversions(LHS);
+
+  if (!S.getLangOptions().CPlusPlus) {
+    S.DefaultFunctionArrayLvalueConversion(RHS);
+    if (!RHS->getType()->isVoidType())
+      S.RequireCompleteType(Loc, RHS->getType(), diag::err_incomplete_type);
+  }
 
   return RHS->getType();
 }
 
 /// CheckIncrementDecrementOperand - unlike most "Check" methods, this routine
 /// doesn't need to call UsualUnaryConversions or UsualArithmeticConversions.
-QualType Sema::CheckIncrementDecrementOperand(Expr *Op, SourceLocation OpLoc,
-                                              bool isInc, bool isPrefix) {
+static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
+                                               ExprValueKind &VK,
+                                               SourceLocation OpLoc,
+                                               bool isInc, bool isPrefix) {
   if (Op->isTypeDependent())
-    return Context.DependentTy;
+    return S.Context.DependentTy;
 
   QualType ResType = Op->getType();
   assert(!ResType.isNull() && "no type for increment/decrement expression");
 
-  if (getLangOptions().CPlusPlus && ResType->isBooleanType()) {
+  if (S.getLangOptions().CPlusPlus && ResType->isBooleanType()) {
     // Decrement of bool is not allowed.
     if (!isInc) {
-      Diag(OpLoc, diag::err_decrement_bool) << Op->getSourceRange();
+      S.Diag(OpLoc, diag::err_decrement_bool) << Op->getSourceRange();
       return QualType();
     }
     // Increment of bool sets it to true, but is deprecated.
-    Diag(OpLoc, diag::warn_increment_bool) << Op->getSourceRange();
+    S.Diag(OpLoc, diag::warn_increment_bool) << Op->getSourceRange();
   } else if (ResType->isRealType()) {
     // OK!
   } else if (ResType->isAnyPointerType()) {
@@ -6122,53 +7451,128 @@ QualType Sema::CheckIncrementDecrementOperand(Expr *Op, SourceLocation OpLoc,
 
     // C99 6.5.2.4p2, 6.5.6p2
     if (PointeeTy->isVoidType()) {
-      if (getLangOptions().CPlusPlus) {
-        Diag(OpLoc, diag::err_typecheck_pointer_arith_void_type)
+      if (S.getLangOptions().CPlusPlus) {
+        S.Diag(OpLoc, diag::err_typecheck_pointer_arith_void_type)
           << Op->getSourceRange();
         return QualType();
       }
 
       // Pointer to void is a GNU extension in C.
-      Diag(OpLoc, diag::ext_gnu_void_ptr) << Op->getSourceRange();
+      S.Diag(OpLoc, diag::ext_gnu_void_ptr) << Op->getSourceRange();
     } else if (PointeeTy->isFunctionType()) {
-      if (getLangOptions().CPlusPlus) {
-        Diag(OpLoc, diag::err_typecheck_pointer_arith_function_type)
+      if (S.getLangOptions().CPlusPlus) {
+        S.Diag(OpLoc, diag::err_typecheck_pointer_arith_function_type)
           << Op->getType() << Op->getSourceRange();
         return QualType();
       }
 
-      Diag(OpLoc, diag::ext_gnu_ptr_func_arith)
+      S.Diag(OpLoc, diag::ext_gnu_ptr_func_arith)
         << ResType << Op->getSourceRange();
-    } else if (RequireCompleteType(OpLoc, PointeeTy,
-                           PDiag(diag::err_typecheck_arithmetic_incomplete_type)
+    } else if (S.RequireCompleteType(OpLoc, PointeeTy,
+                 S.PDiag(diag::err_typecheck_arithmetic_incomplete_type)
                              << Op->getSourceRange()
                              << ResType))
       return QualType();
     // Diagnose bad cases where we step over interface counts.
-    else if (PointeeTy->isObjCObjectType() && LangOpts.ObjCNonFragileABI) {
-      Diag(OpLoc, diag::err_arithmetic_nonfragile_interface)
+    else if (PointeeTy->isObjCObjectType() && S.LangOpts.ObjCNonFragileABI) {
+      S.Diag(OpLoc, diag::err_arithmetic_nonfragile_interface)
         << PointeeTy << Op->getSourceRange();
       return QualType();
     }
   } else if (ResType->isAnyComplexType()) {
     // C99 does not support ++/-- on complex types, we allow as an extension.
-    Diag(OpLoc, diag::ext_integer_increment_complex)
+    S.Diag(OpLoc, diag::ext_integer_increment_complex)
       << ResType << Op->getSourceRange();
+  } else if (ResType->isPlaceholderType()) {
+    ExprResult PR = S.CheckPlaceholderExpr(Op, OpLoc);
+    if (PR.isInvalid()) return QualType();
+    return CheckIncrementDecrementOperand(S, PR.take(), VK, OpLoc,
+                                          isInc, isPrefix);
+  } else if (S.getLangOptions().AltiVec && ResType->isVectorType()) {
+    // OK! ( C/C++ Language Extensions for CBEA(Version 2.6) 10.3 )
   } else {
-    Diag(OpLoc, diag::err_typecheck_illegal_increment_decrement)
+    S.Diag(OpLoc, diag::err_typecheck_illegal_increment_decrement)
       << ResType << int(isInc) << Op->getSourceRange();
     return QualType();
   }
   // At this point, we know we have a real, complex or pointer type.
   // Now make sure the operand is a modifiable lvalue.
-  if (CheckForModifiableLvalue(Op, OpLoc, *this))
+  if (CheckForModifiableLvalue(Op, OpLoc, S))
     return QualType();
   // In C++, a prefix increment is the same type as the operand. Otherwise
   // (in C or with postfix), the increment is the unqualified type of the
   // operand.
-  return isPrefix && getLangOptions().CPlusPlus
-    ? ResType : ResType.getUnqualifiedType();
+  if (isPrefix && S.getLangOptions().CPlusPlus) {
+    VK = VK_LValue;
+    return ResType;
+  } else {
+    VK = VK_RValue;
+    return ResType.getUnqualifiedType();
+  }
+}
+
+void Sema::ConvertPropertyForRValue(Expr *&E) {
+  assert(E->getValueKind() == VK_LValue &&
+         E->getObjectKind() == OK_ObjCProperty);
+  const ObjCPropertyRefExpr *PRE = E->getObjCProperty();
+
+  ExprValueKind VK = VK_RValue;
+  if (PRE->isImplicitProperty()) {
+    if (const ObjCMethodDecl *GetterMethod = 
+          PRE->getImplicitPropertyGetter()) {
+      QualType Result = GetterMethod->getResultType();
+      VK = Expr::getValueKindForType(Result);
+    }
+    else {
+      Diag(PRE->getLocation(), diag::err_getter_not_found)
+            << PRE->getBase()->getType();
+    }
+  }
+
+  E = ImplicitCastExpr::Create(Context, E->getType(), CK_GetObjCProperty,
+                               E, 0, VK);
+  
+  ExprResult Result = MaybeBindToTemporary(E);
+  if (!Result.isInvalid())
+    E = Result.take();
+}
+
+void Sema::ConvertPropertyForLValue(Expr *&LHS, Expr *&RHS, QualType &LHSTy) {
+  assert(LHS->getValueKind() == VK_LValue &&
+         LHS->getObjectKind() == OK_ObjCProperty);
+  const ObjCPropertyRefExpr *PRE = LHS->getObjCProperty();
+
+  if (PRE->isImplicitProperty()) {
+    // If using property-dot syntax notation for assignment, and there is a
+    // setter, RHS expression is being passed to the setter argument. So,
+    // type conversion (and comparison) is RHS to setter's argument type.
+    if (const ObjCMethodDecl *SetterMD = PRE->getImplicitPropertySetter()) {
+      ObjCMethodDecl::param_iterator P = SetterMD->param_begin();
+      LHSTy = (*P)->getType();
+
+    // Otherwise, if the getter returns an l-value, just call that.
+    } else {
+      QualType Result = PRE->getImplicitPropertyGetter()->getResultType();
+      ExprValueKind VK = Expr::getValueKindForType(Result);
+      if (VK == VK_LValue) {
+        LHS = ImplicitCastExpr::Create(Context, LHS->getType(),
+                                       CK_GetObjCProperty, LHS, 0, VK);
+        return;
+      }
+    }
+  }
+
+  if (getLangOptions().CPlusPlus && LHSTy->isRecordType()) {
+    InitializedEntity Entity = 
+    InitializedEntity::InitializeParameter(Context, LHSTy);
+    Expr *Arg = RHS;
+    ExprResult ArgE = PerformCopyInitialization(Entity, SourceLocation(),
+                                                Owned(Arg));
+    if (!ArgE.isInvalid())
+      RHS = ArgE.takeAs<Expr>(); 
+  }
 }
+  
 
 /// getPrimaryDecl - Helper function for CheckAddressOfOperand().
 /// This routine allows us to typecheck complex/recursive expressions
@@ -6182,7 +7586,7 @@ QualType Sema::CheckIncrementDecrementOperand(Expr *Op, SourceLocation OpLoc,
 ///  - *(x + 1) -> x, if x is an array
 ///  - &"123"[2] -> 0
 ///  - & __real__ x -> x
-static NamedDecl *getPrimaryDecl(Expr *E) {
+static ValueDecl *getPrimaryDecl(Expr *E) {
   switch (E->getStmtClass()) {
   case Stmt::DeclRefExprClass:
     return cast<DeclRefExpr>(E)->getDecl();
@@ -6234,16 +7638,21 @@ static NamedDecl *getPrimaryDecl(Expr *E) {
 /// operator (C99 6.3.2.1p[2-4]), and its result is never an lvalue.
 /// In C++, the operand might be an overloaded function name, in which case
 /// we allow the '&' but retain the overloaded-function type.
-QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
+static QualType CheckAddressOfOperand(Sema &S, Expr *OrigOp,
+                                      SourceLocation OpLoc) {
   if (OrigOp->isTypeDependent())
-    return Context.DependentTy;
-  if (OrigOp->getType() == Context.OverloadTy)
-    return Context.OverloadTy;
+    return S.Context.DependentTy;
+  if (OrigOp->getType() == S.Context.OverloadTy)
+    return S.Context.OverloadTy;
+
+  ExprResult PR = S.CheckPlaceholderExpr(OrigOp, OpLoc);
+  if (PR.isInvalid()) return QualType();
+  OrigOp = PR.take();
 
   // Make sure to ignore parentheses in subsequent checks
   Expr *op = OrigOp->IgnoreParens();
 
-  if (getLangOptions().C99) {
+  if (S.getLangOptions().C99) {
     // Implement C99-only parts of addressof rules.
     if (UnaryOperator* uOp = dyn_cast<UnaryOperator>(op)) {
       if (uOp->getOpcode() == UO_Deref)
@@ -6254,24 +7663,25 @@ QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
     // Technically, there should be a check for array subscript
     // expressions here, but the result of one is always an lvalue anyway.
   }
-  NamedDecl *dcl = getPrimaryDecl(op);
-  Expr::isLvalueResult lval = op->isLvalue(Context);
+  ValueDecl *dcl = getPrimaryDecl(op);
+  Expr::LValueClassification lval = op->ClassifyLValue(S.Context);
 
   if (lval == Expr::LV_ClassTemporary) {
-    Diag(OpLoc, isSFINAEContext()? diag::err_typecheck_addrof_class_temporary
-                                 : diag::ext_typecheck_addrof_class_temporary)
+    bool sfinae = S.isSFINAEContext();
+    S.Diag(OpLoc, sfinae ? diag::err_typecheck_addrof_class_temporary
+                         : diag::ext_typecheck_addrof_class_temporary)
       << op->getType() << op->getSourceRange();
-    if (isSFINAEContext())
+    if (sfinae)
       return QualType();
   } else if (isa<ObjCSelectorExpr>(op)) {
-    return Context.getPointerType(op->getType());
+    return S.Context.getPointerType(op->getType());
   } else if (lval == Expr::LV_MemberFunction) {
     // If it's an instance method, make a member pointer.
     // The expression must have exactly the form &A::foo.
 
     // If the underlying expression isn't a decl ref, give up.
     if (!isa<DeclRefExpr>(op)) {
-      Diag(OpLoc, diag::err_invalid_form_pointer_member_function)
+      S.Diag(OpLoc, diag::err_invalid_form_pointer_member_function)
         << OrigOp->getSourceRange();
       return QualType();
     }
@@ -6280,45 +7690,41 @@ QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
 
     // The id-expression was parenthesized.
     if (OrigOp != DRE) {
-      Diag(OpLoc, diag::err_parens_pointer_member_function)
+      S.Diag(OpLoc, diag::err_parens_pointer_member_function)
         << OrigOp->getSourceRange();
 
     // The method was named without a qualifier.
     } else if (!DRE->getQualifier()) {
-      Diag(OpLoc, diag::err_unqualified_pointer_member_function)
+      S.Diag(OpLoc, diag::err_unqualified_pointer_member_function)
         << op->getSourceRange();
     }
 
-    return Context.getMemberPointerType(op->getType(),
-              Context.getTypeDeclType(MD->getParent()).getTypePtr());
+    return S.Context.getMemberPointerType(op->getType(),
+              S.Context.getTypeDeclType(MD->getParent()).getTypePtr());
   } else if (lval != Expr::LV_Valid && lval != Expr::LV_IncompleteVoidType) {
     // C99 6.5.3.2p1
     // The operand must be either an l-value or a function designator
     if (!op->getType()->isFunctionType()) {
       // FIXME: emit more specific diag...
-      Diag(OpLoc, diag::err_typecheck_invalid_lvalue_addrof)
+      S.Diag(OpLoc, diag::err_typecheck_invalid_lvalue_addrof)
         << op->getSourceRange();
       return QualType();
     }
-  } else if (op->getBitField()) { // C99 6.5.3.2p1
+  } else if (op->getObjectKind() == OK_BitField) { // C99 6.5.3.2p1
     // The operand cannot be a bit-field
-    Diag(OpLoc, diag::err_typecheck_address_of)
+    S.Diag(OpLoc, diag::err_typecheck_address_of)
       << "bit-field" << op->getSourceRange();
         return QualType();
-  } else if (op->refersToVectorElement()) {
+  } else if (op->getObjectKind() == OK_VectorComponent) {
     // The operand cannot be an element of a vector
-    Diag(OpLoc, diag::err_typecheck_address_of)
+    S.Diag(OpLoc, diag::err_typecheck_address_of)
       << "vector element" << op->getSourceRange();
     return QualType();
-  } else if (isa<ObjCPropertyRefExpr>(op)) {
+  } else if (op->getObjectKind() == OK_ObjCProperty) {
     // cannot take address of a property expression.
-    Diag(OpLoc, diag::err_typecheck_address_of)
+    S.Diag(OpLoc, diag::err_typecheck_address_of)
       << "property expression" << op->getSourceRange();
     return QualType();
-  } else if (ConditionalOperator *CO = dyn_cast<ConditionalOperator>(op)) {
-    // FIXME: Can LHS ever be null here?
-    if (!CheckAddressOfOperand(CO->getTrueExpr(), OpLoc).isNull())
-      return CheckAddressOfOperand(CO->getFalseExpr(), OpLoc);
   } else if (dcl) { // C99 6.5.3.2p1
     // We have an lvalue with a decl. Make sure the decl is not declared
     // with the register storage-class specifier.
@@ -6326,29 +7732,31 @@ QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
       // in C++ it is not error to take address of a register
       // variable (c++03 7.1.1P3)
       if (vd->getStorageClass() == SC_Register &&
-          !getLangOptions().CPlusPlus) {
-        Diag(OpLoc, diag::err_typecheck_address_of)
+          !S.getLangOptions().CPlusPlus) {
+        S.Diag(OpLoc, diag::err_typecheck_address_of)
           << "register variable" << op->getSourceRange();
         return QualType();
       }
     } else if (isa<FunctionTemplateDecl>(dcl)) {
-      return Context.OverloadTy;
-    } else if (FieldDecl *FD = dyn_cast<FieldDecl>(dcl)) {
+      return S.Context.OverloadTy;
+    } else if (isa<FieldDecl>(dcl) || isa<IndirectFieldDecl>(dcl)) {
       // Okay: we can take the address of a field.
       // Could be a pointer to member, though, if there is an explicit
       // scope qualifier for the class.
       if (isa<DeclRefExpr>(op) && cast<DeclRefExpr>(op)->getQualifier()) {
         DeclContext *Ctx = dcl->getDeclContext();
         if (Ctx && Ctx->isRecord()) {
-          if (FD->getType()->isReferenceType()) {
-            Diag(OpLoc,
-                 diag::err_cannot_form_pointer_to_member_of_reference_type)
-              << FD->getDeclName() << FD->getType();
+          if (dcl->getType()->isReferenceType()) {
+            S.Diag(OpLoc,
+                   diag::err_cannot_form_pointer_to_member_of_reference_type)
+              << dcl->getDeclName() << dcl->getType();
             return QualType();
           }
 
-          return Context.getMemberPointerType(op->getType(),
-                Context.getTypeDeclType(cast<RecordDecl>(Ctx)).getTypePtr());
+          while (cast<RecordDecl>(Ctx)->isAnonymousStructOrUnion())
+            Ctx = Ctx->getParent();
+          return S.Context.getMemberPointerType(op->getType(),
+                S.Context.getTypeDeclType(cast<RecordDecl>(Ctx)).getTypePtr());
         }
       }
     } else if (!isa<FunctionDecl>(dcl))
@@ -6359,21 +7767,22 @@ QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
     // Taking the address of a void variable is technically illegal, but we
     // allow it in cases which are otherwise valid.
     // Example: "extern void x; void* y = &x;".
-    Diag(OpLoc, diag::ext_typecheck_addrof_void) << op->getSourceRange();
+    S.Diag(OpLoc, diag::ext_typecheck_addrof_void) << op->getSourceRange();
   }
 
   // If the operand has type "type", the result has type "pointer to type".
   if (op->getType()->isObjCObjectType())
-    return Context.getObjCObjectPointerType(op->getType());
-  return Context.getPointerType(op->getType());
+    return S.Context.getObjCObjectPointerType(op->getType());
+  return S.Context.getPointerType(op->getType());
 }
 
 /// CheckIndirectionOperand - Type check unary indirection (prefix '*').
-QualType Sema::CheckIndirectionOperand(Expr *Op, SourceLocation OpLoc) {
+static QualType CheckIndirectionOperand(Sema &S, Expr *Op, ExprValueKind &VK,
+                                        SourceLocation OpLoc) {
   if (Op->isTypeDependent())
-    return Context.DependentTy;
+    return S.Context.DependentTy;
 
-  UsualUnaryConversions(Op);
+  S.UsualUnaryConversions(Op);
   QualType OpTy = Op->getType();
   QualType Result;
   
@@ -6386,12 +7795,26 @@ QualType Sema::CheckIndirectionOperand(Expr *Op, SourceLocation OpLoc) {
   else if (const ObjCObjectPointerType *OPT =
              OpTy->getAs<ObjCObjectPointerType>())
     Result = OPT->getPointeeType();
+  else {
+    ExprResult PR = S.CheckPlaceholderExpr(Op, OpLoc);
+    if (PR.isInvalid()) return QualType();
+    if (PR.take() != Op)
+      return CheckIndirectionOperand(S, PR.take(), VK, OpLoc);
+  }
 
   if (Result.isNull()) {
-    Diag(OpLoc, diag::err_typecheck_indirection_requires_pointer)
+    S.Diag(OpLoc, diag::err_typecheck_indirection_requires_pointer)
       << OpTy << Op->getSourceRange();
     return QualType();
   }
+
+  // Dereferences are usually l-values...
+  VK = VK_LValue;
+
+  // ...except that certain expressions are never l-values in C.
+  if (!S.getLangOptions().CPlusPlus &&
+      IsCForbiddenLValueType(S.Context, Result))
+    VK = VK_RValue;
   
   return Result;
 }
@@ -6457,25 +7880,67 @@ static inline UnaryOperatorKind ConvertTokenKindToUnaryOpcode(
   return Opc;
 }
 
+/// DiagnoseSelfAssignment - Emits a warning if a value is assigned to itself.
+/// This warning is only emitted for builtin assignment operations. It is also
+/// suppressed in the event of macro expansions.
+static void DiagnoseSelfAssignment(Sema &S, Expr *lhs, Expr *rhs,
+                                   SourceLocation OpLoc) {
+  if (!S.ActiveTemplateInstantiations.empty())
+    return;
+  if (OpLoc.isInvalid() || OpLoc.isMacroID())
+    return;
+  lhs = lhs->IgnoreParenImpCasts();
+  rhs = rhs->IgnoreParenImpCasts();
+  const DeclRefExpr *LeftDeclRef = dyn_cast<DeclRefExpr>(lhs);
+  const DeclRefExpr *RightDeclRef = dyn_cast<DeclRefExpr>(rhs);
+  if (!LeftDeclRef || !RightDeclRef ||
+      LeftDeclRef->getLocation().isMacroID() ||
+      RightDeclRef->getLocation().isMacroID())
+    return;
+  const ValueDecl *LeftDecl =
+    cast<ValueDecl>(LeftDeclRef->getDecl()->getCanonicalDecl());
+  const ValueDecl *RightDecl =
+    cast<ValueDecl>(RightDeclRef->getDecl()->getCanonicalDecl());
+  if (LeftDecl != RightDecl)
+    return;
+  if (LeftDecl->getType().isVolatileQualified())
+    return;
+  if (const ReferenceType *RefTy = LeftDecl->getType()->getAs<ReferenceType>())
+    if (RefTy->getPointeeType().isVolatileQualified())
+      return;
+
+  S.Diag(OpLoc, diag::warn_self_assignment)
+      << LeftDeclRef->getType()
+      << lhs->getSourceRange() << rhs->getSourceRange();
+}
+
 /// CreateBuiltinBinOp - Creates a new built-in binary operation with
 /// operator @p Opc at location @c TokLoc. This routine only supports
 /// built-in operations; ActOnBinOp handles overloaded operators.
 ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
-                                    unsigned Op,
+                                    BinaryOperatorKind Opc,
                                     Expr *lhs, Expr *rhs) {
   QualType ResultTy;     // Result type of the binary operator.
-  BinaryOperatorKind Opc = (BinaryOperatorKind) Op;
   // The following two variables are used for compound assignment operators
   QualType CompLHSTy;    // Type of LHS after promotions for computation
   QualType CompResultTy; // Type of computation result
+  ExprValueKind VK = VK_RValue;
+  ExprObjectKind OK = OK_Ordinary;
 
   switch (Opc) {
   case BO_Assign:
     ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, QualType());
+    if (getLangOptions().CPlusPlus &&
+        lhs->getObjectKind() != OK_ObjCProperty) {
+      VK = lhs->getValueKind();
+      OK = lhs->getObjectKind();
+    }
+    if (!ResultTy.isNull())
+      DiagnoseSelfAssignment(*this, lhs, rhs, OpLoc);
     break;
   case BO_PtrMemD:
   case BO_PtrMemI:
-    ResultTy = CheckPointerToMemberOperands(lhs, rhs, OpLoc,
+    ResultTy = CheckPointerToMemberOperands(lhs, rhs, VK, OpLoc,
                                             Opc == BO_PtrMemI);
     break;
   case BO_Mul:
@@ -6518,7 +7983,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   case BO_MulAssign:
   case BO_DivAssign:
     CompResultTy = CheckMultiplyDivideOperands(lhs, rhs, OpLoc, true,
-                                              Opc == BO_DivAssign);
+                                               Opc == BO_DivAssign);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
@@ -6555,22 +8020,26 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
   case BO_Comma:
-    ResultTy = CheckCommaOperands(lhs, rhs, OpLoc);
+    ResultTy = CheckCommaOperands(*this, lhs, rhs, OpLoc);
+    if (getLangOptions().CPlusPlus) {
+      VK = rhs->getValueKind();
+      OK = rhs->getObjectKind();
+    }
     break;
   }
   if (ResultTy.isNull())
     return ExprError();
-  if (ResultTy->isObjCObjectType() && LangOpts.ObjCNonFragileABI) {
-    if (Opc >= BO_Assign && Opc <= BO_OrAssign) 
-          Diag(OpLoc, diag::err_assignment_requires_nonfragile_object)
-                << ResultTy;
-  }
   if (CompResultTy.isNull())
-    return Owned(new (Context) BinaryOperator(lhs, rhs, Opc, ResultTy, OpLoc));
-  else
-    return Owned(new (Context) CompoundAssignOperator(lhs, rhs, Opc, ResultTy,
-                                                      CompLHSTy, CompResultTy,
-                                                      OpLoc));
+    return Owned(new (Context) BinaryOperator(lhs, rhs, Opc, ResultTy,
+                                              VK, OK, OpLoc));
+
+  if (getLangOptions().CPlusPlus && lhs->getObjectKind() != OK_ObjCProperty) {
+    VK = VK_LValue;
+    OK = lhs->getObjectKind();
+  }
+  return Owned(new (Context) CompoundAssignOperator(lhs, rhs, Opc, ResultTy,
+                                                    VK, OK, CompLHSTy,
+                                                    CompResultTy, OpLoc));
 }
 
 /// SuggestParentheses - Emit a diagnostic together with a fixit hint that wraps
@@ -6660,13 +8129,87 @@ static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperatorKind Opc,
                        rhs->getSourceRange());
 }
 
+/// \brief It accepts a '&&' expr that is inside a '||' one.
+/// Emit a diagnostic together with a fixit hint that wraps the '&&' expression
+/// in parentheses.
+static void
+EmitDiagnosticForLogicalAndInLogicalOr(Sema &Self, SourceLocation OpLoc,
+                                       Expr *E) {
+  assert(isa<BinaryOperator>(E) &&
+         cast<BinaryOperator>(E)->getOpcode() == BO_LAnd);
+  SuggestParentheses(Self, OpLoc,
+    Self.PDiag(diag::warn_logical_and_in_logical_or)
+        << E->getSourceRange(),
+    Self.PDiag(diag::note_logical_and_in_logical_or_silence),
+    E->getSourceRange(),
+    Self.PDiag(0), SourceRange());
+}
+
+/// \brief Returns true if the given expression can be evaluated as a constant
+/// 'true'.
+static bool EvaluatesAsTrue(Sema &S, Expr *E) {
+  bool Res;
+  return E->EvaluateAsBooleanCondition(Res, S.getASTContext()) && Res;
+}
+
+/// \brief Returns true if the given expression can be evaluated as a constant
+/// 'false'.
+static bool EvaluatesAsFalse(Sema &S, Expr *E) {
+  bool Res;
+  return E->EvaluateAsBooleanCondition(Res, S.getASTContext()) && !Res;
+}
+
+/// \brief Look for '&&' in the left hand of a '||' expr.
+static void DiagnoseLogicalAndInLogicalOrLHS(Sema &S, SourceLocation OpLoc,
+                                             Expr *OrLHS, Expr *OrRHS) {
+  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(OrLHS)) {
+    if (Bop->getOpcode() == BO_LAnd) {
+      // If it's "a && b || 0" don't warn since the precedence doesn't matter.
+      if (EvaluatesAsFalse(S, OrRHS))
+        return;
+      // If it's "1 && a || b" don't warn since the precedence doesn't matter.
+      if (!EvaluatesAsTrue(S, Bop->getLHS()))
+        return EmitDiagnosticForLogicalAndInLogicalOr(S, OpLoc, Bop);
+    } else if (Bop->getOpcode() == BO_LOr) {
+      if (BinaryOperator *RBop = dyn_cast<BinaryOperator>(Bop->getRHS())) {
+        // If it's "a || b && 1 || c" we didn't warn earlier for
+        // "a || b && 1", but warn now.
+        if (RBop->getOpcode() == BO_LAnd && EvaluatesAsTrue(S, RBop->getRHS()))
+          return EmitDiagnosticForLogicalAndInLogicalOr(S, OpLoc, RBop);
+      }
+    }
+  }
+}
+
+/// \brief Look for '&&' in the right hand of a '||' expr.
+static void DiagnoseLogicalAndInLogicalOrRHS(Sema &S, SourceLocation OpLoc,
+                                             Expr *OrLHS, Expr *OrRHS) {
+  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(OrRHS)) {
+    if (Bop->getOpcode() == BO_LAnd) {
+      // If it's "0 || a && b" don't warn since the precedence doesn't matter.
+      if (EvaluatesAsFalse(S, OrLHS))
+        return;
+      // If it's "a || b && 1" don't warn since the precedence doesn't matter.
+      if (!EvaluatesAsTrue(S, Bop->getRHS()))
+        return EmitDiagnosticForLogicalAndInLogicalOr(S, OpLoc, Bop);
+    }
+  }
+}
+
 /// DiagnoseBinOpPrecedence - Emit warnings for expressions with tricky
-/// precedence. This currently diagnoses only "arg1 'bitwise' arg2 'eq' arg3".
-/// But it could also warn about arg1 && arg2 || arg3, as GCC 4.3+ does.
+/// precedence.
 static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
                                     SourceLocation OpLoc, Expr *lhs, Expr *rhs){
+  // Diagnose "arg1 'bitwise' arg2 'eq' arg3".
   if (BinaryOperator::isBitwiseOp(Opc))
-    DiagnoseBitwisePrecedence(Self, Opc, OpLoc, lhs, rhs);
+    return DiagnoseBitwisePrecedence(Self, Opc, OpLoc, lhs, rhs);
+
+  // Warn about arg1 || arg2 && arg3, as GCC 4.3+ does.
+  // We don't warn for 'assert(a || b && "bad")' since this is safe.
+  if (Opc == BO_LOr && !OpLoc.isMacroID()/* Don't warn in macros. */) {
+    DiagnoseLogicalAndInLogicalOrLHS(Self, OpLoc, lhs, rhs);
+    DiagnoseLogicalAndInLogicalOrRHS(Self, OpLoc, lhs, rhs);
+  }
 }
 
 // Binary Operators.  'Tok' is the token for the operator.
@@ -6686,22 +8229,34 @@ ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
 ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
                             BinaryOperatorKind Opc,
                             Expr *lhs, Expr *rhs) {
-  if (getLangOptions().CPlusPlus &&
-      (lhs->getType()->isOverloadableType() ||
-       rhs->getType()->isOverloadableType())) {
-    // Find all of the overloaded operators visible from this
-    // point. We perform both an operator-name lookup from the local
-    // scope and an argument-dependent lookup based on the types of
-    // the arguments.
-    UnresolvedSet<16> Functions;
-    OverloadedOperatorKind OverOp = BinaryOperator::getOverloadedOperator(Opc);
-    if (S && OverOp != OO_None)
-      LookupOverloadedOperatorName(OverOp, S, lhs->getType(), rhs->getType(),
-                                   Functions);
+  if (getLangOptions().CPlusPlus) {
+    bool UseBuiltinOperator;
 
-    // Build the (potentially-overloaded, potentially-dependent)
-    // binary operation.
-    return CreateOverloadedBinOp(OpLoc, Opc, Functions, lhs, rhs);
+    if (lhs->isTypeDependent() || rhs->isTypeDependent()) {
+      UseBuiltinOperator = false;
+    } else if (Opc == BO_Assign && lhs->getObjectKind() == OK_ObjCProperty) {
+      UseBuiltinOperator = true;
+    } else {
+      UseBuiltinOperator = !lhs->getType()->isOverloadableType() &&
+                           !rhs->getType()->isOverloadableType();
+    }
+
+    if (!UseBuiltinOperator) {
+      // Find all of the overloaded operators visible from this
+      // point. We perform both an operator-name lookup from the local
+      // scope and an argument-dependent lookup based on the types of
+      // the arguments.
+      UnresolvedSet<16> Functions;
+      OverloadedOperatorKind OverOp
+        = BinaryOperator::getOverloadedOperator(Opc);
+      if (S && OverOp != OO_None)
+        LookupOverloadedOperatorName(OverOp, S, lhs->getType(), rhs->getType(),
+                                     Functions);
+
+      // Build the (potentially-overloaded, potentially-dependent)
+      // binary operation.
+      return CreateOverloadedBinOp(OpLoc, Opc, Functions, lhs, rhs);
+    }
   }
 
   // Build a built-in binary operation.
@@ -6709,28 +8264,28 @@ ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
 }
 
 ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
-                                                    unsigned OpcIn,
-                                                    Expr *Input) {
-  UnaryOperatorKind Opc = static_cast<UnaryOperatorKind>(OpcIn);
-
+                                      UnaryOperatorKind Opc,
+                                      Expr *Input) {
+  ExprValueKind VK = VK_RValue;
+  ExprObjectKind OK = OK_Ordinary;
   QualType resultType;
   switch (Opc) {
   case UO_PreInc:
   case UO_PreDec:
   case UO_PostInc:
   case UO_PostDec:
-    resultType = CheckIncrementDecrementOperand(Input, OpLoc,
+    resultType = CheckIncrementDecrementOperand(*this, Input, VK, OpLoc,
                                                 Opc == UO_PreInc ||
                                                 Opc == UO_PostInc,
                                                 Opc == UO_PreInc ||
                                                 Opc == UO_PreDec);
     break;
   case UO_AddrOf:
-    resultType = CheckAddressOfOperand(Input, OpLoc);
+    resultType = CheckAddressOfOperand(*this, Input, OpLoc);
     break;
   case UO_Deref:
     DefaultFunctionArrayLvalueConversion(Input);
-    resultType = CheckIndirectionOperand(Input, OpLoc);
+    resultType = CheckIndirectionOperand(*this, Input, VK, OpLoc);
     break;
   case UO_Plus:
   case UO_Minus:
@@ -6748,6 +8303,11 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
              Opc == UO_Plus &&
              resultType->isPointerType())
       break;
+    else if (resultType->isPlaceholderType()) {
+      ExprResult PR = CheckPlaceholderExpr(Input, OpLoc);
+      if (PR.isInvalid()) return ExprError();
+      return CreateBuiltinUnaryOp(OpLoc, Opc, PR.take());
+    }
 
     return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
       << resultType << Input->getSourceRange());
@@ -6761,9 +8321,16 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
       // C99 does not support '~' for complex conjugation.
       Diag(OpLoc, diag::ext_integer_complement_complex)
         << resultType << Input->getSourceRange();
-    else if (!resultType->hasIntegerRepresentation())
+    else if (resultType->hasIntegerRepresentation())
+      break;
+    else if (resultType->isPlaceholderType()) {
+      ExprResult PR = CheckPlaceholderExpr(Input, OpLoc);
+      if (PR.isInvalid()) return ExprError();
+      return CreateBuiltinUnaryOp(OpLoc, Opc, PR.take());
+    } else {
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
         << resultType << Input->getSourceRange());
+    }
     break;
   case UO_LNot: // logical negation
     // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
@@ -6771,25 +8338,40 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     resultType = Input->getType();
     if (resultType->isDependentType())
       break;
-    if (!resultType->isScalarType()) // C99 6.5.3.3p1
+    if (resultType->isScalarType()) { // C99 6.5.3.3p1
+      // ok, fallthrough
+    } else if (resultType->isPlaceholderType()) {
+      ExprResult PR = CheckPlaceholderExpr(Input, OpLoc);
+      if (PR.isInvalid()) return ExprError();
+      return CreateBuiltinUnaryOp(OpLoc, Opc, PR.take());
+    } else {
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
         << resultType << Input->getSourceRange());
+    }
+    
     // LNot always has type int. C99 6.5.3.3p5.
     // In C++, it's bool. C++ 5.3.1p8
-    resultType = getLangOptions().CPlusPlus ? Context.BoolTy : Context.IntTy;
+    resultType = Context.getLogicalOperationType();
     break;
   case UO_Real:
   case UO_Imag:
-    resultType = CheckRealImagOperand(Input, OpLoc, Opc == UO_Real);
+    resultType = CheckRealImagOperand(*this, Input, OpLoc, Opc == UO_Real);
+    // _Real and _Imag map ordinary l-values into ordinary l-values.
+    if (Input->getValueKind() != VK_RValue &&
+        Input->getObjectKind() == OK_Ordinary)
+      VK = Input->getValueKind();
     break;
   case UO_Extension:
     resultType = Input->getType();
+    VK = Input->getValueKind();
+    OK = Input->getObjectKind();
     break;
   }
   if (resultType.isNull())
     return ExprError();
 
-  return Owned(new (Context) UnaryOperator(Input, Opc, resultType, OpLoc));
+  return Owned(new (Context) UnaryOperator(Input, Opc, resultType,
+                                           VK, OK, OpLoc));
 }
 
 ExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
@@ -6815,24 +8397,16 @@ ExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
 
 // Unary Operators.  'Tok' is the token for the operator.
 ExprResult Sema::ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
-                                            tok::TokenKind Op, Expr *Input) {
+                              tok::TokenKind Op, Expr *Input) {
   return BuildUnaryOp(S, OpLoc, ConvertTokenKindToUnaryOpcode(Op), Input);
 }
 
 /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo".
-ExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc,
-                                            SourceLocation LabLoc,
-                                            IdentifierInfo *LabelII) {
-  // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[LabelII];
-
-  // If we haven't seen this label yet, create a forward reference. It
-  // will be validated and/or cleaned up in ActOnFinishFunctionBody.
-  if (LabelDecl == 0)
-    LabelDecl = new (Context) LabelStmt(LabLoc, LabelII, 0);
-
+ExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc, SourceLocation LabLoc,
+                                LabelDecl *TheDecl) {
+  TheDecl->setUsed();
   // Create the AST node.  The address of a label always has type 'void*'.
-  return Owned(new (Context) AddrLabelExpr(OpLoc, LabLoc, LabelDecl,
+  return Owned(new (Context) AddrLabelExpr(OpLoc, LabLoc, TheDecl,
                                        Context.getPointerType(Context.VoidTy)));
 }
 
@@ -6854,28 +8428,54 @@ Sema::ActOnStmtExpr(SourceLocation LPLoc, Stmt *SubStmt,
   // If there are sub stmts in the compound stmt, take the type of the last one
   // as the type of the stmtexpr.
   QualType Ty = Context.VoidTy;
-
+  bool StmtExprMayBindToTemp = false;
   if (!Compound->body_empty()) {
     Stmt *LastStmt = Compound->body_back();
+    LabelStmt *LastLabelStmt = 0;
     // If LastStmt is a label, skip down through into the body.
-    while (LabelStmt *Label = dyn_cast<LabelStmt>(LastStmt))
+    while (LabelStmt *Label = dyn_cast<LabelStmt>(LastStmt)) {
+      LastLabelStmt = Label;
       LastStmt = Label->getSubStmt();
+    }
+    if (Expr *LastExpr = dyn_cast<Expr>(LastStmt)) {
+      // Do function/array conversion on the last expression, but not
+      // lvalue-to-rvalue.  However, initialize an unqualified type.
+      DefaultFunctionArrayConversion(LastExpr);
+      Ty = LastExpr->getType().getUnqualifiedType();
 
-    if (Expr *LastExpr = dyn_cast<Expr>(LastStmt))
-      Ty = LastExpr->getType();
+      if (!Ty->isDependentType() && !LastExpr->isTypeDependent()) {
+        ExprResult Res = PerformCopyInitialization(
+                            InitializedEntity::InitializeResult(LPLoc, 
+                                                                Ty,
+                                                                false),
+                                                   SourceLocation(),
+                                                   Owned(LastExpr));
+        if (Res.isInvalid())
+          return ExprError();
+        if ((LastExpr = Res.takeAs<Expr>())) {
+          if (!LastLabelStmt)
+            Compound->setLastStmt(LastExpr);
+          else
+            LastLabelStmt->setSubStmt(LastExpr);
+          StmtExprMayBindToTemp = true;
+        }
+      }
+    }
   }
 
   // FIXME: Check that expression type is complete/non-abstract; statement
   // expressions are not lvalues.
-
-  return Owned(new (Context) StmtExpr(Compound, Ty, LPLoc, RPLoc));
+  Expr *ResStmtExpr = new (Context) StmtExpr(Compound, Ty, LPLoc, RPLoc);
+  if (StmtExprMayBindToTemp)
+    return MaybeBindToTemporary(ResStmtExpr);
+  return Owned(ResStmtExpr);
 }
 
 ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
-                                                  TypeSourceInfo *TInfo,
-                                                  OffsetOfComponent *CompPtr,
-                                                  unsigned NumComponents,
-                                                  SourceLocation RParenLoc) {
+                                      TypeSourceInfo *TInfo,
+                                      OffsetOfComponent *CompPtr,
+                                      unsigned NumComponents,
+                                      SourceLocation RParenLoc) {
   QualType ArgTy = TInfo->getType();
   bool Dependent = ArgTy->isDependentType();
   SourceRange TypeRange = TInfo->getTypeLoc().getLocalSourceRange();
@@ -6974,6 +8574,12 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
     LookupResult R(*this, OC.U.IdentInfo, OC.LocStart, LookupMemberName);
     LookupQualifiedName(R, RD);
     FieldDecl *MemberDecl = R.getAsSingle<FieldDecl>();
+    IndirectFieldDecl *IndirectMemberDecl = 0;
+    if (!MemberDecl) {
+      if ((IndirectMemberDecl = R.getAsSingle<IndirectFieldDecl>()))
+        MemberDecl = IndirectMemberDecl->getAnonField();
+    }
+
     if (!MemberDecl)
       return ExprError(Diag(BuiltinLoc, diag::err_no_member)
                        << OC.U.IdentInfo << RD << SourceRange(OC.LocStart, 
@@ -6992,12 +8598,8 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
     }
 
     RecordDecl *Parent = MemberDecl->getParent();
-    bool AnonStructUnion = Parent->isAnonymousStructOrUnion();
-    if (AnonStructUnion) {
-      do {
-        Parent = cast<RecordDecl>(Parent->getParent());
-      } while (Parent->isAnonymousStructOrUnion());
-    }
+    if (IndirectMemberDecl)
+      Parent = cast<RecordDecl>(IndirectMemberDecl->getDeclContext());
 
     // If the member was found in a base class, introduce OffsetOfNodes for
     // the base class indirections.
@@ -7010,15 +8612,17 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
         Comps.push_back(OffsetOfNode(B->Base));
     }
 
-    if (AnonStructUnion) {
-      llvm::SmallVector<FieldDecl*, 4> Path;
-      BuildAnonymousStructUnionMemberPath(MemberDecl, Path);
-      unsigned n = Path.size();
-      for (int j = n - 1; j > -1; --j)
-        Comps.push_back(OffsetOfNode(OC.LocStart, Path[j], OC.LocEnd));
-    } else {
+    if (IndirectMemberDecl) {
+      for (IndirectFieldDecl::chain_iterator FI =
+           IndirectMemberDecl->chain_begin(),
+           FEnd = IndirectMemberDecl->chain_end(); FI != FEnd; FI++) {
+        assert(isa<FieldDecl>(*FI));
+        Comps.push_back(OffsetOfNode(OC.LocStart,
+                                     cast<FieldDecl>(*FI), OC.LocEnd));
+      }
+    } else
       Comps.push_back(OffsetOfNode(OC.LocStart, MemberDecl, OC.LocEnd));
-    }
+
     CurrentType = MemberDecl->getType().getNonReferenceType(); 
   }
   
@@ -7028,13 +8632,13 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
 }
 
 ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
-                                                  SourceLocation BuiltinLoc,
-                                                  SourceLocation TypeLoc,
-                                                  ParsedType argty,
-                                                  OffsetOfComponent *CompPtr,
-                                                  unsigned NumComponents,
-                                                  SourceLocation RPLoc) {
-
+                                      SourceLocation BuiltinLoc,
+                                      SourceLocation TypeLoc,
+                                      ParsedType argty,
+                                      OffsetOfComponent *CompPtr,
+                                      unsigned NumComponents,
+                                      SourceLocation RPLoc) {
+  
   TypeSourceInfo *ArgTInfo;
   QualType ArgTy = GetTypeFromParser(argty, &ArgTInfo);
   if (ArgTy.isNull())
@@ -7048,41 +8652,14 @@ ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
 }
 
 
-ExprResult Sema::ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                                      ParsedType arg1,ParsedType arg2,
-                                                      SourceLocation RPLoc) {
-  TypeSourceInfo *argTInfo1;
-  QualType argT1 = GetTypeFromParser(arg1, &argTInfo1);
-  TypeSourceInfo *argTInfo2;
-  QualType argT2 = GetTypeFromParser(arg2, &argTInfo2);
-
-  assert((!argT1.isNull() && !argT2.isNull()) && "Missing type argument(s)");
-
-  return BuildTypesCompatibleExpr(BuiltinLoc, argTInfo1, argTInfo2, RPLoc);
-}
-
-ExprResult
-Sema::BuildTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                               TypeSourceInfo *argTInfo1,
-                               TypeSourceInfo *argTInfo2,
-                               SourceLocation RPLoc) {
-  if (getLangOptions().CPlusPlus) {
-    Diag(BuiltinLoc, diag::err_types_compatible_p_in_cplusplus)
-      << SourceRange(BuiltinLoc, RPLoc);
-    return ExprError();
-  }
-
-  return Owned(new (Context) TypesCompatibleExpr(Context.IntTy, BuiltinLoc,
-                                                 argTInfo1, argTInfo2, RPLoc));
-}
-
-
 ExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
-                                             Expr *CondExpr,
-                                             Expr *LHSExpr, Expr *RHSExpr,
-                                             SourceLocation RPLoc) {
+                                 Expr *CondExpr,
+                                 Expr *LHSExpr, Expr *RHSExpr,
+                                 SourceLocation RPLoc) {
   assert((CondExpr && LHSExpr && RHSExpr) && "Missing type argument(s)");
 
+  ExprValueKind VK = VK_RValue;
+  ExprObjectKind OK = OK_Ordinary;
   QualType resType;
   bool ValueDependent = false;
   if (CondExpr->isTypeDependent() || CondExpr->isValueDependent()) {
@@ -7098,13 +8675,16 @@ ExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
         << CondExpr->getSourceRange());
 
     // If the condition is > zero, then the AST type is the same as the LSHExpr.
-    resType = condEval.getZExtValue() ? LHSExpr->getType() : RHSExpr->getType();
-    ValueDependent = condEval.getZExtValue() ? LHSExpr->isValueDependent()
-                                             : RHSExpr->isValueDependent();
+    Expr *ActiveExpr = condEval.getZExtValue() ? LHSExpr : RHSExpr;
+
+    resType = ActiveExpr->getType();
+    ValueDependent = ActiveExpr->isValueDependent();
+    VK = ActiveExpr->getValueKind();
+    OK = ActiveExpr->getObjectKind();
   }
 
   return Owned(new (Context) ChooseExpr(BuiltinLoc, CondExpr, LHSExpr, RHSExpr,
-                                        resType, RPLoc,
+                                        resType, VK, OK, RPLoc,
                                         resType->isDependentType(),
                                         ValueDependent));
 }
@@ -7126,32 +8706,51 @@ void Sema::ActOnBlockStart(SourceLocation CaretLoc, Scope *BlockScope) {
 
 void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
   assert(ParamInfo.getIdentifier()==0 && "block-id should have no identifier!");
+  assert(ParamInfo.getContext() == Declarator::BlockLiteralContext);
   BlockScopeInfo *CurBlock = getCurBlock();
 
   TypeSourceInfo *Sig = GetTypeForDeclarator(ParamInfo, CurScope);
-  CurBlock->TheDecl->setSignatureAsWritten(Sig);
   QualType T = Sig->getType();
 
-  bool isVariadic;
-  QualType RetTy;
-  if (const FunctionType *Fn = T->getAs<FunctionType>()) {
-    CurBlock->FunctionType = T;
-    RetTy = Fn->getResultType();
-    isVariadic =
-      !isa<FunctionProtoType>(Fn) || cast<FunctionProtoType>(Fn)->isVariadic();
-  } else {
-    RetTy = T;
-    isVariadic = false;
-  }
+  // GetTypeForDeclarator always produces a function type for a block
+  // literal signature.  Furthermore, it is always a FunctionProtoType
+  // unless the function was written with a typedef.
+  assert(T->isFunctionType() &&
+         "GetTypeForDeclarator made a non-function block signature");
 
-  CurBlock->TheDecl->setIsVariadic(isVariadic);
+  // Look for an explicit signature in that function type.
+  FunctionProtoTypeLoc ExplicitSignature;
 
-  // Don't allow returning an array by value.
-  if (RetTy->isArrayType()) {
-    Diag(ParamInfo.getSourceRange().getBegin(), diag::err_block_returns_array);
-    return;
+  TypeLoc tmp = Sig->getTypeLoc().IgnoreParens();
+  if (isa<FunctionProtoTypeLoc>(tmp)) {
+    ExplicitSignature = cast<FunctionProtoTypeLoc>(tmp);
+
+    // Check whether that explicit signature was synthesized by
+    // GetTypeForDeclarator.  If so, don't save that as part of the
+    // written signature.
+    if (ExplicitSignature.getLParenLoc() ==
+        ExplicitSignature.getRParenLoc()) {
+      // This would be much cheaper if we stored TypeLocs instead of
+      // TypeSourceInfos.
+      TypeLoc Result = ExplicitSignature.getResultLoc();
+      unsigned Size = Result.getFullDataSize();
+      Sig = Context.CreateTypeSourceInfo(Result.getType(), Size);
+      Sig->getTypeLoc().initializeFullCopy(Result, Size);
+
+      ExplicitSignature = FunctionProtoTypeLoc();
+    }
   }
 
+  CurBlock->TheDecl->setSignatureAsWritten(Sig);
+  CurBlock->FunctionType = T;
+
+  const FunctionType *Fn = T->getAs<FunctionType>();
+  QualType RetTy = Fn->getResultType();
+  bool isVariadic =
+    (isa<FunctionProtoType>(Fn) && cast<FunctionProtoType>(Fn)->isVariadic());
+
+  CurBlock->TheDecl->setIsVariadic(isVariadic);
+
   // Don't allow returning a objc interface by value.
   if (RetTy->isObjCObjectType()) {
     Diag(ParamInfo.getSourceRange().getBegin(),
@@ -7168,10 +8767,9 @@ void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
 
   // Push block parameters from the declarator if we had them.
   llvm::SmallVector<ParmVarDecl*, 8> Params;
-  if (isa<FunctionProtoType>(T)) {
-    FunctionProtoTypeLoc TL = cast<FunctionProtoTypeLoc>(Sig->getTypeLoc());
-    for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
-      ParmVarDecl *Param = TL.getArg(I);
+  if (ExplicitSignature) {
+    for (unsigned I = 0, E = ExplicitSignature.getNumArgs(); I != E; ++I) {
+      ParmVarDecl *Param = ExplicitSignature.getArg(I);
       if (Param->getIdentifier() == 0 &&
           !Param->isImplicit() &&
           !Param->isInvalidDecl() &&
@@ -7194,9 +8792,13 @@ void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
   }
 
   // Set the parameters on the block decl.
-  if (!Params.empty())
+  if (!Params.empty()) {
     CurBlock->TheDecl->setParams(Params.data(), Params.size());
-
+    CheckParmsForFunctionDef(CurBlock->TheDecl->param_begin(),
+                             CurBlock->TheDecl->param_end(),
+                             /*CheckParameterNames=*/false);
+  }
+  
   // Finally we can process decl attributes.
   ProcessDeclAttributes(CurScope, CurBlock->TheDecl, ParamInfo);
 
@@ -7211,17 +8813,13 @@ void Sema::ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
   if (Params.empty())
     return;
 
-  bool ShouldCheckShadow =
-    Diags.getDiagnosticLevel(diag::warn_decl_shadow) != Diagnostic::Ignored;
-
   for (BlockDecl::param_iterator AI = CurBlock->TheDecl->param_begin(),
          E = CurBlock->TheDecl->param_end(); AI != E; ++AI) {
     (*AI)->setOwningFunction(CurBlock->TheDecl);
 
     // If this has an identifier, add it to the scope stack.
     if ((*AI)->getIdentifier()) {
-      if (ShouldCheckShadow)
-        CheckShadow(CurBlock->TheScope, *AI);
+      CheckShadow(CurBlock->TheScope, *AI);
 
       PushOnScopeChains(*AI, CurBlock->TheScope);
     }
@@ -7234,13 +8832,12 @@ void Sema::ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope) {
   // Pop off CurBlock, handle nested blocks.
   PopDeclContext();
   PopFunctionOrBlockScope();
-  // FIXME: Delete the ParmVarDecl objects as well???
 }
 
 /// ActOnBlockStmtExpr - This is called when the body of a block statement
 /// literal was successfully completed.  ^(int x){...}
 ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
-                                                Stmt *Body, Scope *CurScope) {
+                                    Stmt *Body, Scope *CurScope) {
   // If blocks are disabled, emit an error.
   if (!LangOpts.Blocks)
     Diag(CaretLoc, diag::err_blocks_disable);
@@ -7256,6 +8853,10 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   bool NoReturn = BSI->TheDecl->getAttr<NoReturnAttr>();
   QualType BlockTy;
 
+  // Set the captured variables on the block.
+  BSI->TheDecl->setCaptures(Context, BSI->Captures.begin(), BSI->Captures.end(),
+                            BSI->CapturesCXXThis);
+
   // If the user wrote a function type in some form, try to use that.
   if (!BSI->FunctionType.isNull()) {
     const FunctionType *FTy = BSI->FunctionType->getAs<FunctionType>();
@@ -7265,8 +8866,9 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
     
     // Turn protoless block types into nullary block types.
     if (isa<FunctionNoProtoType>(FTy)) {
-      BlockTy = Context.getFunctionType(RetTy, 0, 0, false, 0,
-                                        false, false, 0, 0, Ext);
+      FunctionProtoType::ExtProtoInfo EPI;
+      EPI.ExtInfo = Ext;
+      BlockTy = Context.getFunctionType(RetTy, 0, 0, EPI);
 
     // Otherwise, if we don't need to change anything about the function type,
     // preserve its sugar structure.
@@ -7277,26 +8879,22 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
     // Otherwise, make the minimal modifications to the function type.
     } else {
       const FunctionProtoType *FPT = cast<FunctionProtoType>(FTy);
+      FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+      EPI.TypeQuals = 0; // FIXME: silently?
+      EPI.ExtInfo = Ext;
       BlockTy = Context.getFunctionType(RetTy,
                                         FPT->arg_type_begin(),
                                         FPT->getNumArgs(),
-                                        FPT->isVariadic(),
-                                        /*quals*/ 0,
-                                        FPT->hasExceptionSpec(),
-                                        FPT->hasAnyExceptionSpec(),
-                                        FPT->getNumExceptions(),
-                                        FPT->exception_begin(),
-                                        Ext);
+                                        EPI);
     }
 
   // If we don't have a function type, just build one from nothing.
   } else {
-    BlockTy = Context.getFunctionType(RetTy, 0, 0, false, 0,
-                                      false, false, 0, 0,
-                             FunctionType::ExtInfo(NoReturn, 0, CC_Default));
+    FunctionProtoType::ExtProtoInfo EPI;
+    EPI.ExtInfo = FunctionType::ExtInfo(NoReturn, 0, CC_Default);
+    BlockTy = Context.getFunctionType(RetTy, 0, 0, EPI);
   }
 
-  // FIXME: Check that return/parameter types are complete/non-abstract
   DiagnoseUnusedParameters(BSI->TheDecl->param_begin(),
                            BSI->TheDecl->param_end());
   BlockTy = Context.getBlockPointerType(BlockTy);
@@ -7307,28 +8905,7 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
 
   BSI->TheDecl->setBody(cast<CompoundStmt>(Body));
 
-  bool Good = true;
-  // Check goto/label use.
-  for (llvm::DenseMap<IdentifierInfo*, LabelStmt*>::iterator
-         I = BSI->LabelMap.begin(), E = BSI->LabelMap.end(); I != E; ++I) {
-    LabelStmt *L = I->second;
-
-    // Verify that we have no forward references left.  If so, there was a goto
-    // or address of a label taken, but no definition of it.
-    if (L->getSubStmt() != 0)
-      continue;
-
-    // Emit error.
-    Diag(L->getIdentLoc(), diag::err_undeclared_label_use) << L->getName();
-    Good = false;
-  }
-  if (!Good) {
-    PopFunctionOrBlockScope();
-    return ExprError();
-  }
-
-  BlockExpr *Result = new (Context) BlockExpr(BSI->TheDecl, BlockTy,
-                                              BSI->hasBlockDeclRefExprs);
+  BlockExpr *Result = new (Context) BlockExpr(BSI->TheDecl, BlockTy);
 
   // Issue any analysis-based warnings.
   const sema::AnalysisBasedWarnings::Policy &WP =
@@ -7343,17 +8920,15 @@ ExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
                                         Expr *expr, ParsedType type,
                                         SourceLocation RPLoc) {
   TypeSourceInfo *TInfo;
-  QualType T = GetTypeFromParser(type, &TInfo);
+  GetTypeFromParser(type, &TInfo);
   return BuildVAArgExpr(BuiltinLoc, expr, TInfo, RPLoc);
 }
 
 ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
-                                            Expr *E, TypeSourceInfo *TInfo,
-                                            SourceLocation RPLoc) {
+                                Expr *E, TypeSourceInfo *TInfo,
+                                SourceLocation RPLoc) {
   Expr *OrigExpr = E;
 
-  InitBuiltinVaListType();
-
   // Get the va_list type
   QualType VaListType = Context.getBuiltinVaListType();
   if (VaListType->isArrayType()) {
@@ -7389,10 +8964,17 @@ ExprResult Sema::ActOnGNUNullExpr(SourceLocation TokenLoc) {
   // The type of __null will be int or long, depending on the size of
   // pointers on the target.
   QualType Ty;
-  if (Context.Target.getPointerWidth(0) == Context.Target.getIntWidth())
+  unsigned pw = Context.Target.getPointerWidth(0);
+  if (pw == Context.Target.getIntWidth())
     Ty = Context.IntTy;
-  else
+  else if (pw == Context.Target.getLongWidth())
     Ty = Context.LongTy;
+  else if (pw == Context.Target.getLongLongWidth())
+    Ty = Context.LongLongTy;
+  else {
+    assert(!"I don't know size of pointer!");
+    Ty = Context.IntTy;
+  }
 
   return Owned(new (Context) GNUNullExpr(Ty, TokenLoc));
 }
@@ -7454,15 +9036,29 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
   case FunctionVoidPointer:
     DiagKind = diag::ext_typecheck_convert_pointer_void_func;
     break;
+  case IncompatiblePointerDiscardsQualifiers: {
+    // Perform array-to-pointer decay if necessary.
+    if (SrcType->isArrayType()) SrcType = Context.getArrayDecayedType(SrcType);
+
+    Qualifiers lhq = SrcType->getPointeeType().getQualifiers();
+    Qualifiers rhq = DstType->getPointeeType().getQualifiers();
+    if (lhq.getAddressSpace() != rhq.getAddressSpace()) {
+      DiagKind = diag::err_typecheck_incompatible_address_space;
+      break;
+    }
+
+    llvm_unreachable("unknown error case for discarding qualifiers!");
+    // fallthrough
+  }
   case CompatiblePointerDiscardsQualifiers:
     // If the qualifiers lost were because we were applying the
     // (deprecated) C++ conversion from a string literal to a char*
     // (or wchar_t*), then there was no error (C++ 4.2p2).  FIXME:
     // Ideally, this check would be performed in
-    // CheckPointerTypesForAssignment. However, that would require a
+    // checkPointerTypesForAssignment. However, that would require a
     // bit of refactoring (so that the second argument is an
     // expression, rather than a type), which should be done as part
-    // of a larger effort to fix CheckPointerTypesForAssignment for
+    // of a larger effort to fix checkPointerTypesForAssignment for
     // C++ semantics.
     if (getLangOptions().CPlusPlus &&
         IsStringLiteralToNonConstPointerConversion(SrcExpr, DstType))
@@ -7548,7 +9144,8 @@ bool Sema::VerifyIntegerConstantExpression(const Expr *E, llvm::APSInt *Result){
     E->getSourceRange();
 
   if (EvalResult.Diag &&
-      Diags.getDiagnosticLevel(diag::ext_expr_not_ice) != Diagnostic::Ignored)
+      Diags.getDiagnosticLevel(diag::ext_expr_not_ice, EvalResult.DiagLoc)
+          != Diagnostic::Ignored)
     Diag(EvalResult.DiagLoc, EvalResult.Diag);
 
   if (Result)
@@ -7625,7 +9222,7 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
   // -Wunused-parameters)
   if (isa<ParmVarDecl>(D) ||
       (isa<VarDecl>(D) && D->getDeclContext()->isFunctionOrMethod())) {
-    D->setUsed(true);
+    D->setUsed();
     return;
   }
 
@@ -7653,6 +9250,11 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
       // potentially evaluated.
       ExprEvalContexts.back().addReferencedDecl(Loc, D);
       return;
+      
+    case PotentiallyEvaluatedIfUsed:
+      // Referenced declarations will only be used if the construct in the
+      // containing expression is used.
+      return;
   }
 
   // Note that this declaration has been used.
@@ -7684,6 +9286,9 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
       MarkVTableUsed(Loc, MethodDecl->getParent());
   }
   if (FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
+    // Recursive functions should be marked when used from another function.
+    if (CurContext == Function) return;
+
     // Implicit instantiation of function templates and member functions of
     // class templates.
     if (Function->isImplicitlyInstantiable()) {
@@ -7712,19 +9317,23 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
         else
           PendingInstantiations.push_back(std::make_pair(Function, Loc));
       }
-    } else // Walk redefinitions, as some of them may be instantiable.
+    } else {
+      // Walk redefinitions, as some of them may be instantiable.
       for (FunctionDecl::redecl_iterator i(Function->redecls_begin()),
            e(Function->redecls_end()); i != e; ++i) {
         if (!i->isUsed(false) && i->isImplicitlyInstantiable())
           MarkDeclarationReferenced(Loc, *i);
       }
+    }
 
-    // FIXME: keep track of references to static functions
-
-    // Recursive functions should be marked when used from another function.
-    if (CurContext != Function)
-      Function->setUsed(true);
+    // Keep track of used but undefined functions.
+    if (!Function->isPure() && !Function->hasBody() &&
+        Function->getLinkage() != ExternalLinkage) {
+      SourceLocation &old = UndefinedInternals[Function->getCanonicalDecl()];
+      if (old.isInvalid()) old = Loc;
+    }
 
+    Function->setUsed(true);
     return;
   }
 
@@ -7741,7 +9350,12 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
       }
     }
 
-    // FIXME: keep track of references to static data?
+    // Keep track of used but undefined variables.
+    if (Var->hasDefinition() == VarDecl::DeclarationOnly
+        && Var->getLinkage() != ExternalLinkage) {
+      SourceLocation &old = UndefinedInternals[Var->getCanonicalDecl()];
+      if (old.isInvalid()) old = Loc;
+    }
 
     D->setUsed(true);
     return;
@@ -7779,8 +9393,7 @@ bool MarkReferencedDecls::TraverseRecordType(RecordType *T) {
   if (ClassTemplateSpecializationDecl *Spec
                   = dyn_cast<ClassTemplateSpecializationDecl>(T->getDecl())) {
     const TemplateArgumentList &Args = Spec->getTemplateArgs();
-    return TraverseTemplateArguments(Args.getFlatArgumentList(),
-                                     Args.flat_size());
+    return TraverseTemplateArguments(Args.data(), Args.size());
   }
 
   return true;
@@ -7791,6 +9404,70 @@ void Sema::MarkDeclarationsReferencedInType(SourceLocation Loc, QualType T) {
   Marker.TraverseType(Context.getCanonicalType(T));
 }
 
+namespace {
+  /// \brief Helper class that marks all of the declarations referenced by
+  /// potentially-evaluated subexpressions as "referenced".
+  class EvaluatedExprMarker : public EvaluatedExprVisitor<EvaluatedExprMarker> {
+    Sema &S;
+    
+  public:
+    typedef EvaluatedExprVisitor<EvaluatedExprMarker> Inherited;
+    
+    explicit EvaluatedExprMarker(Sema &S) : Inherited(S.Context), S(S) { }
+    
+    void VisitDeclRefExpr(DeclRefExpr *E) {
+      S.MarkDeclarationReferenced(E->getLocation(), E->getDecl());
+    }
+    
+    void VisitMemberExpr(MemberExpr *E) {
+      S.MarkDeclarationReferenced(E->getMemberLoc(), E->getMemberDecl());
+      Inherited::VisitMemberExpr(E);
+    }
+    
+    void VisitCXXNewExpr(CXXNewExpr *E) {
+      if (E->getConstructor())
+        S.MarkDeclarationReferenced(E->getLocStart(), E->getConstructor());
+      if (E->getOperatorNew())
+        S.MarkDeclarationReferenced(E->getLocStart(), E->getOperatorNew());
+      if (E->getOperatorDelete())
+        S.MarkDeclarationReferenced(E->getLocStart(), E->getOperatorDelete());
+      Inherited::VisitCXXNewExpr(E);
+    }
+    
+    void VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+      if (E->getOperatorDelete())
+        S.MarkDeclarationReferenced(E->getLocStart(), E->getOperatorDelete());
+      QualType Destroyed = S.Context.getBaseElementType(E->getDestroyedType());
+      if (const RecordType *DestroyedRec = Destroyed->getAs<RecordType>()) {
+        CXXRecordDecl *Record = cast<CXXRecordDecl>(DestroyedRec->getDecl());
+        S.MarkDeclarationReferenced(E->getLocStart(), 
+                                    S.LookupDestructor(Record));
+      }
+      
+      Inherited::VisitCXXDeleteExpr(E);
+    }
+    
+    void VisitCXXConstructExpr(CXXConstructExpr *E) {
+      S.MarkDeclarationReferenced(E->getLocStart(), E->getConstructor());
+      Inherited::VisitCXXConstructExpr(E);
+    }
+    
+    void VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
+      S.MarkDeclarationReferenced(E->getLocation(), E->getDecl());
+    }
+    
+    void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
+      Visit(E->getExpr());
+    }
+  };
+}
+
+/// \brief Mark any declarations that appear within this expression or any
+/// potentially-evaluated subexpressions as "referenced".
+void Sema::MarkDeclarationsReferencedInExpr(Expr *E) {
+  EvaluatedExprMarker(*this).Visit(E);
+}
+
 /// \brief Emit a diagnostic that describes an effect on the run-time behavior
 /// of the program being compiled.
 ///
@@ -7815,6 +9492,7 @@ bool Sema::DiagRuntimeBehavior(SourceLocation Loc,
     break;
 
   case PotentiallyEvaluated:
+  case PotentiallyEvaluatedIfUsed:
     Diag(Loc, PD);
     return true;
 
@@ -7848,40 +9526,42 @@ bool Sema::CheckCallReturnType(QualType ReturnType, SourceLocation Loc,
   return false;
 }
 
-// Diagnose the common s/=/==/ typo.  Note that adding parentheses
+// Diagnose the s/=/==/ and s/\|=/!=/ typos. Note that adding parentheses
 // will prevent this condition from triggering, which is what we want.
 void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
   SourceLocation Loc;
 
   unsigned diagnostic = diag::warn_condition_is_assignment;
+  bool IsOrAssign = false;
 
   if (isa<BinaryOperator>(E)) {
     BinaryOperator *Op = cast<BinaryOperator>(E);
-    if (Op->getOpcode() != BO_Assign)
+    if (Op->getOpcode() != BO_Assign && Op->getOpcode() != BO_OrAssign)
       return;
 
+    IsOrAssign = Op->getOpcode() == BO_OrAssign;
+
     // Greylist some idioms by putting them into a warning subcategory.
     if (ObjCMessageExpr *ME
           = dyn_cast<ObjCMessageExpr>(Op->getRHS()->IgnoreParenCasts())) {
       Selector Sel = ME->getSelector();
 
       // self = [<foo> init...]
-      if (isSelfExpr(Op->getLHS())
-          && Sel.getIdentifierInfoForSlot(0)->getName().startswith("init"))
+      if (isSelfExpr(Op->getLHS()) && Sel.getNameForSlot(0).startswith("init"))
         diagnostic = diag::warn_condition_is_idiomatic_assignment;
 
       // <foo> = [<bar> nextObject]
-      else if (Sel.isUnarySelector() &&
-               Sel.getIdentifierInfoForSlot(0)->getName() == "nextObject")
+      else if (Sel.isUnarySelector() && Sel.getNameForSlot(0) == "nextObject")
         diagnostic = diag::warn_condition_is_idiomatic_assignment;
     }
 
     Loc = Op->getOperatorLoc();
   } else if (isa<CXXOperatorCallExpr>(E)) {
     CXXOperatorCallExpr *Op = cast<CXXOperatorCallExpr>(E);
-    if (Op->getOperator() != OO_Equal)
+    if (Op->getOperator() != OO_Equal && Op->getOperator() != OO_PipeEqual)
       return;
 
+    IsOrAssign = Op->getOperator() == OO_PipeEqual;
     Loc = Op->getOperatorLoc();
   } else {
     // Not an assignment.
@@ -7892,29 +9572,63 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
   SourceLocation Close = PP.getLocForEndOfToken(E->getSourceRange().getEnd());
 
   Diag(Loc, diagnostic) << E->getSourceRange();
-  Diag(Loc, diag::note_condition_assign_to_comparison)
-    << FixItHint::CreateReplacement(Loc, "==");
+
+  if (IsOrAssign)
+    Diag(Loc, diag::note_condition_or_assign_to_comparison)
+      << FixItHint::CreateReplacement(Loc, "!=");
+  else
+    Diag(Loc, diag::note_condition_assign_to_comparison)
+      << FixItHint::CreateReplacement(Loc, "==");
+
   Diag(Loc, diag::note_condition_assign_silence)
     << FixItHint::CreateInsertion(Open, "(")
     << FixItHint::CreateInsertion(Close, ")");
 }
 
+/// \brief Redundant parentheses over an equality comparison can indicate
+/// that the user intended an assignment used as condition.
+void Sema::DiagnoseEqualityWithExtraParens(ParenExpr *parenE) {
+  // Don't warn if the parens came from a macro.
+  SourceLocation parenLoc = parenE->getLocStart();
+  if (parenLoc.isInvalid() || parenLoc.isMacroID())
+    return;
+
+  Expr *E = parenE->IgnoreParens();
+
+  if (BinaryOperator *opE = dyn_cast<BinaryOperator>(E))
+    if (opE->getOpcode() == BO_EQ &&
+        opE->getLHS()->IgnoreParenImpCasts()->isModifiableLvalue(Context)
+                                                           == Expr::MLV_Valid) {
+      SourceLocation Loc = opE->getOperatorLoc();
+      
+      Diag(Loc, diag::warn_equality_with_extra_parens) << E->getSourceRange();
+      Diag(Loc, diag::note_equality_comparison_to_assign)
+        << FixItHint::CreateReplacement(Loc, "=");
+      Diag(Loc, diag::note_equality_comparison_silence)
+        << FixItHint::CreateRemoval(parenE->getSourceRange().getBegin())
+        << FixItHint::CreateRemoval(parenE->getSourceRange().getEnd());
+    }
+}
+
 bool Sema::CheckBooleanCondition(Expr *&E, SourceLocation Loc) {
   DiagnoseAssignmentAsCondition(E);
+  if (ParenExpr *parenE = dyn_cast<ParenExpr>(E))
+    DiagnoseEqualityWithExtraParens(parenE);
 
   if (!E->isTypeDependent()) {
+    if (E->isBoundMemberFunction(Context))
+      return Diag(E->getLocStart(), diag::err_invalid_use_of_bound_member_func)
+        << E->getSourceRange();
+
+    if (getLangOptions().CPlusPlus)
+      return CheckCXXBooleanCondition(E); // C++ 6.4p4
+
     DefaultFunctionArrayLvalueConversion(E);
 
     QualType T = E->getType();
-
-    if (getLangOptions().CPlusPlus) {
-      if (CheckCXXBooleanCondition(E)) // C++ 6.4p4
-        return true;
-    } else if (!T->isScalarType()) { // C99 6.8.4.1p1
-      Diag(Loc, diag::err_typecheck_statement_requires_scalar)
-        << T << E->getSourceRange();
-      return true;
-    }
+    if (!T->isScalarType()) // C99 6.8.4.1p1
+      return Diag(Loc, diag::err_typecheck_statement_requires_scalar)
+               << T << E->getSourceRange();
   }
 
   return false;
@@ -7930,3 +9644,26 @@ ExprResult Sema::ActOnBooleanCondition(Scope *S, SourceLocation Loc,
   
   return Owned(Sub);
 }
+
+/// Check for operands with placeholder types and complain if found.
+/// Returns true if there was an error and no recovery was possible.
+ExprResult Sema::CheckPlaceholderExpr(Expr *E, SourceLocation Loc) {
+  const BuiltinType *BT = E->getType()->getAs<BuiltinType>();
+  if (!BT || !BT->isPlaceholderType()) return Owned(E);
+
+  // If this is overload, check for a single overload.
+  assert(BT->getKind() == BuiltinType::Overload);
+
+  if (FunctionDecl *Specialization
+        = ResolveSingleFunctionTemplateSpecialization(E)) {
+    // The access doesn't really matter in this case.
+    DeclAccessPair Found = DeclAccessPair::make(Specialization,
+                                                Specialization->getAccess());
+    E = FixOverloadedFunctionReference(E, Found, Specialization);
+    if (!E) return ExprError();
+    return Owned(E);
+  }
+
+  Diag(Loc, diag::err_ovl_unresolvable) << E->getSourceRange();
+  return ExprError();
+}
diff --git a/lib/Sema/SemaExprCXX.cpp b/lib/Sema/SemaExprCXX.cpp
index 5720d931b66b..f9c2c9a62ea3 100644
--- a/lib/Sema/SemaExprCXX.cpp
+++ b/lib/Sema/SemaExprCXX.cpp
@@ -16,6 +16,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
@@ -31,7 +32,7 @@ using namespace clang;
 using namespace sema;
 
 ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
-                                   IdentifierInfo &II, 
+                                   IdentifierInfo &II,
                                    SourceLocation NameLoc,
                                    Scope *S, CXXScopeSpec &SS,
                                    ParsedType ObjectTypePtr,
@@ -71,26 +72,26 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
 
   if (SS.isSet()) {
     NestedNameSpecifier *NNS = (NestedNameSpecifier *)SS.getScopeRep();
-    
+
     bool AlreadySearched = false;
     bool LookAtPrefix = true;
     // C++ [basic.lookup.qual]p6:
-    //   If a pseudo-destructor-name (5.2.4) contains a nested-name-specifier, 
+    //   If a pseudo-destructor-name (5.2.4) contains a nested-name-specifier,
     //   the type-names are looked up as types in the scope designated by the
     //   nested-name-specifier. In a qualified-id of the form:
-    // 
-    //     ::[opt] nested-name-specifier  ̃ class-name 
+    //
+    //     ::[opt] nested-name-specifier  ~ class-name
     //
     //   where the nested-name-specifier designates a namespace scope, and in
     //   a qualified-id of the form:
     //
-    //     ::opt nested-name-specifier class-name ::  ̃ class-name 
+    //     ::opt nested-name-specifier class-name ::  ~ class-name
     //
-    //   the class-names are looked up as types in the scope designated by 
+    //   the class-names are looked up as types in the scope designated by
     //   the nested-name-specifier.
     //
     // Here, we check the first case (completely) and determine whether the
-    // code below is permitted to look at the prefix of the 
+    // code below is permitted to look at the prefix of the
     // nested-name-specifier.
     DeclContext *DC = computeDeclContext(SS, EnteringContext);
     if (DC && DC->isFileContext()) {
@@ -99,7 +100,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
       isDependent = false;
     } else if (DC && isa<CXXRecordDecl>(DC))
       LookAtPrefix = false;
-    
+
     // The second case from the C++03 rules quoted further above.
     NestedNameSpecifier *Prefix = 0;
     if (AlreadySearched) {
@@ -116,7 +117,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
       LookupCtx = computeDeclContext(SS, EnteringContext);
       isDependent = LookupCtx && LookupCtx->isDependentContext();
     }
-    
+
     LookInScope = false;
   } else if (ObjectTypePtr) {
     // C++ [basic.lookup.classref]p3:
@@ -128,7 +129,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
     //   cv-qualified) T.
     LookupCtx = computeDeclContext(SearchType);
     isDependent = SearchType->isDependentType();
-    assert((isDependent || !SearchType->isIncompleteType()) && 
+    assert((isDependent || !SearchType->isIncompleteType()) &&
            "Caller should have completed object type");
 
     LookInScope = true;
@@ -170,7 +171,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
     // nested-name-specifier (if present) or the object type, then
     // this is the destructor for that class.
     // FIXME: This is a workaround until we get real drafting for core
-    // issue 399, for which there isn't even an obvious direction. 
+    // issue 399, for which there isn't even an obvious direction.
     if (ClassTemplateDecl *Template = Found.getAsSingle<ClassTemplateDecl>()) {
       QualType MemberOfType;
       if (SS.isSet()) {
@@ -182,7 +183,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
       }
       if (MemberOfType.isNull())
         MemberOfType = SearchType;
-      
+
       if (MemberOfType.isNull())
         continue;
 
@@ -199,7 +200,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
 
         continue;
       }
-      
+
       // We're referring to an unresolved class template
       // specialization. Determine whether we class template we found
       // is the same as the template being specialized or, if we don't
@@ -220,7 +221,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
 
         // The class template we found has the same name as the
         // (dependent) template name being specialized.
-        if (DependentTemplateName *DepTemplate 
+        if (DependentTemplateName *DepTemplate
                                     = SpecName.getAsDependentTemplateName()) {
           if (DepTemplate->isIdentifier() &&
               DepTemplate->getIdentifier() == Template->getIdentifier())
@@ -253,7 +254,7 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
 
   if (ObjectTypePtr)
     Diag(NameLoc, diag::err_ident_in_pseudo_dtor_not_a_type)
-      << &II;        
+      << &II;
   else
     Diag(NameLoc, diag::err_destructor_class_name);
 
@@ -262,13 +263,13 @@ ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
 
 /// \brief Build a C++ typeid expression with a type operand.
 ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
-                                            SourceLocation TypeidLoc,
-                                            TypeSourceInfo *Operand,
-                                            SourceLocation RParenLoc) {
+                                SourceLocation TypeidLoc,
+                                TypeSourceInfo *Operand,
+                                SourceLocation RParenLoc) {
   // C++ [expr.typeid]p4:
-  //   The top-level cv-qualifiers of the lvalue expression or the type-id 
+  //   The top-level cv-qualifiers of the lvalue expression or the type-id
   //   that is the operand of typeid are always ignored.
-  //   If the type of the type-id is a class type or a reference to a class 
+  //   If the type of the type-id is a class type or a reference to a class
   //   type, the class shall be completely-defined.
   Qualifiers Quals;
   QualType T
@@ -277,7 +278,7 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
   if (T->getAs<RecordType>() &&
       RequireCompleteType(TypeidLoc, T, diag::err_incomplete_typeid))
     return ExprError();
-  
+
   return Owned(new (Context) CXXTypeidExpr(TypeInfoType.withConst(),
                                            Operand,
                                            SourceRange(TypeidLoc, RParenLoc)));
@@ -285,9 +286,9 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
 
 /// \brief Build a C++ typeid expression with an expression operand.
 ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
-                                            SourceLocation TypeidLoc,
-                                            Expr *E,
-                                            SourceLocation RParenLoc) {
+                                SourceLocation TypeidLoc,
+                                Expr *E,
+                                SourceLocation RParenLoc) {
   bool isUnevaluatedOperand = true;
   if (E && !E->isTypeDependent()) {
     QualType T = E->getType();
@@ -298,7 +299,7 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
       //   shall be completely-defined.
       if (RequireCompleteType(TypeidLoc, T, diag::err_incomplete_typeid))
         return ExprError();
-      
+
       // C++ [expr.typeid]p3:
       //   When typeid is applied to an expression other than an glvalue of a
       //   polymorphic class type [...] [the] expression is an unevaluated
@@ -310,11 +311,11 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
         MarkVTableUsed(TypeidLoc, RecordD);
       }
     }
-    
+
     // C++ [expr.typeid]p4:
     //   [...] If the type of the type-id is a reference to a possibly
-    //   cv-qualified type, the result of the typeid expression refers to a 
-    //   std::type_info object representing the cv-unqualified referenced 
+    //   cv-qualified type, the result of the typeid expression refers to a
+    //   std::type_info object representing the cv-unqualified referenced
     //   type.
     Qualifiers Quals;
     QualType UnqualT = Context.getUnqualifiedArrayType(T, Quals);
@@ -323,16 +324,16 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
       ImpCastExprToType(E, UnqualT, CK_NoOp, CastCategory(E));
     }
   }
-  
+
   // If this is an unevaluated operand, clear out the set of
   // declaration references we have been computing and eliminate any
   // temporaries introduced in its computation.
   if (isUnevaluatedOperand)
     ExprEvalContexts.back().Context = Unevaluated;
-  
+
   return Owned(new (Context) CXXTypeidExpr(TypeInfoType.withConst(),
                                            E,
-                                           SourceRange(TypeidLoc, RParenLoc)));  
+                                           SourceRange(TypeidLoc, RParenLoc)));
 }
 
 /// ActOnCXXTypeidOfType - Parse typeid( type-id ) or typeid (expression);
@@ -343,15 +344,17 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
   if (!StdNamespace)
     return ExprError(Diag(OpLoc, diag::err_need_header_before_typeid));
 
-  IdentifierInfo *TypeInfoII = &PP.getIdentifierTable().get("type_info");
-  LookupResult R(*this, TypeInfoII, SourceLocation(), LookupTagName);
-  LookupQualifiedName(R, getStdNamespace());
-  RecordDecl *TypeInfoRecordDecl = R.getAsSingle<RecordDecl>();
-  if (!TypeInfoRecordDecl)
-    return ExprError(Diag(OpLoc, diag::err_need_header_before_typeid));
-  
-  QualType TypeInfoType = Context.getTypeDeclType(TypeInfoRecordDecl);
-  
+  if (!CXXTypeInfoDecl) {
+    IdentifierInfo *TypeInfoII = &PP.getIdentifierTable().get("type_info");
+    LookupResult R(*this, TypeInfoII, SourceLocation(), LookupTagName);
+    LookupQualifiedName(R, getStdNamespace());
+    CXXTypeInfoDecl = R.getAsSingle<RecordDecl>();
+    if (!CXXTypeInfoDecl)
+      return ExprError(Diag(OpLoc, diag::err_need_header_before_typeid));
+  }
+
+  QualType TypeInfoType = Context.getTypeDeclType(CXXTypeInfoDecl);
+
   if (isType) {
     // The operand is a type; handle it as such.
     TypeSourceInfo *TInfo = 0;
@@ -359,17 +362,102 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
                                    &TInfo);
     if (T.isNull())
       return ExprError();
-    
+
     if (!TInfo)
       TInfo = Context.getTrivialTypeSourceInfo(T, OpLoc);
 
     return BuildCXXTypeId(TypeInfoType, OpLoc, TInfo, RParenLoc);
   }
 
-  // The operand is an expression.  
+  // The operand is an expression.
   return BuildCXXTypeId(TypeInfoType, OpLoc, (Expr*)TyOrExpr, RParenLoc);
 }
 
+/// Retrieve the UuidAttr associated with QT.
+static UuidAttr *GetUuidAttrOfType(QualType QT) {
+  // Optionally remove one level of pointer, reference or array indirection.
+  const Type *Ty = QT.getTypePtr();;
+  if (QT->isPointerType() || QT->isReferenceType())
+    Ty = QT->getPointeeType().getTypePtr();
+  else if (QT->isArrayType())
+    Ty = cast<ArrayType>(QT)->getElementType().getTypePtr();
+
+  // Loop all class definition and declaration looking for an uuid attribute.
+  CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
+  while (RD) {
+    if (UuidAttr *Uuid = RD->getAttr<UuidAttr>())
+      return Uuid;
+    RD = RD->getPreviousDeclaration();
+  }
+  return 0;
+}
+
+/// \brief Build a Microsoft __uuidof expression with a type operand.
+ExprResult Sema::BuildCXXUuidof(QualType TypeInfoType,
+                                SourceLocation TypeidLoc,
+                                TypeSourceInfo *Operand,
+                                SourceLocation RParenLoc) {
+  if (!Operand->getType()->isDependentType()) {
+    if (!GetUuidAttrOfType(Operand->getType()))
+      return ExprError(Diag(TypeidLoc, diag::err_uuidof_without_guid));
+  }
+
+  // FIXME: add __uuidof semantic analysis for type operand.
+  return Owned(new (Context) CXXUuidofExpr(TypeInfoType.withConst(),
+                                           Operand,
+                                           SourceRange(TypeidLoc, RParenLoc)));
+}
+
+/// \brief Build a Microsoft __uuidof expression with an expression operand.
+ExprResult Sema::BuildCXXUuidof(QualType TypeInfoType,
+                                SourceLocation TypeidLoc,
+                                Expr *E,
+                                SourceLocation RParenLoc) {
+  if (!E->getType()->isDependentType()) {
+    if (!GetUuidAttrOfType(E->getType()) &&
+        !E->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull))
+      return ExprError(Diag(TypeidLoc, diag::err_uuidof_without_guid));
+  }
+  // FIXME: add __uuidof semantic analysis for type operand.
+  return Owned(new (Context) CXXUuidofExpr(TypeInfoType.withConst(),
+                                           E,
+                                           SourceRange(TypeidLoc, RParenLoc)));
+}
+
+/// ActOnCXXUuidof - Parse __uuidof( type-id ) or __uuidof (expression);
+ExprResult
+Sema::ActOnCXXUuidof(SourceLocation OpLoc, SourceLocation LParenLoc,
+                     bool isType, void *TyOrExpr, SourceLocation RParenLoc) {
+  // If MSVCGuidDecl has not been cached, do the lookup.
+  if (!MSVCGuidDecl) {
+    IdentifierInfo *GuidII = &PP.getIdentifierTable().get("_GUID");
+    LookupResult R(*this, GuidII, SourceLocation(), LookupTagName);
+    LookupQualifiedName(R, Context.getTranslationUnitDecl());
+    MSVCGuidDecl = R.getAsSingle<RecordDecl>();
+    if (!MSVCGuidDecl)
+      return ExprError(Diag(OpLoc, diag::err_need_header_before_ms_uuidof));
+  }
+
+  QualType GuidType = Context.getTypeDeclType(MSVCGuidDecl);
+
+  if (isType) {
+    // The operand is a type; handle it as such.
+    TypeSourceInfo *TInfo = 0;
+    QualType T = GetTypeFromParser(ParsedType::getFromOpaquePtr(TyOrExpr),
+                                   &TInfo);
+    if (T.isNull())
+      return ExprError();
+
+    if (!TInfo)
+      TInfo = Context.getTrivialTypeSourceInfo(T, OpLoc);
+
+    return BuildCXXUuidof(GuidType, OpLoc, TInfo, RParenLoc);
+  }
+
+  // The operand is an expression.
+  return BuildCXXUuidof(GuidType, OpLoc, (Expr*)TyOrExpr, RParenLoc);
+}
+
 /// ActOnCXXBoolLiteral - Parse {true,false} literals.
 ExprResult
 Sema::ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind) {
@@ -388,6 +476,9 @@ Sema::ActOnCXXNullPtrLiteral(SourceLocation Loc) {
 /// ActOnCXXThrow - Parse throw expressions.
 ExprResult
 Sema::ActOnCXXThrow(SourceLocation OpLoc, Expr *Ex) {
+  if (!getLangOptions().Exceptions)
+    Diag(OpLoc, diag::err_exceptions_disabled) << "throw";
+
   if (Ex && !Ex->isTypeDependent() && CheckCXXThrowOperand(OpLoc, Ex))
     return ExprError();
   return Owned(new (Context) CXXThrowExpr(Ex, Context.VoidTy, OpLoc));
@@ -399,12 +490,12 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   //   A throw-expression initializes a temporary object, called the exception
   //   object, the type of which is determined by removing any top-level
   //   cv-qualifiers from the static type of the operand of throw and adjusting
-  //   the type from "array of T" or "function returning T" to "pointer to T" 
+  //   the type from "array of T" or "function returning T" to "pointer to T"
   //   or "pointer to function returning T", [...]
   if (E->getType().hasQualifiers())
     ImpCastExprToType(E, E->getType().getUnqualifiedType(), CK_NoOp,
                       CastCategory(E));
-  
+
   DefaultFunctionArrayConversion(E);
 
   //   If the type of the exception would be an incomplete type or a pointer
@@ -430,13 +521,14 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
 
   // Initialize the exception result.  This implicitly weeds out
   // abstract types or types with inaccessible copy constructors.
-  // FIXME: Determine whether we can elide this copy per C++0x [class.copy]p34.
+  const VarDecl *NRVOVariable = getCopyElisionCandidate(QualType(), E, false);
+
+  // FIXME: Determine whether we can elide this copy per C++0x [class.copy]p32.
   InitializedEntity Entity =
-    InitializedEntity::InitializeException(ThrowLoc, E->getType(),
-                                           /*NRVO=*/false);
-  ExprResult Res = PerformCopyInitialization(Entity,
-                                                   SourceLocation(),
-                                                   Owned(E));
+      InitializedEntity::InitializeException(ThrowLoc, E->getType(),
+                                             /*NRVO=*/false);
+  ExprResult Res = PerformMoveOrCopyInitialization(Entity, NRVOVariable,
+                                                   QualType(), E);
   if (Res.isInvalid())
     return true;
   E = Res.takeAs<Expr>();
@@ -451,11 +543,15 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   // exception handling will make use of the vtable.
   MarkVTableUsed(ThrowLoc, RD);
 
+  // If a pointer is thrown, the referenced object will not be destroyed.
+  if (isPointer)
+    return false;
+
   // If the class has a non-trivial destructor, we must be able to call it.
   if (RD->hasTrivialDestructor())
     return false;
 
-  CXXDestructorDecl *Destructor 
+  CXXDestructorDecl *Destructor
     = const_cast<CXXDestructorDecl*>(LookupDestructor(RD));
   if (!Destructor)
     return false;
@@ -466,30 +562,48 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   return false;
 }
 
-ExprResult Sema::ActOnCXXThis(SourceLocation ThisLoc) {
+CXXMethodDecl *Sema::tryCaptureCXXThis() {
+  // Ignore block scopes: we can capture through them.
+  // Ignore nested enum scopes: we'll diagnose non-constant expressions
+  // where they're invalid, and other uses are legitimate.
+  // Don't ignore nested class scopes: you can't use 'this' in a local class.
+  DeclContext *DC = CurContext;
+  while (true) {
+    if (isa<BlockDecl>(DC)) DC = cast<BlockDecl>(DC)->getDeclContext();
+    else if (isa<EnumDecl>(DC)) DC = cast<EnumDecl>(DC)->getDeclContext();
+    else break;
+  }
+
+  // If we're not in an instance method, error out.
+  CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(DC);
+  if (!method || !method->isInstance())
+    return 0;
+
+  // Mark that we're closing on 'this' in all the block scopes, if applicable.
+  for (unsigned idx = FunctionScopes.size() - 1;
+       isa<BlockScopeInfo>(FunctionScopes[idx]);
+       --idx)
+    cast<BlockScopeInfo>(FunctionScopes[idx])->CapturesCXXThis = true;
+
+  return method;
+}
+
+ExprResult Sema::ActOnCXXThis(SourceLocation loc) {
   /// C++ 9.3.2: In the body of a non-static member function, the keyword this
   /// is a non-lvalue expression whose value is the address of the object for
   /// which the function is called.
 
-  DeclContext *DC = getFunctionLevelDeclContext();
-  if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(DC))
-    if (MD->isInstance())
-      return Owned(new (Context) CXXThisExpr(ThisLoc,
-                                             MD->getThisType(Context),
-                                             /*isImplicit=*/false));
+  CXXMethodDecl *method = tryCaptureCXXThis();
+  if (!method) return Diag(loc, diag::err_invalid_this_use);
 
-  return ExprError(Diag(ThisLoc, diag::err_invalid_this_use));
+  return Owned(new (Context) CXXThisExpr(loc, method->getThisType(Context),
+                                         /*isImplicit=*/false));
 }
 
-/// ActOnCXXTypeConstructExpr - Parse construction of a specified type.
-/// Can be interpreted either as function-style casting ("int(x)")
-/// or class type construction ("ClassType(x,y,z)")
-/// or creation of a value-initialized type ("int()").
 ExprResult
-Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
+Sema::ActOnCXXTypeConstructExpr(ParsedType TypeRep,
                                 SourceLocation LParenLoc,
                                 MultiExprArg exprs,
-                                SourceLocation *CommaLocs,
                                 SourceLocation RParenLoc) {
   if (!TypeRep)
     return ExprError();
@@ -498,17 +612,30 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
   QualType Ty = GetTypeFromParser(TypeRep, &TInfo);
   if (!TInfo)
     TInfo = Context.getTrivialTypeSourceInfo(Ty, SourceLocation());
+
+  return BuildCXXTypeConstructExpr(TInfo, LParenLoc, exprs, RParenLoc);
+}
+
+/// ActOnCXXTypeConstructExpr - Parse construction of a specified type.
+/// Can be interpreted either as function-style casting ("int(x)")
+/// or class type construction ("ClassType(x,y,z)")
+/// or creation of a value-initialized type ("int()").
+ExprResult
+Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
+                                SourceLocation LParenLoc,
+                                MultiExprArg exprs,
+                                SourceLocation RParenLoc) {
+  QualType Ty = TInfo->getType();
   unsigned NumExprs = exprs.size();
   Expr **Exprs = (Expr**)exprs.get();
-  SourceLocation TyBeginLoc = TypeRange.getBegin();
+  SourceLocation TyBeginLoc = TInfo->getTypeLoc().getBeginLoc();
   SourceRange FullRange = SourceRange(TyBeginLoc, RParenLoc);
 
   if (Ty->isDependentType() ||
       CallExpr::hasAnyTypeDependentArguments(Exprs, NumExprs)) {
     exprs.release();
 
-    return Owned(CXXUnresolvedConstructExpr::Create(Context,
-                                                    TypeRange.getBegin(), Ty,
+    return Owned(CXXUnresolvedConstructExpr::Create(Context, TInfo,
                                                     LParenLoc,
                                                     Exprs, NumExprs,
                                                     RParenLoc));
@@ -522,7 +649,7 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
                           PDiag(diag::err_invalid_incomplete_type_use)
                             << FullRange))
     return ExprError();
-  
+
   if (RequireNonAbstractType(TyBeginLoc, Ty,
                              diag::err_allocation_of_abstract_type))
     return ExprError();
@@ -534,55 +661,91 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
   // corresponding cast expression.
   //
   if (NumExprs == 1) {
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Invalid;
+    ExprValueKind VK = VK_RValue;
     CXXCastPath BasePath;
-    if (CheckCastTypes(TypeRange, Ty, Exprs[0], Kind, BasePath,
+    if (CheckCastTypes(TInfo->getTypeLoc().getSourceRange(), Ty, Exprs[0],
+                       Kind, VK, BasePath,
                        /*FunctionalStyle=*/true))
       return ExprError();
 
     exprs.release();
 
     return Owned(CXXFunctionalCastExpr::Create(Context,
-                                              Ty.getNonLValueExprType(Context),
-                                               TInfo, TyBeginLoc, Kind,
+                                               Ty.getNonLValueExprType(Context),
+                                               VK, TInfo, TyBeginLoc, Kind,
                                                Exprs[0], &BasePath,
                                                RParenLoc));
   }
 
-  if (Ty->isRecordType()) {
-    InitializedEntity Entity = InitializedEntity::InitializeTemporary(Ty);
-    InitializationKind Kind
-      = NumExprs ? InitializationKind::CreateDirect(TypeRange.getBegin(), 
-                                                    LParenLoc, RParenLoc)
-                 : InitializationKind::CreateValue(TypeRange.getBegin(), 
-                                                   LParenLoc, RParenLoc);
-    InitializationSequence InitSeq(*this, Entity, Kind, Exprs, NumExprs);
-    ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                              move(exprs));
+  InitializedEntity Entity = InitializedEntity::InitializeTemporary(TInfo);
+  InitializationKind Kind
+    = NumExprs ? InitializationKind::CreateDirect(TyBeginLoc,
+                                                  LParenLoc, RParenLoc)
+               : InitializationKind::CreateValue(TyBeginLoc,
+                                                 LParenLoc, RParenLoc);
+  InitializationSequence InitSeq(*this, Entity, Kind, Exprs, NumExprs);
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(exprs));
+
+  // FIXME: Improve AST representation?
+  return move(Result);
+}
+
+/// doesUsualArrayDeleteWantSize - Answers whether the usual
+/// operator delete[] for the given type has a size_t parameter.
+static bool doesUsualArrayDeleteWantSize(Sema &S, SourceLocation loc,
+                                         QualType allocType) {
+  const RecordType *record =
+    allocType->getBaseElementTypeUnsafe()->getAs<RecordType>();
+  if (!record) return false;
+
+  // Try to find an operator delete[] in class scope.
+
+  DeclarationName deleteName =
+    S.Context.DeclarationNames.getCXXOperatorName(OO_Array_Delete);
+  LookupResult ops(S, deleteName, loc, Sema::LookupOrdinaryName);
+  S.LookupQualifiedName(ops, record->getDecl());
+
+  // We're just doing this for information.
+  ops.suppressDiagnostics();
+
+  // Very likely: there's no operator delete[].
+  if (ops.empty()) return false;
+
+  // If it's ambiguous, it should be illegal to call operator delete[]
+  // on this thing, so it doesn't matter if we allocate extra space or not.
+  if (ops.isAmbiguous()) return false;
+
+  LookupResult::Filter filter = ops.makeFilter();
+  while (filter.hasNext()) {
+    NamedDecl *del = filter.next()->getUnderlyingDecl();
 
-    // FIXME: Improve AST representation?
-    return move(Result);
+    // C++0x [basic.stc.dynamic.deallocation]p2:
+    //   A template instance is never a usual deallocation function,
+    //   regardless of its signature.
+    if (isa<FunctionTemplateDecl>(del)) {
+      filter.erase();
+      continue;
+    }
+
+    // C++0x [basic.stc.dynamic.deallocation]p2:
+    //   If class T does not declare [an operator delete[] with one
+    //   parameter] but does declare a member deallocation function
+    //   named operator delete[] with exactly two parameters, the
+    //   second of which has type std::size_t, then this function
+    //   is a usual deallocation function.
+    if (!cast<CXXMethodDecl>(del)->isUsualDeallocationFunction()) {
+      filter.erase();
+      continue;
+    }
   }
+  filter.done();
 
-  // C++ [expr.type.conv]p1:
-  // If the expression list specifies more than a single value, the type shall
-  // be a class with a suitably declared constructor.
-  //
-  if (NumExprs > 1)
-    return ExprError(Diag(CommaLocs[0],
-                          diag::err_builtin_func_cast_more_than_one_arg)
-      << FullRange);
-
-  assert(NumExprs == 0 && "Expected 0 expressions");
-  // C++ [expr.type.conv]p2:
-  // The expression T(), where T is a simple-type-specifier for a non-array
-  // complete object type or the (possibly cv-qualified) void type, creates an
-  // rvalue of the specified type, which is value-initialized.
-  //
-  exprs.release();
-  return Owned(new (Context) CXXScalarValueInitExpr(Ty, TyBeginLoc, RParenLoc));
-}
+  if (!ops.isSingleResult()) return false;
 
+  const FunctionDecl *del = cast<FunctionDecl>(ops.getFoundDecl());
+  return (del->getNumParams() == 2);
+}
 
 /// ActOnCXXNew - Parsed a C++ 'new' expression (C++ 5.3.4), as in e.g.:
 /// @code new (memory) int[size][4] @endcode
@@ -592,15 +755,20 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
 ExprResult
 Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   SourceLocation PlacementLParen, MultiExprArg PlacementArgs,
-                  SourceLocation PlacementRParen, SourceRange TypeIdParens, 
+                  SourceLocation PlacementRParen, SourceRange TypeIdParens,
                   Declarator &D, SourceLocation ConstructorLParen,
                   MultiExprArg ConstructorArgs,
                   SourceLocation ConstructorRParen) {
+  bool TypeContainsAuto = D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto;
+
   Expr *ArraySize = 0;
   // If the specified type is an array, unwrap it and save the expression.
   if (D.getNumTypeObjects() > 0 &&
       D.getTypeObject(0).Kind == DeclaratorChunk::Array) {
     DeclaratorChunk &Chunk = D.getTypeObject(0);
+    if (TypeContainsAuto)
+      return ExprError(Diag(Chunk.Loc, diag::err_new_array_of_auto)
+        << D.getSourceRange());
     if (Chunk.Arr.hasStatic)
       return ExprError(Diag(Chunk.Loc, diag::err_static_illegal_in_new)
         << D.getSourceRange());
@@ -630,25 +798,24 @@ Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
     }
   }
 
-  //FIXME: Store TypeSourceInfo in CXXNew expression.
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, /*Scope=*/0);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, /*Scope=*/0, /*OwnedDecl=*/0,
+                                               /*AllowAuto=*/true);
   QualType AllocType = TInfo->getType();
   if (D.isInvalidType())
     return ExprError();
-  
-  SourceRange R = TInfo->getTypeLoc().getSourceRange();    
+
   return BuildCXXNew(StartLoc, UseGlobal,
                      PlacementLParen,
                      move(PlacementArgs),
                      PlacementRParen,
                      TypeIdParens,
                      AllocType,
-                     D.getSourceRange().getBegin(),
-                     R,
+                     TInfo,
                      ArraySize,
                      ConstructorLParen,
                      move(ConstructorArgs),
-                     ConstructorRParen);
+                     ConstructorRParen,
+                     TypeContainsAuto);
 }
 
 ExprResult
@@ -658,15 +825,37 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   SourceLocation PlacementRParen,
                   SourceRange TypeIdParens,
                   QualType AllocType,
-                  SourceLocation TypeLoc,
-                  SourceRange TypeRange,
+                  TypeSourceInfo *AllocTypeInfo,
                   Expr *ArraySize,
                   SourceLocation ConstructorLParen,
                   MultiExprArg ConstructorArgs,
-                  SourceLocation ConstructorRParen) {
-  if (CheckAllocatedType(AllocType, TypeLoc, TypeRange))
-    return ExprError();
+                  SourceLocation ConstructorRParen,
+                  bool TypeMayContainAuto) {
+  SourceRange TypeRange = AllocTypeInfo->getTypeLoc().getSourceRange();
+
+  // C++0x [decl.spec.auto]p6. Deduce the type which 'auto' stands in for.
+  if (TypeMayContainAuto && AllocType->getContainedAutoType()) {
+    if (ConstructorArgs.size() == 0)
+      return ExprError(Diag(StartLoc, diag::err_auto_new_requires_ctor_arg)
+                       << AllocType << TypeRange);
+    if (ConstructorArgs.size() != 1) {
+      Expr *FirstBad = ConstructorArgs.get()[1];
+      return ExprError(Diag(FirstBad->getSourceRange().getBegin(),
+                            diag::err_auto_new_ctor_multiple_expressions)
+                       << AllocType << TypeRange);
+    }
+    QualType DeducedType;
+    if (!DeduceAutoType(AllocType, ConstructorArgs.get()[0], DeducedType))
+      return ExprError(Diag(StartLoc, diag::err_auto_new_deduction_failure)
+                       << AllocType
+                       << ConstructorArgs.get()[0]->getType()
+                       << TypeRange
+                       << ConstructorArgs.get()[0]->getSourceRange());
 
+    AllocType = DeducedType;
+    AllocTypeInfo = Context.getTrivialTypeSourceInfo(AllocType, StartLoc);
+  }
+  
   // Per C++0x [expr.new]p5, the type being constructed may be a
   // typedef of an array type.
   if (!ArraySize) {
@@ -679,14 +868,17 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     }
   }
 
+  if (CheckAllocatedType(AllocType, TypeRange.getBegin(), TypeRange))
+    return ExprError();
+
   QualType ResultType = Context.getPointerType(AllocType);
 
   // C++ 5.3.4p6: "The expression in a direct-new-declarator shall have integral
   //   or enumeration type with a non-negative value."
   if (ArraySize && !ArraySize->isTypeDependent()) {
-    
+
     QualType SizeType = ArraySize->getType();
-    
+
     ExprResult ConvertedSize
       = ConvertToIntegralOrEnumerationType(StartLoc, ArraySize,
                                        PDiag(diag::err_array_size_not_integral),
@@ -696,16 +888,16 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                                        PDiag(diag::note_array_size_conversion),
                                PDiag(diag::err_array_size_ambiguous_conversion),
                                        PDiag(diag::note_array_size_conversion),
-                          PDiag(getLangOptions().CPlusPlus0x? 0 
+                          PDiag(getLangOptions().CPlusPlus0x? 0
                                             : diag::ext_array_size_conversion));
     if (ConvertedSize.isInvalid())
       return ExprError();
-    
+
     ArraySize = ConvertedSize.take();
     SizeType = ArraySize->getType();
-    if (!SizeType->isIntegralOrEnumerationType())
+    if (!SizeType->isIntegralOrUnscopedEnumerationType())
       return ExprError();
-    
+
     // Let's see if this is a constant < 0. If so, we reject it out of hand.
     // We don't care about special rules, so we tell the machinery it's not
     // evaluated - it gives us a result in more cases.
@@ -713,17 +905,17 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
       llvm::APSInt Value;
       if (ArraySize->isIntegerConstantExpr(Value, Context, 0, false)) {
         if (Value < llvm::APSInt(
-                        llvm::APInt::getNullValue(Value.getBitWidth()), 
+                        llvm::APInt::getNullValue(Value.getBitWidth()),
                                  Value.isUnsigned()))
           return ExprError(Diag(ArraySize->getSourceRange().getBegin(),
                                 diag::err_typecheck_negative_array_size)
             << ArraySize->getSourceRange());
-        
+
         if (!AllocType->isDependentType()) {
           unsigned ActiveSizeBits
             = ConstantArrayType::getNumAddressingBits(Context, AllocType, Value);
           if (ActiveSizeBits > ConstantArrayType::getMaxSizeBits(Context)) {
-            Diag(ArraySize->getSourceRange().getBegin(), 
+            Diag(ArraySize->getSourceRange().getBegin(),
                  diag::err_array_too_large)
               << Value.toString(10)
               << ArraySize->getSourceRange();
@@ -736,11 +928,11 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
           << ArraySize->getSourceRange()
           << FixItHint::CreateRemoval(TypeIdParens.getBegin())
           << FixItHint::CreateRemoval(TypeIdParens.getEnd());
-          
+
         TypeIdParens = SourceRange();
       }
     }
-    
+
     ImpCastExprToType(ArraySize, Context.getSizeType(),
                       CK_IntegralCast);
   }
@@ -749,7 +941,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   FunctionDecl *OperatorDelete = 0;
   Expr **PlaceArgs = (Expr**)PlacementArgs.get();
   unsigned NumPlaceArgs = PlacementArgs.size();
-  
+
   if (!AllocType->isDependentType() &&
       !Expr::hasAnyTypeDependentArguments(PlaceArgs, NumPlaceArgs) &&
       FindAllocationFunctions(StartLoc,
@@ -757,24 +949,32 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                               UseGlobal, AllocType, ArraySize, PlaceArgs,
                               NumPlaceArgs, OperatorNew, OperatorDelete))
     return ExprError();
+
+  // If this is an array allocation, compute whether the usual array
+  // deallocation function for the type has a size_t parameter.
+  bool UsualArrayDeleteWantsSize = false;
+  if (ArraySize && !AllocType->isDependentType())
+    UsualArrayDeleteWantsSize
+      = doesUsualArrayDeleteWantSize(*this, StartLoc, AllocType);
+
   llvm::SmallVector<Expr *, 8> AllPlaceArgs;
   if (OperatorNew) {
     // Add default arguments, if any.
-    const FunctionProtoType *Proto = 
+    const FunctionProtoType *Proto =
       OperatorNew->getType()->getAs<FunctionProtoType>();
-    VariadicCallType CallType = 
+    VariadicCallType CallType =
       Proto->isVariadic() ? VariadicFunction : VariadicDoesNotApply;
-    
+
     if (GatherArgumentsForCall(PlacementLParen, OperatorNew,
-                               Proto, 1, PlaceArgs, NumPlaceArgs, 
+                               Proto, 1, PlaceArgs, NumPlaceArgs,
                                AllPlaceArgs, CallType))
       return ExprError();
-    
+
     NumPlaceArgs = AllPlaceArgs.size();
     if (NumPlaceArgs > 0)
       PlaceArgs = &AllPlaceArgs[0];
   }
-  
+
   bool Init = ConstructorLParen.isValid();
   // --- Choosing a constructor ---
   CXXConstructorDecl *Constructor = 0;
@@ -786,7 +986,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   if (NumConsArgs && (ResultType->isArrayType() || ArraySize)) {
     SourceRange InitRange(ConsArgs[0]->getLocStart(),
                           ConsArgs[NumConsArgs - 1]->getLocEnd());
-    
+
     Diag(StartLoc, diag::err_new_array_init_args) << InitRange;
     return ExprError();
   }
@@ -800,22 +1000,22 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     //     - If the new-initializer is omitted, the object is default-
     //       initialized (8.5); if no initialization is performed,
     //       the object has indeterminate value
-      = !Init? InitializationKind::CreateDefault(TypeLoc)
-    //     - Otherwise, the new-initializer is interpreted according to the 
+      = !Init? InitializationKind::CreateDefault(TypeRange.getBegin())
+    //     - Otherwise, the new-initializer is interpreted according to the
     //       initialization rules of 8.5 for direct-initialization.
-             : InitializationKind::CreateDirect(TypeLoc,
-                                                ConstructorLParen, 
+             : InitializationKind::CreateDirect(TypeRange.getBegin(),
+                                                ConstructorLParen,
                                                 ConstructorRParen);
-    
+
     InitializedEntity Entity
       = InitializedEntity::InitializeNew(StartLoc, AllocType);
     InitializationSequence InitSeq(*this, Entity, Kind, ConsArgs, NumConsArgs);
-    ExprResult FullInit = InitSeq.Perform(*this, Entity, Kind, 
+    ExprResult FullInit = InitSeq.Perform(*this, Entity, Kind,
                                                 move(ConstructorArgs));
     if (FullInit.isInvalid())
       return ExprError();
-    
-    // FullInit is our initializer; walk through it to determine if it's a 
+
+    // FullInit is our initializer; walk through it to determine if it's a
     // constructor call, which CXXNewExpr handles directly.
     if (Expr *FullInitExpr = (Expr *)FullInit.get()) {
       if (CXXBindTemporaryExpr *Binder
@@ -827,7 +1027,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
         for (CXXConstructExpr::arg_iterator A = Construct->arg_begin(),
                                          AEnd = Construct->arg_end();
              A != AEnd; ++A)
-          ConvertedConstructorArgs.push_back(A->Retain());
+          ConvertedConstructorArgs.push_back(*A);
       } else {
         // Take the converted initializer.
         ConvertedConstructorArgs.push_back(FullInit.release());
@@ -835,12 +1035,12 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     } else {
       // No initialization required.
     }
-    
+
     // Take the converted arguments and use them for the new expression.
     NumConsArgs = ConvertedConstructorArgs.size();
     ConsArgs = (Expr **)ConvertedConstructorArgs.take();
   }
-  
+
   // Mark the new and delete operators as referenced.
   if (OperatorNew)
     MarkDeclarationReferenced(StartLoc, OperatorNew);
@@ -848,18 +1048,20 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     MarkDeclarationReferenced(StartLoc, OperatorDelete);
 
   // FIXME: Also check that the destructor is accessible. (C++ 5.3.4p16)
-  
+
   PlacementArgs.release();
   ConstructorArgs.release();
-  
-  // FIXME: The TypeSourceInfo should also be included in CXXNewExpr.
+
   return Owned(new (Context) CXXNewExpr(Context, UseGlobal, OperatorNew,
                                         PlaceArgs, NumPlaceArgs, TypeIdParens,
                                         ArraySize, Constructor, Init,
                                         ConsArgs, NumConsArgs, OperatorDelete,
-                                        ResultType, StartLoc,
+                                        UsualArrayDeleteWantsSize,
+                                        ResultType, AllocTypeInfo,
+                                        StartLoc,
                                         Init ? ConstructorRParen :
-                                               TypeRange.getEnd()));
+                                               TypeRange.getEnd(),
+                                        ConstructorLParen, ConstructorRParen));
 }
 
 /// CheckAllocatedType - Checks that a type is suitable as the allocated type
@@ -883,6 +1085,9 @@ bool Sema::CheckAllocatedType(QualType AllocType, SourceLocation Loc,
   else if (RequireNonAbstractType(Loc, AllocType,
                                   diag::err_allocation_of_abstract_type))
     return true;
+  else if (AllocType->isVariablyModifiedType())
+    return Diag(Loc, diag::err_variably_modified_new_type)
+             << AllocType;
 
   return false;
 }
@@ -931,10 +1136,10 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
 
   // C++ [expr.new]p8:
   //   If the allocated type is a non-array type, the allocation
-  //   function’s name is operator new and the deallocation function’s
+  //   function's name is operator new and the deallocation function's
   //   name is operator delete. If the allocated type is an array
-  //   type, the allocation function’s name is operator new[] and the
-  //   deallocation function’s name is operator delete[].
+  //   type, the allocation function's name is operator new[] and the
+  //   deallocation function's name is operator delete[].
   DeclarationName NewName = Context.DeclarationNames.getCXXOperatorName(
                                         IsArray ? OO_Array_New : OO_New);
   DeclarationName DeleteName = Context.DeclarationNames.getCXXOperatorName(
@@ -975,12 +1180,12 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
   // C++ [expr.new]p19:
   //
   //   If the new-expression begins with a unary :: operator, the
-  //   deallocation function’s name is looked up in the global
+  //   deallocation function's name is looked up in the global
   //   scope. Otherwise, if the allocated type is a class type T or an
-  //   array thereof, the deallocation function’s name is looked up in
+  //   array thereof, the deallocation function's name is looked up in
   //   the scope of T. If this lookup fails to find the name, or if
   //   the allocated type is not a class type or array thereof, the
-  //   deallocation function’s name is looked up in the global scope.
+  //   deallocation function's name is looked up in the global scope.
   LookupResult FoundDelete(*this, DeleteName, StartLoc, LookupOrdinaryName);
   if (AllocElemType->isRecordType() && !UseGlobal) {
     CXXRecordDecl *RD
@@ -999,39 +1204,49 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
 
   llvm::SmallVector<std::pair<DeclAccessPair,FunctionDecl*>, 2> Matches;
 
-  if (NumPlaceArgs > 0) {
+  // Whether we're looking for a placement operator delete is dictated
+  // by whether we selected a placement operator new, not by whether
+  // we had explicit placement arguments.  This matters for things like
+  //   struct A { void *operator new(size_t, int = 0); ... };
+  //   A *a = new A()
+  bool isPlacementNew = (NumPlaceArgs > 0 || OperatorNew->param_size() != 1);
+
+  if (isPlacementNew) {
     // C++ [expr.new]p20:
     //   A declaration of a placement deallocation function matches the
     //   declaration of a placement allocation function if it has the
     //   same number of parameters and, after parameter transformations
     //   (8.3.5), all parameter types except the first are
     //   identical. [...]
-    // 
+    //
     // To perform this comparison, we compute the function type that
     // the deallocation function should have, and use that type both
     // for template argument deduction and for comparison purposes.
+    //
+    // FIXME: this comparison should ignore CC and the like.
     QualType ExpectedFunctionType;
     {
       const FunctionProtoType *Proto
         = OperatorNew->getType()->getAs<FunctionProtoType>();
+
       llvm::SmallVector<QualType, 4> ArgTypes;
-      ArgTypes.push_back(Context.VoidPtrTy); 
+      ArgTypes.push_back(Context.VoidPtrTy);
       for (unsigned I = 1, N = Proto->getNumArgs(); I < N; ++I)
         ArgTypes.push_back(Proto->getArgType(I));
 
+      FunctionProtoType::ExtProtoInfo EPI;
+      EPI.Variadic = Proto->isVariadic();
+
       ExpectedFunctionType
         = Context.getFunctionType(Context.VoidTy, ArgTypes.data(),
-                                  ArgTypes.size(),
-                                  Proto->isVariadic(),
-                                  0, false, false, 0, 0,
-                                  FunctionType::ExtInfo());
+                                  ArgTypes.size(), EPI);
     }
 
-    for (LookupResult::iterator D = FoundDelete.begin(), 
+    for (LookupResult::iterator D = FoundDelete.begin(),
                              DEnd = FoundDelete.end();
          D != DEnd; ++D) {
       FunctionDecl *Fn = 0;
-      if (FunctionTemplateDecl *FnTmpl 
+      if (FunctionTemplateDecl *FnTmpl
             = dyn_cast<FunctionTemplateDecl>((*D)->getUnderlyingDecl())) {
         // Perform template argument deduction to try to match the
         // expected function type.
@@ -1048,7 +1263,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
     // C++ [expr.new]p20:
     //   [...] Any non-placement deallocation function matches a
     //   non-placement allocation function. [...]
-    for (LookupResult::iterator D = FoundDelete.begin(), 
+    for (LookupResult::iterator D = FoundDelete.begin(),
                              DEnd = FoundDelete.end();
          D != DEnd; ++D) {
       if (FunctionDecl *Fn = dyn_cast<FunctionDecl>((*D)->getUnderlyingDecl()))
@@ -1073,7 +1288,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
     if (NumPlaceArgs && getLangOptions().CPlusPlus0x &&
         isNonPlacementDeallocationFunction(OperatorDelete)) {
       Diag(StartLoc, diag::err_placement_new_non_placement_delete)
-        << SourceRange(PlaceArgs[0]->getLocStart(), 
+        << SourceRange(PlaceArgs[0]->getLocStart(),
                        PlaceArgs[NumPlaceArgs - 1]->getLocEnd());
       Diag(OperatorDelete->getLocation(), diag::note_previous_decl)
         << DeleteName;
@@ -1107,7 +1322,7 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
   R.suppressDiagnostics();
 
   OverloadCandidateSet Candidates(StartLoc);
-  for (LookupResult::iterator Alloc = R.begin(), AllocEnd = R.end(); 
+  for (LookupResult::iterator Alloc = R.begin(), AllocEnd = R.end();
        Alloc != AllocEnd; ++Alloc) {
     // Even member operator new/delete are implicitly treated as
     // static, so don't use AddMemberCandidate.
@@ -1140,12 +1355,13 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
     for (unsigned i = 0; (i < NumArgs && i < NumArgsInFnDecl); ++i) {
       ExprResult Result
         = PerformCopyInitialization(InitializedEntity::InitializeParameter(
+                                                       Context,
                                                        FnDecl->getParamDecl(i)),
                                     SourceLocation(),
-                                    Owned(Args[i]->Retain()));
+                                    Owned(Args[i]));
       if (Result.isInvalid())
         return true;
-      
+
       Args[i] = Result.takeAs<Expr>();
     }
     Operator = FnDecl;
@@ -1190,18 +1406,18 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
 void Sema::DeclareGlobalNewDelete() {
   if (GlobalNewDeleteDeclared)
     return;
-  
+
   // C++ [basic.std.dynamic]p2:
-  //   [...] The following allocation and deallocation functions (18.4) are 
-  //   implicitly declared in global scope in each translation unit of a 
+  //   [...] The following allocation and deallocation functions (18.4) are
+  //   implicitly declared in global scope in each translation unit of a
   //   program
-  //   
+  //
   //     void* operator new(std::size_t) throw(std::bad_alloc);
-  //     void* operator new[](std::size_t) throw(std::bad_alloc); 
-  //     void  operator delete(void*) throw(); 
+  //     void* operator new[](std::size_t) throw(std::bad_alloc);
+  //     void  operator delete(void*) throw();
   //     void  operator delete[](void*) throw();
   //
-  //   These implicit declarations introduce only the function names operator 
+  //   These implicit declarations introduce only the function names operator
   //   new, operator new[], operator delete, operator delete[].
   //
   // Here, we need to refer to std::bad_alloc, so we will implicitly declare
@@ -1211,14 +1427,14 @@ void Sema::DeclareGlobalNewDelete() {
   if (!StdBadAlloc) {
     // The "std::bad_alloc" class has not yet been declared, so build it
     // implicitly.
-    StdBadAlloc = CXXRecordDecl::Create(Context, TTK_Class, 
-                                        getOrCreateStdNamespace(), 
-                                        SourceLocation(), 
-                                      &PP.getIdentifierTable().get("bad_alloc"), 
+    StdBadAlloc = CXXRecordDecl::Create(Context, TTK_Class,
+                                        getOrCreateStdNamespace(),
+                                        SourceLocation(),
+                                      &PP.getIdentifierTable().get("bad_alloc"),
                                         SourceLocation(), 0);
     getStdBadAlloc()->setImplicit(true);
   }
-  
+
   GlobalNewDeleteDeclared = true;
 
   QualType VoidPtr = Context.getPointerType(Context.VoidTy);
@@ -1268,28 +1484,31 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
   }
 
   QualType BadAllocType;
-  bool HasBadAllocExceptionSpec 
+  bool HasBadAllocExceptionSpec
     = (Name.getCXXOverloadedOperator() == OO_New ||
        Name.getCXXOverloadedOperator() == OO_Array_New);
   if (HasBadAllocExceptionSpec) {
     assert(StdBadAlloc && "Must have std::bad_alloc declared");
     BadAllocType = Context.getTypeDeclType(getStdBadAlloc());
   }
-  
-  QualType FnType = Context.getFunctionType(Return, &Argument, 1, false, 0,
-                                            true, false,
-                                            HasBadAllocExceptionSpec? 1 : 0,
-                                            &BadAllocType,
-                                            FunctionType::ExtInfo());
+
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.HasExceptionSpec = true;
+  if (HasBadAllocExceptionSpec) {
+    EPI.NumExceptions = 1;
+    EPI.Exceptions = &BadAllocType;
+  }
+
+  QualType FnType = Context.getFunctionType(Return, &Argument, 1, EPI);
   FunctionDecl *Alloc =
     FunctionDecl::Create(Context, GlobalCtx, SourceLocation(), Name,
                          FnType, /*TInfo=*/0, SC_None,
                          SC_None, false, true);
   Alloc->setImplicit();
-  
+
   if (AddMallocAttr)
     Alloc->addAttr(::new (Context) MallocAttr(SourceLocation(), Context));
-  
+
   ParmVarDecl *Param = ParmVarDecl::Create(Context, Alloc, SourceLocation(),
                                            0, Argument, /*TInfo=*/0,
                                            SC_None,
@@ -1308,7 +1527,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
   LookupResult Found(*this, Name, StartLoc, LookupOrdinaryName);
   // Try to find operator delete/operator delete[] in class scope.
   LookupQualifiedName(Found, RD);
-  
+
   if (Found.isAmbiguous())
     return true;
 
@@ -1352,7 +1571,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
   if (!Found.empty()) {
     Diag(StartLoc, diag::err_no_suitable_delete_member_function_found)
       << Name << RD;
-        
+
     for (LookupResult::iterator F = Found.begin(), FEnd = Found.end();
          F != FEnd; ++F)
       Diag((*F)->getUnderlyingDecl()->getLocation(),
@@ -1364,7 +1583,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
   // Look for a global declaration.
   DeclareGlobalNewDelete();
   DeclContext *TUDecl = Context.getTranslationUnitDecl();
-  
+
   CXXNullPtrLiteralExpr Null(Context.VoidPtrTy, SourceLocation());
   Expr* DeallocArgs[1];
   DeallocArgs[0] = &Null;
@@ -1391,19 +1610,21 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
   // DR599 amends "pointer type" to "pointer to object type" in both cases.
 
   FunctionDecl *OperatorDelete = 0;
+  bool ArrayFormAsWritten = ArrayForm;
+  bool UsualArrayDeleteWantsSize = false;
 
   if (!Ex->isTypeDependent()) {
     QualType Type = Ex->getType();
 
     if (const RecordType *Record = Type->getAs<RecordType>()) {
-      if (RequireCompleteType(StartLoc, Type, 
+      if (RequireCompleteType(StartLoc, Type,
                               PDiag(diag::err_delete_incomplete_class_type)))
         return ExprError();
-      
+
       llvm::SmallVector<CXXConversionDecl*, 4> ObjectPtrConversions;
 
       CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
-      const UnresolvedSetImpl *Conversions = RD->getVisibleConversionFunctions();      
+      const UnresolvedSetImpl *Conversions = RD->getVisibleConversionFunctions();
       for (UnresolvedSetImpl::iterator I = Conversions->begin(),
              E = Conversions->end(); I != E; ++I) {
         NamedDecl *D = I.getDecl();
@@ -1413,9 +1634,9 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
         // Skip over templated conversion functions; they aren't considered.
         if (isa<FunctionTemplateDecl>(D))
           continue;
-        
+
         CXXConversionDecl *Conv = cast<CXXConversionDecl>(D);
-        
+
         QualType ConvType = Conv->getConversionType().getNonReferenceType();
         if (const PointerType *ConvPtrType = ConvType->getAs<PointerType>())
           if (ConvPtrType->getPointeeType()->isIncompleteOrObjectType())
@@ -1446,7 +1667,7 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
 
     QualType Pointee = Type->getAs<PointerType>()->getPointeeType();
     if (Pointee->isVoidType() && !isSFINAEContext()) {
-      // The C++ standard bans deleting a pointer to a non-object type, which 
+      // The C++ standard bans deleting a pointer to a non-object type, which
       // effectively bans deletion of "void*". However, most compilers support
       // this, so we treat it as a warning unless we're in a SFINAE context.
       Diag(StartLoc, diag::ext_delete_void_ptr_operand)
@@ -1461,13 +1682,20 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
       return ExprError();
 
     // C++ [expr.delete]p2:
-    //   [Note: a pointer to a const type can be the operand of a 
-    //   delete-expression; it is not necessary to cast away the constness 
-    //   (5.2.11) of the pointer expression before it is used as the operand 
+    //   [Note: a pointer to a const type can be the operand of a
+    //   delete-expression; it is not necessary to cast away the constness
+    //   (5.2.11) of the pointer expression before it is used as the operand
     //   of the delete-expression. ]
-    ImpCastExprToType(Ex, Context.getPointerType(Context.VoidTy), 
+    ImpCastExprToType(Ex, Context.getPointerType(Context.VoidTy),
                       CK_NoOp);
-    
+
+    if (Pointee->isArrayType() && !ArrayForm) {
+      Diag(StartLoc, diag::warn_delete_array_type)
+          << Type << Ex->getSourceRange()
+          << FixItHint::CreateInsertion(PP.getLocForEndOfToken(StartLoc), "[]");
+      ArrayForm = true;
+    }
+
     DeclarationName DeleteName = Context.DeclarationNames.getCXXOperatorName(
                                       ArrayForm ? OO_Array_Delete : OO_Delete);
 
@@ -1475,16 +1703,33 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     if (const RecordType *RT = PointeeElem->getAs<RecordType>()) {
       CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
 
-      if (!UseGlobal && 
+      if (!UseGlobal &&
           FindDeallocationFunction(StartLoc, RD, DeleteName, OperatorDelete))
         return ExprError();
-      
+
+      // If we're allocating an array of records, check whether the
+      // usual operator delete[] has a size_t parameter.
+      if (ArrayForm) {
+        // If the user specifically asked to use the global allocator,
+        // we'll need to do the lookup into the class.
+        if (UseGlobal)
+          UsualArrayDeleteWantsSize =
+            doesUsualArrayDeleteWantSize(*this, StartLoc, PointeeElem);
+
+        // Otherwise, the usual operator delete[] should be the
+        // function we just found.
+        else if (isa<CXXMethodDecl>(OperatorDelete))
+          UsualArrayDeleteWantsSize = (OperatorDelete->getNumParams() == 2);
+      }
+
       if (!RD->hasTrivialDestructor())
-        if (const CXXDestructorDecl *Dtor = LookupDestructor(RD))
+        if (CXXDestructorDecl *Dtor = LookupDestructor(RD)) {
           MarkDeclarationReferenced(StartLoc,
                                     const_cast<CXXDestructorDecl*>(Dtor));
+          DiagnoseUseOfDecl(Dtor, StartLoc);
+        }
     }
-    
+
     if (!OperatorDelete) {
       // Look for a global declaration.
       DeclareGlobalNewDelete();
@@ -1496,38 +1741,49 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     }
 
     MarkDeclarationReferenced(StartLoc, OperatorDelete);
+    
+    // Check access and ambiguity of operator delete and destructor.
+    if (const RecordType *RT = PointeeElem->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (CXXDestructorDecl *Dtor = LookupDestructor(RD)) {
+          CheckDestructorAccess(Ex->getExprLoc(), Dtor, 
+                      PDiag(diag::err_access_dtor) << PointeeElem);
+      }
+    }
 
-    // FIXME: Check access and ambiguity of operator delete and destructor.
   }
 
   return Owned(new (Context) CXXDeleteExpr(Context.VoidTy, UseGlobal, ArrayForm,
+                                           ArrayFormAsWritten,
+                                           UsualArrayDeleteWantsSize,
                                            OperatorDelete, Ex, StartLoc));
 }
 
 /// \brief Check the use of the given variable as a C++ condition in an if,
 /// while, do-while, or switch statement.
 ExprResult Sema::CheckConditionVariable(VarDecl *ConditionVar,
-                                                      SourceLocation StmtLoc,
-                                                      bool ConvertToBoolean) {
+                                        SourceLocation StmtLoc,
+                                        bool ConvertToBoolean) {
   QualType T = ConditionVar->getType();
-  
+
   // C++ [stmt.select]p2:
   //   The declarator shall not specify a function or an array.
   if (T->isFunctionType())
-    return ExprError(Diag(ConditionVar->getLocation(), 
+    return ExprError(Diag(ConditionVar->getLocation(),
                           diag::err_invalid_use_of_function_type)
                        << ConditionVar->getSourceRange());
   else if (T->isArrayType())
-    return ExprError(Diag(ConditionVar->getLocation(), 
+    return ExprError(Diag(ConditionVar->getLocation(),
                           diag::err_invalid_use_of_array_type)
                      << ConditionVar->getSourceRange());
 
   Expr *Condition = DeclRefExpr::Create(Context, 0, SourceRange(), ConditionVar,
-                                        ConditionVar->getLocation(), 
-                                 ConditionVar->getType().getNonReferenceType());
+                                        ConditionVar->getLocation(),
+                            ConditionVar->getType().getNonReferenceType(),
+                                        VK_LValue);
   if (ConvertToBoolean && CheckBooleanCondition(Condition, StmtLoc))
     return ExprError();
-  
+
   return Owned(Condition);
 }
 
@@ -1575,42 +1831,46 @@ Sema::IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType) {
   return false;
 }
 
-static ExprResult BuildCXXCastArgument(Sema &S, 
+static ExprResult BuildCXXCastArgument(Sema &S,
                                        SourceLocation CastLoc,
                                        QualType Ty,
                                        CastKind Kind,
                                        CXXMethodDecl *Method,
+                                       NamedDecl *FoundDecl,
                                        Expr *From) {
   switch (Kind) {
   default: assert(0 && "Unhandled cast kind!");
   case CK_ConstructorConversion: {
     ASTOwningVector<Expr*> ConstructorArgs(S);
-    
+
     if (S.CompleteConstructorCall(cast<CXXConstructorDecl>(Method),
                                   MultiExprArg(&From, 1),
                                   CastLoc, ConstructorArgs))
       return ExprError();
-    
-    ExprResult Result = 
-    S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method), 
+
+    ExprResult Result =
+    S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method),
                             move_arg(ConstructorArgs),
-                            /*ZeroInit*/ false, CXXConstructExpr::CK_Complete);
+                            /*ZeroInit*/ false, CXXConstructExpr::CK_Complete,
+                            SourceRange());
     if (Result.isInvalid())
       return ExprError();
-    
+
     return S.MaybeBindToTemporary(Result.takeAs<Expr>());
   }
-    
+
   case CK_UserDefinedConversion: {
     assert(!From->getType()->isPointerType() && "Arg can't have pointer type!");
-    
+
     // Create an implicit call expr that calls it.
-    // FIXME: pass the FoundDecl for the user-defined conversion here
-    CXXMemberCallExpr *CE = S.BuildCXXMemberCallExpr(From, Method, Method);
-    return S.MaybeBindToTemporary(CE);
+    ExprResult Result = S.BuildCXXMemberCallExpr(From, FoundDecl, Method);
+    if (Result.isInvalid())
+      return ExprError();
+
+    return S.MaybeBindToTemporary(Result.get());
   }
   }
-}    
+}
 
 /// PerformImplicitConversion - Perform an implicit conversion of the
 /// expression From to the type ToType using the pre-computed implicit
@@ -1621,52 +1881,51 @@ static ExprResult BuildCXXCastArgument(Sema &S,
 bool
 Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                                 const ImplicitConversionSequence &ICS,
-                                AssignmentAction Action, bool IgnoreBaseAccess) {
+                                AssignmentAction Action, bool CStyle) {
   switch (ICS.getKind()) {
   case ImplicitConversionSequence::StandardConversion:
     if (PerformImplicitConversion(From, ToType, ICS.Standard, Action,
-                                  IgnoreBaseAccess))
+                                  CStyle))
       return true;
     break;
 
   case ImplicitConversionSequence::UserDefinedConversion: {
-    
+
       FunctionDecl *FD = ICS.UserDefined.ConversionFunction;
-      CastKind CastKind = CK_Unknown;
+      CastKind CastKind;
       QualType BeforeToType;
       if (const CXXConversionDecl *Conv = dyn_cast<CXXConversionDecl>(FD)) {
         CastKind = CK_UserDefinedConversion;
-        
+
         // If the user-defined conversion is specified by a conversion function,
         // the initial standard conversion sequence converts the source type to
         // the implicit object parameter of the conversion function.
         BeforeToType = Context.getTagDeclType(Conv->getParent());
-      } else if (const CXXConstructorDecl *Ctor = 
-                  dyn_cast<CXXConstructorDecl>(FD)) {
+      } else {
+        const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(FD);
         CastKind = CK_ConstructorConversion;
         // Do no conversion if dealing with ... for the first conversion.
         if (!ICS.UserDefined.EllipsisConversion) {
-          // If the user-defined conversion is specified by a constructor, the 
+          // If the user-defined conversion is specified by a constructor, the
           // initial standard conversion sequence converts the source type to the
           // type required by the argument of the constructor
           BeforeToType = Ctor->getParamDecl(0)->getType().getNonReferenceType();
         }
-      }    
-      else
-        assert(0 && "Unknown conversion function kind!");
-      // Whatch out for elipsis conversion.
+      }
+      // Watch out for elipsis conversion.
       if (!ICS.UserDefined.EllipsisConversion) {
-        if (PerformImplicitConversion(From, BeforeToType, 
+        if (PerformImplicitConversion(From, BeforeToType,
                                       ICS.UserDefined.Before, AA_Converting,
-                                      IgnoreBaseAccess))
+                                      CStyle))
           return true;
       }
-    
-      ExprResult CastArg 
+
+      ExprResult CastArg
         = BuildCXXCastArgument(*this,
                                From->getLocStart(),
                                ToType.getNonReferenceType(),
-                               CastKind, cast<CXXMethodDecl>(FD), 
+                               CastKind, cast<CXXMethodDecl>(FD),
+                               ICS.UserDefined.FoundConversionFunction,
                                From);
 
       if (CastArg.isInvalid())
@@ -1675,7 +1934,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
       From = CastArg.takeAs<Expr>();
 
       return PerformImplicitConversion(From, ToType, ICS.UserDefined.After,
-                                       AA_Converting, IgnoreBaseAccess);
+                                       AA_Converting, CStyle);
   }
 
   case ImplicitConversionSequence::AmbiguousConversion:
@@ -1683,7 +1942,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                           PDiag(diag::err_typecheck_ambiguous_condition)
                             << From->getSourceRange());
      return true;
-      
+
   case ImplicitConversionSequence::EllipsisConversion:
     assert(false && "Cannot perform an ellipsis conversion");
     return false;
@@ -1705,7 +1964,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
 bool
 Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                                 const StandardConversionSequence& SCS,
-                                AssignmentAction Action, bool IgnoreBaseAccess) {
+                                AssignmentAction Action, bool CStyle) {
   // Overall FIXME: we are recomputing too many types here and doing far too
   // much extra work. What this means is that we need to keep track of more
   // information that is computed when we try the implicit conversion initially,
@@ -1719,7 +1978,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
       ASTOwningVector<Expr*> ConstructorArgs(*this);
       if (CompleteConstructorCall(cast<CXXConstructorDecl>(SCS.CopyConstructor),
                                   MultiExprArg(*this, &From, 1),
-                                  /*FIXME:ConstructLoc*/SourceLocation(), 
+                                  /*FIXME:ConstructLoc*/SourceLocation(),
                                   ConstructorArgs))
         return true;
       ExprResult FromResult =
@@ -1727,7 +1986,8 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                               ToType, SCS.CopyConstructor,
                               move_arg(ConstructorArgs),
                               /*ZeroInit*/ false,
-                              CXXConstructExpr::CK_Complete);
+                              CXXConstructExpr::CK_Complete,
+                              SourceRange());
       if (FromResult.isInvalid())
         return true;
       From = FromResult.takeAs<Expr>();
@@ -1738,7 +1998,8 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                             ToType, SCS.CopyConstructor,
                             MultiExprArg(*this, &From, 1),
                             /*ZeroInit*/ false,
-                            CXXConstructExpr::CK_Complete);
+                            CXXConstructExpr::CK_Complete,
+                            SourceRange());
 
     if (FromResult.isInvalid())
       return true;
@@ -1765,10 +2026,25 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   // Perform the first implicit conversion.
   switch (SCS.First) {
   case ICK_Identity:
-  case ICK_Lvalue_To_Rvalue:
     // Nothing to do.
     break;
 
+  case ICK_Lvalue_To_Rvalue:
+    // Should this get its own ICK?
+    if (From->getObjectKind() == OK_ObjCProperty) {
+      ConvertPropertyForRValue(From);
+      if (!From->isGLValue()) break;
+    }
+
+    // Check for trivial buffer overflows.
+    if (const ArraySubscriptExpr *AE = dyn_cast<ArraySubscriptExpr>(From))
+      CheckArrayAccess(AE);
+
+    FromType = FromType.getUnqualifiedType();
+    From = ImplicitCastExpr::Create(Context, FromType, CK_LValueToRValue,
+                                    From, 0, VK_RValue);
+    break;
+
   case ICK_Array_To_Pointer:
     FromType = Context.getArrayDecayedType(FromType);
     ImpCastExprToType(From, FromType, CK_ArrayToPointerDecay);
@@ -1798,12 +2074,11 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     // If both sides are functions (or pointers/references to them), there could
     // be incompatible exception declarations.
     if (CheckExceptionSpecCompatibility(From, ToType))
-      return true;      
-      
-    ImpCastExprToType(From, Context.getNoReturnType(From->getType(), false),
-                      CK_NoOp);
+      return true;
+
+    ImpCastExprToType(From, ToType, CK_NoOp);
     break;
-      
+
   case ICK_Integral_Promotion:
   case ICK_Integral_Conversion:
     ImpCastExprToType(From, ToType, CK_IntegralCast);
@@ -1815,9 +2090,23 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     break;
 
   case ICK_Complex_Promotion:
-  case ICK_Complex_Conversion:
-    ImpCastExprToType(From, ToType, CK_Unknown);
+  case ICK_Complex_Conversion: {
+    QualType FromEl = From->getType()->getAs<ComplexType>()->getElementType();
+    QualType ToEl = ToType->getAs<ComplexType>()->getElementType();
+    CastKind CK;
+    if (FromEl->isRealFloatingType()) {
+      if (ToEl->isRealFloatingType())
+        CK = CK_FloatingComplexCast;
+      else
+        CK = CK_FloatingComplexToIntegralComplex;
+    } else if (ToEl->isRealFloatingType()) {
+      CK = CK_IntegralComplexToFloatingComplex;
+    } else {
+      CK = CK_IntegralComplexCast;
+    }
+    ImpCastExprToType(From, ToType, CK);
     break;
+  }
 
   case ICK_Floating_Integral:
     if (ToType->isRealFloatingType())
@@ -1831,7 +2120,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     break;
 
   case ICK_Pointer_Conversion: {
-    if (SCS.IncompatibleObjC) {
+    if (SCS.IncompatibleObjC && Action != AA_Casting) {
       // Diagnose incompatible Objective-C conversions
       Diag(From->getSourceRange().getBegin(),
            diag::ext_typecheck_convert_incompatible_pointer)
@@ -1839,20 +2128,18 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
         << From->getSourceRange();
     }
 
-    
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Invalid;
     CXXCastPath BasePath;
-    if (CheckPointerConversion(From, ToType, Kind, BasePath, IgnoreBaseAccess))
+    if (CheckPointerConversion(From, ToType, Kind, BasePath, CStyle))
       return true;
     ImpCastExprToType(From, ToType, Kind, VK_RValue, &BasePath);
     break;
   }
-  
+
   case ICK_Pointer_Member: {
-    CastKind Kind = CK_Unknown;
+    CastKind Kind = CK_Invalid;
     CXXCastPath BasePath;
-    if (CheckMemberPointerConversion(From, ToType, Kind, BasePath,
-                                     IgnoreBaseAccess))
+    if (CheckMemberPointerConversion(From, ToType, Kind, BasePath, CStyle))
       return true;
     if (CheckExceptionSpecCompatibility(From, ToType))
       return true;
@@ -1860,22 +2147,29 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     break;
   }
   case ICK_Boolean_Conversion: {
-    CastKind Kind = CK_Unknown;
-    if (FromType->isMemberPointerType())
-      Kind = CK_MemberPointerToBoolean;
-    
+    CastKind Kind = CK_Invalid;
+    switch (FromType->getScalarTypeKind()) {
+    case Type::STK_Pointer: Kind = CK_PointerToBoolean; break;
+    case Type::STK_MemberPointer: Kind = CK_MemberPointerToBoolean; break;
+    case Type::STK_Bool: llvm_unreachable("bool -> bool conversion?");
+    case Type::STK_Integral: Kind = CK_IntegralToBoolean; break;
+    case Type::STK_Floating: Kind = CK_FloatingToBoolean; break;
+    case Type::STK_IntegralComplex: Kind = CK_IntegralComplexToBoolean; break;
+    case Type::STK_FloatingComplex: Kind = CK_FloatingComplexToBoolean; break;
+    }
+
     ImpCastExprToType(From, Context.BoolTy, Kind);
     break;
   }
 
   case ICK_Derived_To_Base: {
     CXXCastPath BasePath;
-    if (CheckDerivedToBaseConversion(From->getType(), 
+    if (CheckDerivedToBaseConversion(From->getType(),
                                      ToType.getNonReferenceType(),
                                      From->getLocStart(),
-                                     From->getSourceRange(), 
+                                     From->getSourceRange(),
                                      &BasePath,
-                                     IgnoreBaseAccess))
+                                     CStyle))
       return true;
 
     ImpCastExprToType(From, ToType.getNonReferenceType(),
@@ -1891,10 +2185,60 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   case ICK_Vector_Splat:
     ImpCastExprToType(From, ToType, CK_VectorSplat);
     break;
-      
+
   case ICK_Complex_Real:
-    ImpCastExprToType(From, ToType, CK_Unknown);
+    // Case 1.  x -> _Complex y
+    if (const ComplexType *ToComplex = ToType->getAs<ComplexType>()) {
+      QualType ElType = ToComplex->getElementType();
+      bool isFloatingComplex = ElType->isRealFloatingType();
+
+      // x -> y
+      if (Context.hasSameUnqualifiedType(ElType, From->getType())) {
+        // do nothing
+      } else if (From->getType()->isRealFloatingType()) {
+        ImpCastExprToType(From, ElType,
+                isFloatingComplex ? CK_FloatingCast : CK_FloatingToIntegral);
+      } else {
+        assert(From->getType()->isIntegerType());
+        ImpCastExprToType(From, ElType,
+                isFloatingComplex ? CK_IntegralToFloating : CK_IntegralCast);
+      }
+      // y -> _Complex y
+      ImpCastExprToType(From, ToType,
+                   isFloatingComplex ? CK_FloatingRealToComplex
+                                     : CK_IntegralRealToComplex);
+
+    // Case 2.  _Complex x -> y
+    } else {
+      const ComplexType *FromComplex = From->getType()->getAs<ComplexType>();
+      assert(FromComplex);
+
+      QualType ElType = FromComplex->getElementType();
+      bool isFloatingComplex = ElType->isRealFloatingType();
+
+      // _Complex x -> x
+      ImpCastExprToType(From, ElType,
+                   isFloatingComplex ? CK_FloatingComplexToReal
+                                     : CK_IntegralComplexToReal);
+
+      // x -> y
+      if (Context.hasSameUnqualifiedType(ElType, ToType)) {
+        // do nothing
+      } else if (ToType->isRealFloatingType()) {
+        ImpCastExprToType(From, ToType,
+                isFloatingComplex ? CK_FloatingCast : CK_IntegralToFloating);
+      } else {
+        assert(ToType->isIntegerType());
+        ImpCastExprToType(From, ToType,
+                isFloatingComplex ? CK_FloatingToIntegral : CK_IntegralCast);
+      }
+    }
     break;
+  
+  case ICK_Block_Pointer_Conversion: {
+      ImpCastExprToType(From, ToType.getUnqualifiedType(), CK_BitCast, VK_RValue);
+      break;
+    }
       
   case ICK_Lvalue_To_Rvalue:
   case ICK_Array_To_Pointer:
@@ -1933,31 +2277,409 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   return false;
 }
 
-ExprResult Sema::ActOnUnaryTypeTrait(UnaryTypeTrait OTT,
-                                                 SourceLocation KWLoc,
-                                                 SourceLocation LParen,
-                                                 ParsedType Ty,
-                                                 SourceLocation RParen) {
-  QualType T = GetTypeFromParser(Ty);
+ExprResult Sema::ActOnUnaryTypeTrait(UnaryTypeTrait UTT,
+                                     SourceLocation KWLoc,
+                                     ParsedType Ty,
+                                     SourceLocation RParen) {
+  TypeSourceInfo *TSInfo;
+  QualType T = GetTypeFromParser(Ty, &TSInfo);
+
+  if (!TSInfo)
+    TSInfo = Context.getTrivialTypeSourceInfo(T);
+  return BuildUnaryTypeTrait(UTT, KWLoc, TSInfo, RParen);
+}
+
+static bool EvaluateUnaryTypeTrait(Sema &Self, UnaryTypeTrait UTT, QualType T,
+                                   SourceLocation KeyLoc) {
+  // FIXME: For many of these traits, we need a complete type before we can 
+  // check these properties.
+  assert(!T->isDependentType() &&
+         "Cannot evaluate traits for dependent types.");
+  ASTContext &C = Self.Context;
+  switch(UTT) {
+  default: assert(false && "Unknown type trait or not implemented");
+  case UTT_IsPOD: return T->isPODType();
+  case UTT_IsLiteral: return T->isLiteralType();
+  case UTT_IsClass: // Fallthrough
+  case UTT_IsUnion:
+    if (const RecordType *Record = T->getAs<RecordType>()) {
+      bool Union = Record->getDecl()->isUnion();
+      return UTT == UTT_IsUnion ? Union : !Union;
+    }
+    return false;
+  case UTT_IsEnum: return T->isEnumeralType();
+  case UTT_IsPolymorphic:
+    if (const RecordType *Record = T->getAs<RecordType>()) {
+      // Type traits are only parsed in C++, so we've got CXXRecords.
+      return cast<CXXRecordDecl>(Record->getDecl())->isPolymorphic();
+    }
+    return false;
+  case UTT_IsAbstract:
+    if (const RecordType *RT = T->getAs<RecordType>())
+      return cast<CXXRecordDecl>(RT->getDecl())->isAbstract();
+    return false;
+  case UTT_IsEmpty:
+    if (const RecordType *Record = T->getAs<RecordType>()) {
+      return !Record->getDecl()->isUnion()
+          && cast<CXXRecordDecl>(Record->getDecl())->isEmpty();
+    }
+    return false;
+  case UTT_HasTrivialConstructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __is_pod (type) is true then the trait is true, else if type is
+    //   a cv class or union type (or array thereof) with a trivial default
+    //   constructor ([class.ctor]) then the trait is true, else it is false.
+    if (T->isPODType())
+      return true;
+    if (const RecordType *RT =
+          C.getBaseElementType(T)->getAs<RecordType>())
+      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialConstructor();
+    return false;
+  case UTT_HasTrivialCopy:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __is_pod (type) is true or type is a reference type then
+    //   the trait is true, else if type is a cv class or union type
+    //   with a trivial copy constructor ([class.copy]) then the trait
+    //   is true, else it is false.
+    if (T->isPODType() || T->isReferenceType())
+      return true;
+    if (const RecordType *RT = T->getAs<RecordType>())
+      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialCopyConstructor();
+    return false;
+  case UTT_HasTrivialAssign:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If type is const qualified or is a reference type then the
+    //   trait is false. Otherwise if __is_pod (type) is true then the
+    //   trait is true, else if type is a cv class or union type with
+    //   a trivial copy assignment ([class.copy]) then the trait is
+    //   true, else it is false.
+    // Note: the const and reference restrictions are interesting,
+    // given that const and reference members don't prevent a class
+    // from having a trivial copy assignment operator (but do cause
+    // errors if the copy assignment operator is actually used, q.v.
+    // [class.copy]p12).
+
+    if (C.getBaseElementType(T).isConstQualified())
+      return false;
+    if (T->isPODType())
+      return true;
+    if (const RecordType *RT = T->getAs<RecordType>())
+      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialCopyAssignment();
+    return false;
+  case UTT_HasTrivialDestructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __is_pod (type) is true or type is a reference type
+    //   then the trait is true, else if type is a cv class or union
+    //   type (or array thereof) with a trivial destructor
+    //   ([class.dtor]) then the trait is true, else it is
+    //   false.
+    if (T->isPODType() || T->isReferenceType())
+      return true;
+    if (const RecordType *RT =
+          C.getBaseElementType(T)->getAs<RecordType>())
+      return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialDestructor();
+    return false;
+  // TODO: Propagate nothrowness for implicitly declared special members.
+  case UTT_HasNothrowAssign:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If type is const qualified or is a reference type then the
+    //   trait is false. Otherwise if __has_trivial_assign (type)
+    //   is true then the trait is true, else if type is a cv class
+    //   or union type with copy assignment operators that are known
+    //   not to throw an exception then the trait is true, else it is
+    //   false.
+    if (C.getBaseElementType(T).isConstQualified())
+      return false;
+    if (T->isReferenceType())
+      return false;
+    if (T->isPODType())
+      return true;
+    if (const RecordType *RT = T->getAs<RecordType>()) {
+      CXXRecordDecl* RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialCopyAssignment())
+        return true;
+
+      bool FoundAssign = false;
+      bool AllNoThrow = true;
+      DeclarationName Name = C.DeclarationNames.getCXXOperatorName(OO_Equal);
+      LookupResult Res(Self, DeclarationNameInfo(Name, KeyLoc),
+                       Sema::LookupOrdinaryName);
+      if (Self.LookupQualifiedName(Res, RD)) {
+        for (LookupResult::iterator Op = Res.begin(), OpEnd = Res.end();
+             Op != OpEnd; ++Op) {
+          CXXMethodDecl *Operator = cast<CXXMethodDecl>(*Op);
+          if (Operator->isCopyAssignmentOperator()) {
+            FoundAssign = true;
+            const FunctionProtoType *CPT
+                = Operator->getType()->getAs<FunctionProtoType>();
+            if (!CPT->hasEmptyExceptionSpec()) {
+              AllNoThrow = false;
+              break;
+            }
+          }
+        }
+      }
+
+      return FoundAssign && AllNoThrow;
+    }
+    return false;
+  case UTT_HasNothrowCopy:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __has_trivial_copy (type) is true then the trait is true, else
+    //   if type is a cv class or union type with copy constructors that are
+    //   known not to throw an exception then the trait is true, else it is
+    //   false.
+    if (T->isPODType() || T->isReferenceType())
+      return true;
+    if (const RecordType *RT = T->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialCopyConstructor())
+        return true;
+
+      bool FoundConstructor = false;
+      bool AllNoThrow = true;
+      unsigned FoundTQs;
+      DeclContext::lookup_const_iterator Con, ConEnd;
+      for (llvm::tie(Con, ConEnd) = Self.LookupConstructors(RD);
+           Con != ConEnd; ++Con) {
+        // A template constructor is never a copy constructor.
+        // FIXME: However, it may actually be selected at the actual overload
+        // resolution point.
+        if (isa<FunctionTemplateDecl>(*Con))
+          continue;
+        CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
+        if (Constructor->isCopyConstructor(FoundTQs)) {
+          FoundConstructor = true;
+          const FunctionProtoType *CPT
+              = Constructor->getType()->getAs<FunctionProtoType>();
+          // TODO: check whether evaluating default arguments can throw.
+          // For now, we'll be conservative and assume that they can throw.
+          if (!CPT->hasEmptyExceptionSpec() || CPT->getNumArgs() > 1) {
+            AllNoThrow = false;
+            break;
+          }
+        }
+      }
+
+      return FoundConstructor && AllNoThrow;
+    }
+    return false;
+  case UTT_HasNothrowConstructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __has_trivial_constructor (type) is true then the trait is
+    //   true, else if type is a cv class or union type (or array
+    //   thereof) with a default constructor that is known not to
+    //   throw an exception then the trait is true, else it is false.
+    if (T->isPODType())
+      return true;
+    if (const RecordType *RT = C.getBaseElementType(T)->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialConstructor())
+        return true;
+
+      DeclContext::lookup_const_iterator Con, ConEnd;
+      for (llvm::tie(Con, ConEnd) = Self.LookupConstructors(RD);
+           Con != ConEnd; ++Con) {
+        // FIXME: In C++0x, a constructor template can be a default constructor.
+        if (isa<FunctionTemplateDecl>(*Con))
+          continue;
+        CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
+        if (Constructor->isDefaultConstructor()) {
+          const FunctionProtoType *CPT
+              = Constructor->getType()->getAs<FunctionProtoType>();
+          // TODO: check whether evaluating default arguments can throw.
+          // For now, we'll be conservative and assume that they can throw.
+          return CPT->hasEmptyExceptionSpec() && CPT->getNumArgs() == 0;
+        }
+      }
+    }
+    return false;
+  case UTT_HasVirtualDestructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If type is a class type with a virtual destructor ([class.dtor])
+    //   then the trait is true, else it is false.
+    if (const RecordType *Record = T->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
+      if (CXXDestructorDecl *Destructor = Self.LookupDestructor(RD))
+        return Destructor->isVirtual();
+    }
+    return false;
+  }
+}
+
+ExprResult Sema::BuildUnaryTypeTrait(UnaryTypeTrait UTT,
+                                     SourceLocation KWLoc,
+                                     TypeSourceInfo *TSInfo,
+                                     SourceLocation RParen) {
+  QualType T = TSInfo->getType();
 
   // According to http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html
   // all traits except __is_class, __is_enum and __is_union require a the type
-  // to be complete.
-  if (OTT != UTT_IsClass && OTT != UTT_IsEnum && OTT != UTT_IsUnion) {
-    if (RequireCompleteType(KWLoc, T,
+  // to be complete, an array of unknown bound, or void.
+  if (UTT != UTT_IsClass && UTT != UTT_IsEnum && UTT != UTT_IsUnion) {
+    QualType E = T;
+    if (T->isIncompleteArrayType())
+      E = Context.getAsArrayType(T)->getElementType();
+    if (!T->isVoidType() &&
+        RequireCompleteType(KWLoc, E,
                             diag::err_incomplete_type_used_in_type_trait_expr))
       return ExprError();
   }
 
-  // There is no point in eagerly computing the value. The traits are designed
-  // to be used from type trait templates, so Ty will be a template parameter
-  // 99% of the time.
-  return Owned(new (Context) UnaryTypeTraitExpr(KWLoc, OTT, T,
+  bool Value = false;
+  if (!T->isDependentType())
+    Value = EvaluateUnaryTypeTrait(*this, UTT, T, KWLoc);
+
+  return Owned(new (Context) UnaryTypeTraitExpr(KWLoc, UTT, TSInfo, Value,
                                                 RParen, Context.BoolTy));
 }
 
-QualType Sema::CheckPointerToMemberOperands(
-  Expr *&lex, Expr *&rex, SourceLocation Loc, bool isIndirect) {
+ExprResult Sema::ActOnBinaryTypeTrait(BinaryTypeTrait BTT,
+                                      SourceLocation KWLoc,
+                                      ParsedType LhsTy,
+                                      ParsedType RhsTy,
+                                      SourceLocation RParen) {
+  TypeSourceInfo *LhsTSInfo;
+  QualType LhsT = GetTypeFromParser(LhsTy, &LhsTSInfo);
+  if (!LhsTSInfo)
+    LhsTSInfo = Context.getTrivialTypeSourceInfo(LhsT);
+
+  TypeSourceInfo *RhsTSInfo;
+  QualType RhsT = GetTypeFromParser(RhsTy, &RhsTSInfo);
+  if (!RhsTSInfo)
+    RhsTSInfo = Context.getTrivialTypeSourceInfo(RhsT);
+
+  return BuildBinaryTypeTrait(BTT, KWLoc, LhsTSInfo, RhsTSInfo, RParen);
+}
+
+static bool EvaluateBinaryTypeTrait(Sema &Self, BinaryTypeTrait BTT,
+                                    QualType LhsT, QualType RhsT,
+                                    SourceLocation KeyLoc) {
+  assert((!LhsT->isDependentType() || RhsT->isDependentType()) &&
+         "Cannot evaluate traits for dependent types.");
+
+  switch(BTT) {
+  case BTT_IsBaseOf: {
+    // C++0x [meta.rel]p2
+    // Base is a base class of Derived without regard to cv-qualifiers or
+    // Base and Derived are not unions and name the same class type without
+    // regard to cv-qualifiers.
+
+    const RecordType *lhsRecord = LhsT->getAs<RecordType>();
+    if (!lhsRecord) return false;
+
+    const RecordType *rhsRecord = RhsT->getAs<RecordType>();
+    if (!rhsRecord) return false;
+
+    assert(Self.Context.hasSameUnqualifiedType(LhsT, RhsT)
+             == (lhsRecord == rhsRecord));
+
+    if (lhsRecord == rhsRecord)
+      return !lhsRecord->getDecl()->isUnion();
+
+    // C++0x [meta.rel]p2:
+    //   If Base and Derived are class types and are different types
+    //   (ignoring possible cv-qualifiers) then Derived shall be a
+    //   complete type.
+    if (Self.RequireCompleteType(KeyLoc, RhsT, 
+                          diag::err_incomplete_type_used_in_type_trait_expr))
+      return false;
+
+    return cast<CXXRecordDecl>(rhsRecord->getDecl())
+      ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
+  }
+
+  case BTT_TypeCompatible:
+    return Self.Context.typesAreCompatible(LhsT.getUnqualifiedType(),
+                                           RhsT.getUnqualifiedType());
+      
+  case BTT_IsConvertibleTo: {
+    // C++0x [meta.rel]p4:
+    //   Given the following function prototype:
+    //
+    //     template <class T> 
+    //       typename add_rvalue_reference<T>::type create();
+    //
+    //   the predicate condition for a template specialization 
+    //   is_convertible<From, To> shall be satisfied if and only if 
+    //   the return expression in the following code would be 
+    //   well-formed, including any implicit conversions to the return
+    //   type of the function:
+    //
+    //     To test() { 
+    //       return create<From>();
+    //     }
+    //
+    //   Access checking is performed as if in a context unrelated to To and 
+    //   From. Only the validity of the immediate context of the expression 
+    //   of the return-statement (including conversions to the return type)
+    //   is considered.
+    //
+    // We model the initialization as a copy-initialization of a temporary
+    // of the appropriate type, which for this expression is identical to the
+    // return statement (since NRVO doesn't apply).
+    if (LhsT->isObjectType() || LhsT->isFunctionType())
+      LhsT = Self.Context.getRValueReferenceType(LhsT);
+    
+    InitializedEntity To(InitializedEntity::InitializeTemporary(RhsT));
+    OpaqueValueExpr From(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
+                         Expr::getValueKindForType(LhsT));
+    Expr *FromPtr = &From;
+    InitializationKind Kind(InitializationKind::CreateCopy(KeyLoc, 
+                                                           SourceLocation()));
+    
+    // Perform the initialization within a SFINAE trap at translation unit 
+    // scope.
+    Sema::SFINAETrap SFINAE(Self, /*AccessCheckingSFINAE=*/true);
+    Sema::ContextRAII TUContext(Self, Self.Context.getTranslationUnitDecl());
+    InitializationSequence Init(Self, To, Kind, &FromPtr, 1);
+    if (Init.getKind() == InitializationSequence::FailedSequence)
+      return false;
+
+    ExprResult Result = Init.Perform(Self, To, Kind, MultiExprArg(&FromPtr, 1));
+    return !Result.isInvalid() && !SFINAE.hasErrorOccurred();
+  }
+  }
+  llvm_unreachable("Unknown type trait or not implemented");
+}
+
+ExprResult Sema::BuildBinaryTypeTrait(BinaryTypeTrait BTT,
+                                      SourceLocation KWLoc,
+                                      TypeSourceInfo *LhsTSInfo,
+                                      TypeSourceInfo *RhsTSInfo,
+                                      SourceLocation RParen) {
+  QualType LhsT = LhsTSInfo->getType();
+  QualType RhsT = RhsTSInfo->getType();
+
+  if (BTT == BTT_TypeCompatible) {
+    if (getLangOptions().CPlusPlus) {
+      Diag(KWLoc, diag::err_types_compatible_p_in_cplusplus)
+        << SourceRange(KWLoc, RParen);
+      return ExprError();
+    }
+  }
+
+  bool Value = false;
+  if (!LhsT->isDependentType() && !RhsT->isDependentType())
+    Value = EvaluateBinaryTypeTrait(*this, BTT, LhsT, RhsT, KWLoc);
+
+  // Select trait result type.
+  QualType ResultType;
+  switch (BTT) {
+  case BTT_IsBaseOf:       ResultType = Context.BoolTy; break;
+  case BTT_TypeCompatible: ResultType = Context.IntTy; break;
+  case BTT_IsConvertibleTo: ResultType = Context.BoolTy; break;
+  }
+
+  return Owned(new (Context) BinaryTypeTraitExpr(KWLoc, BTT, LhsTSInfo,
+                                                 RhsTSInfo, Value, RParen,
+                                                 ResultType));
+}
+
+QualType Sema::CheckPointerToMemberOperands(Expr *&lex, Expr *&rex,
+                                            ExprValueKind &VK,
+                                            SourceLocation Loc,
+                                            bool isIndirect) {
   const char *OpSpelling = isIndirect ? "->*" : ".*";
   // C++ 5.5p2
   //   The binary operator .* [p3: ->*] binds its second operand, which shall
@@ -1973,8 +2695,11 @@ QualType Sema::CheckPointerToMemberOperands(
 
   QualType Class(MemPtr->getClass(), 0);
 
-  if (RequireCompleteType(Loc, Class, diag::err_memptr_rhs_to_incomplete))
-    return QualType();
+  // Note: C++ [expr.mptr.oper]p2-3 says that the class type into which the
+  // member pointer points must be completely-defined. However, there is no
+  // reason for this semantic distinction, and the rule is not enforced by
+  // other compilers. Therefore, we do not check this property, as it is
+  // likely to be considered a defect.
 
   // C++ 5.5p2
   //   [...] to its first operand, which shall be of class T or of a class of
@@ -1983,7 +2708,7 @@ QualType Sema::CheckPointerToMemberOperands(
   QualType LType = lex->getType();
   if (isIndirect) {
     if (const PointerType *Ptr = LType->getAs<PointerType>())
-      LType = Ptr->getPointeeType().getNonReferenceType();
+      LType = Ptr->getPointeeType();
     else {
       Diag(Loc, diag::err_bad_memptr_lhs)
         << OpSpelling << 1 << LType
@@ -2024,6 +2749,7 @@ QualType Sema::CheckPointerToMemberOperands(
     Diag(Loc, diag::err_pointer_to_member_type) << isIndirect;
      return QualType();
   }
+
   // C++ 5.5p2
   //   The result is an object or a function of the type specified by the
   //   second operand.
@@ -2038,6 +2764,47 @@ QualType Sema::CheckPointerToMemberOperands(
   // We probably need a "MemberFunctionClosureType" or something like that.
   QualType Result = MemPtr->getPointeeType();
   Result = Context.getCVRQualifiedType(Result, LType.getCVRQualifiers());
+
+  // C++0x [expr.mptr.oper]p6:
+  //   In a .* expression whose object expression is an rvalue, the program is
+  //   ill-formed if the second operand is a pointer to member function with
+  //   ref-qualifier &. In a ->* expression or in a .* expression whose object
+  //   expression is an lvalue, the program is ill-formed if the second operand
+  //   is a pointer to member function with ref-qualifier &&.
+  if (const FunctionProtoType *Proto = Result->getAs<FunctionProtoType>()) {
+    switch (Proto->getRefQualifier()) {
+    case RQ_None:
+      // Do nothing
+      break;
+
+    case RQ_LValue:
+      if (!isIndirect && !lex->Classify(Context).isLValue())
+        Diag(Loc, diag::err_pointer_to_member_oper_value_classify)
+          << RType << 1 << lex->getSourceRange();
+      break;
+
+    case RQ_RValue:
+      if (isIndirect || !lex->Classify(Context).isRValue())
+        Diag(Loc, diag::err_pointer_to_member_oper_value_classify)
+          << RType << 0 << lex->getSourceRange();
+      break;
+    }
+  }
+
+  // C++ [expr.mptr.oper]p6:
+  //   The result of a .* expression whose second operand is a pointer
+  //   to a data member is of the same value category as its
+  //   first operand. The result of a .* expression whose second
+  //   operand is a pointer to a member function is a prvalue. The
+  //   result of an ->* expression is an lvalue if its second operand
+  //   is a pointer to data member and a prvalue otherwise.
+  if (Result->isFunctionType())
+    VK = VK_RValue;
+  else if (isIndirect)
+    VK = VK_LValue;
+  else
+    VK = lex->getValueKind();
+
   return Result;
 }
 
@@ -2054,29 +2821,29 @@ static bool TryClassUnification(Sema &Self, Expr *From, Expr *To,
                                 QualType &ToType) {
   HaveConversion = false;
   ToType = To->getType();
-  
-  InitializationKind Kind = InitializationKind::CreateCopy(To->getLocStart(), 
+
+  InitializationKind Kind = InitializationKind::CreateCopy(To->getLocStart(),
                                                            SourceLocation());
   // C++0x 5.16p3
   //   The process for determining whether an operand expression E1 of type T1
   //   can be converted to match an operand expression E2 of type T2 is defined
   //   as follows:
   //   -- If E2 is an lvalue:
-  bool ToIsLvalue = (To->isLvalue(Self.Context) == Expr::LV_Valid);
+  bool ToIsLvalue = To->isLValue();
   if (ToIsLvalue) {
     //   E1 can be converted to match E2 if E1 can be implicitly converted to
     //   type "lvalue reference to T2", subject to the constraint that in the
     //   conversion the reference must bind directly to E1.
     QualType T = Self.Context.getLValueReferenceType(ToType);
     InitializedEntity Entity = InitializedEntity::InitializeTemporary(T);
-    
+
     InitializationSequence InitSeq(Self, Entity, Kind, &From, 1);
     if (InitSeq.isDirectReferenceBinding()) {
       ToType = T;
       HaveConversion = true;
       return false;
     }
-    
+
     if (InitSeq.isAmbiguous())
       return InitSeq.Diagnose(Self, Entity, Kind, &From, 1);
   }
@@ -2088,9 +2855,9 @@ static bool TryClassUnification(Sema &Self, Expr *From, Expr *To,
   QualType TTy = To->getType();
   const RecordType *FRec = FTy->getAs<RecordType>();
   const RecordType *TRec = TTy->getAs<RecordType>();
-  bool FDerivedFromT = FRec && TRec && FRec != TRec && 
+  bool FDerivedFromT = FRec && TRec && FRec != TRec &&
                        Self.IsDerivedFrom(FTy, TTy);
-  if (FRec && TRec && 
+  if (FRec && TRec &&
       (FRec == TRec || FDerivedFromT || Self.IsDerivedFrom(TTy, FTy))) {
     //         E1 can be converted to match E2 if the class of T2 is the
     //         same type as, or a base class of, the class of T1, and
@@ -2103,28 +2870,28 @@ static bool TryClassUnification(Sema &Self, Expr *From, Expr *To,
           HaveConversion = true;
           return false;
         }
-        
+
         if (InitSeq.isAmbiguous())
           return InitSeq.Diagnose(Self, Entity, Kind, &From, 1);
-      } 
+      }
     }
-    
+
     return false;
   }
-  
+
   //     -- Otherwise: E1 can be converted to match E2 if E1 can be
   //        implicitly converted to the type that expression E2 would have
-  //        if E2 were converted to an rvalue (or the type it has, if E2 is 
+  //        if E2 were converted to an rvalue (or the type it has, if E2 is
   //        an rvalue).
   //
   // This actually refers very narrowly to the lvalue-to-rvalue conversion, not
   // to the array-to-pointer or function-to-pointer conversions.
   if (!TTy->getAs<TagType>())
     TTy = TTy.getUnqualifiedType();
-  
+
   InitializedEntity Entity = InitializedEntity::InitializeTemporary(TTy);
   InitializationSequence InitSeq(Self, Entity, Kind, &From, 1);
-  HaveConversion = InitSeq.getKind() != InitializationSequence::FailedSequence;  
+  HaveConversion = InitSeq.getKind() != InitializationSequence::FailedSequence;
   ToType = TTy;
   if (InitSeq.isAmbiguous())
     return InitSeq.Diagnose(Self, Entity, Kind, &From, 1);
@@ -2138,13 +2905,14 @@ static bool TryClassUnification(Sema &Self, Expr *From, Expr *To,
 /// value operand is a class type, overload resolution is used to find a
 /// conversion to a common type.
 static bool FindConditionalOverload(Sema &Self, Expr *&LHS, Expr *&RHS,
-                                    SourceLocation Loc) {
+                                    SourceLocation QuestionLoc) {
   Expr *Args[2] = { LHS, RHS };
-  OverloadCandidateSet CandidateSet(Loc);
-  Self.AddBuiltinOperatorCandidates(OO_Conditional, Loc, Args, 2, CandidateSet);
+  OverloadCandidateSet CandidateSet(QuestionLoc);
+  Self.AddBuiltinOperatorCandidates(OO_Conditional, QuestionLoc, Args, 2,
+                                    CandidateSet);
 
   OverloadCandidateSet::iterator Best;
-  switch (CandidateSet.BestViableFunction(Self, Loc, Best)) {
+  switch (CandidateSet.BestViableFunction(Self, QuestionLoc, Best)) {
     case OR_Success:
       // We found a match. Perform the conversions on the arguments and move on.
       if (Self.PerformImplicitConversion(LHS, Best->BuiltinTypes.ParamTypes[0],
@@ -2155,13 +2923,20 @@ static bool FindConditionalOverload(Sema &Self, Expr *&LHS, Expr *&RHS,
       return false;
 
     case OR_No_Viable_Function:
-      Self.Diag(Loc, diag::err_typecheck_cond_incompatible_operands)
+
+      // Emit a better diagnostic if one of the expressions is a null pointer
+      // constant and the other is a pointer type. In this case, the user most
+      // likely forgot to take the address of the other expression.
+      if (Self.DiagnoseConditionalForNull(LHS, RHS, QuestionLoc))
+        return true;
+
+      Self.Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
         << LHS->getType() << RHS->getType()
         << LHS->getSourceRange() << RHS->getSourceRange();
       return true;
 
     case OR_Ambiguous:
-      Self.Diag(Loc, diag::err_conditional_ambiguous_ovl)
+      Self.Diag(QuestionLoc, diag::err_conditional_ambiguous_ovl)
         << LHS->getType() << RHS->getType()
         << LHS->getSourceRange() << RHS->getSourceRange();
       // FIXME: Print the possible common types by printing the return types of
@@ -2185,7 +2960,7 @@ static bool ConvertForConditional(Sema &Self, Expr *&E, QualType T) {
   ExprResult Result = InitSeq.Perform(Self, Entity, Kind, MultiExprArg(&E, 1));
   if (Result.isInvalid())
     return true;
-  
+
   E = Result.takeAs<Expr>();
   return false;
 }
@@ -2195,6 +2970,7 @@ static bool ConvertForConditional(Sema &Self, Expr *&E, QualType T) {
 /// See C++ [expr.cond]. Note that LHS is never null, even for the GNU x ?: y
 /// extension. In this case, LHS == Cond. (But they're not aliases.)
 QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
+                                           ExprValueKind &VK, ExprObjectKind &OK,
                                            SourceLocation QuestionLoc) {
   // FIXME: Handle C99's complex types, vector types, block pointers and Obj-C++
   // interface pointers.
@@ -2206,6 +2982,10 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
       return QualType();
   }
 
+  // Assume r-value.
+  VK = VK_RValue;
+  OK = OK_Ordinary;
+
   // Either of the arguments dependent?
   if (LHS->isTypeDependent() || RHS->isTypeDependent())
     return Context.DependentTy;
@@ -2252,7 +3032,7 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   //   Otherwise, if the second and third operand have different types, and
   //   either has (cv) class type, and attempt is made to convert each of those
   //   operands to the other.
-  if (!Context.hasSameType(LTy, RTy) && 
+  if (!Context.hasSameType(LTy, RTy) &&
       (LTy->isRecordType() || RTy->isRecordType())) {
     ImplicitConversionSequence ICSLeftToRight, ICSRightToLeft;
     // These return true if a single direction is already ambiguous.
@@ -2262,7 +3042,7 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
       return QualType();
     if (TryClassUnification(*this, RHS, LHS, QuestionLoc, HaveR2L, R2LType))
       return QualType();
-    
+
     //   If both can be converted, [...] the program is ill-formed.
     if (HaveL2R && HaveR2L) {
       Diag(QuestionLoc, diag::err_conditional_ambiguous)
@@ -2285,12 +3065,24 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   }
 
   // C++0x 5.16p4
-  //   If the second and third operands are lvalues and have the same type,
-  //   the result is of that type [...]
+  //   If the second and third operands are glvalues of the same value
+  //   category and have the same type, the result is of that type and
+  //   value category and it is a bit-field if the second or the third
+  //   operand is a bit-field, or if both are bit-fields.
+  // We only extend this to bitfields, not to the crazy other kinds of
+  // l-values.
   bool Same = Context.hasSameType(LTy, RTy);
-  if (Same && LHS->isLvalue(Context) == Expr::LV_Valid &&
-      RHS->isLvalue(Context) == Expr::LV_Valid)
+  if (Same &&
+      LHS->isGLValue() &&
+      LHS->getValueKind() == RHS->getValueKind() &&
+      LHS->isOrdinaryOrBitFieldObject() &&
+      RHS->isOrdinaryOrBitFieldObject()) {
+    VK = LHS->getValueKind();
+    if (LHS->getObjectKind() == OK_BitField ||
+        RHS->getObjectKind() == OK_BitField)
+      OK = OK_BitField;
     return LTy;
+  }
 
   // C++0x 5.16p5
   //   Otherwise, the result is an rvalue. If the second and third operands
@@ -2321,18 +3113,18 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     if (LTy->isRecordType()) {
       // The operands have class type. Make a temporary copy.
       InitializedEntity Entity = InitializedEntity::InitializeTemporary(LTy);
-      ExprResult LHSCopy = PerformCopyInitialization(Entity, 
-                                                           SourceLocation(), 
-                                                           Owned(LHS));
+      ExprResult LHSCopy = PerformCopyInitialization(Entity,
+                                                     SourceLocation(),
+                                                     Owned(LHS));
       if (LHSCopy.isInvalid())
         return QualType();
-        
-      ExprResult RHSCopy = PerformCopyInitialization(Entity, 
-                                                           SourceLocation(), 
-                                                           Owned(RHS));
+
+      ExprResult RHSCopy = PerformCopyInitialization(Entity,
+                                                     SourceLocation(),
+                                                     Owned(RHS));
       if (RHSCopy.isInvalid())
         return QualType();
-      
+
       LHS = LHSCopy.takeAs<Expr>();
       RHS = RHSCopy.takeAs<Expr>();
     }
@@ -2341,7 +3133,7 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   }
 
   // Extension: conditional operator involving vector types.
-  if (LTy->isVectorType() || RTy->isVectorType()) 
+  if (LTy->isVectorType() || RTy->isVectorType())
     return CheckVectorOperands(QuestionLoc, LHS, RHS);
 
   //   -- The second and third operands have arithmetic or enumeration type;
@@ -2367,19 +3159,23 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
                               isSFINAEContext()? 0 : &NonStandardCompositeType);
   if (!Composite.isNull()) {
     if (NonStandardCompositeType)
-      Diag(QuestionLoc, 
+      Diag(QuestionLoc,
            diag::ext_typecheck_cond_incompatible_operands_nonstandard)
         << LTy << RTy << Composite
         << LHS->getSourceRange() << RHS->getSourceRange();
-      
+
     return Composite;
   }
-  
+
   // Similarly, attempt to find composite type of two objective-c pointers.
   Composite = FindCompositeObjCPointerType(LHS, RHS, QuestionLoc);
   if (!Composite.isNull())
     return Composite;
 
+  // Check if we are using a null with a non-pointer type.
+  if (DiagnoseConditionalForNull(LHS, RHS, QuestionLoc))
+    return QualType();
+
   Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
     << LHS->getType() << RHS->getType()
     << LHS->getSourceRange() << RHS->getSourceRange();
@@ -2400,12 +3196,12 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
 /// a non-standard (but still sane) composite type to which both expressions
 /// can be converted. When such a type is chosen, \c *NonStandardCompositeType
 /// will be set true.
-QualType Sema::FindCompositePointerType(SourceLocation Loc, 
+QualType Sema::FindCompositePointerType(SourceLocation Loc,
                                         Expr *&E1, Expr *&E2,
                                         bool *NonStandardCompositeType) {
   if (NonStandardCompositeType)
     *NonStandardCompositeType = false;
-  
+
   assert(getLangOptions().CPlusPlus && "This function assumes C++");
   QualType T1 = E1->getType(), T2 = E2->getType();
 
@@ -2422,14 +3218,14 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
     if (T2->isMemberPointerType())
       ImpCastExprToType(E1, T2, CK_NullToMemberPointer);
     else
-      ImpCastExprToType(E1, T2, CK_IntegralToPointer);
+      ImpCastExprToType(E1, T2, CK_NullToPointer);
     return T2;
   }
   if (E2->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
     if (T1->isMemberPointerType())
       ImpCastExprToType(E2, T1, CK_NullToMemberPointer);
     else
-      ImpCastExprToType(E2, T1, CK_IntegralToPointer);
+      ImpCastExprToType(E2, T1, CK_NullToPointer);
     return T1;
   }
 
@@ -2456,20 +3252,20 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
   ContainingClassVector MemberOfClass;
   QualType Composite1 = Context.getCanonicalType(T1),
            Composite2 = Context.getCanonicalType(T2);
-  unsigned NeedConstBefore = 0;  
+  unsigned NeedConstBefore = 0;
   do {
     const PointerType *Ptr1, *Ptr2;
     if ((Ptr1 = Composite1->getAs<PointerType>()) &&
         (Ptr2 = Composite2->getAs<PointerType>())) {
       Composite1 = Ptr1->getPointeeType();
       Composite2 = Ptr2->getPointeeType();
-      
+
       // If we're allowed to create a non-standard composite type, keep track
-      // of where we need to fill in additional 'const' qualifiers. 
+      // of where we need to fill in additional 'const' qualifiers.
       if (NonStandardCompositeType &&
           Composite1.getCVRQualifiers() != Composite2.getCVRQualifiers())
         NeedConstBefore = QualifierUnion.size();
-      
+
       QualifierUnion.push_back(
                  Composite1.getCVRQualifiers() | Composite2.getCVRQualifiers());
       MemberOfClass.push_back(std::make_pair((const Type *)0, (const Type *)0));
@@ -2481,13 +3277,13 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
         (MemPtr2 = Composite2->getAs<MemberPointerType>())) {
       Composite1 = MemPtr1->getPointeeType();
       Composite2 = MemPtr2->getPointeeType();
-      
+
       // If we're allowed to create a non-standard composite type, keep track
-      // of where we need to fill in additional 'const' qualifiers. 
+      // of where we need to fill in additional 'const' qualifiers.
       if (NonStandardCompositeType &&
           Composite1.getCVRQualifiers() != Composite2.getCVRQualifiers())
         NeedConstBefore = QualifierUnion.size();
-      
+
       QualifierUnion.push_back(
                  Composite1.getCVRQualifiers() | Composite2.getCVRQualifiers());
       MemberOfClass.push_back(std::make_pair(MemPtr1->getClass(),
@@ -2503,7 +3299,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
 
   if (NeedConstBefore && NonStandardCompositeType) {
     // Extension: Add 'const' to qualifiers that come before the first qualifier
-    // mismatch, so that our (non-standard!) composite type meets the 
+    // mismatch, so that our (non-standard!) composite type meets the
     // requirements of C++ [conv.qual]p4 bullet 3.
     for (unsigned I = 0; I != NeedConstBefore; ++I) {
       if ((QualifierUnion[I] & Qualifiers::Const) == 0) {
@@ -2512,7 +3308,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
       }
     }
   }
-  
+
   // Rewrap the composites as pointers or member pointers with the union CVRs.
   ContainingClassVector::reverse_iterator MOC
     = MemberOfClass.rbegin();
@@ -2575,7 +3371,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
     if (E2Result.isInvalid())
       return QualType();
     E2 = E2Result.takeAs<Expr>();
-    
+
     return Composite1;
   }
 
@@ -2586,25 +3382,28 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
   InitializationSequence E2ToC2(*this, Entity2, Kind, &E2, 1);
   if (!E1ToC2 || !E2ToC2)
     return QualType();
-  
+
   // Convert E1 to Composite2
   ExprResult E1Result
     = E1ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E1, 1));
   if (E1Result.isInvalid())
     return QualType();
   E1 = E1Result.takeAs<Expr>();
-  
+
   // Convert E2 to Composite2
   ExprResult E2Result
     = E2ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E2, 1));
   if (E2Result.isInvalid())
     return QualType();
   E2 = E2Result.takeAs<Expr>();
-  
+
   return Composite2;
 }
 
 ExprResult Sema::MaybeBindToTemporary(Expr *E) {
+  if (!E)
+    return ExprError();
+
   if (!Context.getLangOptions().CPlusPlus)
     return Owned(E);
 
@@ -2614,17 +3413,9 @@ ExprResult Sema::MaybeBindToTemporary(Expr *E) {
   if (!RT)
     return Owned(E);
 
-  // If this is the result of a call or an Objective-C message send expression,
-  // our source might actually be a reference, in which case we shouldn't bind.
-  if (CallExpr *CE = dyn_cast<CallExpr>(E)) {
-    if (CE->getCallReturnType()->isReferenceType())
-      return Owned(E);
-  } else if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
-    if (const ObjCMethodDecl *MD = ME->getMethodDecl()) {
-      if (MD->getResultType()->isReferenceType())
-        return Owned(E);    
-    }
-  }
+  // If the result is a glvalue, we shouldn't bind it.
+  if (E->Classify(Context).isGLValue())
+    return Owned(E);
 
   // That should be enough to guarantee that this type is complete.
   // If it has a trivial destructor, we can avoid the extra copy.
@@ -2644,48 +3435,49 @@ ExprResult Sema::MaybeBindToTemporary(Expr *E) {
   return Owned(CXXBindTemporaryExpr::Create(Context, Temp, E));
 }
 
-Expr *Sema::MaybeCreateCXXExprWithTemporaries(Expr *SubExpr) {
+Expr *Sema::MaybeCreateExprWithCleanups(Expr *SubExpr) {
   assert(SubExpr && "sub expression can't be null!");
 
-  // Check any implicit conversions within the expression.
-  CheckImplicitConversions(SubExpr);
-
   unsigned FirstTemporary = ExprEvalContexts.back().NumTemporaries;
   assert(ExprTemporaries.size() >= FirstTemporary);
   if (ExprTemporaries.size() == FirstTemporary)
     return SubExpr;
 
-  Expr *E = CXXExprWithTemporaries::Create(Context, SubExpr,
-                                           &ExprTemporaries[FirstTemporary],
-                                       ExprTemporaries.size() - FirstTemporary);
+  Expr *E = ExprWithCleanups::Create(Context, SubExpr,
+                                     &ExprTemporaries[FirstTemporary],
+                                     ExprTemporaries.size() - FirstTemporary);
   ExprTemporaries.erase(ExprTemporaries.begin() + FirstTemporary,
                         ExprTemporaries.end());
 
   return E;
 }
 
-ExprResult 
-Sema::MaybeCreateCXXExprWithTemporaries(ExprResult SubExpr) {
+ExprResult
+Sema::MaybeCreateExprWithCleanups(ExprResult SubExpr) {
   if (SubExpr.isInvalid())
     return ExprError();
-  
-  return Owned(MaybeCreateCXXExprWithTemporaries(SubExpr.takeAs<Expr>()));
+
+  return Owned(MaybeCreateExprWithCleanups(SubExpr.take()));
 }
 
-FullExpr Sema::CreateFullExpr(Expr *SubExpr) {
+Stmt *Sema::MaybeCreateStmtWithCleanups(Stmt *SubStmt) {
+  assert(SubStmt && "sub statement can't be null!");
+
   unsigned FirstTemporary = ExprEvalContexts.back().NumTemporaries;
   assert(ExprTemporaries.size() >= FirstTemporary);
-  
-  unsigned NumTemporaries = ExprTemporaries.size() - FirstTemporary;
-  CXXTemporary **Temporaries = 
-    NumTemporaries == 0 ? 0 : &ExprTemporaries[FirstTemporary];
-  
-  FullExpr E = FullExpr::Create(Context, SubExpr, Temporaries, NumTemporaries);
-
-  ExprTemporaries.erase(ExprTemporaries.begin() + FirstTemporary,
-                        ExprTemporaries.end());
-
-  return E;
+  if (ExprTemporaries.size() == FirstTemporary)
+    return SubStmt;
+
+  // FIXME: In order to attach the temporaries, wrap the statement into
+  // a StmtExpr; currently this is only used for asm statements.
+  // This is hacky, either create a new CXXStmtWithTemporaries statement or
+  // a new AsmStmtWithTemporaries.
+  CompoundStmt *CompStmt = new (Context) CompoundStmt(Context, &SubStmt, 1,
+                                                      SourceLocation(),
+                                                      SourceLocation());
+  Expr *E = new (Context) StmtExpr(CompStmt, Context.VoidTy, SourceLocation(),
+                                   SourceLocation());
+  return MaybeCreateExprWithCleanups(E);
 }
 
 ExprResult
@@ -2706,7 +3498,7 @@ Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
     if (OpKind == tok::arrow)
       if (const PointerType *Ptr = BaseType->getAs<PointerType>())
         BaseType = Ptr->getPointeeType();
-    
+
     ObjectType = ParsedType::make(BaseType);
     MayBePseudoDestructor = true;
     return Owned(Base);
@@ -2720,7 +3512,7 @@ Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
     llvm::SmallPtrSet<CanQualType,8> CTypes;
     llvm::SmallVector<SourceLocation, 8> Locations;
     CTypes.insert(Context.getCanonicalType(BaseType));
-    
+
     while (BaseType->isRecordType()) {
       Result = BuildOverloadedArrowExpr(S, Base, OpLoc);
       if (Result.isInvalid())
@@ -2760,10 +3552,10 @@ Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
 
   // The object type must be complete (or dependent).
   if (!BaseType->isDependentType() &&
-      RequireCompleteType(OpLoc, BaseType, 
+      RequireCompleteType(OpLoc, BaseType,
                           PDiag(diag::err_incomplete_member_access)))
     return ExprError();
-  
+
   // C++ [basic.lookup.classref]p2:
   //   If the id-expression in a class member access (5.2.5) is an
   //   unqualified-id, and the type of the object expression is of a class
@@ -2779,12 +3571,11 @@ ExprResult Sema::DiagnoseDtorReference(SourceLocation NameLoc,
   Diag(MemExpr->getLocStart(), diag::err_dtor_expr_without_call)
     << isa<CXXPseudoDestructorExpr>(MemExpr)
     << FixItHint::CreateInsertion(ExpectedLParenLoc, "()");
-  
+
   return ActOnCallExpr(/*Scope*/ 0,
                        MemExpr,
                        /*LPLoc*/ ExpectedLParenLoc,
                        MultiExprArg(),
-                       /*CommaLocs*/ 0,
                        /*RPLoc*/ ExpectedLParenLoc);
 }
 
@@ -2798,11 +3589,11 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
                                          PseudoDestructorTypeStorage Destructed,
                                                        bool HasTrailingLParen) {
   TypeSourceInfo *DestructedTypeInfo = Destructed.getTypeSourceInfo();
-  
+
   // C++ [expr.pseudo]p2:
-  //   The left-hand side of the dot operator shall be of scalar type. The 
+  //   The left-hand side of the dot operator shall be of scalar type. The
   //   left-hand side of the arrow operator shall be of pointer to scalar type.
-  //   This scalar type is the object type. 
+  //   This scalar type is the object type.
   QualType ObjectType = Base->getType();
   if (OpKind == tok::arrow) {
     if (const PointerType *Ptr = ObjectType->getAs<PointerType>()) {
@@ -2814,11 +3605,11 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
         << FixItHint::CreateReplacement(OpLoc, ".");
       if (isSFINAEContext())
         return ExprError();
-      
+
       OpKind = tok::period;
     }
   }
-  
+
   if (!ObjectType->isDependentType() && !ObjectType->isScalarType()) {
     Diag(OpLoc, diag::err_pseudo_dtor_base_not_scalar)
       << ObjectType << Base->getSourceRange();
@@ -2826,7 +3617,7 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
   }
 
   // C++ [expr.pseudo]p2:
-  //   [...] The cv-unqualified versions of the object type and of the type 
+  //   [...] The cv-unqualified versions of the object type and of the type
   //   designated by the pseudo-destructor-name shall be the same type.
   if (DestructedTypeInfo) {
     QualType DestructedType = DestructedTypeInfo->getType();
@@ -2837,7 +3628,7 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
       Diag(DestructedTypeStart, diag::err_pseudo_dtor_type_mismatch)
         << ObjectType << DestructedType << Base->getSourceRange()
         << DestructedTypeInfo->getTypeLoc().getLocalSourceRange();
-      
+
       // Recover by setting the destructed type to the object type.
       DestructedType = ObjectType;
       DestructedTypeInfo = Context.getTrivialTypeSourceInfo(ObjectType,
@@ -2845,29 +3636,29 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
       Destructed = PseudoDestructorTypeStorage(DestructedTypeInfo);
     }
   }
-  
+
   // C++ [expr.pseudo]p2:
   //   [...] Furthermore, the two type-names in a pseudo-destructor-name of the
   //   form
   //
-  //     ::[opt] nested-name-specifier[opt] type-name :: ~ type-name 
+  //     ::[opt] nested-name-specifier[opt] type-name :: ~ type-name
   //
   //   shall designate the same scalar type.
   if (ScopeTypeInfo) {
     QualType ScopeType = ScopeTypeInfo->getType();
     if (!ScopeType->isDependentType() && !ObjectType->isDependentType() &&
         !Context.hasSameUnqualifiedType(ScopeType, ObjectType)) {
-      
+
       Diag(ScopeTypeInfo->getTypeLoc().getLocalSourceRange().getBegin(),
            diag::err_pseudo_dtor_type_mismatch)
         << ObjectType << ScopeType << Base->getSourceRange()
         << ScopeTypeInfo->getTypeLoc().getLocalSourceRange();
-  
+
       ScopeType = QualType();
       ScopeTypeInfo = 0;
     }
   }
-  
+
   Expr *Result
     = new (Context) CXXPseudoDestructorExpr(Context, Base,
                                             OpKind == tok::arrow, OpLoc,
@@ -2876,10 +3667,10 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
                                             CCLoc,
                                             TildeLoc,
                                             Destructed);
-            
+
   if (HasTrailingLParen)
     return Owned(Result);
-  
+
   return DiagnoseDtorReference(Destructed.getLocation(), Result);
 }
 
@@ -2900,9 +3691,9 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
          "Invalid second type name in pseudo-destructor");
 
   // C++ [expr.pseudo]p2:
-  //   The left-hand side of the dot operator shall be of scalar type. The 
+  //   The left-hand side of the dot operator shall be of scalar type. The
   //   left-hand side of the arrow operator shall be of pointer to scalar type.
-  //   This scalar type is the object type. 
+  //   This scalar type is the object type.
   QualType ObjectType = Base->getType();
   if (OpKind == tok::arrow) {
     if (const PointerType *Ptr = ObjectType->getAs<PointerType>()) {
@@ -2914,7 +3705,7 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
         << FixItHint::CreateReplacement(OpLoc, ".");
       if (isSFINAEContext())
         return ExprError();
-      
+
       OpKind = tok::period;
     }
   }
@@ -2928,32 +3719,32 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
     else if (ObjectType->isDependentType())
       ObjectTypePtrForLookup = ParsedType::make(Context.DependentTy);
   }
-  
-  // Convert the name of the type being destructed (following the ~) into a 
+
+  // Convert the name of the type being destructed (following the ~) into a
   // type (with source-location information).
   QualType DestructedType;
   TypeSourceInfo *DestructedTypeInfo = 0;
   PseudoDestructorTypeStorage Destructed;
   if (SecondTypeName.getKind() == UnqualifiedId::IK_Identifier) {
-    ParsedType T = getTypeName(*SecondTypeName.Identifier, 
+    ParsedType T = getTypeName(*SecondTypeName.Identifier,
                                SecondTypeName.StartLocation,
-                               S, &SS, true, ObjectTypePtrForLookup);
-    if (!T && 
+                               S, &SS, true, false, ObjectTypePtrForLookup);
+    if (!T &&
         ((SS.isSet() && !computeDeclContext(SS, false)) ||
          (!SS.isSet() && ObjectType->isDependentType()))) {
-      // The name of the type being destroyed is a dependent name, and we 
+      // The name of the type being destroyed is a dependent name, and we
       // couldn't find anything useful in scope. Just store the identifier and
       // it's location, and we'll perform (qualified) name lookup again at
       // template instantiation time.
       Destructed = PseudoDestructorTypeStorage(SecondTypeName.Identifier,
                                                SecondTypeName.StartLocation);
     } else if (!T) {
-      Diag(SecondTypeName.StartLocation, 
+      Diag(SecondTypeName.StartLocation,
            diag::err_pseudo_dtor_destructor_non_type)
         << SecondTypeName.Identifier << ObjectType;
       if (isSFINAEContext())
         return ExprError();
-      
+
       // Recover by assuming we had the right type all along.
       DestructedType = ObjectType;
     } else
@@ -2975,8 +3766,8 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
     } else
       DestructedType = GetTypeFromParser(T.get(), &DestructedTypeInfo);
   }
-  
-  // If we've performed some kind of recovery, (re-)build the type source 
+
+  // If we've performed some kind of recovery, (re-)build the type source
   // information.
   if (!DestructedType.isNull()) {
     if (!DestructedTypeInfo)
@@ -2984,24 +3775,24 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
                                                   SecondTypeName.StartLocation);
     Destructed = PseudoDestructorTypeStorage(DestructedTypeInfo);
   }
-  
+
   // Convert the name of the scope type (the type prior to '::') into a type.
   TypeSourceInfo *ScopeTypeInfo = 0;
   QualType ScopeType;
-  if (FirstTypeName.getKind() == UnqualifiedId::IK_TemplateId || 
+  if (FirstTypeName.getKind() == UnqualifiedId::IK_TemplateId ||
       FirstTypeName.Identifier) {
     if (FirstTypeName.getKind() == UnqualifiedId::IK_Identifier) {
-      ParsedType T = getTypeName(*FirstTypeName.Identifier, 
+      ParsedType T = getTypeName(*FirstTypeName.Identifier,
                                  FirstTypeName.StartLocation,
-                                 S, &SS, false, ObjectTypePtrForLookup);
+                                 S, &SS, false, false, ObjectTypePtrForLookup);
       if (!T) {
-        Diag(FirstTypeName.StartLocation, 
+        Diag(FirstTypeName.StartLocation,
              diag::err_pseudo_dtor_destructor_non_type)
           << FirstTypeName.Identifier << ObjectType;
-        
+
         if (isSFINAEContext())
           return ExprError();
-        
+
         // Just drop this type. It's unnecessary anyway.
         ScopeType = QualType();
       } else
@@ -3021,39 +3812,100 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
         // Recover by dropping this type.
         ScopeType = QualType();
       } else
-        ScopeType = GetTypeFromParser(T.get(), &ScopeTypeInfo);      
+        ScopeType = GetTypeFromParser(T.get(), &ScopeTypeInfo);
     }
   }
-      
+
   if (!ScopeType.isNull() && !ScopeTypeInfo)
     ScopeTypeInfo = Context.getTrivialTypeSourceInfo(ScopeType,
                                                   FirstTypeName.StartLocation);
 
-    
+
   return BuildPseudoDestructorExpr(Base, OpLoc, OpKind, SS,
                                    ScopeTypeInfo, CCLoc, TildeLoc,
                                    Destructed, HasTrailingLParen);
 }
 
-CXXMemberCallExpr *Sema::BuildCXXMemberCallExpr(Expr *Exp, 
-                                                NamedDecl *FoundDecl,
-                                                CXXMethodDecl *Method) {
+ExprResult Sema::BuildCXXMemberCallExpr(Expr *Exp, NamedDecl *FoundDecl,
+                                        CXXMethodDecl *Method) {
   if (PerformObjectArgumentInitialization(Exp, /*Qualifier=*/0,
                                           FoundDecl, Method))
-    assert(0 && "Calling BuildCXXMemberCallExpr with invalid call?");
+    return true;
 
-  MemberExpr *ME = 
+  MemberExpr *ME =
       new (Context) MemberExpr(Exp, /*IsArrow=*/false, Method,
-                               SourceLocation(), Method->getType());
-  QualType ResultType = Method->getCallResultType();
+                               SourceLocation(), Method->getType(),
+                               VK_RValue, OK_Ordinary);
+  QualType ResultType = Method->getResultType();
+  ExprValueKind VK = Expr::getValueKindForType(ResultType);
+  ResultType = ResultType.getNonLValueExprType(Context);
+
   MarkDeclarationReferenced(Exp->getLocStart(), Method);
   CXXMemberCallExpr *CE =
-    new (Context) CXXMemberCallExpr(Context, ME, 0, 0, ResultType,
+    new (Context) CXXMemberCallExpr(Context, ME, 0, 0, ResultType, VK,
                                     Exp->getLocEnd());
   return CE;
 }
 
+ExprResult Sema::BuildCXXNoexceptExpr(SourceLocation KeyLoc, Expr *Operand,
+                                      SourceLocation RParen) {
+  return Owned(new (Context) CXXNoexceptExpr(Context.BoolTy, Operand,
+                                             Operand->CanThrow(Context),
+                                             KeyLoc, RParen));
+}
+
+ExprResult Sema::ActOnNoexceptExpr(SourceLocation KeyLoc, SourceLocation,
+                                   Expr *Operand, SourceLocation RParen) {
+  return BuildCXXNoexceptExpr(KeyLoc, Operand, RParen);
+}
+
+/// Perform the conversions required for an expression used in a
+/// context that ignores the result.
+void Sema::IgnoredValueConversions(Expr *&E) {
+  // C99 6.3.2.1:
+  //   [Except in specific positions,] an lvalue that does not have
+  //   array type is converted to the value stored in the
+  //   designated object (and is no longer an lvalue).
+  if (E->isRValue()) return;
+
+  // We always want to do this on ObjC property references.
+  if (E->getObjectKind() == OK_ObjCProperty) {
+    ConvertPropertyForRValue(E);
+    if (E->isRValue()) return;
+  }
+
+  // Otherwise, this rule does not apply in C++, at least not for the moment.
+  if (getLangOptions().CPlusPlus) return;
+
+  // GCC seems to also exclude expressions of incomplete enum type.
+  if (const EnumType *T = E->getType()->getAs<EnumType>()) {
+    if (!T->getDecl()->isComplete()) {
+      // FIXME: stupid workaround for a codegen bug!
+      ImpCastExprToType(E, Context.VoidTy, CK_ToVoid);
+      return;
+    }
+  }
+
+  DefaultFunctionArrayLvalueConversion(E);
+  if (!E->getType()->isVoidType())
+    RequireCompleteType(E->getExprLoc(), E->getType(),
+                        diag::err_incomplete_type);
+}
+
 ExprResult Sema::ActOnFinishFullExpr(Expr *FullExpr) {
-  if (!FullExpr) return ExprError();
-  return MaybeCreateCXXExprWithTemporaries(FullExpr);
+  if (!FullExpr)
+    return ExprError();
+
+  if (DiagnoseUnexpandedParameterPack(FullExpr))
+    return ExprError();
+
+  IgnoredValueConversions(FullExpr);
+  CheckImplicitConversions(FullExpr);
+  return MaybeCreateExprWithCleanups(FullExpr);
+}
+
+StmtResult Sema::ActOnFinishFullStmt(Stmt *FullStmt) {
+  if (!FullStmt) return StmtError();
+
+  return MaybeCreateStmtWithCleanups(FullStmt);
 }
diff --git a/lib/Sema/SemaExprObjC.cpp b/lib/Sema/SemaExprObjC.cpp
index b56159c453b5..4d03b068ca85 100644
--- a/lib/Sema/SemaExprObjC.cpp
+++ b/lib/Sema/SemaExprObjC.cpp
@@ -14,6 +14,7 @@
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
@@ -23,6 +24,7 @@
 #include "clang/Lex/Preprocessor.h"
 
 using namespace clang;
+using namespace sema;
 
 ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
                                         Expr **strings,
@@ -77,7 +79,14 @@ ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
   if (!Ty.isNull()) {
     Ty = Context.getObjCObjectPointerType(Ty);
   } else if (getLangOptions().NoConstantCFStrings) {
-    IdentifierInfo *NSIdent = &Context.Idents.get("NSConstantString");
+    IdentifierInfo *NSIdent=0;
+    std::string StringClass(getLangOptions().ObjCConstantStringClass);
+    
+    if (StringClass.empty())
+      NSIdent = &Context.Idents.get("NSConstantString");
+    else
+      NSIdent = &Context.Idents.get(StringClass);
+    
     NamedDecl *IF = LookupSingleName(TUScope, NSIdent, AtLocs[0],
                                      LookupOrdinaryName);
     if (ObjCInterfaceDecl *StrIF = dyn_cast_or_null<ObjCInterfaceDecl>(IF)) {
@@ -188,11 +197,49 @@ ExprResult Sema::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
   return new (Context) ObjCProtocolExpr(Ty, PDecl, AtLoc, RParenLoc);
 }
 
+/// Try to capture an implicit reference to 'self'.
+ObjCMethodDecl *Sema::tryCaptureObjCSelf() {
+  // Ignore block scopes: we can capture through them.
+  DeclContext *DC = CurContext;
+  while (true) {
+    if (isa<BlockDecl>(DC)) DC = cast<BlockDecl>(DC)->getDeclContext();
+    else if (isa<EnumDecl>(DC)) DC = cast<EnumDecl>(DC)->getDeclContext();
+    else break;
+  }
+
+  // If we're not in an ObjC method, error out.  Note that, unlike the
+  // C++ case, we don't require an instance method --- class methods
+  // still have a 'self', and we really do still need to capture it!
+  ObjCMethodDecl *method = dyn_cast<ObjCMethodDecl>(DC);
+  if (!method)
+    return 0;
+
+  ImplicitParamDecl *self = method->getSelfDecl();
+  assert(self && "capturing 'self' in non-definition?");
+
+  // Mark that we're closing on 'this' in all the block scopes, if applicable.
+  for (unsigned idx = FunctionScopes.size() - 1;
+       isa<BlockScopeInfo>(FunctionScopes[idx]);
+       --idx) {
+    BlockScopeInfo *blockScope = cast<BlockScopeInfo>(FunctionScopes[idx]);
+    unsigned &captureIndex = blockScope->CaptureMap[self];
+    if (captureIndex) break;
+
+    bool nested = isa<BlockScopeInfo>(FunctionScopes[idx-1]);
+    blockScope->Captures.push_back(
+              BlockDecl::Capture(self, /*byref*/ false, nested, /*copy*/ 0));
+    captureIndex = blockScope->Captures.size(); // +1
+  }
+
+  return method;
+}
+
+
 bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
                                      Selector Sel, ObjCMethodDecl *Method,
                                      bool isClassMessage,
                                      SourceLocation lbrac, SourceLocation rbrac,
-                                     QualType &ReturnType) {
+                                     QualType &ReturnType, ExprValueKind &VK) {
   if (!Method) {
     // Apply default argument promotion as for (C99 6.5.2.2p6).
     for (unsigned i = 0; i != NumArgs; i++) {
@@ -207,10 +254,12 @@ bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
     Diag(lbrac, DiagID)
       << Sel << isClassMessage << SourceRange(lbrac, rbrac);
     ReturnType = Context.getObjCIdType();
+    VK = VK_RValue;
     return false;
   }
 
   ReturnType = Method->getSendResultType();
+  VK = Expr::getValueKindForType(Method->getResultType());
 
   unsigned NumNamedArgs = Sel.getNumArgs();
   // Method might have more arguments than selector indicates. This is due
@@ -219,8 +268,8 @@ bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
     NumNamedArgs = Method->param_size();
   // FIXME. This need be cleaned up.
   if (NumArgs < NumNamedArgs) {
-    Diag(lbrac, diag::err_typecheck_call_too_few_args) << 2
-    << NumNamedArgs << NumArgs;
+    Diag(lbrac, diag::err_typecheck_call_too_few_args)
+      << 2 << NumNamedArgs << NumArgs;
     return false;
   }
 
@@ -241,10 +290,9 @@ bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
                               << argExpr->getSourceRange()))
       return true;
 
-    InitializedEntity Entity = InitializedEntity::InitializeParameter(Param);
-    ExprResult ArgE = PerformCopyInitialization(Entity,
-                                                      SourceLocation(),
-                                                      Owned(argExpr->Retain()));
+    InitializedEntity Entity = InitializedEntity::InitializeParameter(Context,
+                                                                      Param);
+    ExprResult ArgE = PerformCopyInitialization(Entity, lbrac, Owned(argExpr));
     if (ArgE.isInvalid())
       IsError = true;
     else
@@ -276,6 +324,9 @@ bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
 }
 
 bool Sema::isSelfExpr(Expr *RExpr) {
+  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(RExpr))
+    if (ICE->getCastKind() == CK_LValueToRValue)
+      RExpr = ICE->getSubExpr();
   if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(RExpr))
     if (DRE->getDecl()->getIdentifier() == &Context.Idents.get("self"))
       return true;
@@ -335,11 +386,19 @@ ObjCMethodDecl *Sema::LookupPrivateInstanceMethod(Selector Sel,
 ExprResult Sema::
 HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
                           Expr *BaseExpr, DeclarationName MemberName,
-                          SourceLocation MemberLoc) {
+                          SourceLocation MemberLoc,
+                          SourceLocation SuperLoc, QualType SuperType,
+                          bool Super) {
   const ObjCInterfaceType *IFaceT = OPT->getInterfaceType();
   ObjCInterfaceDecl *IFace = IFaceT->getDecl();
   IdentifierInfo *Member = MemberName.getAsIdentifierInfo();
 
+  if (IFace->isForwardDecl()) {
+    Diag(MemberLoc, diag::err_property_not_found_forward_class)
+         << MemberName << QualType(OPT, 0);
+    Diag(IFace->getLocation(), diag::note_forward_class);
+    return ExprError();
+  }
   // Search for a declared property first.
   if (ObjCPropertyDecl *PD = IFace->FindPropertyDeclaration(Member)) {
     // Check whether we can reference this property.
@@ -349,9 +408,17 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
     Selector Sel = PP.getSelectorTable().getNullarySelector(Member);
     ObjCMethodDecl *Getter = IFace->lookupInstanceMethod(Sel);
     if (DiagnosePropertyAccessorMismatch(PD, Getter, MemberLoc))
-      ResTy = Getter->getSendResultType();
-    return Owned(new (Context) ObjCPropertyRefExpr(PD, ResTy,
-                                                   MemberLoc, BaseExpr));
+      ResTy = Getter->getResultType();
+
+    if (Super)
+      return Owned(new (Context) ObjCPropertyRefExpr(PD, ResTy,
+                                                     VK_LValue, OK_ObjCProperty,
+                                                     MemberLoc, 
+                                                     SuperLoc, SuperType));
+    else
+      return Owned(new (Context) ObjCPropertyRefExpr(PD, ResTy,
+                                                     VK_LValue, OK_ObjCProperty,
+                                                     MemberLoc, BaseExpr));
   }
   // Check protocols on qualified interfaces.
   for (ObjCObjectPointerType::qual_iterator I = OPT->qual_begin(),
@@ -360,9 +427,18 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
       // Check whether we can reference this property.
       if (DiagnoseUseOfDecl(PD, MemberLoc))
         return ExprError();
-
-      return Owned(new (Context) ObjCPropertyRefExpr(PD, PD->getType(),
-                                                     MemberLoc, BaseExpr));
+      if (Super)
+        return Owned(new (Context) ObjCPropertyRefExpr(PD, PD->getType(),
+                                                       VK_LValue,
+                                                       OK_ObjCProperty,
+                                                       MemberLoc, 
+                                                       SuperLoc, SuperType));
+      else
+        return Owned(new (Context) ObjCPropertyRefExpr(PD, PD->getType(),
+                                                       VK_LValue,
+                                                       OK_ObjCProperty,
+                                                       MemberLoc,
+                                                       BaseExpr));
     }
   // If that failed, look for an "implicit" property by seeing if the nullary
   // selector is implemented.
@@ -375,7 +451,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
 
   // If this reference is in an @implementation, check for 'private' methods.
   if (!Getter)
-    Getter = IFace->lookupPrivateInstanceMethod(Sel);
+    Getter = IFace->lookupPrivateMethod(Sel);
 
   // Look through local category implementations associated with the class.
   if (!Getter)
@@ -394,7 +470,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
   if (!Setter) {
     // If this reference is in an @implementation, also check for 'private'
     // methods.
-    Setter = IFace->lookupPrivateInstanceMethod(SetterSel);
+    Setter = IFace->lookupPrivateMethod(SetterSel);
   }
   // Look through local category implementations associated with the class.
   if (!Setter)
@@ -403,11 +479,31 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
   if (Setter && DiagnoseUseOfDecl(Setter, MemberLoc))
     return ExprError();
 
-  if (Getter) {
+  if (Getter || Setter) {
     QualType PType;
-    PType = Getter->getSendResultType();
-    return Owned(new (Context) ObjCImplicitSetterGetterRefExpr(Getter, PType,
-                                    Setter, MemberLoc, BaseExpr));
+    if (Getter)
+      PType = Getter->getSendResultType();
+    else {
+      ParmVarDecl *ArgDecl = *Setter->param_begin();
+      PType = ArgDecl->getType();
+    }
+    
+    ExprValueKind VK = VK_LValue;
+    ExprObjectKind OK = OK_ObjCProperty;
+    if (!getLangOptions().CPlusPlus && !PType.hasQualifiers() &&
+        PType->isVoidType())
+      VK = VK_RValue, OK = OK_Ordinary;
+
+    if (Super)
+      return Owned(new (Context) ObjCPropertyRefExpr(Getter, Setter,
+                                                     PType, VK, OK,
+                                                     MemberLoc,
+                                                     SuperLoc, SuperType));
+    else
+      return Owned(new (Context) ObjCPropertyRefExpr(Getter, Setter,
+                                                     PType, VK, OK,
+                                                     MemberLoc, BaseExpr));
+
   }
 
   // Attempt to correct for typos in property names.
@@ -421,14 +517,31 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
     ObjCPropertyDecl *Property = Res.getAsSingle<ObjCPropertyDecl>();
     Diag(Property->getLocation(), diag::note_previous_decl)
       << Property->getDeclName();
-    return HandleExprPropertyRefExpr(OPT, BaseExpr, TypoResult, MemberLoc);
+    return HandleExprPropertyRefExpr(OPT, BaseExpr, TypoResult, MemberLoc,
+                                     SuperLoc, SuperType, Super);
+  }
+  ObjCInterfaceDecl *ClassDeclared;
+  if (ObjCIvarDecl *Ivar = 
+      IFace->lookupInstanceVariable(Member, ClassDeclared)) {
+    QualType T = Ivar->getType();
+    if (const ObjCObjectPointerType * OBJPT = 
+        T->getAsObjCInterfacePointerType()) {
+      const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
+      if (ObjCInterfaceDecl *IFace = IFaceT->getDecl())
+        if (IFace->isForwardDecl()) {
+          Diag(MemberLoc, diag::err_property_not_as_forward_class)
+          << MemberName << IFace;
+          Diag(IFace->getLocation(), diag::note_forward_class);
+          return ExprError();
+        }
+    }
   }
   
   Diag(MemberLoc, diag::err_property_not_found)
     << MemberName << QualType(OPT, 0);
-  if (Setter && !Getter)
+  if (Setter)
     Diag(Setter->getLocation(), diag::note_getter_unavailable)
-      << MemberName << BaseExpr->getSourceRange();
+          << MemberName << BaseExpr->getSourceRange();
   return ExprError();
 }
 
@@ -446,23 +559,24 @@ ActOnClassPropertyRefExpr(IdentifierInfo &receiverName,
   if (IFace == 0) {
     // If the "receiver" is 'super' in a method, handle it as an expression-like
     // property reference.
-    if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
-      if (receiverNamePtr->isStr("super")) {
+    if (receiverNamePtr->isStr("super")) {
+      if (ObjCMethodDecl *CurMethod = tryCaptureObjCSelf()) {
         if (CurMethod->isInstanceMethod()) {
           QualType T = 
             Context.getObjCInterfaceType(CurMethod->getClassInterface());
           T = Context.getObjCObjectPointerType(T);
-          Expr *SuperExpr = new (Context) ObjCSuperExpr(receiverNameLoc, T);
         
           return HandleExprPropertyRefExpr(T->getAsObjCInterfacePointerType(),
-                                           SuperExpr, &propertyName,
-                                           propertyNameLoc);
+                                           /*BaseExpr*/0, &propertyName,
+                                           propertyNameLoc,
+                                           receiverNameLoc, T, true);
         }
 
         // Otherwise, if this is a class method, try dispatching to our
         // superclass.
         IFace = CurMethod->getClassInterface()->getSuperClass();
       }
+    }
     
     if (IFace == 0) {
       Diag(receiverNameLoc, diag::err_expected_ident_or_lparen);
@@ -512,16 +626,25 @@ ActOnClassPropertyRefExpr(IdentifierInfo &receiverName,
   if (Getter || Setter) {
     QualType PType;
 
-    if (Getter)
+    ExprValueKind VK = VK_LValue;
+    if (Getter) {
       PType = Getter->getSendResultType();
-    else {
+      if (!getLangOptions().CPlusPlus &&
+          !PType.hasQualifiers() && PType->isVoidType())
+        VK = VK_RValue;
+    } else {
       for (ObjCMethodDecl::param_iterator PI = Setter->param_begin(),
            E = Setter->param_end(); PI != E; ++PI)
         PType = (*PI)->getType();
+      VK = VK_LValue;
     }
-    return Owned(new (Context) ObjCImplicitSetterGetterRefExpr(
-                                  Getter, PType, Setter,
-                                  propertyNameLoc, IFace, receiverNameLoc));
+
+    ExprObjectKind OK = (VK == VK_RValue ? OK_Ordinary : OK_ObjCProperty);
+
+    return Owned(new (Context) ObjCPropertyRefExpr(Getter, Setter,
+                                                   PType, VK, OK,
+                                                   propertyNameLoc,
+                                                   receiverNameLoc, IFace));
   }
   return ExprError(Diag(propertyNameLoc, diag::err_property_not_found)
                      << &propertyName << Context.getObjCInterfaceType(IFace));
@@ -536,9 +659,10 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
   ReceiverType = ParsedType();
 
   // If the identifier is "super" and there is no trailing dot, we're
-  // messaging super.
-  if (IsSuper && !HasTrailingDot && S->isInObjcMethodScope())
-    return ObjCSuperMessage;
+  // messaging super. If the identifier is "super" and there is a
+  // trailing dot, it's an instance message.
+  if (IsSuper && S->isInObjcMethodScope())
+    return HasTrailingDot? ObjCInstanceMessage : ObjCSuperMessage;
   
   LookupResult Result(*this, Name, NameLoc, LookupOrdinaryName);
   LookupName(Result, S);
@@ -547,14 +671,15 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
   case LookupResult::NotFound:
     // Normal name lookup didn't find anything. If we're in an
     // Objective-C method, look for ivars. If we find one, we're done!
-    // FIXME: This is a hack. Ivar lookup should be part of normal lookup.
+    // FIXME: This is a hack. Ivar lookup should be part of normal
+    // lookup.
     if (ObjCMethodDecl *Method = getCurMethodDecl()) {
       ObjCInterfaceDecl *ClassDeclared;
       if (Method->getClassInterface()->lookupInstanceVariable(Name, 
                                                               ClassDeclared))
         return ObjCInstanceMessage;
     }
-      
+  
     // Break out; we'll perform typo correction below.
     break;
 
@@ -566,6 +691,10 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
     return ObjCInstanceMessage;
 
   case LookupResult::Found: {
+    // If the identifier is a class or not, and there is a trailing dot,
+    // it's an instance message.
+    if (HasTrailingDot)
+      return ObjCInstanceMessage;
     // We found something. If it's a type, then we have a class
     // message. Otherwise, it's an instance message.
     NamedDecl *ND = Result.getFoundDecl();
@@ -616,7 +745,6 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
       Diag(NameLoc, diag::err_unknown_receiver_suggest)
         << Name << Corrected
         << FixItHint::CreateReplacement(SourceRange(NameLoc), "super");
-      Name = Corrected.getAsIdentifierInfo();
       return ObjCSuperMessage;
     }
   }
@@ -626,14 +754,14 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
 }
 
 ExprResult Sema::ActOnSuperMessage(Scope *S, 
-                                               SourceLocation SuperLoc,
-                                               Selector Sel,
-                                               SourceLocation LBracLoc,
-                                               SourceLocation SelectorLoc,
-                                               SourceLocation RBracLoc,
-                                               MultiExprArg Args) {
+                                   SourceLocation SuperLoc,
+                                   Selector Sel,
+                                   SourceLocation LBracLoc,
+                                   SourceLocation SelectorLoc,
+                                   SourceLocation RBracLoc,
+                                   MultiExprArg Args) {
   // Determine whether we are inside a method or not.
-  ObjCMethodDecl *Method = getCurMethodDecl();
+  ObjCMethodDecl *Method = tryCaptureObjCSelf();
   if (!Method) {
     Diag(SuperLoc, diag::err_invalid_receiver_to_message_super);
     return ExprError();
@@ -649,7 +777,8 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
   ObjCInterfaceDecl *Super = Class->getSuperClass();
   if (!Super) {
     // The current class does not have a superclass.
-    Diag(SuperLoc, diag::error_no_super_class) << Class->getIdentifier();
+    Diag(SuperLoc, diag::error_root_class_cannot_use_super)
+      << Class->getIdentifier();
     return ExprError();
   }
 
@@ -661,16 +790,16 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
     QualType SuperTy = Context.getObjCInterfaceType(Super);
     SuperTy = Context.getObjCObjectPointerType(SuperTy);
     return BuildInstanceMessage(0, SuperTy, SuperLoc,
-                                Sel, /*Method=*/0, LBracLoc, RBracLoc, 
-                                move(Args));
+                                Sel, /*Method=*/0,
+                                LBracLoc, SelectorLoc, RBracLoc, move(Args));
   }
   
   // Since we are in a class method, this is a class message to
   // the superclass.
   return BuildClassMessage(/*ReceiverTypeInfo=*/0,
                            Context.getObjCInterfaceType(Super),
-                           SuperLoc, Sel, /*Method=*/0, LBracLoc, RBracLoc, 
-                           move(Args));
+                           SuperLoc, Sel, /*Method=*/0,
+                           LBracLoc, SelectorLoc, RBracLoc, move(Args));
 }
 
 /// \brief Build an Objective-C class message expression.
@@ -702,27 +831,34 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
 ///
 /// \param Args The message arguments.
 ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
-                                               QualType ReceiverType,
-                                               SourceLocation SuperLoc,
-                                               Selector Sel,
-                                               ObjCMethodDecl *Method,
-                                               SourceLocation LBracLoc, 
-                                               SourceLocation RBracLoc,
-                                               MultiExprArg ArgsIn) {
+                                   QualType ReceiverType,
+                                   SourceLocation SuperLoc,
+                                   Selector Sel,
+                                   ObjCMethodDecl *Method,
+                                   SourceLocation LBracLoc, 
+                                   SourceLocation SelectorLoc,
+                                   SourceLocation RBracLoc,
+                                   MultiExprArg ArgsIn) {
+  SourceLocation Loc = SuperLoc.isValid()? SuperLoc
+    : ReceiverTypeInfo->getTypeLoc().getSourceRange().getBegin();
+  if (LBracLoc.isInvalid()) {
+    Diag(Loc, diag::err_missing_open_square_message_send)
+      << FixItHint::CreateInsertion(Loc, "[");
+    LBracLoc = Loc;
+  }
+  
   if (ReceiverType->isDependentType()) {
     // If the receiver type is dependent, we can't type-check anything
     // at this point. Build a dependent expression.
     unsigned NumArgs = ArgsIn.size();
     Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
     assert(SuperLoc.isInvalid() && "Message to super with dependent type");
-    return Owned(ObjCMessageExpr::Create(Context, ReceiverType, LBracLoc,
-                                         ReceiverTypeInfo, Sel, /*Method=*/0, 
+    return Owned(ObjCMessageExpr::Create(Context, ReceiverType,
+                                         VK_RValue, LBracLoc, ReceiverTypeInfo,
+                                         Sel, SelectorLoc, /*Method=*/0,
                                          Args, NumArgs, RBracLoc));
   }
   
-  SourceLocation Loc = SuperLoc.isValid()? SuperLoc
-             : ReceiverTypeInfo->getTypeLoc().getLocalSourceRange().getBegin();
-
   // Find the class to which we are sending this message.
   ObjCInterfaceDecl *Class = 0;
   const ObjCObjectType *ClassType = ReceiverType->getAs<ObjCObjectType>();
@@ -757,23 +893,30 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
 
   // Check the argument types and determine the result type.
   QualType ReturnType;
+  ExprValueKind VK = VK_RValue;
+
   unsigned NumArgs = ArgsIn.size();
   Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
   if (CheckMessageArgumentTypes(Args, NumArgs, Sel, Method, true,
-                                LBracLoc, RBracLoc, ReturnType))
+                                LBracLoc, RBracLoc, ReturnType, VK))
+    return ExprError();
+
+  if (Method && !Method->getResultType()->isVoidType() &&
+      RequireCompleteType(LBracLoc, Method->getResultType(), 
+                          diag::err_illegal_message_expr_incomplete_type))
     return ExprError();
 
   // Construct the appropriate ObjCMessageExpr.
   Expr *Result;
   if (SuperLoc.isValid())
-    Result = ObjCMessageExpr::Create(Context, ReturnType, LBracLoc, 
+    Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc, 
                                      SuperLoc, /*IsInstanceSuper=*/false, 
-                                     ReceiverType, Sel, Method, Args, 
-                                     NumArgs, RBracLoc);
+                                     ReceiverType, Sel, SelectorLoc,
+                                     Method, Args, NumArgs, RBracLoc);
   else
-    Result = ObjCMessageExpr::Create(Context, ReturnType, LBracLoc, 
-                                     ReceiverTypeInfo, Sel, Method, Args, 
-                                     NumArgs, RBracLoc);
+    Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc, 
+                                     ReceiverTypeInfo, Sel, SelectorLoc,
+                                     Method, Args, NumArgs, RBracLoc);
   return MaybeBindToTemporary(Result);
 }
 
@@ -781,12 +924,12 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
 // ArgExprs is optional - if it is present, the number of expressions
 // is obtained from Sel.getNumArgs().
 ExprResult Sema::ActOnClassMessage(Scope *S, 
-                                               ParsedType Receiver,
-                                               Selector Sel,
-                                               SourceLocation LBracLoc,
-                                               SourceLocation SelectorLoc,
-                                               SourceLocation RBracLoc,
-                                               MultiExprArg Args) {
+                                   ParsedType Receiver,
+                                   Selector Sel,
+                                   SourceLocation LBracLoc,
+                                   SourceLocation SelectorLoc,
+                                   SourceLocation RBracLoc,
+                                   MultiExprArg Args) {
   TypeSourceInfo *ReceiverTypeInfo;
   QualType ReceiverType = GetTypeFromParser(Receiver, &ReceiverTypeInfo);
   if (ReceiverType.isNull())
@@ -798,7 +941,7 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
 
   return BuildClassMessage(ReceiverTypeInfo, ReceiverType, 
                            /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0,
-                           LBracLoc, RBracLoc, move(Args));
+                           LBracLoc, SelectorLoc, RBracLoc, move(Args));
 }
 
 /// \brief Build an Objective-C instance message expression.
@@ -830,13 +973,23 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
 ///
 /// \param Args The message arguments.
 ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
-                                                  QualType ReceiverType,
-                                                  SourceLocation SuperLoc,
-                                                  Selector Sel,
-                                                  ObjCMethodDecl *Method,
-                                                  SourceLocation LBracLoc, 
-                                                  SourceLocation RBracLoc,
-                                                  MultiExprArg ArgsIn) {
+                                      QualType ReceiverType,
+                                      SourceLocation SuperLoc,
+                                      Selector Sel,
+                                      ObjCMethodDecl *Method,
+                                      SourceLocation LBracLoc, 
+                                      SourceLocation SelectorLoc,
+                                      SourceLocation RBracLoc,
+                                      MultiExprArg ArgsIn) {
+  // The location of the receiver.
+  SourceLocation Loc = SuperLoc.isValid()? SuperLoc : Receiver->getLocStart();
+  
+  if (LBracLoc.isInvalid()) {
+    Diag(Loc, diag::err_missing_open_square_message_send)
+      << FixItHint::CreateInsertion(Loc, "[");
+    LBracLoc = Loc;
+  }
+
   // If we have a receiver expression, perform appropriate promotions
   // and determine receiver type.
   if (Receiver) {
@@ -847,9 +1000,9 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
       Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
       assert(SuperLoc.isInvalid() && "Message to super with dependent type");
       return Owned(ObjCMessageExpr::Create(Context, Context.DependentTy,
-                                           LBracLoc, Receiver, Sel, 
-                                           /*Method=*/0, Args, NumArgs, 
-                                           RBracLoc));
+                                           VK_RValue, LBracLoc, Receiver, Sel, 
+                                           SelectorLoc, /*Method=*/0,
+                                           Args, NumArgs, RBracLoc));
     }
 
     // If necessary, apply function/array conversion to the receiver.
@@ -858,9 +1011,6 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     ReceiverType = Receiver->getType();
   }
 
-  // The location of the receiver.
-  SourceLocation Loc = SuperLoc.isValid()? SuperLoc : Receiver->getLocStart();
-
   if (!Method) {
     // Handle messages to id.
     bool receiverIsId = ReceiverType->isObjCIdType();
@@ -946,6 +1096,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
               break;
           }
         }
+        bool forwardClass = false;
         if (!Method) {
           // If we have implementations in scope, check "private" methods.
           Method = LookupPrivateInstanceMethod(Sel, ClassDecl);
@@ -956,14 +1107,15 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
             // compatibility. FIXME: should we deviate??
             if (OCIType->qual_empty()) {
               Method = LookupInstanceMethodInGlobalPool(Sel,
-                                                 SourceRange(LBracLoc, RBracLoc)); 
-              if (Method && !OCIType->getInterfaceDecl()->isForwardDecl())
+                                                 SourceRange(LBracLoc, RBracLoc));
+              forwardClass = OCIType->getInterfaceDecl()->isForwardDecl();
+              if (Method && !forwardClass)
                 Diag(Loc, diag::warn_maynot_respond)
                   << OCIType->getInterfaceDecl()->getIdentifier() << Sel;
             }
           }
         }
-        if (Method && DiagnoseUseOfDecl(Method, Loc))
+        if (Method && DiagnoseUseOfDecl(Method, Loc, forwardClass))
           return ExprError();
       } else if (!Context.getObjCIdType().isNull() &&
                  (ReceiverType->isPointerType() || 
@@ -975,9 +1127,13 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
         if (ReceiverType->isPointerType())
           ImpCastExprToType(Receiver, Context.getObjCIdType(), 
                             CK_BitCast);
-        else
+        else {
+          // TODO: specialized warning on null receivers?
+          bool IsNull = Receiver->isNullPointerConstant(Context,
+                                              Expr::NPC_ValueDependentIsNull);
           ImpCastExprToType(Receiver, Context.getObjCIdType(),
-                            CK_IntegralToPointer);
+                            IsNull ? CK_NullToPointer : CK_IntegralToPointer);
+        }
         ReceiverType = Receiver->getType();
       } 
       else if (getLangOptions().CPlusPlus &&
@@ -991,7 +1147,8 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                                     SuperLoc,
                                     Sel,
                                     Method,
-                                    LBracLoc, 
+                                    LBracLoc,
+                                    SelectorLoc,
                                     RBracLoc,
                                     move(ArgsIn));
       } else {
@@ -1007,26 +1164,29 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
   unsigned NumArgs = ArgsIn.size();
   Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
   QualType ReturnType;
-  if (CheckMessageArgumentTypes(Args, NumArgs, Sel, Method, false,
-                                LBracLoc, RBracLoc, ReturnType))
+  ExprValueKind VK = VK_RValue;
+  bool ClassMessage = (ReceiverType->isObjCClassType() ||
+                       ReceiverType->isObjCQualifiedClassType());
+  if (CheckMessageArgumentTypes(Args, NumArgs, Sel, Method, ClassMessage,
+                                LBracLoc, RBracLoc, ReturnType, VK))
     return ExprError();
   
-  if (!ReturnType->isVoidType()) {
-    if (RequireCompleteType(LBracLoc, ReturnType, 
-                            diag::err_illegal_message_expr_incomplete_type))
-      return ExprError();
-  }
+  if (Method && !Method->getResultType()->isVoidType() &&
+      RequireCompleteType(LBracLoc, Method->getResultType(), 
+                          diag::err_illegal_message_expr_incomplete_type))
+    return ExprError();
 
   // Construct the appropriate ObjCMessageExpr instance.
   Expr *Result;
   if (SuperLoc.isValid())
-    Result = ObjCMessageExpr::Create(Context, ReturnType, LBracLoc,
+    Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc,
                                      SuperLoc,  /*IsInstanceSuper=*/true,
-                                     ReceiverType, Sel, Method, 
+                                     ReceiverType, Sel, SelectorLoc, Method, 
                                      Args, NumArgs, RBracLoc);
   else
-    Result = ObjCMessageExpr::Create(Context, ReturnType, LBracLoc, Receiver, 
-                                     Sel, Method, Args, NumArgs, RBracLoc);
+    Result = ObjCMessageExpr::Create(Context, ReturnType, VK, LBracLoc,
+                                     Receiver, Sel, SelectorLoc, Method,
+                                     Args, NumArgs, RBracLoc);
   return MaybeBindToTemporary(Result);
 }
 
@@ -1045,6 +1205,6 @@ ExprResult Sema::ActOnInstanceMessage(Scope *S,
 
   return BuildInstanceMessage(Receiver, Receiver->getType(),
                               /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0, 
-                              LBracLoc, RBracLoc, move(Args));
+                              LBracLoc, SelectorLoc, RBracLoc, move(Args));
 }
 
diff --git a/lib/Sema/SemaInit.cpp b/lib/Sema/SemaInit.cpp
index a28fd7fe12bd..b9a6a5713b8b 100644
--- a/lib/Sema/SemaInit.cpp
+++ b/lib/Sema/SemaInit.cpp
@@ -11,8 +11,6 @@
 // point is Sema::CheckInitList(), but all of the work is performed
 // within the InitListChecker class.
 //
-// This file also implements Sema::CheckInitializerTypes.
-//
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/Designator.h"
@@ -122,7 +120,7 @@ static void CheckStringInit(Expr *Str, QualType &DeclT, Sema &S) {
 /// responsible for moving that Index forward as it consumes elements.
 ///
 /// Each Check* routine also has a StructuredList/StructuredIndex
-/// arguments, which contains the current the "structured" (semantic)
+/// arguments, which contains the current "structured" (semantic)
 /// initializer list and the index into that initializer list where we
 /// are copying initializers as we map them over to the semantic
 /// list. Once we have completed our recursive walk of the subobject
@@ -231,11 +229,11 @@ public:
 
 void InitListChecker::FillInValueInitForField(unsigned Init, FieldDecl *Field,
                                         const InitializedEntity &ParentEntity,
-                                              InitListExpr *ILE, 
+                                              InitListExpr *ILE,
                                               bool &RequiresSecondPass) {
   SourceLocation Loc = ILE->getSourceRange().getBegin();
   unsigned NumInits = ILE->getNumInits();
-  InitializedEntity MemberEntity 
+  InitializedEntity MemberEntity
     = InitializedEntity::InitializeMember(Field, &ParentEntity);
   if (Init >= NumInits || !ILE->getInit(Init)) {
     // FIXME: We probably don't need to handle references
@@ -254,7 +252,7 @@ void InitListChecker::FillInValueInitForField(unsigned Init, FieldDecl *Field,
       hadError = true;
       return;
     }
-    
+
     InitializationKind Kind = InitializationKind::CreateValue(Loc, Loc, Loc,
                                                               true);
     InitializationSequence InitSeq(SemaRef, MemberEntity, Kind, 0, 0);
@@ -263,14 +261,14 @@ void InitListChecker::FillInValueInitForField(unsigned Init, FieldDecl *Field,
       hadError = true;
       return;
     }
-    
+
     ExprResult MemberInit
       = InitSeq.Perform(SemaRef, MemberEntity, Kind, MultiExprArg());
     if (MemberInit.isInvalid()) {
       hadError = true;
       return;
     }
-    
+
     if (hadError) {
       // Do nothing
     } else if (Init < NumInits) {
@@ -286,14 +284,14 @@ void InitListChecker::FillInValueInitForField(unsigned Init, FieldDecl *Field,
     }
   } else if (InitListExpr *InnerILE
                = dyn_cast<InitListExpr>(ILE->getInit(Init)))
-    FillInValueInitializations(MemberEntity, InnerILE, 
-                               RequiresSecondPass);  
+    FillInValueInitializations(MemberEntity, InnerILE,
+                               RequiresSecondPass);
 }
 
 /// Recursively replaces NULL values within the given initializer list
 /// with expressions that perform value-initialization of the
 /// appropriate type.
-void 
+void
 InitListChecker::FillInValueInitializations(const InitializedEntity &Entity,
                                             InitListExpr *ILE,
                                             bool &RequiresSecondPass) {
@@ -344,17 +342,17 @@ InitListChecker::FillInValueInitializations(const InitializedEntity &Entity,
     ElementType = AType->getElementType();
     if (const ConstantArrayType *CAType = dyn_cast<ConstantArrayType>(AType))
       NumElements = CAType->getSize().getZExtValue();
-    ElementEntity = InitializedEntity::InitializeElement(SemaRef.Context, 
+    ElementEntity = InitializedEntity::InitializeElement(SemaRef.Context,
                                                          0, Entity);
   } else if (const VectorType *VType = ILE->getType()->getAs<VectorType>()) {
     ElementType = VType->getElementType();
     NumElements = VType->getNumElements();
-    ElementEntity = InitializedEntity::InitializeElement(SemaRef.Context, 
+    ElementEntity = InitializedEntity::InitializeElement(SemaRef.Context,
                                                          0, Entity);
   } else
     ElementType = ILE->getType();
 
-  
+
   for (unsigned Init = 0; Init != NumElements; ++Init) {
     if (hadError)
       return;
@@ -409,7 +407,7 @@ InitListChecker::InitListChecker(Sema &S, const InitializedEntity &Entity,
   unsigned newStructuredIndex = 0;
   FullyStructuredList
     = getStructuredSubobjectInit(IL, newIndex, T, 0, 0, IL->getSourceRange());
-  CheckExplicitInitList(Entity, IL, T, newIndex, 
+  CheckExplicitInitList(Entity, IL, T, newIndex,
                         FullyStructuredList, newStructuredIndex,
                         /*TopLevelObject=*/true);
 
@@ -417,7 +415,7 @@ InitListChecker::InitListChecker(Sema &S, const InitializedEntity &Entity,
     bool RequiresSecondPass = false;
     FillInValueInitializations(Entity, FullyStructuredList, RequiresSecondPass);
     if (RequiresSecondPass && !hadError)
-      FillInValueInitializations(Entity, FullyStructuredList, 
+      FillInValueInitializations(Entity, FullyStructuredList,
                                  RequiresSecondPass);
   }
 }
@@ -482,7 +480,7 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
 
   // Check the element types and build the structural subobject.
   unsigned StartIndex = Index;
-  CheckListElementTypes(Entity, ParentIList, T, 
+  CheckListElementTypes(Entity, ParentIList, T,
                         /*SubobjectIsDesignatorContext=*/false, Index,
                         StructuredSubobjectInitList,
                         StructuredSubobjectInitIndex,
@@ -497,16 +495,16 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
       = ParentIList->getInit(EndIndex)->getSourceRange().getEnd();
     StructuredSubobjectInitList->setRBraceLoc(EndLoc);
   }
-  
+
   // Warn about missing braces.
   if (T->isArrayType() || T->isRecordType()) {
     SemaRef.Diag(StructuredSubobjectInitList->getLocStart(),
                  diag::warn_missing_braces)
     << StructuredSubobjectInitList->getSourceRange()
-    << FixItHint::CreateInsertion(StructuredSubobjectInitList->getLocStart(), 
+    << FixItHint::CreateInsertion(StructuredSubobjectInitList->getLocStart(),
                                   "{")
     << FixItHint::CreateInsertion(SemaRef.PP.getLocForEndOfToken(
-                                      StructuredSubobjectInitList->getLocEnd()), 
+                                      StructuredSubobjectInitList->getLocEnd()),
                                   "}");
   }
 }
@@ -520,7 +518,7 @@ void InitListChecker::CheckExplicitInitList(const InitializedEntity &Entity,
   assert(IList->isExplicit() && "Illegal Implicit InitListExpr");
   SyntacticToSemantic[IList] = StructuredList;
   StructuredList->setSyntacticForm(IList);
-  CheckListElementTypes(Entity, IList, T, /*SubobjectIsDesignatorContext=*/true, 
+  CheckListElementTypes(Entity, IList, T, /*SubobjectIsDesignatorContext=*/true,
                         Index, StructuredList, StructuredIndex, TopLevelObject);
   QualType ExprTy = T.getNonLValueExprType(SemaRef.Context);
   IList->setType(ExprTy);
@@ -585,7 +583,7 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
     CheckScalarType(Entity, IList, DeclType, Index,
                     StructuredList, StructuredIndex);
   } else if (DeclType->isVectorType()) {
-    CheckVectorType(Entity, IList, DeclType, Index, 
+    CheckVectorType(Entity, IList, DeclType, Index,
                     StructuredList, StructuredIndex);
   } else if (DeclType->isAggregateType()) {
     if (DeclType->isRecordType()) {
@@ -598,7 +596,7 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
       llvm::APSInt Zero(
                       SemaRef.Context.getTypeSize(SemaRef.Context.getSizeType()),
                       false);
-      CheckArrayType(Entity, IList, DeclType, Zero, 
+      CheckArrayType(Entity, IList, DeclType, Zero,
                      SubobjectIsDesignatorContext, Index,
                      StructuredList, StructuredIndex);
     } else
@@ -658,7 +656,7 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
     UpdateStructuredListElement(StructuredList, StructuredIndex, Str);
     ++Index;
   } else if (ElemType->isScalarType()) {
-    CheckScalarType(Entity, IList, ElemType, Index, 
+    CheckScalarType(Entity, IList, ElemType, Index,
                     StructuredList, StructuredIndex);
   } else if (ElemType->isReferenceType()) {
     CheckReferenceType(Entity, IList, ElemType, Index,
@@ -672,17 +670,17 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
       //   member, the member is initialized. [...]
 
       // FIXME: Better EqualLoc?
-      InitializationKind Kind = 
+      InitializationKind Kind =
         InitializationKind::CreateCopy(expr->getLocStart(), SourceLocation());
       InitializationSequence Seq(SemaRef, Entity, Kind, &expr, 1);
-      
+
       if (Seq) {
-        ExprResult Result = 
+        ExprResult Result =
           Seq.Perform(SemaRef, Entity, Kind, MultiExprArg(&expr, 1));
         if (Result.isInvalid())
           hadError = true;
-        
-        UpdateStructuredListElement(StructuredList, StructuredIndex, 
+
+        UpdateStructuredListElement(StructuredList, StructuredIndex,
                                     Result.takeAs<Expr>());
         ++Index;
         return;
@@ -699,7 +697,9 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
       //   initial value of the object, including unnamed members, is
       //   that of the expression.
       if ((ElemType->isRecordType() || ElemType->isVectorType()) &&
-          SemaRef.Context.hasSameUnqualifiedType(expr->getType(), ElemType)) {
+          SemaRef.CheckSingleAssignmentConstraints(ElemType, expr)
+              == Sema::Compatible) {
+        SemaRef.DefaultFunctionArrayLvalueConversion(expr);
         UpdateStructuredListElement(StructuredList, StructuredIndex, expr);
         ++Index;
         return;
@@ -721,9 +721,8 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
     } else {
       // We cannot initialize this element, so let
       // PerformCopyInitialization produce the appropriate diagnostic.
-      SemaRef.PerformCopyInitialization(Entity, SourceLocation(), 
+      SemaRef.PerformCopyInitialization(Entity, SourceLocation(),
                                         SemaRef.Owned(expr));
-      IList->setInit(Index, 0);
       hadError = true;
       ++Index;
       ++StructuredIndex;
@@ -736,48 +735,7 @@ void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
                                       unsigned &Index,
                                       InitListExpr *StructuredList,
                                       unsigned &StructuredIndex) {
-  if (Index < IList->getNumInits()) {
-    Expr *expr = IList->getInit(Index);
-    if (InitListExpr *SubIList = dyn_cast<InitListExpr>(expr)) {
-      SemaRef.Diag(SubIList->getLocStart(),
-                    diag::warn_many_braces_around_scalar_init)
-        << SubIList->getSourceRange();
-
-      CheckScalarType(Entity, SubIList, DeclType, Index, StructuredList,
-                      StructuredIndex);
-      return;
-    } else if (isa<DesignatedInitExpr>(expr)) {
-      SemaRef.Diag(expr->getSourceRange().getBegin(),
-                    diag::err_designator_for_scalar_init)
-        << DeclType << expr->getSourceRange();
-      hadError = true;
-      ++Index;
-      ++StructuredIndex;
-      return;
-    }
-
-    ExprResult Result =
-      SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
-                                        SemaRef.Owned(expr));
-
-    Expr *ResultExpr = 0;
-
-    if (Result.isInvalid())
-      hadError = true; // types weren't compatible.
-    else {
-      ResultExpr = Result.takeAs<Expr>();
-      
-      if (ResultExpr != expr) {
-        // The type was promoted, update initializer list.
-        IList->setInit(Index, ResultExpr);
-      }
-    }
-    if (hadError)
-      ++StructuredIndex;
-    else
-      UpdateStructuredListElement(StructuredList, StructuredIndex, ResultExpr);
-    ++Index;
-  } else {
+  if (Index >= IList->getNumInits()) {
     SemaRef.Diag(IList->getLocStart(), diag::err_empty_scalar_initializer)
       << IList->getSourceRange();
     hadError = true;
@@ -785,6 +743,47 @@ void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
     ++StructuredIndex;
     return;
   }
+
+  Expr *expr = IList->getInit(Index);
+  if (InitListExpr *SubIList = dyn_cast<InitListExpr>(expr)) {
+    SemaRef.Diag(SubIList->getLocStart(),
+                 diag::warn_many_braces_around_scalar_init)
+      << SubIList->getSourceRange();
+
+    CheckScalarType(Entity, SubIList, DeclType, Index, StructuredList,
+                    StructuredIndex);
+    return;
+  } else if (isa<DesignatedInitExpr>(expr)) {
+    SemaRef.Diag(expr->getSourceRange().getBegin(),
+                 diag::err_designator_for_scalar_init)
+      << DeclType << expr->getSourceRange();
+    hadError = true;
+    ++Index;
+    ++StructuredIndex;
+    return;
+  }
+
+  ExprResult Result =
+    SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
+                                      SemaRef.Owned(expr));
+
+  Expr *ResultExpr = 0;
+
+  if (Result.isInvalid())
+    hadError = true; // types weren't compatible.
+  else {
+    ResultExpr = Result.takeAs<Expr>();
+
+    if (ResultExpr != expr) {
+      // The type was promoted, update initializer list.
+      IList->setInit(Index, ResultExpr);
+    }
+  }
+  if (hadError)
+    ++StructuredIndex;
+  else
+    UpdateStructuredListElement(StructuredList, StructuredIndex, ResultExpr);
+  ++Index;
 }
 
 void InitListChecker::CheckReferenceType(const InitializedEntity &Entity,
@@ -839,66 +838,95 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
                                       unsigned &Index,
                                       InitListExpr *StructuredList,
                                       unsigned &StructuredIndex) {
-  if (Index < IList->getNumInits()) {
-    const VectorType *VT = DeclType->getAs<VectorType>();
-    unsigned maxElements = VT->getNumElements();
-    unsigned numEltsInit = 0;
-    QualType elementType = VT->getElementType();
-
-    if (!SemaRef.getLangOptions().OpenCL) {
-      InitializedEntity ElementEntity =
-        InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
-
-      for (unsigned i = 0; i < maxElements; ++i, ++numEltsInit) {
-        // Don't attempt to go past the end of the init list
-        if (Index >= IList->getNumInits())
-          break;
-        
-        ElementEntity.setElementIndex(Index);
-        CheckSubElementType(ElementEntity, IList, elementType, Index,
-                            StructuredList, StructuredIndex);
-      }
-    } else {
-      InitializedEntity ElementEntity =
-        InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
-      
-      // OpenCL initializers allows vectors to be constructed from vectors.
-      for (unsigned i = 0; i < maxElements; ++i) {
-        // Don't attempt to go past the end of the init list
-        if (Index >= IList->getNumInits())
-          break;
-        
-        ElementEntity.setElementIndex(Index);
-
-        QualType IType = IList->getInit(Index)->getType();
-        if (!IType->isVectorType()) {
-          CheckSubElementType(ElementEntity, IList, elementType, Index,
-                              StructuredList, StructuredIndex);
-          ++numEltsInit;
-        } else {
-          QualType VecType;
-          const VectorType *IVT = IType->getAs<VectorType>();
-          unsigned numIElts = IVT->getNumElements();
-          
-          if (IType->isExtVectorType())
-            VecType = SemaRef.Context.getExtVectorType(elementType, numIElts);
-          else
-            VecType = SemaRef.Context.getVectorType(elementType, numIElts,
-                                                    IVT->getAltiVecSpecific());
-          CheckSubElementType(ElementEntity, IList, VecType, Index,
-                              StructuredList, StructuredIndex);
-          numEltsInit += numIElts;
+  if (Index >= IList->getNumInits())
+    return;
+
+  const VectorType *VT = DeclType->getAs<VectorType>();
+  unsigned maxElements = VT->getNumElements();
+  unsigned numEltsInit = 0;
+  QualType elementType = VT->getElementType();
+
+  if (!SemaRef.getLangOptions().OpenCL) {
+    // If the initializing element is a vector, try to copy-initialize
+    // instead of breaking it apart (which is doomed to failure anyway).
+    Expr *Init = IList->getInit(Index);
+    if (!isa<InitListExpr>(Init) && Init->getType()->isVectorType()) {
+      ExprResult Result =
+        SemaRef.PerformCopyInitialization(Entity, Init->getLocStart(),
+                                          SemaRef.Owned(Init));
+
+      Expr *ResultExpr = 0;
+      if (Result.isInvalid())
+        hadError = true; // types weren't compatible.
+      else {
+        ResultExpr = Result.takeAs<Expr>();
+
+        if (ResultExpr != Init) {
+          // The type was promoted, update initializer list.
+          IList->setInit(Index, ResultExpr);
         }
       }
+      if (hadError)
+        ++StructuredIndex;
+      else
+        UpdateStructuredListElement(StructuredList, StructuredIndex, ResultExpr);
+      ++Index;
+      return;
     }
 
-    // OpenCL requires all elements to be initialized.
-    if (numEltsInit != maxElements)
-      if (SemaRef.getLangOptions().OpenCL)
-        SemaRef.Diag(IList->getSourceRange().getBegin(),
-                     diag::err_vector_incorrect_num_initializers)
-          << (numEltsInit < maxElements) << maxElements << numEltsInit;
+    InitializedEntity ElementEntity =
+      InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
+
+    for (unsigned i = 0; i < maxElements; ++i, ++numEltsInit) {
+      // Don't attempt to go past the end of the init list
+      if (Index >= IList->getNumInits())
+        break;
+
+      ElementEntity.setElementIndex(Index);
+      CheckSubElementType(ElementEntity, IList, elementType, Index,
+                          StructuredList, StructuredIndex);
+    }
+    return;
+  }
+
+  InitializedEntity ElementEntity =
+    InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
+
+  // OpenCL initializers allows vectors to be constructed from vectors.
+  for (unsigned i = 0; i < maxElements; ++i) {
+    // Don't attempt to go past the end of the init list
+    if (Index >= IList->getNumInits())
+      break;
+
+    ElementEntity.setElementIndex(Index);
+
+    QualType IType = IList->getInit(Index)->getType();
+    if (!IType->isVectorType()) {
+      CheckSubElementType(ElementEntity, IList, elementType, Index,
+                          StructuredList, StructuredIndex);
+      ++numEltsInit;
+    } else {
+      QualType VecType;
+      const VectorType *IVT = IType->getAs<VectorType>();
+      unsigned numIElts = IVT->getNumElements();
+
+      if (IType->isExtVectorType())
+        VecType = SemaRef.Context.getExtVectorType(elementType, numIElts);
+      else
+        VecType = SemaRef.Context.getVectorType(elementType, numIElts,
+                                                IVT->getVectorKind());
+      CheckSubElementType(ElementEntity, IList, VecType, Index,
+                          StructuredList, StructuredIndex);
+      numEltsInit += numIElts;
+    }
   }
+
+  // OpenCL requires all elements to be initialized.
+  if (numEltsInit != maxElements)
+    if (SemaRef.getLangOptions().OpenCL)
+      SemaRef.Diag(IList->getSourceRange().getBegin(),
+                   diag::err_vector_incorrect_num_initializers)
+        << (numEltsInit < maxElements) << maxElements << numEltsInit;
 }
 
 void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
@@ -945,7 +973,7 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
   if (const ConstantArrayType *CAT =
         SemaRef.Context.getAsConstantArrayType(DeclType)) {
     maxElements = CAT->getSize();
-    elementIndex.extOrTrunc(maxElements.getBitWidth());
+    elementIndex = elementIndex.extOrTrunc(maxElements.getBitWidth());
     elementIndex.setIsUnsigned(maxElements.isUnsigned());
     maxElementsKnown = true;
   }
@@ -972,9 +1000,9 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
       }
 
       if (elementIndex.getBitWidth() > maxElements.getBitWidth())
-        maxElements.extend(elementIndex.getBitWidth());
+        maxElements = maxElements.extend(elementIndex.getBitWidth());
       else if (elementIndex.getBitWidth() < maxElements.getBitWidth())
-        elementIndex.extend(maxElements.getBitWidth());
+        elementIndex = elementIndex.extend(maxElements.getBitWidth());
       elementIndex.setIsUnsigned(maxElements.isUnsigned());
 
       // If the array is of incomplete type, keep track of the number of
@@ -991,7 +1019,7 @@ void InitListChecker::CheckArrayType(const InitializedEntity &Entity,
       break;
 
     InitializedEntity ElementEntity =
-      InitializedEntity::InitializeElement(SemaRef.Context, StructuredIndex, 
+      InitializedEntity::InitializeElement(SemaRef.Context, StructuredIndex,
                                            Entity);
     // Check this element.
     CheckSubElementType(ElementEntity, IList, elementType, Index,
@@ -1118,7 +1146,7 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
   }
 
   // Emit warnings for missing struct field initializers.
-  if (InitializedSomething && CheckForMissingFields && Field != FieldEnd && 
+  if (InitializedSomething && CheckForMissingFields && Field != FieldEnd &&
       !Field->getType()->isIncompleteArrayType() && !DeclType->isUnionType()) {
     // It is possible we have one or more unnamed bitfields remaining.
     // Find first (if any) named field and emit warning.
@@ -1158,12 +1186,12 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
 
   InitializedEntity MemberEntity =
     InitializedEntity::InitializeMember(*Field, &Entity);
-    
+
   if (isa<InitListExpr>(IList->getInit(Index)))
-    CheckSubElementType(MemberEntity, IList, Field->getType(), Index, 
+    CheckSubElementType(MemberEntity, IList, Field->getType(), Index,
                         StructuredList, StructuredIndex);
   else
-    CheckImplicitInitList(MemberEntity, IList, Field->getType(), Index, 
+    CheckImplicitInitList(MemberEntity, IList, Field->getType(), Index,
                           StructuredList, StructuredIndex);
 }
 
@@ -1171,34 +1199,25 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
 /// anonymous struct or union into a series of field designators that
 /// refers to the field within the appropriate subobject.
 ///
-/// Field/FieldIndex will be updated to point to the (new)
-/// currently-designated field.
 static void ExpandAnonymousFieldDesignator(Sema &SemaRef,
                                            DesignatedInitExpr *DIE,
                                            unsigned DesigIdx,
-                                           FieldDecl *Field,
-                                        RecordDecl::field_iterator &FieldIter,
-                                           unsigned &FieldIndex) {
+                                           IndirectFieldDecl *IndirectField) {
   typedef DesignatedInitExpr::Designator Designator;
 
-  // Build the path from the current object to the member of the
-  // anonymous struct/union (backwards).
-  llvm::SmallVector<FieldDecl *, 4> Path;
-  SemaRef.BuildAnonymousStructUnionMemberPath(Field, Path);
-
   // Build the replacement designators.
   llvm::SmallVector<Designator, 4> Replacements;
-  for (llvm::SmallVector<FieldDecl *, 4>::reverse_iterator
-         FI = Path.rbegin(), FIEnd = Path.rend();
-       FI != FIEnd; ++FI) {
-    if (FI + 1 == FIEnd)
+  for (IndirectFieldDecl::chain_iterator PI = IndirectField->chain_begin(),
+       PE = IndirectField->chain_end(); PI != PE; ++PI) {
+    if (PI + 1 == PE)
       Replacements.push_back(Designator((IdentifierInfo *)0,
                                     DIE->getDesignator(DesigIdx)->getDotLoc(),
                                 DIE->getDesignator(DesigIdx)->getFieldLoc()));
     else
       Replacements.push_back(Designator((IdentifierInfo *)0, SourceLocation(),
                                         SourceLocation()));
-    Replacements.back().setField(*FI);
+    assert(isa<FieldDecl>(*PI));
+    Replacements.back().setField(cast<FieldDecl>(*PI));
   }
 
   // Expand the current designator into the set of replacement
@@ -1206,23 +1225,20 @@ static void ExpandAnonymousFieldDesignator(Sema &SemaRef,
   // member of the anonymous struct/union is actually stored.
   DIE->ExpandDesignator(SemaRef.Context, DesigIdx, &Replacements[0],
                         &Replacements[0] + Replacements.size());
+}
 
-  // Update FieldIter/FieldIndex;
-  RecordDecl *Record = cast<RecordDecl>(Path.back()->getDeclContext());
-  FieldIter = Record->field_begin();
-  FieldIndex = 0;
-  for (RecordDecl::field_iterator FEnd = Record->field_end();
-       FieldIter != FEnd; ++FieldIter) {
-    if (FieldIter->isUnnamedBitfield())
-        continue;
-
-    if (*FieldIter == Path.back())
-      return;
-
-    ++FieldIndex;
+/// \brief Given an implicit anonymous field, search the IndirectField that
+///  corresponds to FieldName.
+static IndirectFieldDecl *FindIndirectFieldDesignator(FieldDecl *AnonField,
+                                                 IdentifierInfo *FieldName) {
+  assert(AnonField->isAnonymousStructOrUnion());
+  Decl *NextDecl = AnonField->getNextDeclInContext();
+  while (IndirectFieldDecl *IF = dyn_cast<IndirectFieldDecl>(NextDecl)) {
+    if (FieldName && FieldName == IF->getAnonField()->getIdentifier())
+      return IF;
+    NextDecl = NextDecl->getNextDeclInContext();
   }
-
-  assert(false && "Unable to find anonymous struct/union field");
+  return 0;
 }
 
 /// @brief Check the well-formedness of a C99 designated initializer.
@@ -1343,7 +1359,19 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       if (Field->isUnnamedBitfield())
         continue;
 
-      if (KnownField == *Field || Field->getIdentifier() == FieldName)
+      // If we find a field representing an anonymous field, look in the
+      // IndirectFieldDecl that follow for the designated initializer.
+      if (!KnownField && Field->isAnonymousStructOrUnion()) {
+        if (IndirectFieldDecl *IF =
+            FindIndirectFieldDesignator(*Field, FieldName)) {
+          ExpandAnonymousFieldDesignator(SemaRef, DIE, DesigIdx, IF);
+          D = DIE->getDesignator(DesigIdx);
+          break;
+        }
+      }
+      if (KnownField && KnownField == *Field)
+        break;
+      if (FieldName && FieldName == Field->getIdentifier())
         break;
 
       ++FieldIndex;
@@ -1360,19 +1388,19 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       if (Lookup.first == Lookup.second) {
         // Name lookup didn't find anything. Determine whether this
         // was a typo for another field name.
-        LookupResult R(SemaRef, FieldName, D->getFieldLoc(), 
+        LookupResult R(SemaRef, FieldName, D->getFieldLoc(),
                        Sema::LookupMemberName);
         if (SemaRef.CorrectTypo(R, /*Scope=*/0, /*SS=*/0, RT->getDecl(), false,
-                                Sema::CTC_NoKeywords) && 
+                                Sema::CTC_NoKeywords) &&
             (ReplacementField = R.getAsSingle<FieldDecl>()) &&
             ReplacementField->getDeclContext()->getRedeclContext()
                                                       ->Equals(RT->getDecl())) {
-          SemaRef.Diag(D->getFieldLoc(), 
+          SemaRef.Diag(D->getFieldLoc(),
                        diag::err_field_designator_unknown_suggest)
             << FieldName << CurrentObjectType << R.getLookupName()
             << FixItHint::CreateReplacement(D->getFieldLoc(),
                                             R.getLookupName().getAsString());
-          SemaRef.Diag(ReplacementField->getLocation(), 
+          SemaRef.Diag(ReplacementField->getLocation(),
                        diag::note_previous_decl)
             << ReplacementField->getDeclName();
         } else {
@@ -1381,9 +1409,6 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
           ++Index;
           return true;
         }
-      } else if (!KnownField) {
-        // Determine whether we found a field at all.
-        ReplacementField = dyn_cast<FieldDecl>(*Lookup.first);
       }
 
       if (!ReplacementField) {
@@ -1396,16 +1421,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
         return true;
       }
 
-      if (!KnownField && 
-          cast<RecordDecl>((ReplacementField)->getDeclContext())
-                                                 ->isAnonymousStructOrUnion()) {
-        // Handle an field designator that refers to a member of an
-        // anonymous struct or union.
-        ExpandAnonymousFieldDesignator(SemaRef, DIE, DesigIdx,
-                                       ReplacementField,
-                                       Field, FieldIndex);
-        D = DIE->getDesignator(DesigIdx);
-      } else if (!KnownField) {
+      if (!KnownField) {
         // The replacement field comes from typo correction; find it
         // in the list of fields.
         FieldIndex = 0;
@@ -1414,19 +1430,13 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
           if (Field->isUnnamedBitfield())
             continue;
 
-          if (ReplacementField == *Field || 
+          if (ReplacementField == *Field ||
               Field->getIdentifier() == ReplacementField->getIdentifier())
             break;
 
           ++FieldIndex;
         }
       }
-    } else if (!KnownField &&
-               cast<RecordDecl>((*Field)->getDeclContext())
-                 ->isAnonymousStructOrUnion()) {
-      ExpandAnonymousFieldDesignator(SemaRef, DIE, DesigIdx, *Field,
-                                     Field, FieldIndex);
-      D = DIE->getDesignator(DesigIdx);
     }
 
     // All of the fields of a union are located at the same place in
@@ -1461,7 +1471,8 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
         Invalid = true;
       }
 
-      if (!hadError && !isa<InitListExpr>(DIE->getInit())) {
+      if (!hadError && !isa<InitListExpr>(DIE->getInit()) &&
+          !isa<StringLiteral>(DIE->getInit())) {
         // The initializer is not an initializer list.
         SemaRef.Diag(DIE->getInit()->getSourceRange().getBegin(),
                       diag::err_flexible_array_init_needs_braces)
@@ -1511,11 +1522,11 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       // Recurse to check later designated subobjects.
       QualType FieldType = (*Field)->getType();
       unsigned newStructuredIndex = FieldIndex;
-      
+
       InitializedEntity MemberEntity =
         InitializedEntity::InitializeMember(*Field, &Entity);
-      if (CheckDesignatedInitializer(MemberEntity, IList, DIE, DesigIdx + 1, 
-                                     FieldType, 0, 0, Index, 
+      if (CheckDesignatedInitializer(MemberEntity, IList, DIE, DesigIdx + 1,
+                                     FieldType, 0, 0, Index,
                                      StructuredList, newStructuredIndex,
                                      true, false))
         return true;
@@ -1544,7 +1555,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
 
     // Check the remaining fields within this class/struct/union subobject.
     bool prevHadError = hadError;
-    
+
     CheckStructUnionTypes(Entity, IList, CurrentObjectType, Field, false, Index,
                           StructuredList, FieldIndex);
     return hadError && !prevHadError;
@@ -1582,22 +1593,29 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
   } else {
     assert(D->isArrayRangeDesignator() && "Need array-range designator");
 
-
     DesignatedStartIndex =
       DIE->getArrayRangeStart(*D)->EvaluateAsInt(SemaRef.Context);
     DesignatedEndIndex =
       DIE->getArrayRangeEnd(*D)->EvaluateAsInt(SemaRef.Context);
     IndexExpr = DIE->getArrayRangeEnd(*D);
 
-    if (DesignatedStartIndex.getZExtValue() !=DesignatedEndIndex.getZExtValue())
+    // Codegen can't handle evaluating array range designators that have side
+    // effects, because we replicate the AST value for each initialized element.
+    // As such, set the sawArrayRangeDesignator() bit if we initialize multiple
+    // elements with something that has a side effect, so codegen can emit an
+    // "error unsupported" error instead of miscompiling the app.
+    if (DesignatedStartIndex.getZExtValue()!=DesignatedEndIndex.getZExtValue()&&
+        DIE->getInit()->HasSideEffects(SemaRef.Context))
       FullyStructuredList->sawArrayRangeDesignator();
   }
 
   if (isa<ConstantArrayType>(AT)) {
     llvm::APSInt MaxElements(cast<ConstantArrayType>(AT)->getSize(), false);
-    DesignatedStartIndex.extOrTrunc(MaxElements.getBitWidth());
+    DesignatedStartIndex
+      = DesignatedStartIndex.extOrTrunc(MaxElements.getBitWidth());
     DesignatedStartIndex.setIsUnsigned(MaxElements.isUnsigned());
-    DesignatedEndIndex.extOrTrunc(MaxElements.getBitWidth());
+    DesignatedEndIndex
+      = DesignatedEndIndex.extOrTrunc(MaxElements.getBitWidth());
     DesignatedEndIndex.setIsUnsigned(MaxElements.isUnsigned());
     if (DesignatedEndIndex >= MaxElements) {
       SemaRef.Diag(IndexExpr->getSourceRange().getBegin(),
@@ -1610,10 +1628,12 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
   } else {
     // Make sure the bit-widths and signedness match.
     if (DesignatedStartIndex.getBitWidth() > DesignatedEndIndex.getBitWidth())
-      DesignatedEndIndex.extend(DesignatedStartIndex.getBitWidth());
+      DesignatedEndIndex
+        = DesignatedEndIndex.extend(DesignatedStartIndex.getBitWidth());
     else if (DesignatedStartIndex.getBitWidth() <
              DesignatedEndIndex.getBitWidth())
-      DesignatedStartIndex.extend(DesignatedEndIndex.getBitWidth());
+      DesignatedStartIndex
+        = DesignatedStartIndex.extend(DesignatedEndIndex.getBitWidth());
     DesignatedStartIndex.setIsUnsigned(true);
     DesignatedEndIndex.setIsUnsigned(true);
   }
@@ -1630,7 +1650,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
   // Move to the next designator
   unsigned ElementIndex = DesignatedStartIndex.getZExtValue();
   unsigned OldIndex = Index;
-  
+
   InitializedEntity ElementEntity =
     InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
 
@@ -1638,10 +1658,10 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     // Recurse to check later designated subobjects.
     QualType ElementType = AT->getElementType();
     Index = OldIndex;
-    
+
     ElementEntity.setElementIndex(ElementIndex);
-    if (CheckDesignatedInitializer(ElementEntity, IList, DIE, DesigIdx + 1, 
-                                   ElementType, 0, 0, Index, 
+    if (CheckDesignatedInitializer(ElementEntity, IList, DIE, DesigIdx + 1,
+                                   ElementType, 0, 0, Index,
                                    StructuredList, ElementIndex,
                                    (DesignatedStartIndex == DesignatedEndIndex),
                                    false))
@@ -1666,7 +1686,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
 
   // Check the remaining elements within this array subobject.
   bool prevHadError = hadError;
-  CheckArrayType(Entity, IList, CurrentObjectType, DesignatedStartIndex, 
+  CheckArrayType(Entity, IList, CurrentObjectType, DesignatedStartIndex,
                  /*SubobjectIsDesignatorContext=*/false, Index,
                  StructuredList, ElementIndex);
   return hadError && !prevHadError;
@@ -1812,9 +1832,9 @@ CheckArrayDesignatorExpr(Sema &S, Expr *Index, llvm::APSInt &Value) {
 }
 
 ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
-                                                        SourceLocation Loc,
-                                                        bool GNUSyntax,
-                                                        ExprResult Init) {
+                                            SourceLocation Loc,
+                                            bool GNUSyntax,
+                                            ExprResult Init) {
   typedef DesignatedInitExpr::Designator ASTDesignator;
 
   bool Invalid = false;
@@ -1865,9 +1885,9 @@ ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
         if (StartDependent || EndDependent) {
           // Nothing to compute.
         } else if (StartValue.getBitWidth() > EndValue.getBitWidth())
-          EndValue.extend(StartValue.getBitWidth());
+          EndValue = EndValue.extend(StartValue.getBitWidth());
         else if (StartValue.getBitWidth() < EndValue.getBitWidth())
-          StartValue.extend(EndValue.getBitWidth());
+          StartValue = StartValue.extend(EndValue.getBitWidth());
 
         if (!StartDependent && !EndDependent && EndValue < StartValue) {
           Diag(D.getEllipsisLoc(), diag::err_array_designator_empty_range)
@@ -1899,6 +1919,11 @@ ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
                                  Designators.data(), Designators.size(),
                                  InitExpressions.data(), InitExpressions.size(),
                                  Loc, GNUSyntax, Init.takeAs<Expr>());
+
+  if (getLangOptions().CPlusPlus)
+    Diag(DIE->getLocStart(), diag::ext_designated_init)
+      << DIE->getSourceRange();
+
   return Owned(DIE);
 }
 
@@ -1915,9 +1940,9 @@ bool Sema::CheckInitList(const InitializedEntity &Entity,
 // Initialization entity
 //===----------------------------------------------------------------------===//
 
-InitializedEntity::InitializedEntity(ASTContext &Context, unsigned Index, 
+InitializedEntity::InitializedEntity(ASTContext &Context, unsigned Index,
                                      const InitializedEntity &Parent)
-  : Parent(&Parent), Index(Index) 
+  : Parent(&Parent), Index(Index)
 {
   if (const ArrayType *AT = Context.getAsArrayType(Parent.getType())) {
     Kind = EK_ArrayElement;
@@ -1928,7 +1953,7 @@ InitializedEntity::InitializedEntity(ASTContext &Context, unsigned Index,
   }
 }
 
-InitializedEntity InitializedEntity::InitializeBase(ASTContext &Context, 
+InitializedEntity InitializedEntity::InitializeBase(ASTContext &Context,
                                                     CXXBaseSpecifier *Base,
                                                     bool IsInheritedVirtualBase)
 {
@@ -1937,7 +1962,7 @@ InitializedEntity InitializedEntity::InitializeBase(ASTContext &Context,
   Result.Base = reinterpret_cast<uintptr_t>(Base);
   if (IsInheritedVirtualBase)
     Result.Base |= 0x01;
-  
+
   Result.Type = Base->getType();
   return Result;
 }
@@ -1963,7 +1988,7 @@ DeclarationName InitializedEntity::getName() const {
   case EK_BlockElement:
     return DeclarationName();
   }
-  
+
   // Silence GCC warning
   return DeclarationName();
 }
@@ -1985,7 +2010,7 @@ DeclaratorDecl *InitializedEntity::getDecl() const {
   case EK_BlockElement:
     return 0;
   }
-  
+
   // Silence GCC warning
   return 0;
 }
@@ -1995,7 +2020,7 @@ bool InitializedEntity::allowsNRVO() const {
   case EK_Result:
   case EK_Exception:
     return LocAndNRVO.NRVO;
-    
+
   case EK_Variable:
   case EK_Parameter:
   case EK_Member:
@@ -2035,7 +2060,7 @@ void InitializationSequence::Step::Destroy() {
   case SK_StringInit:
   case SK_ObjCObjectConversion:
     break;
-    
+
   case SK_ConversionSequence:
     delete ICS;
   }
@@ -2048,7 +2073,7 @@ bool InitializationSequence::isDirectReferenceBinding() const {
 bool InitializationSequence::isAmbiguous() const {
   if (getKind() != FailedSequence)
     return false;
-  
+
   switch (getFailureKind()) {
   case FK_TooManyInitsForReference:
   case FK_ArrayNeedsInitList:
@@ -2066,13 +2091,13 @@ bool InitializationSequence::isAmbiguous() const {
   case FK_DefaultInitOfConst:
   case FK_Incomplete:
     return false;
-    
+
   case FK_ReferenceInitOverloadFailed:
   case FK_UserConversionOverloadFailed:
   case FK_ConstructorOverloadFailed:
     return FailedOverloadResult == OR_Ambiguous;
   }
-  
+
   return false;
 }
 
@@ -2091,7 +2116,7 @@ void InitializationSequence::AddAddressOverloadResolutionStep(
   Steps.push_back(S);
 }
 
-void InitializationSequence::AddDerivedToBaseCastStep(QualType BaseType, 
+void InitializationSequence::AddDerivedToBaseCastStep(QualType BaseType,
                                                       ExprValueKind VK) {
   Step S;
   switch (VK) {
@@ -2104,7 +2129,7 @@ void InitializationSequence::AddDerivedToBaseCastStep(QualType BaseType,
   Steps.push_back(S);
 }
 
-void InitializationSequence::AddReferenceBindingStep(QualType T, 
+void InitializationSequence::AddReferenceBindingStep(QualType T,
                                                      bool BindingTemporary) {
   Step S;
   S.Kind = BindingTemporary? SK_BindReferenceToTemporary : SK_BindReference;
@@ -2166,7 +2191,7 @@ void InitializationSequence::AddListInitializationStep(QualType T) {
   Steps.push_back(S);
 }
 
-void 
+void
 InitializationSequence::AddConstructorInitializationStep(
                                               CXXConstructorDecl *Constructor,
                                                        AccessSpecifier Access,
@@ -2207,7 +2232,7 @@ void InitializationSequence::AddObjCObjectConversionStep(QualType T) {
   Steps.push_back(S);
 }
 
-void InitializationSequence::SetOverloadFailure(FailureKind Failure, 
+void InitializationSequence::SetOverloadFailure(FailureKind Failure,
                                                 OverloadingResult Result) {
   SequenceKind = FailedSequence;
   this->Failure = Failure;
@@ -2218,8 +2243,8 @@ void InitializationSequence::SetOverloadFailure(FailureKind Failure,
 // Attempt initialization
 //===----------------------------------------------------------------------===//
 
-/// \brief Attempt list initialization (C++0x [dcl.init.list]) 
-static void TryListInitialization(Sema &S, 
+/// \brief Attempt list initialization (C++0x [dcl.init.list])
+static void TryListInitialization(Sema &S,
                                   const InitializedEntity &Entity,
                                   const InitializationKind &Kind,
                                   InitListExpr *InitList,
@@ -2235,7 +2260,7 @@ static void TryListInitialization(Sema &S,
   QualType DestType = Entity.getType();
 
   // C++ [dcl.init]p13:
-  //   If T is a scalar type, then a declaration of the form 
+  //   If T is a scalar type, then a declaration of the form
   //
   //     T x = { a };
   //
@@ -2282,7 +2307,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
 
   bool DerivedToBase;
   bool ObjCConversion;
-  assert(!S.CompareReferenceRelationship(Initializer->getLocStart(), 
+  assert(!S.CompareReferenceRelationship(Initializer->getLocStart(),
                                          T1, T2, DerivedToBase,
                                          ObjCConversion) &&
          "Must have incompatible references when binding via conversion");
@@ -2297,7 +2322,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
   // Determine whether we are allowed to call explicit constructors or
   // explicit conversion operators.
   bool AllowExplicit = Kind.getKind() == InitializationKind::IK_Direct;
-  
+
   const RecordType *T1RecordType = 0;
   if (AllowRValues && (T1RecordType = T1->getAs<RecordType>()) &&
       !S.RequireCompleteType(Kind.getLocation(), T1, 0)) {
@@ -2319,22 +2344,24 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
                                          ConstructorTmpl->getTemplatedDecl());
       else
         Constructor = cast<CXXConstructorDecl>(D);
-      
+
       if (!Constructor->isInvalidDecl() &&
           Constructor->isConvertingConstructor(AllowExplicit)) {
         if (ConstructorTmpl)
           S.AddTemplateOverloadCandidate(ConstructorTmpl, FoundDecl,
                                          /*ExplicitArgs*/ 0,
-                                         &Initializer, 1, CandidateSet);
+                                         &Initializer, 1, CandidateSet,
+                                         /*SuppressUserConversions=*/true);
         else
           S.AddOverloadCandidate(Constructor, FoundDecl,
-                                 &Initializer, 1, CandidateSet);
+                                 &Initializer, 1, CandidateSet,
+                                 /*SuppressUserConversions=*/true);
       }
-    }    
+    }
   }
   if (T1RecordType && T1RecordType->getDecl()->isInvalidDecl())
     return OR_No_Viable_Function;
-  
+
   const RecordType *T2RecordType = 0;
   if ((T2RecordType = T2->getAs<RecordType>()) &&
       !S.RequireCompleteType(Kind.getLocation(), T2, 0)) {
@@ -2342,11 +2369,6 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
     // functions.
     CXXRecordDecl *T2RecordDecl = cast<CXXRecordDecl>(T2RecordType->getDecl());
 
-    // Determine the type we are converting to. If we are allowed to
-    // convert to an rvalue, take the type that the destination type
-    // refers to.
-    QualType ToType = AllowRValues? cv1T1 : DestType;
-
     const UnresolvedSetImpl *Conversions
       = T2RecordDecl->getVisibleConversionFunctions();
     for (UnresolvedSetImpl::const_iterator I = Conversions->begin(),
@@ -2355,41 +2377,41 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
       CXXRecordDecl *ActingDC = cast<CXXRecordDecl>(D->getDeclContext());
       if (isa<UsingShadowDecl>(D))
         D = cast<UsingShadowDecl>(D)->getTargetDecl();
-      
+
       FunctionTemplateDecl *ConvTemplate = dyn_cast<FunctionTemplateDecl>(D);
       CXXConversionDecl *Conv;
       if (ConvTemplate)
         Conv = cast<CXXConversionDecl>(ConvTemplate->getTemplatedDecl());
       else
         Conv = cast<CXXConversionDecl>(D);
-      
+
       // If the conversion function doesn't return a reference type,
       // it can't be considered for this conversion unless we're allowed to
       // consider rvalues.
-      // FIXME: Do we need to make sure that we only consider conversion 
-      // candidates with reference-compatible results? That might be needed to 
+      // FIXME: Do we need to make sure that we only consider conversion
+      // candidates with reference-compatible results? That might be needed to
       // break recursion.
       if ((AllowExplicit || !Conv->isExplicit()) &&
           (AllowRValues || Conv->getConversionType()->isLValueReferenceType())){
         if (ConvTemplate)
           S.AddTemplateConversionCandidate(ConvTemplate, I.getPair(),
                                            ActingDC, Initializer,
-                                           ToType, CandidateSet);
+                                           DestType, CandidateSet);
         else
           S.AddConversionCandidate(Conv, I.getPair(), ActingDC,
-                                   Initializer, ToType, CandidateSet);
+                                   Initializer, DestType, CandidateSet);
       }
     }
   }
   if (T2RecordType && T2RecordType->getDecl()->isInvalidDecl())
     return OR_No_Viable_Function;
-  
+
   SourceLocation DeclLoc = Initializer->getLocStart();
 
-  // Perform overload resolution. If it fails, return the failed result.  
+  // Perform overload resolution. If it fails, return the failed result.
   OverloadCandidateSet::iterator Best;
-  if (OverloadingResult Result 
-        = CandidateSet.BestViableFunction(S, DeclLoc, Best))
+  if (OverloadingResult Result
+        = CandidateSet.BestViableFunction(S, DeclLoc, Best, true))
     return Result;
 
   FunctionDecl *Function = Best->Function;
@@ -2404,7 +2426,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
   Sequence.AddUserConversionStep(Function, Best->FoundDecl,
                                  T2.getNonLValueExprType(S.Context));
 
-  // Determine whether we need to perform derived-to-base or 
+  // Determine whether we need to perform derived-to-base or
   // cv-qualification adjustments.
   ExprValueKind VK = VK_RValue;
   if (T2->isLValueReferenceType())
@@ -2415,7 +2437,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
   bool NewDerivedToBase = false;
   bool NewObjCConversion = false;
   Sema::ReferenceCompareResult NewRefRelationship
-    = S.CompareReferenceRelationship(DeclLoc, T1, 
+    = S.CompareReferenceRelationship(DeclLoc, T1,
                                      T2.getNonLValueExprType(S.Context),
                                      NewDerivedToBase, NewObjCConversion);
   if (NewRefRelationship == Sema::Ref_Incompatible) {
@@ -2431,7 +2453,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
   } else if (NewDerivedToBase)
     Sequence.AddDerivedToBaseCastStep(
                                 S.Context.getQualifiedType(T1,
-                                  T2.getNonReferenceType().getQualifiers()), 
+                                  T2.getNonReferenceType().getQualifiers()),
                                       VK);
   else if (NewObjCConversion)
     Sequence.AddObjCObjectConversionStep(
@@ -2440,13 +2462,13 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
 
   if (cv1T1.getQualifiers() != T2.getNonReferenceType().getQualifiers())
     Sequence.AddQualificationConversionStep(cv1T1, VK);
-  
+
   Sequence.AddReferenceBindingStep(cv1T1, !T2->isReferenceType());
   return OR_Success;
 }
-  
-/// \brief Attempt reference initialization (C++0x [dcl.init.ref]) 
-static void TryReferenceInitialization(Sema &S, 
+
+/// \brief Attempt reference initialization (C++0x [dcl.init.ref])
+static void TryReferenceInitialization(Sema &S,
                                        const InitializedEntity &Entity,
                                        const InitializationKind &Kind,
                                        Expr *Initializer,
@@ -2467,18 +2489,17 @@ static void TryReferenceInitialization(Sema &S,
   // type of the resulting function.
   if (S.Context.getCanonicalType(T2) == S.Context.OverloadTy) {
     DeclAccessPair Found;
-    FunctionDecl *Fn = S.ResolveAddressOfOverloadedFunction(Initializer, 
-                                                            T1,
-                                                            false,
-                                                            Found);
-    if (!Fn) {
+    if (FunctionDecl *Fn = S.ResolveAddressOfOverloadedFunction(Initializer,
+                                                                T1,
+                                                                false,
+                                                                Found)) {
+      Sequence.AddAddressOverloadResolutionStep(Fn, Found);
+      cv2T2 = Fn->getType();
+      T2 = cv2T2.getUnqualifiedType();
+    } else if (!T1->isRecordType()) {
       Sequence.SetFailed(InitializationSequence::FK_AddressOfOverloadFailed);
       return;
     }
-
-    Sequence.AddAddressOverloadResolutionStep(Fn, Found);
-    cv2T2 = Fn->getType();
-    T2 = cv2T2.getUnqualifiedType();
   }
 
   // Compute some basic properties of the types and the initializer.
@@ -2492,10 +2513,10 @@ static void TryReferenceInitialization(Sema &S,
                                      ObjCConversion);
 
   // C++0x [dcl.init.ref]p5:
-  //   A reference to type "cv1 T1" is initialized by an expression of type 
+  //   A reference to type "cv1 T1" is initialized by an expression of type
   //   "cv2 T2" as follows:
   //
-  //     - If the reference is an lvalue reference and the initializer 
+  //     - If the reference is an lvalue reference and the initializer
   //       expression
   // Note the analogous bullet points for rvlaue refs to functions. Because
   // there are no function rvalues in C++, rvalue refs to functions are treated
@@ -2503,19 +2524,21 @@ static void TryReferenceInitialization(Sema &S,
   OverloadingResult ConvOvlResult = OR_Success;
   bool T1Function = T1->isFunctionType();
   if (isLValueRef || T1Function) {
-    if (InitCategory.isLValue() && 
-        RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification) {
-      //   - is an lvalue (but is not a bit-field), and "cv1 T1" is 
+    if (InitCategory.isLValue() &&
+        (RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification ||
+         (Kind.isCStyleOrFunctionalCast() &&
+          RefRelationship == Sema::Ref_Related))) {
+      //   - is an lvalue (but is not a bit-field), and "cv1 T1" is
       //     reference-compatible with "cv2 T2," or
       //
-      // Per C++ [over.best.ics]p2, we don't diagnose whether the lvalue is a 
+      // Per C++ [over.best.ics]p2, we don't diagnose whether the lvalue is a
       // bit-field when we're determining whether the reference initialization
       // can occur. However, we do pay attention to whether it is a bit-field
       // to decide whether we're actually binding to a temporary created from
       // the bit-field.
       if (DerivedToBase)
         Sequence.AddDerivedToBaseCastStep(
-                         S.Context.getQualifiedType(T1, T2Quals), 
+                         S.Context.getQualifiedType(T1, T2Quals),
                          VK_LValue);
       else if (ObjCConversion)
         Sequence.AddObjCObjectConversionStep(
@@ -2528,18 +2551,18 @@ static void TryReferenceInitialization(Sema &S,
       Sequence.AddReferenceBindingStep(cv1T1, BindingTemporary);
       return;
     }
-    
-    //     - has a class type (i.e., T2 is a class type), where T1 is not 
-    //       reference-related to T2, and can be implicitly converted to an 
-    //       lvalue of type "cv3 T3," where "cv1 T1" is reference-compatible 
-    //       with "cv3 T3" (this conversion is selected by enumerating the 
+
+    //     - has a class type (i.e., T2 is a class type), where T1 is not
+    //       reference-related to T2, and can be implicitly converted to an
+    //       lvalue of type "cv3 T3," where "cv1 T1" is reference-compatible
+    //       with "cv3 T3" (this conversion is selected by enumerating the
     //       applicable conversion functions (13.3.1.6) and choosing the best
     //       one through overload resolution (13.3)),
     // If we have an rvalue ref to function type here, the rhs must be
     // an rvalue.
     if (RefRelationship == Sema::Ref_Incompatible && T2->isRecordType() &&
         (isLValueRef || InitCategory.isRValue())) {
-      ConvOvlResult = TryRefInitWithConversionFunction(S, Entity, Kind, 
+      ConvOvlResult = TryRefInitWithConversionFunction(S, Entity, Kind,
                                                        Initializer,
                                                    /*AllowRValues=*/isRValueRef,
                                                        Sequence);
@@ -2553,37 +2576,39 @@ static void TryReferenceInitialization(Sema &S,
     }
   }
 
-  //     - Otherwise, the reference shall be an lvalue reference to a 
+  //     - Otherwise, the reference shall be an lvalue reference to a
   //       non-volatile const type (i.e., cv1 shall be const), or the reference
-  //       shall be an rvalue reference and the initializer expression shall 
-  //       be an rvalue or have a function type.
-  // We handled the function type stuff above.
-  if (!((isLValueRef && T1Quals.hasConst() && !T1Quals.hasVolatile()) ||
-        (isRValueRef && InitCategory.isRValue()))) {
-    if (ConvOvlResult && !Sequence.getFailedCandidateSet().empty())
+  //       shall be an rvalue reference.
+  if (isLValueRef && !(T1Quals.hasConst() && !T1Quals.hasVolatile())) {
+    if (S.Context.getCanonicalType(T2) == S.Context.OverloadTy)
+      Sequence.SetFailed(InitializationSequence::FK_AddressOfOverloadFailed);
+    else if (ConvOvlResult && !Sequence.getFailedCandidateSet().empty())
       Sequence.SetOverloadFailure(
                         InitializationSequence::FK_ReferenceInitOverloadFailed,
                                   ConvOvlResult);
-    else if (isLValueRef)
+    else
       Sequence.SetFailed(InitCategory.isLValue()
         ? (RefRelationship == Sema::Ref_Related
              ? InitializationSequence::FK_ReferenceInitDropsQualifiers
              : InitializationSequence::FK_NonConstLValueReferenceBindingToUnrelated)
         : InitializationSequence::FK_NonConstLValueReferenceBindingToTemporary);
-    else
-      Sequence.SetFailed(
-                    InitializationSequence::FK_RValueReferenceBindingToLValue);
 
     return;
   }
 
-  //       - [If T1 is not a function type], if T2 is a class type and
-  if (!T1Function && T2->isRecordType()) {
-    bool isXValue = InitCategory.isXValue();
-    //       - the initializer expression is an rvalue and "cv1 T1" is 
-    //         reference-compatible with "cv2 T2", or
-    if (InitCategory.isRValue() && 
-        RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification) {
+  //    - If the initializer expression
+  //      - is an xvalue, class prvalue, array prvalue, or function lvalue and
+  //        "cv1 T1" is reference-compatible with "cv2 T2"
+  // Note: functions are handled below.
+  if (!T1Function &&
+      (RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification ||
+       (Kind.isCStyleOrFunctionalCast() &&
+        RefRelationship == Sema::Ref_Related)) &&
+      (InitCategory.isXValue() ||
+       (InitCategory.isPRValue() && T2->isRecordType()) ||
+       (InitCategory.isPRValue() && T2->isArrayType()))) {
+    ExprValueKind ValueKind = InitCategory.isXValue()? VK_XValue : VK_RValue;
+    if (InitCategory.isPRValue() && T2->isRecordType()) {
       // The corresponding bullet in C++03 [dcl.init.ref]p5 gives the
       // compiler the freedom to perform a copy here or bind to the
       // object, while C++0x requires that we bind directly to the
@@ -2593,29 +2618,29 @@ static void TryReferenceInitialization(Sema &S,
       //
       //   The constructor that would be used to make the copy shall
       //   be callable whether or not the copy is actually done.
-      if (!S.getLangOptions().CPlusPlus0x)
+      if (!S.getLangOptions().CPlusPlus0x && !S.getLangOptions().Microsoft)
         Sequence.AddExtraneousCopyToTemporary(cv2T2);
-
-      if (DerivedToBase)
-        Sequence.AddDerivedToBaseCastStep(
-                         S.Context.getQualifiedType(T1, T2Quals), 
-                         isXValue ? VK_XValue : VK_RValue);
-      else if (ObjCConversion)
-        Sequence.AddObjCObjectConversionStep(
-                                     S.Context.getQualifiedType(T1, T2Quals));
-
-      if (T1Quals != T2Quals)
-        Sequence.AddQualificationConversionStep(cv1T1,
-                                            isXValue ? VK_XValue : VK_RValue);
-      Sequence.AddReferenceBindingStep(cv1T1, /*bindingTemporary=*/!isXValue);
-      return;
     }
 
-    //       - T1 is not reference-related to T2 and the initializer expression
-    //         can be implicitly converted to an rvalue of type "cv3 T3" (this
-    //         conversion is selected by enumerating the applicable conversion
-    //         functions (13.3.1.6) and choosing the best one through overload 
-    //         resolution (13.3)),
+    if (DerivedToBase)
+      Sequence.AddDerivedToBaseCastStep(S.Context.getQualifiedType(T1, T2Quals),
+                                        ValueKind);
+    else if (ObjCConversion)
+      Sequence.AddObjCObjectConversionStep(
+                                       S.Context.getQualifiedType(T1, T2Quals));
+
+    if (T1Quals != T2Quals)
+      Sequence.AddQualificationConversionStep(cv1T1, ValueKind);
+    Sequence.AddReferenceBindingStep(cv1T1,
+         /*bindingTemporary=*/(InitCategory.isPRValue() && !T2->isArrayType()));
+    return;
+  }
+
+  //       - has a class type (i.e., T2 is a class type), where T1 is not
+  //         reference-related to T2, and can be implicitly converted to an
+  //         xvalue, class prvalue, or function lvalue of type "cv3 T3",
+  //         where "cv1 T1" is reference-compatible with "cv3 T3",
+  if (T2->isRecordType()) {
     if (RefRelationship == Sema::Ref_Incompatible) {
       ConvOvlResult = TryRefInitWithConversionFunction(S, Entity,
                                                        Kind, Initializer,
@@ -2625,23 +2650,17 @@ static void TryReferenceInitialization(Sema &S,
         Sequence.SetOverloadFailure(
                       InitializationSequence::FK_ReferenceInitOverloadFailed,
                                     ConvOvlResult);
-        
+
       return;
     }
-    
+
     Sequence.SetFailed(InitializationSequence::FK_ReferenceInitDropsQualifiers);
     return;
   }
-  
-  //      - If the initializer expression is an rvalue, with T2 an array type,
-  //        and "cv1 T1" is reference-compatible with "cv2 T2," the reference
-  //        is bound to the object represented by the rvalue (see 3.10).
-  // FIXME: How can an array type be reference-compatible with anything?
-  // Don't we mean the element types of T1 and T2?
-  
-  //      - Otherwise, a temporary of type “cv1 T1” is created and initialized
+
+  //      - Otherwise, a temporary of type "cv1 T1" is created and initialized
   //        from the initializer expression using the rules for a non-reference
-  //        copy initialization (8.5). The reference is then bound to the 
+  //        copy initialization (8.5). The reference is then bound to the
   //        temporary. [...]
 
   // Determine whether we are allowed to call explicit constructors or
@@ -2653,7 +2672,8 @@ static void TryReferenceInitialization(Sema &S,
   if (S.TryImplicitConversion(Sequence, TempEntity, Initializer,
                               /*SuppressUserConversions*/ false,
                               AllowExplicit,
-                              /*FIXME:InOverloadResolution=*/false)) {
+                              /*FIXME:InOverloadResolution=*/false,
+                              /*CStyle=*/Kind.isCStyleOrFunctionalCast())) {
     // FIXME: Use the conversion function set stored in ICS to turn
     // this into an overloading ambiguity diagnostic. However, we need
     // to keep that set as an OverloadCandidateSet rather than as some
@@ -2662,6 +2682,8 @@ static void TryReferenceInitialization(Sema &S,
       Sequence.SetOverloadFailure(
                         InitializationSequence::FK_ReferenceInitOverloadFailed,
                                   ConvOvlResult);
+    else if (S.Context.getCanonicalType(T2) == S.Context.OverloadTy)
+      Sequence.SetFailed(InitializationSequence::FK_AddressOfOverloadFailed);
     else
       Sequence.SetFailed(InitializationSequence::FK_ReferenceInitFailed);
     return;
@@ -2672,19 +2694,28 @@ static void TryReferenceInitialization(Sema &S,
   //        than, cv2; otherwise, the program is ill-formed.
   unsigned T1CVRQuals = T1Quals.getCVRQualifiers();
   unsigned T2CVRQuals = T2Quals.getCVRQualifiers();
-  if (RefRelationship == Sema::Ref_Related && 
+  if (RefRelationship == Sema::Ref_Related &&
       (T1CVRQuals | T2CVRQuals) != T1CVRQuals) {
     Sequence.SetFailed(InitializationSequence::FK_ReferenceInitDropsQualifiers);
     return;
   }
 
+  //   [...] If T1 is reference-related to T2 and the reference is an rvalue
+  //   reference, the initializer expression shall not be an lvalue.
+  if (RefRelationship >= Sema::Ref_Related && !isLValueRef &&
+      InitCategory.isLValue()) {
+    Sequence.SetFailed(
+                    InitializationSequence::FK_RValueReferenceBindingToLValue);
+    return;
+  }
+
   Sequence.AddReferenceBindingStep(cv1T1, /*bindingTemporary=*/true);
   return;
 }
 
 /// \brief Attempt character array initialization from a string literal
-/// (C++ [dcl.init.string], C99 6.7.8). 
-static void TryStringLiteralInitialization(Sema &S, 
+/// (C++ [dcl.init.string], C99 6.7.8).
+static void TryStringLiteralInitialization(Sema &S,
                                            const InitializedEntity &Entity,
                                            const InitializationKind &Kind,
                                            Expr *Initializer,
@@ -2696,19 +2727,19 @@ static void TryStringLiteralInitialization(Sema &S,
 /// \brief Attempt initialization by constructor (C++ [dcl.init]), which
 /// enumerates the constructors of the initialized entity and performs overload
 /// resolution to select the best.
-static void TryConstructorInitialization(Sema &S, 
+static void TryConstructorInitialization(Sema &S,
                                          const InitializedEntity &Entity,
                                          const InitializationKind &Kind,
                                          Expr **Args, unsigned NumArgs,
                                          QualType DestType,
                                          InitializationSequence &Sequence) {
   Sequence.setSequenceKind(InitializationSequence::ConstructorInitialization);
-  
+
   // Build the candidate set directly in the initialization sequence
   // structure, so that it will persist if we fail.
   OverloadCandidateSet &CandidateSet = Sequence.getFailedCandidateSet();
   CandidateSet.clear();
-  
+
   // Determine whether we are allowed to call explicit constructors or
   // explicit conversion operators.
   bool AllowExplicit = (Kind.getKind() == InitializationKind::IK_Direct ||
@@ -2720,21 +2751,21 @@ static void TryConstructorInitialization(Sema &S,
     Sequence.SetFailed(InitializationSequence::FK_Incomplete);
     return;
   }
-  
+
   // The type we're converting to is a class type. Enumerate its constructors
   // to see if one is suitable.
   const RecordType *DestRecordType = DestType->getAs<RecordType>();
-  assert(DestRecordType && "Constructor initialization requires record type");  
+  assert(DestRecordType && "Constructor initialization requires record type");
   CXXRecordDecl *DestRecordDecl
     = cast<CXXRecordDecl>(DestRecordType->getDecl());
-    
+
   DeclContext::lookup_iterator Con, ConEnd;
   for (llvm::tie(Con, ConEnd) = S.LookupConstructors(DestRecordDecl);
        Con != ConEnd; ++Con) {
     NamedDecl *D = *Con;
     DeclAccessPair FoundDecl = DeclAccessPair::make(D, D->getAccess());
     bool SuppressUserConversions = false;
-    
+
     // Find the constructor (which may be a template).
     CXXConstructorDecl *Constructor = 0;
     FunctionTemplateDecl *ConstructorTmpl = dyn_cast<FunctionTemplateDecl>(D);
@@ -2744,14 +2775,14 @@ static void TryConstructorInitialization(Sema &S,
     else {
       Constructor = cast<CXXConstructorDecl>(D);
 
-      // If we're performing copy initialization using a copy constructor, we 
+      // If we're performing copy initialization using a copy constructor, we
       // suppress user-defined conversions on the arguments.
       // FIXME: Move constructors?
       if (Kind.getKind() == InitializationKind::IK_Copy &&
           Constructor->isCopyConstructor())
         SuppressUserConversions = true;
     }
-    
+
     if (!Constructor->isInvalidDecl() &&
         (AllowExplicit || !Constructor->isExplicit())) {
       if (ConstructorTmpl)
@@ -2764,16 +2795,16 @@ static void TryConstructorInitialization(Sema &S,
                                Args, NumArgs, CandidateSet,
                                SuppressUserConversions);
     }
-  }    
-    
+  }
+
   SourceLocation DeclLoc = Kind.getLocation();
-  
-  // Perform overload resolution. If it fails, return the failed result.  
+
+  // Perform overload resolution. If it fails, return the failed result.
   OverloadCandidateSet::iterator Best;
-  if (OverloadingResult Result 
+  if (OverloadingResult Result
         = CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
     Sequence.SetOverloadFailure(
-                          InitializationSequence::FK_ConstructorOverloadFailed, 
+                          InitializationSequence::FK_ConstructorOverloadFailed,
                                 Result);
     return;
   }
@@ -2792,13 +2823,13 @@ static void TryConstructorInitialization(Sema &S,
   // Add the constructor initialization step. Any cv-qualification conversion is
   // subsumed by the initialization.
   Sequence.AddConstructorInitializationStep(
-                                      cast<CXXConstructorDecl>(Best->Function), 
+                                      cast<CXXConstructorDecl>(Best->Function),
                                       Best->FoundDecl.getAccess(),
                                       DestType);
 }
 
 /// \brief Attempt value initialization (C++ [dcl.init]p7).
-static void TryValueInitialization(Sema &S, 
+static void TryValueInitialization(Sema &S,
                                    const InitializedEntity &Entity,
                                    const InitializationKind &Kind,
                                    InitializationSequence &Sequence) {
@@ -2806,11 +2837,11 @@ static void TryValueInitialization(Sema &S,
   //
   //   To value-initialize an object of type T means:
   QualType T = Entity.getType();
-  
+
   //     -- if T is an array type, then each element is value-initialized;
   while (const ArrayType *AT = S.Context.getAsArrayType(T))
     T = AT->getElementType();
-  
+
   if (const RecordType *RT = T->getAs<RecordType>()) {
     if (CXXRecordDecl *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
       // -- if T is a class type (clause 9) with a user-declared
@@ -2822,15 +2853,15 @@ static void TryValueInitialization(Sema &S,
       // but Entity doesn't have a way to capture that (yet).
       if (ClassDecl->hasUserDeclaredConstructor())
         return TryConstructorInitialization(S, Entity, Kind, 0, 0, T, Sequence);
-      
+
       // -- if T is a (possibly cv-qualified) non-union class type
       //    without a user-provided constructor, then the object is
-      //    zero-initialized and, if T’s implicitly-declared default
+      //    zero-initialized and, if T's implicitly-declared default
       //    constructor is non-trivial, that constructor is called.
       if ((ClassDecl->getTagKind() == TTK_Class ||
            ClassDecl->getTagKind() == TTK_Struct)) {
         Sequence.AddZeroInitializationStep(Entity.getType());
-        return TryConstructorInitialization(S, Entity, Kind, 0, 0, T, Sequence);        
+        return TryConstructorInitialization(S, Entity, Kind, 0, 0, T, Sequence);
       }
     }
   }
@@ -2845,14 +2876,14 @@ static void TryDefaultInitialization(Sema &S,
                                      const InitializationKind &Kind,
                                      InitializationSequence &Sequence) {
   assert(Kind.getKind() == InitializationKind::IK_Default);
-  
+
   // C++ [dcl.init]p6:
   //   To default-initialize an object of type T means:
   //     - if T is an array type, each element is default-initialized;
   QualType DestType = Entity.getType();
   while (const ArrayType *Array = S.Context.getAsArrayType(DestType))
     DestType = Array->getElementType();
-         
+
   //     - if T is a (possibly cv-qualified) class type (Clause 9), the default
   //       constructor for T is called (and the initialization is ill-formed if
   //       T has no accessible default constructor);
@@ -2860,12 +2891,12 @@ static void TryDefaultInitialization(Sema &S,
     TryConstructorInitialization(S, Entity, Kind, 0, 0, DestType, Sequence);
     return;
   }
-  
+
   //     - otherwise, no initialization is performed.
   Sequence.setSequenceKind(InitializationSequence::NoInitialization);
-  
+
   //   If a program calls for the default initialization of an object of
-  //   a const-qualified type T, T shall be a class type with a user-provided 
+  //   a const-qualified type T, T shall be a class type with a user-provided
   //   default constructor.
   if (DestType.isConstQualified() && S.getLangOptions().CPlusPlus)
     Sequence.SetFailed(InitializationSequence::FK_DefaultInitOfConst);
@@ -2874,42 +2905,42 @@ static void TryDefaultInitialization(Sema &S,
 /// \brief Attempt a user-defined conversion between two types (C++ [dcl.init]),
 /// which enumerates all conversion functions and performs overload resolution
 /// to select the best.
-static void TryUserDefinedConversion(Sema &S, 
+static void TryUserDefinedConversion(Sema &S,
                                      const InitializedEntity &Entity,
                                      const InitializationKind &Kind,
                                      Expr *Initializer,
                                      InitializationSequence &Sequence) {
   Sequence.setSequenceKind(InitializationSequence::UserDefinedConversion);
-  
+
   QualType DestType = Entity.getType();
   assert(!DestType->isReferenceType() && "References are handled elsewhere");
   QualType SourceType = Initializer->getType();
   assert((DestType->isRecordType() || SourceType->isRecordType()) &&
          "Must have a class type to perform a user-defined conversion");
-  
+
   // Build the candidate set directly in the initialization sequence
   // structure, so that it will persist if we fail.
   OverloadCandidateSet &CandidateSet = Sequence.getFailedCandidateSet();
   CandidateSet.clear();
-  
+
   // Determine whether we are allowed to call explicit constructors or
   // explicit conversion operators.
   bool AllowExplicit = Kind.getKind() == InitializationKind::IK_Direct;
-  
+
   if (const RecordType *DestRecordType = DestType->getAs<RecordType>()) {
     // The type we're converting to is a class type. Enumerate its constructors
     // to see if there is a suitable conversion.
     CXXRecordDecl *DestRecordDecl
       = cast<CXXRecordDecl>(DestRecordType->getDecl());
-    
+
     // Try to complete the type we're converting to.
-    if (!S.RequireCompleteType(Kind.getLocation(), DestType, 0)) {    
+    if (!S.RequireCompleteType(Kind.getLocation(), DestType, 0)) {
       DeclContext::lookup_iterator Con, ConEnd;
       for (llvm::tie(Con, ConEnd) = S.LookupConstructors(DestRecordDecl);
            Con != ConEnd; ++Con) {
         NamedDecl *D = *Con;
         DeclAccessPair FoundDecl = DeclAccessPair::make(D, D->getAccess());
-        
+
         // Find the constructor (which may be a template).
         CXXConstructorDecl *Constructor = 0;
         FunctionTemplateDecl *ConstructorTmpl
@@ -2919,7 +2950,7 @@ static void TryUserDefinedConversion(Sema &S,
                                            ConstructorTmpl->getTemplatedDecl());
         else
           Constructor = cast<CXXConstructorDecl>(D);
-        
+
         if (!Constructor->isInvalidDecl() &&
             Constructor->isConvertingConstructor(AllowExplicit)) {
           if (ConstructorTmpl)
@@ -2932,7 +2963,7 @@ static void TryUserDefinedConversion(Sema &S,
                                    &Initializer, 1, CandidateSet,
                                    /*SuppressUserConversions=*/true);
         }
-      }    
+      }
     }
   }
 
@@ -2947,24 +2978,24 @@ static void TryUserDefinedConversion(Sema &S,
     if (!S.RequireCompleteType(DeclLoc, SourceType, 0)) {
       CXXRecordDecl *SourceRecordDecl
         = cast<CXXRecordDecl>(SourceRecordType->getDecl());
-      
+
       const UnresolvedSetImpl *Conversions
         = SourceRecordDecl->getVisibleConversionFunctions();
       for (UnresolvedSetImpl::const_iterator I = Conversions->begin(),
-           E = Conversions->end(); 
+           E = Conversions->end();
            I != E; ++I) {
         NamedDecl *D = *I;
         CXXRecordDecl *ActingDC = cast<CXXRecordDecl>(D->getDeclContext());
         if (isa<UsingShadowDecl>(D))
           D = cast<UsingShadowDecl>(D)->getTargetDecl();
-        
+
         FunctionTemplateDecl *ConvTemplate = dyn_cast<FunctionTemplateDecl>(D);
         CXXConversionDecl *Conv;
         if (ConvTemplate)
           Conv = cast<CXXConversionDecl>(ConvTemplate->getTemplatedDecl());
         else
           Conv = cast<CXXConversionDecl>(D);
-        
+
         if (AllowExplicit || !Conv->isExplicit()) {
           if (ConvTemplate)
             S.AddTemplateConversionCandidate(ConvTemplate, I.getPair(),
@@ -2977,19 +3008,19 @@ static void TryUserDefinedConversion(Sema &S,
       }
     }
   }
-  
-  // Perform overload resolution. If it fails, return the failed result.  
+
+  // Perform overload resolution. If it fails, return the failed result.
   OverloadCandidateSet::iterator Best;
   if (OverloadingResult Result
-        = CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
+        = CandidateSet.BestViableFunction(S, DeclLoc, Best, true)) {
     Sequence.SetOverloadFailure(
-                        InitializationSequence::FK_UserConversionOverloadFailed, 
+                        InitializationSequence::FK_UserConversionOverloadFailed,
                                 Result);
     return;
   }
 
   FunctionDecl *Function = Best->Function;
-  
+
   if (isa<CXXConstructorDecl>(Function)) {
     // Add the user-defined conversion step. Any cv-qualification conversion is
     // subsumed by the initialization.
@@ -3011,7 +3042,7 @@ static void TryUserDefinedConversion(Sema &S,
   }
 
   Sequence.AddUserConversionStep(Function, Best->FoundDecl, ConvType);
-    
+
   // If the conversion following the call to the conversion function
   // is interesting, add it as a separate step.
   if (Best->FinalConversion.First || Best->FinalConversion.Second ||
@@ -3030,12 +3061,12 @@ InitializationSequence::InitializationSequence(Sema &S,
                                                unsigned NumArgs)
     : FailedCandidateSet(Kind.getLocation()) {
   ASTContext &Context = S.Context;
-  
+
   // C++0x [dcl.init]p16:
-  //   The semantics of initializers are as follows. The destination type is 
-  //   the type of the object or reference being initialized and the source 
+  //   The semantics of initializers are as follows. The destination type is
+  //   the type of the object or reference being initialized and the source
   //   type is the type of the initializer expression. The source type is not
-  //   defined when the initializer is a braced-init-list or when it is a 
+  //   defined when the initializer is a braced-init-list or when it is a
   //   parenthesized list of expressions.
   QualType DestType = Entity.getType();
 
@@ -3045,6 +3076,10 @@ InitializationSequence::InitializationSequence(Sema &S,
     return;
   }
 
+  for (unsigned I = 0; I != NumArgs; ++I)
+    if (Args[I]->getObjectKind() == OK_ObjCProperty)
+      S.ConvertPropertyForRValue(Args[I]);
+
   QualType SourceType;
   Expr *Initializer = 0;
   if (NumArgs == 1) {
@@ -3052,14 +3087,14 @@ InitializationSequence::InitializationSequence(Sema &S,
     if (!isa<InitListExpr>(Initializer))
       SourceType = Initializer->getType();
   }
-  
-  //     - If the initializer is a braced-init-list, the object is 
+
+  //     - If the initializer is a braced-init-list, the object is
   //       list-initialized (8.5.4).
   if (InitListExpr *InitList = dyn_cast_or_null<InitListExpr>(Initializer)) {
     TryListInitialization(S, Entity, Kind, InitList, *this);
     return;
   }
-  
+
   //     - If the destination type is a reference type, see 8.5.3.
   if (DestType->isReferenceType()) {
     // C++0x [dcl.init.ref]p1:
@@ -3073,36 +3108,36 @@ InitializationSequence::InitializationSequence(Sema &S,
       TryReferenceInitialization(S, Entity, Kind, Args[0], *this);
     return;
   }
-  
-  //     - If the destination type is an array of characters, an array of 
-  //       char16_t, an array of char32_t, or an array of wchar_t, and the 
+
+  //     - If the destination type is an array of characters, an array of
+  //       char16_t, an array of char32_t, or an array of wchar_t, and the
   //       initializer is a string literal, see 8.5.2.
   if (Initializer && IsStringInit(Initializer, DestType, Context)) {
     TryStringLiteralInitialization(S, Entity, Kind, Initializer, *this);
     return;
   }
-  
+
   //     - If the initializer is (), the object is value-initialized.
   if (Kind.getKind() == InitializationKind::IK_Value ||
       (Kind.getKind() == InitializationKind::IK_Direct && NumArgs == 0)) {
     TryValueInitialization(S, Entity, Kind, *this);
     return;
   }
-  
+
   // Handle default initialization.
-  if (Kind.getKind() == InitializationKind::IK_Default){
+  if (Kind.getKind() == InitializationKind::IK_Default) {
     TryDefaultInitialization(S, Entity, Kind, *this);
     return;
   }
 
-  //     - Otherwise, if the destination type is an array, the program is 
+  //     - Otherwise, if the destination type is an array, the program is
   //       ill-formed.
   if (const ArrayType *AT = Context.getAsArrayType(DestType)) {
     if (AT->getElementType()->isAnyCharacterType())
       SetFailed(FK_ArrayNeedsInitListOrStringLiteral);
     else
       SetFailed(FK_ArrayNeedsInitList);
-    
+
     return;
   }
 
@@ -3112,22 +3147,22 @@ InitializationSequence::InitializationSequence(Sema &S,
     AddCAssignmentStep(DestType);
     return;
   }
-  
+
   //     - If the destination type is a (possibly cv-qualified) class type:
   if (DestType->isRecordType()) {
-    //     - If the initialization is direct-initialization, or if it is 
-    //       copy-initialization where the cv-unqualified version of the 
-    //       source type is the same class as, or a derived class of, the 
+    //     - If the initialization is direct-initialization, or if it is
+    //       copy-initialization where the cv-unqualified version of the
+    //       source type is the same class as, or a derived class of, the
     //       class of the destination, constructors are considered. [...]
     if (Kind.getKind() == InitializationKind::IK_Direct ||
         (Kind.getKind() == InitializationKind::IK_Copy &&
          (Context.hasSameUnqualifiedType(SourceType, DestType) ||
           S.IsDerivedFrom(SourceType, DestType))))
-      TryConstructorInitialization(S, Entity, Kind, Args, NumArgs, 
+      TryConstructorInitialization(S, Entity, Kind, Args, NumArgs,
                                    Entity.getType(), *this);
-    //     - Otherwise (i.e., for the remaining copy-initialization cases), 
+    //     - Otherwise (i.e., for the remaining copy-initialization cases),
     //       user-defined conversion sequences that can convert from the source
-    //       type to the destination type or (when a conversion function is 
+    //       type to the destination type or (when a conversion function is
     //       used) to a derived class thereof are enumerated as described in
     //       13.3.1.4, and the best one is chosen through overload resolution
     //       (13.3).
@@ -3135,30 +3170,39 @@ InitializationSequence::InitializationSequence(Sema &S,
       TryUserDefinedConversion(S, Entity, Kind, Initializer, *this);
     return;
   }
-  
+
   if (NumArgs > 1) {
     SetFailed(FK_TooManyInitsForScalar);
     return;
   }
   assert(NumArgs == 1 && "Zero-argument case handled above");
-  
-  //    - Otherwise, if the source type is a (possibly cv-qualified) class 
+
+  //    - Otherwise, if the source type is a (possibly cv-qualified) class
   //      type, conversion functions are considered.
   if (!SourceType.isNull() && SourceType->isRecordType()) {
     TryUserDefinedConversion(S, Entity, Kind, Initializer, *this);
     return;
   }
-  
+
   //    - Otherwise, the initial value of the object being initialized is the
   //      (possibly converted) value of the initializer expression. Standard
   //      conversions (Clause 4) will be used, if necessary, to convert the
-  //      initializer expression to the cv-unqualified version of the 
+  //      initializer expression to the cv-unqualified version of the
   //      destination type; no user-defined conversions are considered.
   if (S.TryImplicitConversion(*this, Entity, Initializer,
                               /*SuppressUserConversions*/ true,
                               /*AllowExplicitConversions*/ false,
-                              /*InOverloadResolution*/ false))
-    SetFailed(InitializationSequence::FK_ConversionFailed);
+                              /*InOverloadResolution*/ false,
+                              /*CStyle=*/Kind.isCStyleOrFunctionalCast()))
+  {
+    DeclAccessPair dap;
+    if (Initializer->getType() == Context.OverloadTy && 
+          !S.ResolveAddressOfOverloadedFunction(Initializer
+                      , DestType, false, dap))
+      SetFailed(InitializationSequence::FK_AddressOfOverloadFailed);
+    else
+      SetFailed(InitializationSequence::FK_ConversionFailed);
+  }
   else
     setSequenceKind(StandardConversion);
 }
@@ -3173,15 +3217,17 @@ InitializationSequence::~InitializationSequence() {
 //===----------------------------------------------------------------------===//
 // Perform initialization
 //===----------------------------------------------------------------------===//
-static Sema::AssignmentAction 
+static Sema::AssignmentAction
 getAssignmentAction(const InitializedEntity &Entity) {
   switch(Entity.getKind()) {
   case InitializedEntity::EK_Variable:
   case InitializedEntity::EK_New:
+  case InitializedEntity::EK_Exception:
+  case InitializedEntity::EK_Base:
     return Sema::AA_Initializing;
 
   case InitializedEntity::EK_Parameter:
-    if (Entity.getDecl() && 
+    if (Entity.getDecl() &&
         isa<ObjCMethodDecl>(Entity.getDecl()->getDeclContext()))
       return Sema::AA_Sending;
 
@@ -3190,15 +3236,10 @@ getAssignmentAction(const InitializedEntity &Entity) {
   case InitializedEntity::EK_Result:
     return Sema::AA_Returning;
 
-  case InitializedEntity::EK_Exception:
-  case InitializedEntity::EK_Base:
-    llvm_unreachable("No assignment action for C++-specific initialization");
-    break;
-
   case InitializedEntity::EK_Temporary:
     // FIXME: Can we tell apart casting vs. converting?
     return Sema::AA_Casting;
-    
+
   case InitializedEntity::EK_Member:
   case InitializedEntity::EK_ArrayElement:
   case InitializedEntity::EK_VectorElement:
@@ -3223,12 +3264,12 @@ static bool shouldBindAsTemporary(const InitializedEntity &Entity) {
   case InitializedEntity::EK_Exception:
   case InitializedEntity::EK_BlockElement:
     return false;
-    
+
   case InitializedEntity::EK_Parameter:
   case InitializedEntity::EK_Temporary:
     return true;
   }
-  
+
   llvm_unreachable("missed an InitializedEntity kind?");
 }
 
@@ -3243,7 +3284,7 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
     case InitializedEntity::EK_VectorElement:
     case InitializedEntity::EK_BlockElement:
       return false;
-      
+
     case InitializedEntity::EK_Variable:
     case InitializedEntity::EK_Parameter:
     case InitializedEntity::EK_Temporary:
@@ -3251,8 +3292,8 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
     case InitializedEntity::EK_Exception:
       return true;
   }
-  
-  llvm_unreachable("missed an InitializedEntity kind?");  
+
+  llvm_unreachable("missed an InitializedEntity kind?");
 }
 
 /// \brief Make a (potentially elidable) temporary copy of the object
@@ -3261,7 +3302,7 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
 ///
 /// \param S The Sema object used for type-checking.
 ///
-/// \param T The type of the temporary object, which must either by
+/// \param T The type of the temporary object, which must either be
 /// the type of the initializer expression or a superclass thereof.
 ///
 /// \param Enter The entity being initialized.
@@ -3276,19 +3317,19 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
 /// a temporary object, or an error expression if a copy could not be
 /// created.
 static ExprResult CopyObject(Sema &S,
-                                         QualType T,
-                                         const InitializedEntity &Entity,
-                                         ExprResult CurInit,
-                                         bool IsExtraneousCopy) {
+                             QualType T,
+                             const InitializedEntity &Entity,
+                             ExprResult CurInit,
+                             bool IsExtraneousCopy) {
   // Determine which class type we're copying to.
   Expr *CurInitExpr = (Expr *)CurInit.get();
-  CXXRecordDecl *Class = 0; 
+  CXXRecordDecl *Class = 0;
   if (const RecordType *Record = T->getAs<RecordType>())
     Class = cast<CXXRecordDecl>(Record->getDecl());
   if (!Class)
     return move(CurInit);
 
-  // C++0x [class.copy]p34:
+  // C++0x [class.copy]p32:
   //   When certain criteria are met, an implementation is allowed to
   //   omit the copy/move construction of a class object, even if the
   //   copy/move constructor and/or destructor for the object have
@@ -3298,23 +3339,22 @@ static ExprResult CopyObject(Sema &S,
   //       with the same cv-unqualified type, the copy/move operation
   //       can be omitted by constructing the temporary object
   //       directly into the target of the omitted copy/move
-  // 
+  //
   // Note that the other three bullets are handled elsewhere. Copy
   // elision for return statements and throw expressions are handled as part
-  // of constructor initialization, while copy elision for exception handlers 
+  // of constructor initialization, while copy elision for exception handlers
   // is handled by the run-time.
-  bool Elidable = CurInitExpr->isTemporaryObject() &&
-     S.Context.hasSameUnqualifiedType(T, CurInitExpr->getType());
+  bool Elidable = CurInitExpr->isTemporaryObject(S.Context, Class);
   SourceLocation Loc;
   switch (Entity.getKind()) {
   case InitializedEntity::EK_Result:
     Loc = Entity.getReturnLoc();
     break;
-      
+
   case InitializedEntity::EK_Exception:
     Loc = Entity.getThrowLoc();
     break;
-    
+
   case InitializedEntity::EK_Variable:
     Loc = Entity.getDecl()->getLocation();
     break;
@@ -3331,33 +3371,57 @@ static ExprResult CopyObject(Sema &S,
     break;
   }
 
-  // Make sure that the type we are copying is complete.   
+  // Make sure that the type we are copying is complete.
   if (S.RequireCompleteType(Loc, T, S.PDiag(diag::err_temp_copy_incomplete)))
     return move(CurInit);
 
-  // Perform overload resolution using the class's copy constructors.
+  // Perform overload resolution using the class's copy/move constructors.
   DeclContext::lookup_iterator Con, ConEnd;
   OverloadCandidateSet CandidateSet(Loc);
   for (llvm::tie(Con, ConEnd) = S.LookupConstructors(Class);
        Con != ConEnd; ++Con) {
-    // Only consider copy constructors.
-    CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(*Con);
-    if (!Constructor || Constructor->isInvalidDecl() ||
-        !Constructor->isCopyConstructor() ||
-        !Constructor->isConvertingConstructor(/*AllowExplicit=*/false))
+    // Only consider copy/move constructors and constructor templates. Per
+    // C++0x [dcl.init]p16, second bullet to class types, this
+    // initialization is direct-initialization.
+    CXXConstructorDecl *Constructor = 0;
+
+    if ((Constructor = dyn_cast<CXXConstructorDecl>(*Con))) {
+      // Handle copy/moveconstructors, only.
+      if (!Constructor || Constructor->isInvalidDecl() ||
+          !Constructor->isCopyOrMoveConstructor() ||
+          !Constructor->isConvertingConstructor(/*AllowExplicit=*/true))
+        continue;
+
+      DeclAccessPair FoundDecl
+        = DeclAccessPair::make(Constructor, Constructor->getAccess());
+      S.AddOverloadCandidate(Constructor, FoundDecl,
+                             &CurInitExpr, 1, CandidateSet);
+      continue;
+    }
+
+    // Handle constructor templates.
+    FunctionTemplateDecl *ConstructorTmpl = cast<FunctionTemplateDecl>(*Con);
+    if (ConstructorTmpl->isInvalidDecl())
+      continue;
+
+    Constructor = cast<CXXConstructorDecl>(
+                                         ConstructorTmpl->getTemplatedDecl());
+    if (!Constructor->isConvertingConstructor(/*AllowExplicit=*/true))
       continue;
 
+    // FIXME: Do we need to limit this to copy-constructor-like
+    // candidates?
     DeclAccessPair FoundDecl
-      = DeclAccessPair::make(Constructor, Constructor->getAccess());
-    S.AddOverloadCandidate(Constructor, FoundDecl,
-                           &CurInitExpr, 1, CandidateSet);
+      = DeclAccessPair::make(ConstructorTmpl, ConstructorTmpl->getAccess());
+    S.AddTemplateOverloadCandidate(ConstructorTmpl, FoundDecl, 0,
+                                   &CurInitExpr, 1, CandidateSet, true);
   }
-  
+
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(S, Loc, Best)) {
   case OR_Success:
     break;
-      
+
   case OR_No_Viable_Function:
     S.Diag(Loc, IsExtraneousCopy && !S.isSFINAEContext()
            ? diag::ext_rvalue_to_reference_temp_copy_no_viable
@@ -3368,14 +3432,14 @@ static ExprResult CopyObject(Sema &S,
     if (!IsExtraneousCopy || S.isSFINAEContext())
       return ExprError();
     return move(CurInit);
-      
+
   case OR_Ambiguous:
     S.Diag(Loc, diag::err_temp_copy_ambiguous)
       << (int)Entity.getKind() << CurInitExpr->getType()
       << CurInitExpr->getSourceRange();
     CandidateSet.NoteCandidates(S, OCD_ViableCandidates, &CurInitExpr, 1);
     return ExprError();
-    
+
   case OR_Deleted:
     S.Diag(Loc, diag::err_temp_copy_deleted)
       << (int)Entity.getKind() << CurInitExpr->getType()
@@ -3417,7 +3481,7 @@ static ExprResult CopyObject(Sema &S,
 
     return S.Owned(CurInitExpr);
   }
-  
+
   // Determine the arguments required to actually perform the
   // constructor call (we might have derived-to-base conversions, or
   // the copy constructor may have default arguments).
@@ -3429,8 +3493,9 @@ static ExprResult CopyObject(Sema &S,
   CurInit = S.BuildCXXConstructExpr(Loc, T, Constructor, Elidable,
                                     move_arg(ConstructorArgs),
                                     /*ZeroInit*/ false,
-                                    CXXConstructExpr::CK_Complete);
-  
+                                    CXXConstructExpr::CK_Complete,
+                                    SourceRange());
+
   // If we're supposed to bind temporaries, do so.
   if (!CurInit.isInvalid() && shouldBindAsTemporary(Entity))
     CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
@@ -3451,7 +3516,7 @@ void InitializationSequence::PrintInitLocationNote(Sema &S,
   }
 }
 
-ExprResult 
+ExprResult
 InitializationSequence::Perform(Sema &S,
                                 const InitializedEntity &Entity,
                                 const InitializationKind &Kind,
@@ -3462,7 +3527,7 @@ InitializationSequence::Perform(Sema &S,
     Diagnose(S, Entity, Kind, (Expr **)Args.release(), NumArgs);
     return ExprError();
   }
-  
+
   if (SequenceKind == DependentSequence) {
     // If the declaration is a non-dependent, incomplete array type
     // that has an initializer, then its type will be completed once
@@ -3512,14 +3577,14 @@ InitializationSequence::Perform(Sema &S,
     unsigned NumArgs = Args.size();
     return S.Owned(new (S.Context) ParenListExpr(S.Context,
                                                  SourceLocation(),
-                                                 (Expr **)Args.release(), 
+                                                 (Expr **)Args.release(),
                                                  NumArgs,
                                                  SourceLocation()));
   }
 
   if (SequenceKind == NoInitialization)
     return S.Owned((Expr *)0);
-  
+
   QualType DestType = Entity.getType().getNonReferenceType();
   // FIXME: Ugly hack around the fact that Entity.getType() is not
   // the same as Entity.getDecl()->getType() in cases involving type merging,
@@ -3529,10 +3594,10 @@ InitializationSequence::Perform(Sema &S,
                                      Entity.getType();
 
   ExprResult CurInit = S.Owned((Expr *)0);
-  
+
   assert(!Steps.empty() && "Cannot have an empty initialization sequence");
-  
-  // For initialization steps that start with a single initializer, 
+
+  // For initialization steps that start with a single initializer,
   // grab the only argument out the Args and place it into the "current"
   // initializer.
   switch (Steps.front().Kind) {
@@ -3551,32 +3616,38 @@ InitializationSequence::Perform(Sema &S,
   case SK_ListInitialization:
   case SK_CAssignment:
   case SK_StringInit:
-  case SK_ObjCObjectConversion:
+  case SK_ObjCObjectConversion: {
     assert(Args.size() == 1);
-    CurInit = ExprResult(((Expr **)(Args.get()))[0]->Retain());
-    if (CurInit.isInvalid())
-      return ExprError();
+    Expr *CurInitExpr = Args.get()[0];
+    if (!CurInitExpr) return ExprError();
+
+    // Read from a property when initializing something with it.
+    if (CurInitExpr->getObjectKind() == OK_ObjCProperty)
+      S.ConvertPropertyForRValue(CurInitExpr);
+
+    CurInit = ExprResult(CurInitExpr);
     break;
-    
+  }
+
   case SK_ConstructorInitialization:
   case SK_ZeroInitialization:
     break;
   }
-    
-  // Walk through the computed steps for the initialization sequence, 
+
+  // Walk through the computed steps for the initialization sequence,
   // performing the specified conversions along the way.
   bool ConstructorInitRequiresZeroInit = false;
   for (step_iterator Step = step_begin(), StepEnd = step_end();
        Step != StepEnd; ++Step) {
     if (CurInit.isInvalid())
       return ExprError();
-    
-    Expr *CurInitExpr = (Expr *)CurInit.get();
+
+    Expr *CurInitExpr = CurInit.get();
     QualType SourceType = CurInitExpr? CurInitExpr->getType() : QualType();
-    
+
     switch (Step->Kind) {
     case SK_ResolveAddressOfOverloadedFunction:
-      // Overload resolution determined which function invoke; update the 
+      // Overload resolution determined which function invoke; update the
       // initializer to reflect that choice.
       S.CheckAddressOfMemberAccess(CurInitExpr, Step->Function.FoundDecl);
       S.DiagnoseUseOfDecl(Step->Function.FoundDecl, Kind.getLocation());
@@ -3584,29 +3655,29 @@ InitializationSequence::Perform(Sema &S,
                                                  Step->Function.FoundDecl,
                                                  Step->Function.Function);
       break;
-        
+
     case SK_CastDerivedToBaseRValue:
     case SK_CastDerivedToBaseXValue:
     case SK_CastDerivedToBaseLValue: {
       // We have a derived-to-base cast that produces either an rvalue or an
       // lvalue. Perform that cast.
-      
+
       CXXCastPath BasePath;
 
       // Casts to inaccessible base classes are allowed with C-style casts.
       bool IgnoreBaseAccess = Kind.isCStyleOrFunctionalCast();
       if (S.CheckDerivedToBaseConversion(SourceType, Step->Type,
                                          CurInitExpr->getLocStart(),
-                                         CurInitExpr->getSourceRange(), 
+                                         CurInitExpr->getSourceRange(),
                                          &BasePath, IgnoreBaseAccess))
         return ExprError();
-        
+
       if (S.BasePathInvolvesVirtualBase(BasePath)) {
         QualType T = SourceType;
         if (const PointerType *Pointer = T->getAs<PointerType>())
           T = Pointer->getPointeeType();
         if (const RecordType *RecordTy = T->getAs<RecordType>())
-          S.MarkVTableUsed(CurInitExpr->getLocStart(), 
+          S.MarkVTableUsed(CurInitExpr->getLocStart(),
                            cast<CXXRecordDecl>(RecordTy->getDecl()));
       }
 
@@ -3623,7 +3694,7 @@ InitializationSequence::Perform(Sema &S,
                                                  &BasePath, VK));
       break;
     }
-        
+
     case SK_BindReference:
       if (FieldDecl *BitField = CurInitExpr->getBitField()) {
         // References cannot bind to bit fields (C++ [dcl.init.ref]p5).
@@ -3643,7 +3714,7 @@ InitializationSequence::Perform(Sema &S,
         PrintInitLocationNote(S, Entity);
         return ExprError();
       }
-        
+
       // Reference binding does not have any corresponding ASTs.
 
       // Check exception specifications
@@ -3660,16 +3731,16 @@ InitializationSequence::Perform(Sema &S,
         return ExprError();
 
       break;
-        
+
     case SK_ExtraneousCopyToTemporary:
-      CurInit = CopyObject(S, Step->Type, Entity, move(CurInit), 
+      CurInit = CopyObject(S, Step->Type, Entity, move(CurInit),
                            /*IsExtraneousCopy=*/true);
       break;
 
     case SK_UserConversion: {
       // We have a user-defined conversion that invokes either a constructor
       // or a conversion function.
-      CastKind CastKind = CK_Unknown;
+      CastKind CastKind;
       bool IsCopy = false;
       FunctionDecl *Fn = Step->Function.Function;
       DeclAccessPair FoundFn = Step->Function.FoundDecl;
@@ -3687,25 +3758,26 @@ InitializationSequence::Perform(Sema &S,
                                       MultiExprArg(&CurInitExpr, 1),
                                       Loc, ConstructorArgs))
           return ExprError();
-        
+
         // Build the an expression that constructs a temporary.
-        CurInit = S.BuildCXXConstructExpr(Loc, Step->Type, Constructor, 
+        CurInit = S.BuildCXXConstructExpr(Loc, Step->Type, Constructor,
                                           move_arg(ConstructorArgs),
                                           /*ZeroInit*/ false,
-                                          CXXConstructExpr::CK_Complete);
+                                          CXXConstructExpr::CK_Complete,
+                                          SourceRange());
         if (CurInit.isInvalid())
           return ExprError();
 
         S.CheckConstructorAccess(Kind.getLocation(), Constructor, Entity,
                                  FoundFn.getAccess());
         S.DiagnoseUseOfDecl(FoundFn, Kind.getLocation());
-        
+
         CastKind = CK_ConstructorConversion;
         QualType Class = S.Context.getTypeDeclType(Constructor->getParent());
         if (S.Context.hasSameUnqualifiedType(SourceType, Class) ||
             S.IsDerivedFrom(SourceType, Class))
           IsCopy = true;
-        
+
         CreatedObject = true;
       } else {
         // Build a call to the conversion function.
@@ -3714,8 +3786,8 @@ InitializationSequence::Perform(Sema &S,
         S.CheckMemberOperatorAccess(Kind.getLocation(), CurInitExpr, 0,
                                     FoundFn);
         S.DiagnoseUseOfDecl(FoundFn, Kind.getLocation());
-        
-        // FIXME: Should we move this initialization into a separate 
+
+        // FIXME: Should we move this initialization into a separate
         // derived-to-base conversion? I believe the answer is "no", because
         // we don't want to turn off access control here for c-style casts.
         if (S.PerformObjectArgumentInitialization(CurInitExpr, /*Qualifier=*/0,
@@ -3725,19 +3797,18 @@ InitializationSequence::Perform(Sema &S,
         // Do a little dance to make sure that CurInit has the proper
         // pointer.
         CurInit.release();
-        
+
         // Build the actual call to the conversion function.
-        CurInit = S.Owned(S.BuildCXXMemberCallExpr(CurInitExpr, FoundFn,
-                                                   Conversion));
+        CurInit = S.BuildCXXMemberCallExpr(CurInitExpr, FoundFn, Conversion);
         if (CurInit.isInvalid() || !CurInit.get())
           return ExprError();
-        
+
         CastKind = CK_UserDefinedConversion;
-        
+
         CreatedObject = Conversion->getResultType()->isRecordType();
       }
-      
-      bool RequiresCopy = !IsCopy && 
+
+      bool RequiresCopy = !IsCopy &&
         getKind() != InitializationSequence::ReferenceBinding;
       if (RequiresCopy || shouldBindAsTemporary(Entity))
         CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
@@ -3745,21 +3816,22 @@ InitializationSequence::Perform(Sema &S,
         CurInitExpr = static_cast<Expr *>(CurInit.get());
         QualType T = CurInitExpr->getType();
         if (const RecordType *Record = T->getAs<RecordType>()) {
-          CXXDestructorDecl *Destructor 
+          CXXDestructorDecl *Destructor
             = S.LookupDestructor(cast<CXXRecordDecl>(Record->getDecl()));
-          S.CheckDestructorAccess(CurInitExpr->getLocStart(), Destructor, 
+          S.CheckDestructorAccess(CurInitExpr->getLocStart(), Destructor,
                                   S.PDiag(diag::err_access_dtor_temp) << T);
           S.MarkDeclarationReferenced(CurInitExpr->getLocStart(), Destructor);
+          S.DiagnoseUseOfDecl(Destructor, CurInitExpr->getLocStart());
         }
       }
-      
+
       CurInitExpr = CurInit.takeAs<Expr>();
       // FIXME: xvalues
       CurInit = S.Owned(ImplicitCastExpr::Create(S.Context,
                                                  CurInitExpr->getType(),
                                                  CastKind, CurInitExpr, 0,
                                            IsLvalue ? VK_LValue : VK_RValue));
-      
+
       if (RequiresCopy)
         CurInit = CopyObject(S, Entity.getType().getNonReferenceType(), Entity,
                              move(CurInit), /*IsExtraneousCopy=*/false);
@@ -3784,17 +3856,16 @@ InitializationSequence::Perform(Sema &S,
     }
 
     case SK_ConversionSequence: {
-      bool IgnoreBaseAccess = Kind.isCStyleOrFunctionalCast();
-
       if (S.PerformImplicitConversion(CurInitExpr, Step->Type, *Step->ICS,
-                                      Sema::AA_Converting, IgnoreBaseAccess))
+                                      getAssignmentAction(Entity),
+                                      Kind.isCStyleOrFunctionalCast()))
         return ExprError();
-        
+
       CurInit.release();
       CurInit = S.Owned(CurInitExpr);
       break;
     }
-        
+
     case SK_ListInitialization: {
       InitListExpr *InitList = cast<InitListExpr>(CurInitExpr);
       QualType Ty = Step->Type;
@@ -3810,7 +3881,7 @@ InitializationSequence::Perform(Sema &S,
       unsigned NumArgs = Args.size();
       CXXConstructorDecl *Constructor
         = cast<CXXConstructorDecl>(Step->Function.Function);
-      
+
       // Build a call to the selected constructor.
       ASTOwningVector<Expr*> ConstructorArgs(S);
       SourceLocation Loc = (Kind.isCopyInit() && Kind.getEqualLoc().isValid())
@@ -3830,11 +3901,11 @@ InitializationSequence::Perform(Sema &S,
 
       // Determine the arguments required to actually perform the constructor
       // call.
-      if (S.CompleteConstructorCall(Constructor, move(Args), 
+      if (S.CompleteConstructorCall(Constructor, move(Args),
                                     Loc, ConstructorArgs))
         return ExprError();
-          
-      
+
+
       if (Entity.getKind() == InitializedEntity::EK_Temporary &&
           NumArgs != 1 && // FIXME: Hack to work around cast weirdness
           (Kind.getKind() == InitializationKind::IK_Direct ||
@@ -3843,24 +3914,34 @@ InitializationSequence::Perform(Sema &S,
         unsigned NumExprs = ConstructorArgs.size();
         Expr **Exprs = (Expr **)ConstructorArgs.take();
         S.MarkDeclarationReferenced(Loc, Constructor);
+        S.DiagnoseUseOfDecl(Constructor, Loc);
+
+        TypeSourceInfo *TSInfo = Entity.getTypeSourceInfo();
+        if (!TSInfo)
+          TSInfo = S.Context.getTrivialTypeSourceInfo(Entity.getType(), Loc);
+
         CurInit = S.Owned(new (S.Context) CXXTemporaryObjectExpr(S.Context,
                                                                  Constructor,
-                                                              Entity.getType(),
-                                                                 Loc,
-                                                                 Exprs, 
+                                                                 TSInfo,
+                                                                 Exprs,
                                                                  NumExprs,
-                                                Kind.getParenRange().getEnd(),
+                                                         Kind.getParenRange(),
                                              ConstructorInitRequiresZeroInit));
       } else {
         CXXConstructExpr::ConstructionKind ConstructKind =
           CXXConstructExpr::CK_Complete;
-        
+
         if (Entity.getKind() == InitializedEntity::EK_Base) {
           ConstructKind = Entity.getBaseSpecifier()->isVirtual() ?
-            CXXConstructExpr::CK_VirtualBase : 
+            CXXConstructExpr::CK_VirtualBase :
             CXXConstructExpr::CK_NonVirtualBase;
-        }    
-        
+        }
+
+        // Only get the parenthesis range if it is a direct construction.
+        SourceRange parenRange =
+            Kind.getKind() == InitializationKind::IK_Direct ?
+            Kind.getParenRange() : SourceRange();
+
         // If the entity allows NRVO, mark the construction as elidable
         // unconditionally.
         if (Entity.allowsNRVO())
@@ -3868,13 +3949,15 @@ InitializationSequence::Perform(Sema &S,
                                             Constructor, /*Elidable=*/true,
                                             move_arg(ConstructorArgs),
                                             ConstructorInitRequiresZeroInit,
-                                            ConstructKind);
+                                            ConstructKind,
+                                            parenRange);
         else
           CurInit = S.BuildCXXConstructExpr(Loc, Entity.getType(),
-                                            Constructor, 
+                                            Constructor,
                                             move_arg(ConstructorArgs),
                                             ConstructorInitRequiresZeroInit,
-                                            ConstructKind);
+                                            ConstructKind,
+                                            parenRange);
       }
       if (CurInit.isInvalid())
         return ExprError();
@@ -3883,17 +3966,17 @@ InitializationSequence::Perform(Sema &S,
       S.CheckConstructorAccess(Loc, Constructor, Entity,
                                Step->Function.FoundDecl.getAccess());
       S.DiagnoseUseOfDecl(Step->Function.FoundDecl, Loc);
-      
+
       if (shouldBindAsTemporary(Entity))
         CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
-      
+
       break;
     }
-        
+
     case SK_ZeroInitialization: {
       step_iterator NextStep = Step;
       ++NextStep;
-      if (NextStep != StepEnd && 
+      if (NextStep != StepEnd &&
           NextStep->Kind == SK_ConstructorInitialization) {
         // The need for zero-initialization is recorded directly into
         // the call to the object's constructor within the next step.
@@ -3901,8 +3984,14 @@ InitializationSequence::Perform(Sema &S,
       } else if (Kind.getKind() == InitializationKind::IK_Value &&
                  S.getLangOptions().CPlusPlus &&
                  !Kind.isImplicitValueInit()) {
-        CurInit = S.Owned(new (S.Context) CXXScalarValueInitExpr(Step->Type,
-                                                   Kind.getRange().getBegin(),
+        TypeSourceInfo *TSInfo = Entity.getTypeSourceInfo();
+        if (!TSInfo)
+          TSInfo = S.Context.getTrivialTypeSourceInfo(Step->Type,
+                                                    Kind.getRange().getBegin());
+
+        CurInit = S.Owned(new (S.Context) CXXScalarValueInitExpr(
+                              TSInfo->getType().getNonLValueExprType(S.Context),
+                                                                 TSInfo,
                                                     Kind.getRange().getEnd()));
       } else {
         CurInit = S.Owned(new (S.Context) ImplicitValueInitExpr(Step->Type));
@@ -3925,7 +4014,7 @@ InitializationSequence::Perform(Sema &S,
       bool Complained;
       if (S.DiagnoseAssignmentResult(ConvTy, Kind.getLocation(),
                                      Step->Type, SourceType,
-                                     CurInitExpr, 
+                                     CurInitExpr,
                                      getAssignmentAction(Entity),
                                      &Complained)) {
         PrintInitLocationNote(S, Entity);
@@ -3945,7 +4034,7 @@ InitializationSequence::Perform(Sema &S,
     }
 
     case SK_ObjCObjectConversion:
-      S.ImpCastExprToType(CurInitExpr, Step->Type, 
+      S.ImpCastExprToType(CurInitExpr, Step->Type,
                           CK_ObjCObjectLValueCast,
                           S.CastCategory(CurInitExpr));
       CurInit.release();
@@ -3953,20 +4042,27 @@ InitializationSequence::Perform(Sema &S,
       break;
     }
   }
-  
+
+  // Diagnose non-fatal problems with the completed initialization.
+  if (Entity.getKind() == InitializedEntity::EK_Member &&
+      cast<FieldDecl>(Entity.getDecl())->isBitField())
+    S.CheckBitFieldInitialization(Kind.getLocation(),
+                                  cast<FieldDecl>(Entity.getDecl()),
+                                  CurInit.get());
+
   return move(CurInit);
 }
 
 //===----------------------------------------------------------------------===//
 // Diagnose initialization failures
 //===----------------------------------------------------------------------===//
-bool InitializationSequence::Diagnose(Sema &S, 
+bool InitializationSequence::Diagnose(Sema &S,
                                       const InitializedEntity &Entity,
                                       const InitializationKind &Kind,
                                       Expr **Args, unsigned NumArgs) {
   if (SequenceKind != FailedSequence)
     return false;
-  
+
   QualType DestType = Entity.getType();
   switch (Failure) {
   case FK_TooManyInitsForReference:
@@ -3978,22 +4074,22 @@ bool InitializationSequence::Diagnose(Sema &S,
       S.Diag(Kind.getLocation(), diag::err_reference_has_multiple_inits)
         << SourceRange(Args[0]->getLocStart(), Args[NumArgs - 1]->getLocEnd());
     break;
-    
+
   case FK_ArrayNeedsInitList:
   case FK_ArrayNeedsInitListOrStringLiteral:
     S.Diag(Kind.getLocation(), diag::err_array_init_not_init_list)
       << (Failure == FK_ArrayNeedsInitListOrStringLiteral);
     break;
-      
+
   case FK_AddressOfOverloadFailed: {
     DeclAccessPair Found;
-    S.ResolveAddressOfOverloadedFunction(Args[0], 
+    S.ResolveAddressOfOverloadedFunction(Args[0],
                                          DestType.getNonReferenceType(),
                                          true,
                                          Found);
     break;
   }
-      
+
   case FK_ReferenceInitOverloadFailed:
   case FK_UserConversionOverloadFailed:
     switch (FailedOverloadResult) {
@@ -4009,21 +4105,22 @@ bool InitializationSequence::Diagnose(Sema &S,
 
       FailedCandidateSet.NoteCandidates(S, OCD_ViableCandidates, Args, NumArgs);
       break;
-        
+
     case OR_No_Viable_Function:
       S.Diag(Kind.getLocation(), diag::err_typecheck_nonviable_condition)
         << Args[0]->getType() << DestType.getNonReferenceType()
         << Args[0]->getSourceRange();
       FailedCandidateSet.NoteCandidates(S, OCD_AllCandidates, Args, NumArgs);
       break;
-        
+
     case OR_Deleted: {
       S.Diag(Kind.getLocation(), diag::err_typecheck_deleted_function)
         << Args[0]->getType() << DestType.getNonReferenceType()
         << Args[0]->getSourceRange();
       OverloadCandidateSet::iterator Best;
       OverloadingResult Ovl
-        = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
+        = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best,
+                                                true);
       if (Ovl == OR_Deleted) {
         S.Diag(Best->Function->getLocation(), diag::note_unavailable_here)
           << Best->Function->isDeleted();
@@ -4032,16 +4129,16 @@ bool InitializationSequence::Diagnose(Sema &S,
       }
       break;
     }
-        
+
     case OR_Success:
       llvm_unreachable("Conversion did not fail!");
       break;
     }
     break;
-      
+
   case FK_NonConstLValueReferenceBindingToTemporary:
   case FK_NonConstLValueReferenceBindingToUnrelated:
-    S.Diag(Kind.getLocation(), 
+    S.Diag(Kind.getLocation(),
            Failure == FK_NonConstLValueReferenceBindingToTemporary
              ? diag::err_lvalue_reference_bind_to_temporary
              : diag::err_lvalue_reference_bind_to_unrelated)
@@ -4050,47 +4147,54 @@ bool InitializationSequence::Diagnose(Sema &S,
       << Args[0]->getType()
       << Args[0]->getSourceRange();
     break;
-      
+
   case FK_RValueReferenceBindingToLValue:
     S.Diag(Kind.getLocation(), diag::err_lvalue_to_rvalue_ref)
+      << DestType.getNonReferenceType() << Args[0]->getType()
       << Args[0]->getSourceRange();
     break;
-      
+
   case FK_ReferenceInitDropsQualifiers:
     S.Diag(Kind.getLocation(), diag::err_reference_bind_drops_quals)
       << DestType.getNonReferenceType()
       << Args[0]->getType()
       << Args[0]->getSourceRange();
     break;
-      
+
   case FK_ReferenceInitFailed:
     S.Diag(Kind.getLocation(), diag::err_reference_bind_failed)
       << DestType.getNonReferenceType()
-      << (Args[0]->isLvalue(S.Context) == Expr::LV_Valid)
+      << Args[0]->isLValue()
       << Args[0]->getType()
       << Args[0]->getSourceRange();
     break;
-      
-  case FK_ConversionFailed:
+
+  case FK_ConversionFailed: {
+    QualType FromType = Args[0]->getType();
     S.Diag(Kind.getLocation(), diag::err_init_conversion_failed)
       << (int)Entity.getKind()
       << DestType
-      << (Args[0]->isLvalue(S.Context) == Expr::LV_Valid)
-      << Args[0]->getType()
+      << Args[0]->isLValue()
+      << FromType
       << Args[0]->getSourceRange();
     break;
-
+  }
   case FK_TooManyInitsForScalar: {
     SourceRange R;
 
     if (InitListExpr *InitList = dyn_cast<InitListExpr>(Args[0]))
-      R = SourceRange(InitList->getInit(1)->getLocStart(),
+      R = SourceRange(InitList->getInit(0)->getLocEnd(),
                       InitList->getLocEnd());
     else
-      R = SourceRange(Args[0]->getLocStart(), Args[NumArgs - 1]->getLocEnd());
+      R = SourceRange(Args[0]->getLocEnd(), Args[NumArgs - 1]->getLocEnd());
 
-    S.Diag(Kind.getLocation(), diag::err_excess_initializers)
-      << /*scalar=*/2 << R;
+    R.setBegin(S.PP.getLocForEndOfToken(R.getBegin()));
+    if (Kind.isCStyleOrFunctionalCast())
+      S.Diag(Kind.getLocation(), diag::err_builtin_func_cast_more_than_one_arg)
+        << R;
+    else
+      S.Diag(Kind.getLocation(), diag::err_excess_initializers)
+        << /*scalar=*/2 << R;
     break;
   }
 
@@ -4103,13 +4207,13 @@ bool InitializationSequence::Diagnose(Sema &S,
     S.Diag(Kind.getLocation(), diag::err_init_list_bad_dest_type)
       << (DestType->isRecordType()) << DestType << Args[0]->getSourceRange();
     break;
-      
+
   case FK_ConstructorOverloadFailed: {
     SourceRange ArgsRange;
     if (NumArgs)
-      ArgsRange = SourceRange(Args[0]->getLocStart(), 
+      ArgsRange = SourceRange(Args[0]->getLocStart(),
                               Args[NumArgs - 1]->getLocEnd());
-    
+
     // FIXME: Using "DestType" for the entity we're printing is probably
     // bad.
     switch (FailedOverloadResult) {
@@ -4119,7 +4223,7 @@ bool InitializationSequence::Diagnose(Sema &S,
         FailedCandidateSet.NoteCandidates(S, OCD_ViableCandidates,
                                           Args, NumArgs);
         break;
-        
+
       case OR_No_Viable_Function:
         if (Kind.getKind() == InitializationKind::IK_Default &&
             (Entity.getKind() == InitializedEntity::EK_Base ||
@@ -4153,7 +4257,7 @@ bool InitializationSequence::Diagnose(Sema &S,
 
             if (const RecordType *Record
                                  = Entity.getType()->getAs<RecordType>())
-              S.Diag(Record->getDecl()->getLocation(), 
+              S.Diag(Record->getDecl()->getLocation(),
                      diag::note_previous_decl)
                 << S.Context.getTagDeclType(Record->getDecl());
           }
@@ -4164,7 +4268,7 @@ bool InitializationSequence::Diagnose(Sema &S,
           << DestType << ArgsRange;
         FailedCandidateSet.NoteCandidates(S, OCD_AllCandidates, Args, NumArgs);
         break;
-        
+
       case OR_Deleted: {
         S.Diag(Kind.getLocation(), diag::err_ovl_deleted_init)
           << true << DestType << ArgsRange;
@@ -4179,14 +4283,14 @@ bool InitializationSequence::Diagnose(Sema &S,
         }
         break;
       }
-        
+
       case OR_Success:
         llvm_unreachable("Conversion did not fail!");
         break;
     }
     break;
   }
-      
+
   case FK_DefaultInitOfConst:
     if (Entity.getKind() == InitializedEntity::EK_Member &&
         isa<CXXConstructorDecl>(S.CurContext)) {
@@ -4206,13 +4310,13 @@ bool InitializationSequence::Diagnose(Sema &S,
         << DestType << (bool)DestType->getAs<RecordType>();
     }
     break;
-      
+
     case FK_Incomplete:
-      S.RequireCompleteType(Kind.getLocation(), DestType, 
+      S.RequireCompleteType(Kind.getLocation(), DestType,
                             diag::err_init_incomplete_type);
-      break;      
+      break;
   }
-  
+
   PrintInitLocationNote(S, Entity);
   return true;
 }
@@ -4225,95 +4329,95 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
     case FK_TooManyInitsForReference:
       OS << "too many initializers for reference";
       break;
-      
+
     case FK_ArrayNeedsInitList:
       OS << "array requires initializer list";
       break;
-      
+
     case FK_ArrayNeedsInitListOrStringLiteral:
       OS << "array requires initializer list or string literal";
       break;
-      
+
     case FK_AddressOfOverloadFailed:
       OS << "address of overloaded function failed";
       break;
-      
+
     case FK_ReferenceInitOverloadFailed:
       OS << "overload resolution for reference initialization failed";
       break;
-      
+
     case FK_NonConstLValueReferenceBindingToTemporary:
       OS << "non-const lvalue reference bound to temporary";
       break;
-      
+
     case FK_NonConstLValueReferenceBindingToUnrelated:
       OS << "non-const lvalue reference bound to unrelated type";
       break;
-      
+
     case FK_RValueReferenceBindingToLValue:
       OS << "rvalue reference bound to an lvalue";
       break;
-      
+
     case FK_ReferenceInitDropsQualifiers:
       OS << "reference initialization drops qualifiers";
       break;
-      
+
     case FK_ReferenceInitFailed:
       OS << "reference initialization failed";
       break;
-      
+
     case FK_ConversionFailed:
       OS << "conversion failed";
       break;
-      
+
     case FK_TooManyInitsForScalar:
       OS << "too many initializers for scalar";
       break;
-      
+
     case FK_ReferenceBindingToInitList:
       OS << "referencing binding to initializer list";
       break;
-      
+
     case FK_InitListBadDestinationType:
       OS << "initializer list for non-aggregate, non-scalar type";
       break;
-      
+
     case FK_UserConversionOverloadFailed:
       OS << "overloading failed for user-defined conversion";
       break;
-      
+
     case FK_ConstructorOverloadFailed:
       OS << "constructor overloading failed";
       break;
-      
+
     case FK_DefaultInitOfConst:
       OS << "default initialization of a const variable";
       break;
-        
+
     case FK_Incomplete:
       OS << "initialization of incomplete type";
       break;
-    }   
+    }
     OS << '\n';
     return;
   }
-      
+
   case DependentSequence:
     OS << "Dependent sequence: ";
     return;
-      
+
   case UserDefinedConversion:
     OS << "User-defined conversion sequence: ";
     break;
-      
+
   case ConstructorInitialization:
     OS << "Constructor initialization sequence: ";
     break;
-      
+
   case ReferenceBinding:
     OS << "Reference binding: ";
     break;
-      
+
   case ListInitialization:
     OS << "List initialization: ";
     break;
@@ -4321,54 +4425,54 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
   case ZeroInitialization:
     OS << "Zero initialization\n";
     return;
-      
+
   case NoInitialization:
     OS << "No initialization\n";
     return;
-      
+
   case StandardConversion:
     OS << "Standard conversion: ";
     break;
-      
+
   case CAssignment:
     OS << "C assignment: ";
     break;
-      
+
   case StringInit:
     OS << "String initialization: ";
     break;
   }
-  
+
   for (step_iterator S = step_begin(), SEnd = step_end(); S != SEnd; ++S) {
     if (S != step_begin()) {
       OS << " -> ";
     }
-    
+
     switch (S->Kind) {
     case SK_ResolveAddressOfOverloadedFunction:
       OS << "resolve address of overloaded function";
       break;
-      
+
     case SK_CastDerivedToBaseRValue:
       OS << "derived-to-base case (rvalue" << S->Type.getAsString() << ")";
       break;
-      
+
     case SK_CastDerivedToBaseXValue:
       OS << "derived-to-base case (xvalue" << S->Type.getAsString() << ")";
       break;
-      
+
     case SK_CastDerivedToBaseLValue:
       OS << "derived-to-base case (lvalue" << S->Type.getAsString() << ")";
       break;
-      
+
     case SK_BindReference:
       OS << "bind reference to lvalue";
       break;
-      
+
     case SK_BindReferenceToTemporary:
       OS << "bind reference to a temporary";
       break;
-      
+
     case SK_ExtraneousCopyToTemporary:
       OS << "extraneous C++03 copy to temporary";
       break;
@@ -4386,29 +4490,29 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
     case SK_QualificationConversionLValue:
       OS << "qualification conversion (lvalue)";
       break;
-      
+
     case SK_ConversionSequence:
       OS << "implicit conversion sequence (";
       S->ICS->DebugPrint(); // FIXME: use OS
       OS << ")";
       break;
-      
+
     case SK_ListInitialization:
       OS << "list initialization";
       break;
-      
+
     case SK_ConstructorInitialization:
       OS << "constructor initialization";
       break;
-      
+
     case SK_ZeroInitialization:
       OS << "zero initialization";
       break;
-      
+
     case SK_CAssignment:
       OS << "C assignment";
       break;
-      
+
     case SK_StringInit:
       OS << "string initialization";
       break;
@@ -4427,14 +4531,14 @@ void InitializationSequence::dump() const {
 //===----------------------------------------------------------------------===//
 // Initialization helper functions
 //===----------------------------------------------------------------------===//
-ExprResult 
+ExprResult
 Sema::PerformCopyInitialization(const InitializedEntity &Entity,
                                 SourceLocation EqualLoc,
                                 ExprResult Init) {
   if (Init.isInvalid())
     return ExprError();
 
-  Expr *InitE = (Expr *)Init.get();
+  Expr *InitE = Init.get();
   assert(InitE && "No initialization expression?");
 
   if (EqualLoc.isInvalid())
diff --git a/lib/Sema/SemaLookup.cpp b/lib/Sema/SemaLookup.cpp
index 306e95a497e8..0fd0e08ac830 100644
--- a/lib/Sema/SemaLookup.cpp
+++ b/lib/Sema/SemaLookup.cpp
@@ -31,7 +31,9 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <limits>
 #include <list>
 #include <set>
 #include <vector>
@@ -89,7 +91,7 @@ namespace {
     UnqualUsingDirectiveSet() {}
 
     void visitScopeChain(Scope *S, Scope *InnermostFileScope) {
-      // C++ [namespace.udir]p1: 
+      // C++ [namespace.udir]p1:
       //   During unqualified name lookup, the names appear as if they
       //   were declared in the nearest enclosing namespace which contains
       //   both the using-directive and the nominated namespace.
@@ -104,7 +106,7 @@ namespace {
         } else {
           Scope::udir_iterator I = S->using_directives_begin(),
                              End = S->using_directives_end();
-          
+
           for (; I != End; ++I)
             visit(*I, InnermostFileDC);
         }
@@ -175,7 +177,7 @@ namespace {
       while (!Common->Encloses(EffectiveDC))
         Common = Common->getParent();
       Common = Common->getPrimaryContext();
-      
+
       list.push_back(UnqualUsingEntry(UD->getNominatedNamespace(), Common));
     }
 
@@ -183,11 +185,8 @@ namespace {
       std::sort(list.begin(), list.end(), UnqualUsingEntry::Comparator());
     }
 
-    typedef ListTy::iterator iterator;
     typedef ListTy::const_iterator const_iterator;
-    
-    iterator begin() { return list.begin(); }
-    iterator end() { return list.end(); }
+
     const_iterator begin() const { return list.begin(); }
     const_iterator end() const { return list.end(); }
 
@@ -211,7 +210,8 @@ static inline unsigned getIDNS(Sema::LookupNameKind NameKind,
     IDNS = Decl::IDNS_Ordinary;
     if (CPlusPlus) {
       IDNS |= Decl::IDNS_Tag | Decl::IDNS_Member | Decl::IDNS_Namespace;
-      if (Redeclaration) IDNS |= Decl::IDNS_TagFriend | Decl::IDNS_OrdinaryFriend;
+      if (Redeclaration)
+        IDNS |= Decl::IDNS_TagFriend | Decl::IDNS_OrdinaryFriend;
     }
     break;
 
@@ -237,7 +237,10 @@ static inline unsigned getIDNS(Sema::LookupNameKind NameKind,
       IDNS = Decl::IDNS_Tag;
     }
     break;
-
+  case Sema::LookupLabel:
+    IDNS = Decl::IDNS_Label;
+    break;
+      
   case Sema::LookupMemberName:
     IDNS = Decl::IDNS_Member;
     if (CPlusPlus)
@@ -260,9 +263,9 @@ static inline unsigned getIDNS(Sema::LookupNameKind NameKind,
   case Sema::LookupObjCProtocolName:
     IDNS = Decl::IDNS_ObjCProtocol;
     break;
-      
+
   case Sema::LookupAnyName:
-    IDNS = Decl::IDNS_Ordinary | Decl::IDNS_Tag | Decl::IDNS_Member 
+    IDNS = Decl::IDNS_Ordinary | Decl::IDNS_Tag | Decl::IDNS_Member
       | Decl::IDNS_Using | Decl::IDNS_Namespace | Decl::IDNS_ObjCProtocol
       | Decl::IDNS_Type;
     break;
@@ -271,8 +274,7 @@ static inline unsigned getIDNS(Sema::LookupNameKind NameKind,
 }
 
 void LookupResult::configure() {
-  IDNS = getIDNS(LookupKind,
-                 SemaRef.getLangOptions().CPlusPlus,
+  IDNS = getIDNS(LookupKind, SemaRef.getLangOptions().CPlusPlus,
                  isForRedeclaration());
 
   // If we're looking for one of the allocation or deallocation
@@ -293,7 +295,6 @@ void LookupResult::configure() {
   }
 }
 
-#ifndef NDEBUG
 void LookupResult::sanity() const {
   assert(ResultKind != NotFound || Decls.size() == 0);
   assert(ResultKind != Found || Decls.size() == 1);
@@ -302,12 +303,12 @@ void LookupResult::sanity() const {
           isa<FunctionTemplateDecl>((*begin())->getUnderlyingDecl())));
   assert(ResultKind != FoundUnresolvedValue || sanityCheckUnresolved());
   assert(ResultKind != Ambiguous || Decls.size() > 1 ||
-         (Decls.size() == 1 && Ambiguity == AmbiguousBaseSubobjects));
+         (Decls.size() == 1 && (Ambiguity == AmbiguousBaseSubobjects ||
+                                Ambiguity == AmbiguousBaseSubobjectTypes)));
   assert((Paths != NULL) == (ResultKind == Ambiguous &&
                              (Ambiguity == AmbiguousBaseSubobjectTypes ||
                               Ambiguity == AmbiguousBaseSubobjects)));
 }
-#endif
 
 // Necessary because CXXBasePaths is not complete in Sema.h
 void LookupResult::deletePaths(CXXBasePaths *Paths) {
@@ -317,7 +318,7 @@ void LookupResult::deletePaths(CXXBasePaths *Paths) {
 /// Resolves the result kind of this lookup.
 void LookupResult::resolveKind() {
   unsigned N = Decls.size();
- 
+
   // Fast case: no possible ambiguity.
   if (N == 0) {
     assert(ResultKind == NotFound || ResultKind == NotFoundInCurrentInstantiation);
@@ -340,13 +341,13 @@ void LookupResult::resolveKind() {
 
   llvm::SmallPtrSet<NamedDecl*, 16> Unique;
   llvm::SmallPtrSet<QualType, 16> UniqueTypes;
-  
+
   bool Ambiguous = false;
   bool HasTag = false, HasFunction = false, HasNonFunction = false;
   bool HasFunctionTemplate = false, HasUnresolved = false;
 
   unsigned UniqueTagIndex = 0;
-  
+
   unsigned I = 0;
   while (I < N) {
     NamedDecl *D = Decls[I]->getUnderlyingDecl();
@@ -367,14 +368,14 @@ void LookupResult::resolveKind() {
         }
       }
     }
-    
+
     if (!Unique.insert(D)) {
       // If it's not unique, pull something off the back (and
       // continue at this index).
       Decls[I] = Decls[--N];
       continue;
-    } 
-    
+    }
+
     // Otherwise, do some decl type analysis and then continue.
 
     if (isa<UnresolvedUsingValueDecl>(D)) {
@@ -407,8 +408,13 @@ void LookupResult::resolveKind() {
   // But it's still an error if there are distinct tag types found,
   // even if they're not visible. (ref?)
   if (HideTags && HasTag && !Ambiguous &&
-      (HasFunction || HasNonFunction || HasUnresolved))
-    Decls[UniqueTagIndex] = Decls[--N];
+      (HasFunction || HasNonFunction || HasUnresolved)) {
+    if (Decls[UniqueTagIndex]->getDeclContext()->getRedeclContext()->Equals(
+         Decls[UniqueTagIndex? 0 : N-1]->getDeclContext()->getRedeclContext()))
+      Decls[UniqueTagIndex] = Decls[--N];
+    else
+      Ambiguous = true;
+  }
 
   Decls.set_size(N);
 
@@ -453,7 +459,7 @@ void LookupResult::print(llvm::raw_ostream &Out) {
   Out << Decls.size() << " result(s)";
   if (isAmbiguous()) Out << ", ambiguous";
   if (Paths) Out << ", base paths present";
-  
+
   for (iterator I = begin(), E = end(); I != E; ++I) {
     Out << "\n";
     (*I)->print(Out, 2);
@@ -480,12 +486,21 @@ static bool LookupBuiltin(Sema &S, LookupResult &R) {
             S.Context.BuiltinInfo.isPredefinedLibFunction(BuiltinID))
           return false;
 
-        NamedDecl *D = S.LazilyCreateBuiltin((IdentifierInfo *)II, BuiltinID,
-                                             S.TUScope, R.isForRedeclaration(),
-                                             R.getNameLoc());
-        if (D) 
+        if (NamedDecl *D = S.LazilyCreateBuiltin((IdentifierInfo *)II,
+                                                 BuiltinID, S.TUScope,
+                                                 R.isForRedeclaration(),
+                                                 R.getNameLoc())) {
           R.addDecl(D);
-        return (D != NULL);
+          return true;
+        }
+
+        if (R.isForRedeclaration()) {
+          // If we're redeclaring this function anyway, forget that
+          // this was a builtin at all.
+          S.Context.BuiltinInfo.ForgetBuiltin(BuiltinID, S.Context.Idents);
+        }
+
+        return false;
       }
     }
   }
@@ -500,16 +515,16 @@ static bool CanDeclareSpecialMemberFunction(ASTContext &Context,
   // Don't do it if the class is invalid.
   if (Class->isInvalidDecl())
     return false;
-  
+
   // We need to have a definition for the class.
   if (!Class->getDefinition() || Class->isDependentContext())
     return false;
-  
+
   // We can't be in the middle of defining the class.
   if (const RecordType *RecordTy
                         = Context.getTypeDeclType(Class)->getAs<RecordType>())
     return !RecordTy->isBeingDefined();
-    
+
   return false;
 }
 
@@ -520,46 +535,46 @@ void Sema::ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class) {
   // If the default constructor has not yet been declared, do so now.
   if (!Class->hasDeclaredDefaultConstructor())
     DeclareImplicitDefaultConstructor(Class);
-  
+
   // If the copy constructor has not yet been declared, do so now.
   if (!Class->hasDeclaredCopyConstructor())
     DeclareImplicitCopyConstructor(Class);
-  
+
   // If the copy assignment operator has not yet been declared, do so now.
   if (!Class->hasDeclaredCopyAssignment())
     DeclareImplicitCopyAssignment(Class);
 
   // If the destructor has not yet been declared, do so now.
   if (!Class->hasDeclaredDestructor())
-    DeclareImplicitDestructor(Class);  
+    DeclareImplicitDestructor(Class);
 }
 
-/// \brief Determine whether this is the name of an implicitly-declared 
+/// \brief Determine whether this is the name of an implicitly-declared
 /// special member function.
 static bool isImplicitlyDeclaredMemberFunctionName(DeclarationName Name) {
   switch (Name.getNameKind()) {
   case DeclarationName::CXXConstructorName:
   case DeclarationName::CXXDestructorName:
     return true;
-    
+
   case DeclarationName::CXXOperatorName:
     return Name.getCXXOverloadedOperator() == OO_Equal;
-    
+
   default:
-    break;      
+    break;
   }
-  
+
   return false;
 }
 
 /// \brief If there are any implicit member functions with the given name
 /// that need to be declared in the given declaration context, do so.
-static void DeclareImplicitMemberFunctionsWithName(Sema &S, 
+static void DeclareImplicitMemberFunctionsWithName(Sema &S,
                                                    DeclarationName Name,
                                                    const DeclContext *DC) {
   if (!DC)
     return;
-  
+
   switch (Name.getNameKind()) {
   case DeclarationName::CXXConstructorName:
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC))
@@ -572,26 +587,26 @@ static void DeclareImplicitMemberFunctionsWithName(Sema &S,
           S.DeclareImplicitCopyConstructor(const_cast<CXXRecordDecl *>(Record));
       }
     break;
-      
+
   case DeclarationName::CXXDestructorName:
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC))
       if (Record->getDefinition() && !Record->hasDeclaredDestructor() &&
           CanDeclareSpecialMemberFunction(S.Context, Record))
         S.DeclareImplicitDestructor(const_cast<CXXRecordDecl *>(Record));
     break;
-    
+
   case DeclarationName::CXXOperatorName:
     if (Name.getCXXOverloadedOperator() != OO_Equal)
       break;
-    
+
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(DC))
       if (Record->getDefinition() && !Record->hasDeclaredCopyAssignment() &&
           CanDeclareSpecialMemberFunction(S.Context, Record))
         S.DeclareImplicitCopyAssignment(const_cast<CXXRecordDecl *>(Record));
     break;
-    
+
   default:
-    break;      
+    break;
   }
 }
 
@@ -603,7 +618,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
   // Lazily declare C++ special member functions.
   if (S.getLangOptions().CPlusPlus)
     DeclareImplicitMemberFunctionsWithName(S, R.getLookupName(), DC);
-  
+
   // Perform lookup into this declaration context.
   DeclContext::lookup_const_iterator I, E;
   for (llvm::tie(I, E) = DC->lookup(R.getLookupName()); I != E; ++I) {
@@ -624,7 +639,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
     return Found;
 
   // C++ [temp.mem]p6:
-  //   A specialization of a conversion function template is not found by 
+  //   A specialization of a conversion function template is not found by
   //   name lookup. Instead, any conversion function templates visible in the
   //   context of the use are considered. [...]
   const CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
@@ -632,50 +647,51 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
     return Found;
 
   const UnresolvedSetImpl *Unresolved = Record->getConversionFunctions();
-  for (UnresolvedSetImpl::iterator U = Unresolved->begin(), 
+  for (UnresolvedSetImpl::iterator U = Unresolved->begin(),
          UEnd = Unresolved->end(); U != UEnd; ++U) {
     FunctionTemplateDecl *ConvTemplate = dyn_cast<FunctionTemplateDecl>(*U);
     if (!ConvTemplate)
       continue;
-    
+
     // When we're performing lookup for the purposes of redeclaration, just
-    // add the conversion function template. When we deduce template 
-    // arguments for specializations, we'll end up unifying the return 
+    // add the conversion function template. When we deduce template
+    // arguments for specializations, we'll end up unifying the return
     // type of the new declaration with the type of the function template.
     if (R.isForRedeclaration()) {
       R.addDecl(ConvTemplate);
       Found = true;
       continue;
     }
-    
+
     // C++ [temp.mem]p6:
-    //   [...] For each such operator, if argument deduction succeeds 
-    //   (14.9.2.3), the resulting specialization is used as if found by 
+    //   [...] For each such operator, if argument deduction succeeds
+    //   (14.9.2.3), the resulting specialization is used as if found by
     //   name lookup.
     //
     // When referencing a conversion function for any purpose other than
     // a redeclaration (such that we'll be building an expression with the
-    // result), perform template argument deduction and place the 
+    // result), perform template argument deduction and place the
     // specialization into the result set. We do this to avoid forcing all
     // callers to perform special deduction for conversion functions.
     TemplateDeductionInfo Info(R.getSema().Context, R.getNameLoc());
     FunctionDecl *Specialization = 0;
-    
-    const FunctionProtoType *ConvProto        
+
+    const FunctionProtoType *ConvProto
       = ConvTemplate->getTemplatedDecl()->getType()->getAs<FunctionProtoType>();
     assert(ConvProto && "Nonsensical conversion function template type");
 
     // Compute the type of the function that we would expect the conversion
     // function to have, if it were to match the name given.
     // FIXME: Calling convention!
-    FunctionType::ExtInfo ConvProtoInfo = ConvProto->getExtInfo();
+    FunctionProtoType::ExtProtoInfo EPI = ConvProto->getExtProtoInfo();
+    EPI.ExtInfo = EPI.ExtInfo.withCallingConv(CC_Default);
+    EPI.HasExceptionSpec = false;
+    EPI.HasAnyExceptionSpec = false;
+    EPI.NumExceptions = 0;
     QualType ExpectedType
       = R.getSema().Context.getFunctionType(R.getLookupName().getCXXNameType(),
-                                            0, 0, ConvProto->isVariadic(),
-                                            ConvProto->getTypeQuals(),
-                                            false, false, 0, 0,
-                                    ConvProtoInfo.withCallingConv(CC_Default));
- 
+                                            0, 0, EPI);
+
     // Perform template argument deduction against the type that we would
     // expect the function to have.
     if (R.getSema().DeduceTemplateArguments(ConvTemplate, 0, ExpectedType,
@@ -691,7 +707,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
 
 // Performs C++ unqualified lookup into the given file context.
 static bool
-CppNamespaceLookup(Sema &S, LookupResult &R, ASTContext &Context, 
+CppNamespaceLookup(Sema &S, LookupResult &R, ASTContext &Context,
                    DeclContext *NS, UnqualUsingDirectiveSet &UDirs) {
 
   assert(NS && NS->isFileContext() && "CppNamespaceLookup() requires namespace!");
@@ -729,7 +745,7 @@ static bool isNamespaceOrTranslationUnitScope(Scope *S) {
 static std::pair<DeclContext *, bool> findOuterContext(Scope *S) {
   DeclContext *DC = static_cast<DeclContext *>(S->getEntity());
   DeclContext *Lexical = 0;
-  for (Scope *OuterS = S->getParent(); OuterS; 
+  for (Scope *OuterS = S->getParent(); OuterS;
        OuterS = OuterS->getParent()) {
     if (OuterS->getEntity()) {
       Lexical = static_cast<DeclContext *>(OuterS->getEntity());
@@ -745,12 +761,12 @@ static std::pair<DeclContext *, bool> findOuterContext(Scope *S) {
   //
   // Example:
   //
-  //   namespace N { 
-  //     class C { }; 
+  //   namespace N {
+  //     class C { };
   //
   //     template<class T> class B {
   //       void f(T);
-  //     }; 
+  //     };
   //   }
   //
   //   template<class C> void N::B<C>::f(C) {
@@ -759,24 +775,24 @@ static std::pair<DeclContext *, bool> findOuterContext(Scope *S) {
   //
   // In this example, the lexical context we return is the
   // TranslationUnit, while the semantic context is the namespace N.
-  if (!Lexical || !DC || !S->getParent() || 
+  if (!Lexical || !DC || !S->getParent() ||
       !S->getParent()->isTemplateParamScope())
     return std::make_pair(Lexical, false);
 
-  // Find the outermost template parameter scope. 
+  // Find the outermost template parameter scope.
   // For the example, this is the scope for the template parameters of
   // template<class C>.
   Scope *OutermostTemplateScope = S->getParent();
   while (OutermostTemplateScope->getParent() &&
          OutermostTemplateScope->getParent()->isTemplateParamScope())
     OutermostTemplateScope = OutermostTemplateScope->getParent();
-  
+
   // Find the namespace context in which the original scope occurs. In
   // the example, this is namespace N.
   DeclContext *Semantic = DC;
   while (!Semantic->isFileContext())
     Semantic = Semantic->getParent();
-  
+
   // Find the declaration context just outside of the template
   // parameter scope. This is the context in which the template is
   // being lexically declaration (a namespace context). In the
@@ -800,7 +816,7 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
       if (DeclContext *DC = static_cast<DeclContext *>(PreS->getEntity()))
         DeclareImplicitMemberFunctionsWithName(*this, Name, DC);
   }
-    
+
   // Implicitly declare member functions with the name we're looking for, if in
   // fact we are in a scope where it matters.
 
@@ -883,7 +899,7 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
               if (ObjCInterfaceDecl *Class = Method->getClassInterface()) {
                 ObjCInterfaceDecl *ClassDeclared;
                 if (ObjCIvarDecl *Ivar = Class->lookupInstanceVariable(
-                                                 Name.getAsIdentifierInfo(), 
+                                                 Name.getAsIdentifierInfo(),
                                                              ClassDeclared)) {
                   if (R.isAcceptableDecl(Ivar)) {
                     R.addDecl(Ivar);
@@ -913,6 +929,10 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   // FIXME:  This really, really shouldn't be happening.
   if (!S) return false;
 
+  // If we are looking for members, no need to look into global/namespace scope.
+  if (R.getLookupKind() == LookupMemberName)
+    return false;
+
   // Collect UsingDirectiveDecls in all scopes, and recursively all
   // nominated namespaces by those using-directives.
   //
@@ -958,7 +978,7 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
       Ctx = OutsideOfTemplateParamDC;
       OutsideOfTemplateParamDC = 0;
     }
-    
+
     if (Ctx) {
       DeclContext *OuterCtx;
       bool SearchAfterTemplateScope;
@@ -972,24 +992,24 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
         // non-transparent context.
         if (Ctx->isTransparentContext())
           continue;
-        
+
         // If we have a context, and it's not a context stashed in the
         // template parameter scope for an out-of-line definition, also
         // look into that context.
         if (!(Found && S && S->isTemplateParamScope())) {
           assert(Ctx->isFileContext() &&
               "We should have been looking only at file context here already.");
-          
+
           // Look into context considering using-directives.
           if (CppNamespaceLookup(*this, R, Context, Ctx, UDirs))
             Found = true;
         }
-        
+
         if (Found) {
           R.resolveKind();
           return true;
         }
-        
+
         if (R.isForRedeclaration() && !Ctx->isTransparentContext())
           return false;
       }
@@ -1042,7 +1062,6 @@ bool Sema::LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation) {
   if (!getLangOptions().CPlusPlus) {
     // Unqualified name lookup in C/Objective-C is purely lexical, so
     // search in the declarations attached to the name.
-
     if (NameKind == Sema::LookupRedeclarationWithLinkage) {
       // Find the nearest non-transparent declaration scope.
       while (!(S->getFlags() & Scope::DeclScope) ||
@@ -1228,16 +1247,48 @@ static bool LookupQualifiedNameInUsingDirectives(Sema &S, LookupResult &R,
 }
 
 /// \brief Callback that looks for any member of a class with the given name.
-static bool LookupAnyMember(const CXXBaseSpecifier *Specifier, 
+static bool LookupAnyMember(const CXXBaseSpecifier *Specifier,
                             CXXBasePath &Path,
                             void *Name) {
   RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
-  
+
   DeclarationName N = DeclarationName::getFromOpaquePtr(Name);
   Path.Decls = BaseRecord->lookup(N);
   return Path.Decls.first != Path.Decls.second;
 }
 
+/// \brief Determine whether the given set of member declarations contains only
+/// static members, nested types, and enumerators.
+template<typename InputIterator>
+static bool HasOnlyStaticMembers(InputIterator First, InputIterator Last) {
+  Decl *D = (*First)->getUnderlyingDecl();
+  if (isa<VarDecl>(D) || isa<TypeDecl>(D) || isa<EnumConstantDecl>(D))
+    return true;
+
+  if (isa<CXXMethodDecl>(D)) {
+    // Determine whether all of the methods are static.
+    bool AllMethodsAreStatic = true;
+    for(; First != Last; ++First) {
+      D = (*First)->getUnderlyingDecl();
+
+      if (!isa<CXXMethodDecl>(D)) {
+        assert(isa<TagDecl>(D) && "Non-function must be a tag decl");
+        break;
+      }
+
+      if (!cast<CXXMethodDecl>(D)->isStatic()) {
+        AllMethodsAreStatic = false;
+        break;
+      }
+    }
+
+    if (AllMethodsAreStatic)
+      return true;
+  }
+
+  return false;
+}
+
 /// \brief Perform qualified name lookup into a given context.
 ///
 /// Qualified name lookup (C++ [basic.lookup.qual]) is used to find
@@ -1256,7 +1307,7 @@ static bool LookupAnyMember(const CXXBaseSpecifier *Specifier,
 /// search. If the lookup criteria permits, name lookup may also search
 /// in the parent contexts or (for C++ classes) base classes.
 ///
-/// \param InUnqualifiedLookup true if this is qualified name lookup that 
+/// \param InUnqualifiedLookup true if this is qualified name lookup that
 /// occurs as part of unqualified name lookup.
 ///
 /// \returns true if lookup succeeded, false if it failed.
@@ -1307,7 +1358,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 
   // If we're performing qualified name lookup into a dependent class,
   // then we are actually looking into a current instantiation. If we have any
-  // dependent base classes, then we either have to delay lookup until 
+  // dependent base classes, then we either have to delay lookup until
   // template instantiation time (at which point all bases will be available)
   // or we have to fail.
   if (!InUnqualifiedLookup && LookupRec->isDependentContext() &&
@@ -1315,7 +1366,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     R.setNotFoundInCurrentInstantiation();
     return false;
   }
-    
+
   // Perform lookup into our base classes.
   CXXBasePaths Paths;
   Paths.setOrigin(LookupRec);
@@ -1328,7 +1379,7 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     case LookupRedeclarationWithLinkage:
       BaseCallback = &CXXRecordDecl::FindOrdinaryMember;
       break;
-      
+
     case LookupTagName:
       BaseCallback = &CXXRecordDecl::FindTagMember;
       break;
@@ -1336,21 +1387,22 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     case LookupAnyName:
       BaseCallback = &LookupAnyMember;
       break;
-      
+
     case LookupUsingDeclName:
       // This lookup is for redeclarations only.
-      
+
     case LookupOperatorName:
     case LookupNamespaceName:
     case LookupObjCProtocolName:
+    case LookupLabel:
       // These lookups will never find a member in a C++ class (or base class).
       return false;
-      
+
     case LookupNestedNameSpecifierName:
       BaseCallback = &CXXRecordDecl::FindNestedNameSpecifierMember;
       break;
   }
-  
+
   if (!LookupRec->lookupInBases(BaseCallback,
                                 R.getLookupName().getAsOpaquePtr(), Paths))
     return false;
@@ -1363,10 +1415,10 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
   //   and includes members from distinct sub-objects, there is an
   //   ambiguity and the program is ill-formed. Otherwise that set is
   //   the result of the lookup.
-  // FIXME: support using declarations!
   QualType SubobjectType;
   int SubobjectNumber = 0;
   AccessSpecifier SubobjectAccess = AS_none;
+
   for (CXXBasePaths::paths_iterator Path = Paths.begin(), PathEnd = Paths.end();
        Path != PathEnd; ++Path) {
     const CXXBasePathElement &PathElement = Path->back();
@@ -1374,51 +1426,54 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
     // Pick the best (i.e. most permissive i.e. numerically lowest) access
     // across all paths.
     SubobjectAccess = std::min(SubobjectAccess, Path->Access);
-    
+
     // Determine whether we're looking at a distinct sub-object or not.
     if (SubobjectType.isNull()) {
       // This is the first subobject we've looked at. Record its type.
       SubobjectType = Context.getCanonicalType(PathElement.Base->getType());
       SubobjectNumber = PathElement.SubobjectNumber;
-    } else if (SubobjectType
+      continue;
+    }
+
+    if (SubobjectType
                  != Context.getCanonicalType(PathElement.Base->getType())) {
       // We found members of the given name in two subobjects of
-      // different types. This lookup is ambiguous.
+      // different types. If the declaration sets aren't the same, this
+      // this lookup is ambiguous.
+      if (HasOnlyStaticMembers(Path->Decls.first, Path->Decls.second)) {
+        CXXBasePaths::paths_iterator FirstPath = Paths.begin();
+        DeclContext::lookup_iterator FirstD = FirstPath->Decls.first;
+        DeclContext::lookup_iterator CurrentD = Path->Decls.first;
+
+        while (FirstD != FirstPath->Decls.second &&
+               CurrentD != Path->Decls.second) {
+         if ((*FirstD)->getUnderlyingDecl()->getCanonicalDecl() !=
+             (*CurrentD)->getUnderlyingDecl()->getCanonicalDecl())
+           break;
+
+          ++FirstD;
+          ++CurrentD;
+        }
+
+        if (FirstD == FirstPath->Decls.second &&
+            CurrentD == Path->Decls.second)
+          continue;
+      }
+
       R.setAmbiguousBaseSubobjectTypes(Paths);
       return true;
-    } else if (SubobjectNumber != PathElement.SubobjectNumber) {
+    }
+
+    if (SubobjectNumber != PathElement.SubobjectNumber) {
       // We have a different subobject of the same type.
 
       // C++ [class.member.lookup]p5:
       //   A static member, a nested type or an enumerator defined in
       //   a base class T can unambiguously be found even if an object
       //   has more than one base class subobject of type T.
-      Decl *FirstDecl = *Path->Decls.first;
-      if (isa<VarDecl>(FirstDecl) ||
-          isa<TypeDecl>(FirstDecl) ||
-          isa<EnumConstantDecl>(FirstDecl))
+      if (HasOnlyStaticMembers(Path->Decls.first, Path->Decls.second))
         continue;
 
-      if (isa<CXXMethodDecl>(FirstDecl)) {
-        // Determine whether all of the methods are static.
-        bool AllMethodsAreStatic = true;
-        for (DeclContext::lookup_iterator Func = Path->Decls.first;
-             Func != Path->Decls.second; ++Func) {
-          if (!isa<CXXMethodDecl>(*Func)) {
-            assert(isa<TagDecl>(*Func) && "Non-function must be a tag decl");
-            break;
-          }
-
-          if (!cast<CXXMethodDecl>(*Func)->isStatic()) {
-            AllMethodsAreStatic = false;
-            break;
-          }
-        }
-
-        if (AllMethodsAreStatic)
-          continue;
-      }
-
       // We have found a nonstatic member name in multiple, distinct
       // subobjects. Name lookup is ambiguous.
       R.setAmbiguousBaseSubobjects(Paths);
@@ -1453,13 +1508,6 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 ///
 /// @param SS       An optional C++ scope-specifier, e.g., "::N::M".
 ///
-/// @param Name     The name of the entity that name lookup will
-/// search for.
-///
-/// @param Loc      If provided, the source location where we're performing
-/// name lookup. At present, this is only used to produce diagnostics when
-/// C library functions (like "malloc") are implicitly declared.
-///
 /// @param EnteringContext Indicates whether we are going to enter the
 /// context of the scope-specifier SS (if present).
 ///
@@ -1525,21 +1573,21 @@ bool Sema::DiagnoseAmbiguousLookup(LookupResult &Result) {
     Diag(NameLoc, diag::err_ambiguous_member_multiple_subobjects)
       << Name << SubobjectType << getAmbiguousPathsDisplayString(*Paths)
       << LookupRange;
-    
+
     DeclContext::lookup_iterator Found = Paths->front().Decls.first;
     while (isa<CXXMethodDecl>(*Found) &&
            cast<CXXMethodDecl>(*Found)->isStatic())
       ++Found;
-    
+
     Diag((*Found)->getLocation(), diag::note_ambiguous_member_found);
-    
+
     return true;
   }
 
   case LookupResult::AmbiguousBaseSubobjectTypes: {
     Diag(NameLoc, diag::err_ambiguous_member_multiple_subobject_types)
       << Name << LookupRange;
-    
+
     CXXBasePaths *Paths = Result.getBasePaths();
     std::set<Decl *> DeclsPrinted;
     for (CXXBasePaths::paths_iterator Path = Paths->begin(),
@@ -1582,7 +1630,7 @@ bool Sema::DiagnoseAmbiguousLookup(LookupResult &Result) {
 
   case LookupResult::AmbiguousReference: {
     Diag(NameLoc, diag::err_ambiguous_reference) << Name << LookupRange;
-  
+
     LookupResult::iterator DI = Result.begin(), DE = Result.end();
     for (; DI != DE; ++DI)
       Diag((*DI)->getLocation(), diag::note_ambiguous_candidate) << *DI;
@@ -1648,11 +1696,12 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
       addAssociatedClassesAndNamespaces(Result, Arg.getAsType());
       break;
 
-    case TemplateArgument::Template: {
+    case TemplateArgument::Template:
+    case TemplateArgument::TemplateExpansion: {
       // [...] the namespaces in which any template template arguments are
       // defined; and the classes in which any member templates used as
       // template template arguments are defined.
-      TemplateName Template = Arg.getAsTemplate();
+      TemplateName Template = Arg.getAsTemplateOrTemplatePattern();
       if (ClassTemplateDecl *ClassTemplate
                  = dyn_cast<ClassTemplateDecl>(Template.getAsTemplateDecl())) {
         DeclContext *Ctx = ClassTemplate->getDeclContext();
@@ -1663,7 +1712,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
       }
       break;
     }
-      
+
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
     case TemplateArgument::Expression:
@@ -1713,7 +1762,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
 
   // -- If T is a template-id, its associated namespaces and classes are
   //    the namespace in which the template is defined; for member
-  //    templates, the member template’s class; the namespaces and classes
+  //    templates, the member template's class; the namespaces and classes
   //    associated with the types of the template arguments provided for
   //    template type parameters (excluding template template parameters); the
   //    namespaces in which any template template arguments are defined; and
@@ -1840,7 +1889,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
 
     //     -- If T is an enumeration type, its associated namespace is
     //        the namespace in which it is defined. If it is class
-    //        member, its associated class is the member’s class; else
+    //        member, its associated class is the member's class; else
     //        it has no associated class.
     case Type::Enum: {
       EnumDecl *Enum = cast<EnumType>(T)->getDecl();
@@ -2032,7 +2081,7 @@ NamedDecl *Sema::LookupSingleName(Scope *S, DeclarationName Name,
 }
 
 /// \brief Find the protocol with the given name, if any.
-ObjCProtocolDecl *Sema::LookupProtocol(IdentifierInfo *II, 
+ObjCProtocolDecl *Sema::LookupProtocol(IdentifierInfo *II,
                                        SourceLocation IdLoc) {
   Decl *D = LookupSingleName(TUScope, II, IdLoc,
                              LookupObjCProtocolName);
@@ -2089,7 +2138,7 @@ DeclContext::lookup_result Sema::LookupConstructors(CXXRecordDecl *Class) {
     if (!Class->hasDeclaredCopyConstructor())
       DeclareImplicitCopyConstructor(Class);
   }
-  
+
   CanQualType T = Context.getCanonicalType(Context.getTypeDeclType(Class));
   DeclarationName Name = Context.DeclarationNames.getCXXConstructorName(T);
   return Class->lookup(Name);
@@ -2097,7 +2146,7 @@ DeclContext::lookup_result Sema::LookupConstructors(CXXRecordDecl *Class) {
 
 /// \brief Look for the destructor of the given class.
 ///
-/// During semantic analysis, this routine should be used in lieu of 
+/// During semantic analysis, this routine should be used in lieu of
 /// CXXRecordDecl::getDestructor().
 ///
 /// \returns The destructor for this class.
@@ -2233,7 +2282,7 @@ public:
   /// of declarations.
   class ShadowMapEntry {
     typedef llvm::SmallVector<NamedDecl *, 4> DeclVector;
-    
+
     /// \brief Contains either the solitary NamedDecl * or a vector
     /// of declarations.
     llvm::PointerUnion<NamedDecl *, DeclVector*> DeclOrVector;
@@ -2245,7 +2294,7 @@ public:
     void Destroy();
 
     // Iteration.
-    typedef NamedDecl **iterator;
+    typedef NamedDecl * const *iterator;
     iterator begin();
     iterator end();
   };
@@ -2315,7 +2364,7 @@ void VisibleDeclsRecord::ShadowMapEntry::Add(NamedDecl *ND) {
     DeclOrVector = ND;
     return;
   }
-  
+
   if (NamedDecl *PrevND = DeclOrVector.dyn_cast<NamedDecl *>()) {
     // 1 -> 2 elements: create the vector of results and push in the
     // existing declaration.
@@ -2335,18 +2384,18 @@ void VisibleDeclsRecord::ShadowMapEntry::Destroy() {
   }
 }
 
-VisibleDeclsRecord::ShadowMapEntry::iterator 
+VisibleDeclsRecord::ShadowMapEntry::iterator
 VisibleDeclsRecord::ShadowMapEntry::begin() {
   if (DeclOrVector.isNull())
     return 0;
 
-  if (DeclOrVector.dyn_cast<NamedDecl *>())
-    return &reinterpret_cast<NamedDecl*&>(DeclOrVector);
+  if (DeclOrVector.is<NamedDecl *>())
+    return DeclOrVector.getAddrOf<NamedDecl *>();
 
   return DeclOrVector.get<DeclVector *>()->begin();
 }
 
-VisibleDeclsRecord::ShadowMapEntry::iterator 
+VisibleDeclsRecord::ShadowMapEntry::iterator
 VisibleDeclsRecord::ShadowMapEntry::end() {
   if (DeclOrVector.isNull())
     return 0;
@@ -2360,7 +2409,7 @@ VisibleDeclsRecord::ShadowMapEntry::end() {
 NamedDecl *VisibleDeclsRecord::checkHidden(NamedDecl *ND) {
   // Look through using declarations.
   ND = ND->getUnderlyingDecl();
-  
+
   unsigned IDNS = ND->getIdentifierNamespace();
   std::list<ShadowMap>::reverse_iterator SM = ShadowMaps.rbegin();
   for (std::list<ShadowMap>::reverse_iterator SMEnd = ShadowMaps.rend();
@@ -2369,12 +2418,12 @@ NamedDecl *VisibleDeclsRecord::checkHidden(NamedDecl *ND) {
     if (Pos == SM->end())
       continue;
 
-    for (ShadowMapEntry::iterator I = Pos->second.begin(), 
+    for (ShadowMapEntry::iterator I = Pos->second.begin(),
                                IEnd = Pos->second.end();
          I != IEnd; ++I) {
       // A tag declaration does not hide a non-tag declaration.
       if ((*I)->hasTagIdentifierNamespace() &&
-          (IDNS & (Decl::IDNS_Member | Decl::IDNS_Ordinary | 
+          (IDNS & (Decl::IDNS_Member | Decl::IDNS_Ordinary |
                    Decl::IDNS_ObjCProtocol)))
         continue;
 
@@ -2391,7 +2440,7 @@ NamedDecl *VisibleDeclsRecord::checkHidden(NamedDecl *ND) {
           ND->isFunctionOrFunctionTemplate() &&
           SM == ShadowMaps.rbegin())
         continue;
-          
+
       // We've found a declaration that hides this one.
       return *I;
     }
@@ -2411,22 +2460,44 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
   // Make sure we don't visit the same context twice.
   if (Visited.visitedContext(Ctx->getPrimaryContext()))
     return;
-  
+
   if (CXXRecordDecl *Class = dyn_cast<CXXRecordDecl>(Ctx))
     Result.getSema().ForceDeclarationOfImplicitMembers(Class);
 
   // Enumerate all of the results in this context.
-  for (DeclContext *CurCtx = Ctx->getPrimaryContext(); CurCtx; 
+  for (DeclContext *CurCtx = Ctx->getPrimaryContext(); CurCtx;
        CurCtx = CurCtx->getNextContext()) {
-    for (DeclContext::decl_iterator D = CurCtx->decls_begin(), 
+    for (DeclContext::decl_iterator D = CurCtx->decls_begin(),
                                  DEnd = CurCtx->decls_end();
          D != DEnd; ++D) {
-      if (NamedDecl *ND = dyn_cast<NamedDecl>(*D))
+      if (NamedDecl *ND = dyn_cast<NamedDecl>(*D)) {
         if (Result.isAcceptableDecl(ND)) {
           Consumer.FoundDecl(ND, Visited.checkHidden(ND), InBaseClass);
           Visited.add(ND);
         }
-
+      } else if (ObjCForwardProtocolDecl *ForwardProto
+                                      = dyn_cast<ObjCForwardProtocolDecl>(*D)) {
+        for (ObjCForwardProtocolDecl::protocol_iterator
+                  P = ForwardProto->protocol_begin(),
+               PEnd = ForwardProto->protocol_end();
+             P != PEnd;
+             ++P) {
+          if (Result.isAcceptableDecl(*P)) {
+            Consumer.FoundDecl(*P, Visited.checkHidden(*P), InBaseClass);
+            Visited.add(*P);
+          }
+        }
+      } else if (ObjCClassDecl *Class = dyn_cast<ObjCClassDecl>(*D)) {
+        for (ObjCClassDecl::iterator I = Class->begin(), IEnd = Class->end();
+             I != IEnd; ++I) {
+          ObjCInterfaceDecl *IFace = I->getInterface();
+          if (Result.isAcceptableDecl(IFace)) {
+            Consumer.FoundDecl(IFace, Visited.checkHidden(IFace), InBaseClass);
+            Visited.add(IFace);
+          }
+        }
+      }
+      
       // Visit transparent contexts and inline namespaces inside this context.
       if (DeclContext *InnerCtx = dyn_cast<DeclContext>(*D)) {
         if (InnerCtx->isTransparentContext() || InnerCtx->isInlineNamespace())
@@ -2441,7 +2512,7 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
     ShadowContextRAII Shadow(Visited);
     DeclContext::udir_iterator I, E;
     for (llvm::tie(I, E) = Ctx->getUsingDirectives(); I != E; ++I) {
-      LookupVisibleDecls((*I)->getNominatedNamespace(), Result, 
+      LookupVisibleDecls((*I)->getNominatedNamespace(), Result,
                          QualifiedNameLookup, InBaseClass, Consumer, Visited);
     }
   }
@@ -2455,16 +2526,16 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
                                          BEnd = Record->bases_end();
          B != BEnd; ++B) {
       QualType BaseType = B->getType();
-      
+
       // Don't look into dependent bases, because name lookup can't look
       // there anyway.
       if (BaseType->isDependentType())
         continue;
-      
+
       const RecordType *Record = BaseType->getAs<RecordType>();
       if (!Record)
         continue;
-      
+
       // FIXME: It would be nice to be able to determine whether referencing
       // a particular member would be ambiguous. For example, given
       //
@@ -2483,21 +2554,21 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
       // or
       //
       //   c->A::member
-      
+
       // Find results in this base class (and its bases).
       ShadowContextRAII Shadow(Visited);
       LookupVisibleDecls(Record->getDecl(), Result, QualifiedNameLookup,
                          true, Consumer, Visited);
     }
   }
-  
+
   // Traverse the contexts of Objective-C classes.
   if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(Ctx)) {
     // Traverse categories.
     for (ObjCCategoryDecl *Category = IFace->getCategoryList();
          Category; Category = Category->getNextClassCategory()) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(Category, Result, QualifiedNameLookup, false, 
+      LookupVisibleDecls(Category, Result, QualifiedNameLookup, false,
                          Consumer, Visited);
     }
 
@@ -2506,7 +2577,7 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
          I = IFace->all_referenced_protocol_begin(),
          E = IFace->all_referenced_protocol_end(); I != E; ++I) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer, 
+      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer,
                          Visited);
     }
 
@@ -2516,35 +2587,35 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
       LookupVisibleDecls(IFace->getSuperClass(), Result, QualifiedNameLookup,
                          true, Consumer, Visited);
     }
-    
+
     // If there is an implementation, traverse it. We do this to find
     // synthesized ivars.
     if (IFace->getImplementation()) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(IFace->getImplementation(), Result, 
+      LookupVisibleDecls(IFace->getImplementation(), Result,
                          QualifiedNameLookup, true, Consumer, Visited);
     }
   } else if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Ctx)) {
     for (ObjCProtocolDecl::protocol_iterator I = Protocol->protocol_begin(),
            E = Protocol->protocol_end(); I != E; ++I) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer, 
+      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer,
                          Visited);
     }
   } else if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(Ctx)) {
     for (ObjCCategoryDecl::protocol_iterator I = Category->protocol_begin(),
            E = Category->protocol_end(); I != E; ++I) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer, 
+      LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer,
                          Visited);
     }
-    
+
     // If there is an implementation, traverse it.
     if (Category->getImplementation()) {
       ShadowContextRAII Shadow(Visited);
-      LookupVisibleDecls(Category->getImplementation(), Result, 
+      LookupVisibleDecls(Category->getImplementation(), Result,
                          QualifiedNameLookup, true, Consumer, Visited);
-    }    
+    }
   }
 }
 
@@ -2555,8 +2626,8 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
   if (!S)
     return;
 
-  if (!S->getEntity() || 
-      (!S->getParent() && 
+  if (!S->getEntity() ||
+      (!S->getParent() &&
        !Visited.alreadyVisitedContext((DeclContext *)S->getEntity())) ||
       ((DeclContext *)S->getEntity())->isFunctionOrMethod()) {
     // Walk through the declarations in this Scope.
@@ -2569,7 +2640,7 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
         }
     }
   }
-  
+
   // FIXME: C++ [temp.local]p8
   DeclContext *Entity = 0;
   if (S->getEntity()) {
@@ -2578,7 +2649,7 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
     // where we hit the context stored in the next outer scope.
     Entity = (DeclContext *)S->getEntity();
     DeclContext *OuterCtx = findOuterContext(S).first; // FIXME
-    
+
     for (DeclContext *Ctx = Entity; Ctx && !Ctx->Equals(OuterCtx);
          Ctx = Ctx->getLookupParent()) {
       if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(Ctx)) {
@@ -2586,9 +2657,27 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
           // For instance methods, look for ivars in the method's interface.
           LookupResult IvarResult(Result.getSema(), Result.getLookupName(),
                                   Result.getNameLoc(), Sema::LookupMemberName);
-          if (ObjCInterfaceDecl *IFace = Method->getClassInterface())
-            LookupVisibleDecls(IFace, IvarResult, /*QualifiedNameLookup=*/false, 
+          if (ObjCInterfaceDecl *IFace = Method->getClassInterface()) {
+            LookupVisibleDecls(IFace, IvarResult, /*QualifiedNameLookup=*/false,
                                /*InBaseClass=*/false, Consumer, Visited);
+
+            // Look for properties from which we can synthesize ivars, if
+            // permitted.
+            if (Result.getSema().getLangOptions().ObjCNonFragileABI2 &&
+                IFace->getImplementation() &&
+                Result.getLookupKind() == Sema::LookupOrdinaryName) {
+              for (ObjCInterfaceDecl::prop_iterator
+                        P = IFace->prop_begin(),
+                     PEnd = IFace->prop_end();
+                   P != PEnd; ++P) {
+                if (Result.getSema().canSynthesizeProvisionalIvar(*P) &&
+                    !IFace->lookupInstanceVariable((*P)->getIdentifier())) {
+                  Consumer.FoundDecl(*P, Visited.checkHidden(*P), false);
+                  Visited.add(*P);
+                }
+              }
+            }
+          }
         }
 
         // We've already performed all of the name lookup that we need
@@ -2599,8 +2688,8 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
 
       if (Ctx->isFunctionOrMethod())
         continue;
-      
-      LookupVisibleDecls(Ctx, Result, /*QualifiedNameLookup=*/false, 
+
+      LookupVisibleDecls(Ctx, Result, /*QualifiedNameLookup=*/false,
                          /*InBaseClass=*/false, Consumer, Visited);
     }
   } else if (!S->getParent()) {
@@ -2610,14 +2699,14 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
     // FIXME: We would like the translation unit's Scope object to point to the
     // translation unit, so we don't need this special "if" branch. However,
     // doing so would force the normal C++ name-lookup code to look into the
-    // translation unit decl when the IdentifierInfo chains would suffice. 
+    // translation unit decl when the IdentifierInfo chains would suffice.
     // Once we fix that problem (which is part of a more general "don't look
     // in DeclContexts unless we have to" optimization), we can eliminate this.
     Entity = Result.getSema().Context.getTranslationUnitDecl();
-    LookupVisibleDecls(Entity, Result, /*QualifiedNameLookup=*/false, 
+    LookupVisibleDecls(Entity, Result, /*QualifiedNameLookup=*/false,
                        /*InBaseClass=*/false, Consumer, Visited);
-  } 
-  
+  }
+
   if (Entity) {
     // Lookup visible declarations in any namespaces found by using
     // directives.
@@ -2625,7 +2714,7 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
     llvm::tie(UI, UEnd) = UDirs.getNamespacesFor(Entity);
     for (; UI != UEnd; ++UI)
       LookupVisibleDecls(const_cast<DeclContext *>(UI->getNominatedNamespace()),
-                         Result, /*QualifiedNameLookup=*/false, 
+                         Result, /*QualifiedNameLookup=*/false,
                          /*InBaseClass=*/false, Consumer, Visited);
   }
 
@@ -2667,13 +2756,41 @@ void Sema::LookupVisibleDecls(DeclContext *Ctx, LookupNameKind Kind,
   if (!IncludeGlobalScope)
     Visited.visitedContext(Context.getTranslationUnitDecl());
   ShadowContextRAII Shadow(Visited);
-  ::LookupVisibleDecls(Ctx, Result, /*QualifiedNameLookup=*/true, 
+  ::LookupVisibleDecls(Ctx, Result, /*QualifiedNameLookup=*/true,
                        /*InBaseClass=*/false, Consumer, Visited);
 }
 
-//----------------------------------------------------------------------------
+/// LookupOrCreateLabel - Do a name lookup of a label with the specified name.
+/// If isLocalLabel is true, then this is a definition of an __label__ label
+/// name, otherwise it is a normal label definition or use.
+LabelDecl *Sema::LookupOrCreateLabel(IdentifierInfo *II, SourceLocation Loc,
+                                     bool isLocalLabel) {
+  // Do a lookup to see if we have a label with this name already.
+  NamedDecl *Res = 0;
+  
+  // Local label definitions always shadow existing labels.
+  if (!isLocalLabel)
+    Res = LookupSingleName(CurScope, II, Loc, LookupLabel, NotForRedeclaration);
+  
+  // If we found a label, check to see if it is in the same context as us.  When
+  // in a Block, we don't want to reuse a label in an enclosing function.
+  if (Res && Res->getDeclContext() != CurContext)
+    Res = 0;
+  
+  if (Res == 0) {
+    // If not forward referenced or defined already, create the backing decl.
+    Res = LabelDecl::Create(Context, CurContext, Loc, II);
+    Scope *S = isLocalLabel ? CurScope : CurScope->getFnParent();
+    assert(S && "Not in a function?");
+    PushOnScopeChains(Res, S, true);
+  }
+  
+  return cast<LabelDecl>(Res);
+}
+
+//===----------------------------------------------------------------------===//
 // Typo correction
-//----------------------------------------------------------------------------
+//===----------------------------------------------------------------------===//
 
 namespace {
 class TypoCorrectionConsumer : public VisibleDeclConsumer {
@@ -2682,46 +2799,44 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
 
   /// \brief The results found that have the smallest edit distance
   /// found (so far) with the typo name.
-  llvm::SmallVector<NamedDecl *, 4> BestResults;
+  ///
+  /// The boolean value indicates whether there is a keyword with this name.
+  llvm::StringMap<bool, llvm::BumpPtrAllocator> BestResults;
 
-  /// \brief The keywords that have the smallest edit distance.
-  llvm::SmallVector<IdentifierInfo *, 4> BestKeywords;
-  
   /// \brief The best edit distance found so far.
   unsigned BestEditDistance;
-  
+
 public:
   explicit TypoCorrectionConsumer(IdentifierInfo *Typo)
-    : Typo(Typo->getName()) { }
+    : Typo(Typo->getName()),
+      BestEditDistance((std::numeric_limits<unsigned>::max)()) { }
 
   virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, bool InBaseClass);
+  void FoundName(llvm::StringRef Name);
   void addKeywordResult(ASTContext &Context, llvm::StringRef Keyword);
 
-  typedef llvm::SmallVector<NamedDecl *, 4>::const_iterator iterator;
-  iterator begin() const { return BestResults.begin(); }
-  iterator end() const { return BestResults.end(); }
-  void clear_decls() { BestResults.clear(); }
-  
-  bool empty() const { return BestResults.empty() && BestKeywords.empty(); }
-
-  typedef llvm::SmallVector<IdentifierInfo *, 4>::const_iterator
-    keyword_iterator;
-  keyword_iterator keyword_begin() const { return BestKeywords.begin(); }
-  keyword_iterator keyword_end() const { return BestKeywords.end(); }
-  bool keyword_empty() const { return BestKeywords.empty(); }
-  unsigned keyword_size() const { return BestKeywords.size(); }
-  
-  unsigned getBestEditDistance() const { return BestEditDistance; }  
+  typedef llvm::StringMap<bool, llvm::BumpPtrAllocator>::iterator iterator;
+  iterator begin() { return BestResults.begin(); }
+  iterator end()  { return BestResults.end(); }
+  void erase(iterator I) { BestResults.erase(I); }
+  unsigned size() const { return BestResults.size(); }
+  bool empty() const { return BestResults.empty(); }
+
+  bool &operator[](llvm::StringRef Name) {
+    return BestResults[Name];
+  }
+
+  unsigned getBestEditDistance() const { return BestEditDistance; }
 };
 
 }
 
-void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding, 
+void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding,
                                        bool InBaseClass) {
   // Don't consider hidden names for typo correction.
   if (Hiding)
     return;
-  
+
   // Only consider entities with identifiers for names, ignoring
   // special names (constructors, overloaded operators, selectors,
   // etc.).
@@ -2729,48 +2844,114 @@ void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding,
   if (!Name)
     return;
 
+  FoundName(Name->getName());
+}
+
+void TypoCorrectionConsumer::FoundName(llvm::StringRef Name) {
+  using namespace std;
+
+  // Use a simple length-based heuristic to determine the minimum possible
+  // edit distance. If the minimum isn't good enough, bail out early.
+  unsigned MinED = abs((int)Name.size() - (int)Typo.size());
+  if (MinED > BestEditDistance || (MinED && Typo.size() / MinED < 3))
+    return;
+
+  // Compute an upper bound on the allowable edit distance, so that the
+  // edit-distance algorithm can short-circuit.
+  unsigned UpperBound = min(unsigned((Typo.size() + 2) / 3), BestEditDistance);
+
   // Compute the edit distance between the typo and the name of this
   // entity. If this edit distance is not worse than the best edit
   // distance we've seen so far, add it to the list of results.
-  unsigned ED = Typo.edit_distance(Name->getName());
-  if (!BestResults.empty() || !BestKeywords.empty()) {
-    if (ED < BestEditDistance) {
-      // This result is better than any we've seen before; clear out
-      // the previous results.
-      BestResults.clear();
-      BestKeywords.clear();
-      BestEditDistance = ED;
-    } else if (ED > BestEditDistance) {
-      // This result is worse than the best results we've seen so far;
-      // ignore it.
-      return;
-    }
-  } else
+  unsigned ED = Typo.edit_distance(Name, true, UpperBound);
+  if (ED == 0)
+    return;
+
+  if (ED < BestEditDistance) {
+    // This result is better than any we've seen before; clear out
+    // the previous results.
+    BestResults.clear();
     BestEditDistance = ED;
+  } else if (ED > BestEditDistance) {
+    // This result is worse than the best results we've seen so far;
+    // ignore it.
+    return;
+  }
 
-  BestResults.push_back(ND);
+  // Add this name to the list of results. By not assigning a value, we
+  // keep the current value if we've seen this name before (either as a
+  // keyword or as a declaration), or get the default value (not a keyword)
+  // if we haven't seen it before.
+  (void)BestResults[Name];
 }
 
-void TypoCorrectionConsumer::addKeywordResult(ASTContext &Context, 
+void TypoCorrectionConsumer::addKeywordResult(ASTContext &Context,
                                               llvm::StringRef Keyword) {
   // Compute the edit distance between the typo and this keyword.
   // If this edit distance is not worse than the best edit
   // distance we've seen so far, add it to the list of results.
   unsigned ED = Typo.edit_distance(Keyword);
-  if (!BestResults.empty() || !BestKeywords.empty()) {
-    if (ED < BestEditDistance) {
-      BestResults.clear();
-      BestKeywords.clear();
-      BestEditDistance = ED;
-    } else if (ED > BestEditDistance) {
-      // This result is worse than the best results we've seen so far;
-      // ignore it.
-      return;
-    }
-  } else
+  if (ED < BestEditDistance) {
+    BestResults.clear();
     BestEditDistance = ED;
-  
-  BestKeywords.push_back(&Context.Idents.get(Keyword));
+  } else if (ED > BestEditDistance) {
+    // This result is worse than the best results we've seen so far;
+    // ignore it.
+    return;
+  }
+
+  BestResults[Keyword] = true;
+}
+
+/// \brief Perform name lookup for a possible result for typo correction.
+static void LookupPotentialTypoResult(Sema &SemaRef,
+                                      LookupResult &Res,
+                                      IdentifierInfo *Name,
+                                      Scope *S, CXXScopeSpec *SS,
+                                      DeclContext *MemberContext,
+                                      bool EnteringContext,
+                                      Sema::CorrectTypoContext CTC) {
+  Res.suppressDiagnostics();
+  Res.clear();
+  Res.setLookupName(Name);
+  if (MemberContext) {
+    if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(MemberContext)) {
+      if (CTC == Sema::CTC_ObjCIvarLookup) {
+        if (ObjCIvarDecl *Ivar = Class->lookupInstanceVariable(Name)) {
+          Res.addDecl(Ivar);
+          Res.resolveKind();
+          return;
+        }
+      }
+
+      if (ObjCPropertyDecl *Prop = Class->FindPropertyDeclaration(Name)) {
+        Res.addDecl(Prop);
+        Res.resolveKind();
+        return;
+      }
+    }
+
+    SemaRef.LookupQualifiedName(Res, MemberContext);
+    return;
+  }
+
+  SemaRef.LookupParsedName(Res, S, SS, /*AllowBuiltinCreation=*/false,
+                           EnteringContext);
+
+  // Fake ivar lookup; this should really be part of
+  // LookupParsedName.
+  if (ObjCMethodDecl *Method = SemaRef.getCurMethodDecl()) {
+    if (Method->isInstanceMethod() && Method->getClassInterface() &&
+        (Res.empty() ||
+         (Res.isSingleResult() &&
+          Res.getFoundDecl()->isDefinedOutsideFunctionOrMethod()))) {
+       if (ObjCIvarDecl *IV
+             = Method->getClassInterface()->lookupInstanceVariable(Name)) {
+         Res.addDecl(IV);
+         Res.resolveKind();
+       }
+     }
+  }
 }
 
 /// \brief Try to "correct" a typo in the source code by finding
@@ -2790,7 +2971,7 @@ void TypoCorrectionConsumer::addKeywordResult(ASTContext &Context,
 /// \param MemberContext if non-NULL, the context in which to look for
 /// a member access expression.
 ///
-/// \param EnteringContext whether we're entering the context described by 
+/// \param EnteringContext whether we're entering the context described by
 /// the nested-name-specifier SS.
 ///
 /// \param CTC The context in which typo correction occurs, which impacts the
@@ -2804,21 +2985,13 @@ void TypoCorrectionConsumer::addKeywordResult(ASTContext &Context,
 /// may contain the results of name lookup for the correct name or it may be
 /// empty.
 DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
-                                  DeclContext *MemberContext, 
+                                  DeclContext *MemberContext,
                                   bool EnteringContext,
                                   CorrectTypoContext CTC,
                                   const ObjCObjectPointerType *OPT) {
   if (Diags.hasFatalErrorOccurred() || !getLangOptions().SpellChecking)
     return DeclarationName();
 
-  // Provide a stop gap for files that are just seriously broken.  Trying
-  // to correct all typos can turn into a HUGE performance penalty, causing
-  // some files to take minutes to get rejected by the parser.
-  // FIXME: Is this the right solution?
-  if (TyposCorrected == 20)
-    return DeclarationName();
-  ++TyposCorrected;
-  
   // We only attempt to correct typos for identifiers.
   IdentifierInfo *Typo = Res.getLookupName().getAsIdentifierInfo();
   if (!Typo)
@@ -2833,17 +3006,18 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
   // instantiation.
   if (!ActiveTemplateInstantiations.empty())
     return DeclarationName();
-  
+
   TypoCorrectionConsumer Consumer(Typo);
-  
+
   // Perform name lookup to find visible, similarly-named entities.
+  bool IsUnqualifiedLookup = false;
   if (MemberContext) {
     LookupVisibleDecls(MemberContext, Res.getLookupKind(), Consumer);
 
     // Look in qualified interfaces.
     if (OPT) {
-      for (ObjCObjectPointerType::qual_iterator 
-             I = OPT->qual_begin(), E = OPT->qual_end(); 
+      for (ObjCObjectPointerType::qual_iterator
+             I = OPT->qual_begin(), E = OPT->qual_end();
            I != E; ++I)
         LookupVisibleDecls(*I, Res.getLookupKind(), Consumer);
     }
@@ -2851,10 +3025,54 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
     DeclContext *DC = computeDeclContext(*SS, EnteringContext);
     if (!DC)
       return DeclarationName();
-    
+
+    // Provide a stop gap for files that are just seriously broken.  Trying
+    // to correct all typos can turn into a HUGE performance penalty, causing
+    // some files to take minutes to get rejected by the parser.
+    if (TyposCorrected + UnqualifiedTyposCorrected.size() >= 20)
+      return DeclarationName();
+    ++TyposCorrected;
+
     LookupVisibleDecls(DC, Res.getLookupKind(), Consumer);
   } else {
-    LookupVisibleDecls(S, Res.getLookupKind(), Consumer);
+    IsUnqualifiedLookup = true;
+    UnqualifiedTyposCorrectedMap::iterator Cached
+      = UnqualifiedTyposCorrected.find(Typo);
+    if (Cached == UnqualifiedTyposCorrected.end()) {
+      // Provide a stop gap for files that are just seriously broken.  Trying
+      // to correct all typos can turn into a HUGE performance penalty, causing
+      // some files to take minutes to get rejected by the parser.
+      if (TyposCorrected + UnqualifiedTyposCorrected.size() >= 20)
+        return DeclarationName();
+
+      // For unqualified lookup, look through all of the names that we have
+      // seen in this translation unit.
+      for (IdentifierTable::iterator I = Context.Idents.begin(),
+                                  IEnd = Context.Idents.end();
+           I != IEnd; ++I)
+        Consumer.FoundName(I->getKey());
+
+      // Walk through identifiers in external identifier sources.
+      if (IdentifierInfoLookup *External
+                              = Context.Idents.getExternalIdentifierLookup()) {
+        llvm::OwningPtr<IdentifierIterator> Iter(External->getIdentifiers());
+        do {
+          llvm::StringRef Name = Iter->Next();
+          if (Name.empty())
+            break;
+
+          Consumer.FoundName(Name);
+        } while (true);
+      }
+    } else {
+      // Use the cached value, unless it's a keyword. In the keyword case, we'll
+      // end up adding the keyword below.
+      if (Cached->second.first.empty())
+        return DeclarationName();
+
+      if (!Cached->second.second)
+        Consumer.FoundName(Cached->second.first);
+    }
   }
 
   // Add context-dependent keywords.
@@ -2868,39 +3086,46 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
       WantExpressionKeywords = true;
       WantCXXNamedCasts = true;
       WantRemainingKeywords = true;
-      
+
       if (ObjCMethodDecl *Method = getCurMethodDecl())
         if (Method->getClassInterface() &&
             Method->getClassInterface()->getSuperClass())
           Consumer.addKeywordResult(Context, "super");
-      
+
       break;
-  
+
     case CTC_NoKeywords:
       break;
-  
+
     case CTC_Type:
       WantTypeSpecifiers = true;
       break;
-      
+
     case CTC_ObjCMessageReceiver:
       Consumer.addKeywordResult(Context, "super");
       // Fall through to handle message receivers like expressions.
-      
+
     case CTC_Expression:
       if (getLangOptions().CPlusPlus)
         WantTypeSpecifiers = true;
       WantExpressionKeywords = true;
       // Fall through to get C++ named casts.
-      
+
     case CTC_CXXCasts:
       WantCXXNamedCasts = true;
       break;
-      
+
+    case CTC_ObjCPropertyLookup:
+      // FIXME: Add "isa"?
+      break;
+
     case CTC_MemberLookup:
       if (getLangOptions().CPlusPlus)
         Consumer.addKeywordResult(Context, "template");
       break;
+
+    case CTC_ObjCIvarLookup:
+      break;
   }
 
   if (WantTypeSpecifiers) {
@@ -2912,67 +3137,67 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
       // storage-specifiers as well
       "extern", "inline", "static", "typedef"
     };
-    
+
     const unsigned NumCTypeSpecs = sizeof(CTypeSpecs) / sizeof(CTypeSpecs[0]);
     for (unsigned I = 0; I != NumCTypeSpecs; ++I)
       Consumer.addKeywordResult(Context, CTypeSpecs[I]);
-    
+
     if (getLangOptions().C99)
       Consumer.addKeywordResult(Context, "restrict");
     if (getLangOptions().Bool || getLangOptions().CPlusPlus)
       Consumer.addKeywordResult(Context, "bool");
-    
+
     if (getLangOptions().CPlusPlus) {
       Consumer.addKeywordResult(Context, "class");
       Consumer.addKeywordResult(Context, "typename");
       Consumer.addKeywordResult(Context, "wchar_t");
-      
+
       if (getLangOptions().CPlusPlus0x) {
         Consumer.addKeywordResult(Context, "char16_t");
         Consumer.addKeywordResult(Context, "char32_t");
         Consumer.addKeywordResult(Context, "constexpr");
         Consumer.addKeywordResult(Context, "decltype");
         Consumer.addKeywordResult(Context, "thread_local");
-      }      
+      }
     }
-        
+
     if (getLangOptions().GNUMode)
       Consumer.addKeywordResult(Context, "typeof");
   }
-  
+
   if (WantCXXNamedCasts && getLangOptions().CPlusPlus) {
     Consumer.addKeywordResult(Context, "const_cast");
     Consumer.addKeywordResult(Context, "dynamic_cast");
     Consumer.addKeywordResult(Context, "reinterpret_cast");
     Consumer.addKeywordResult(Context, "static_cast");
   }
-  
+
   if (WantExpressionKeywords) {
     Consumer.addKeywordResult(Context, "sizeof");
     if (getLangOptions().Bool || getLangOptions().CPlusPlus) {
       Consumer.addKeywordResult(Context, "false");
       Consumer.addKeywordResult(Context, "true");
     }
-    
+
     if (getLangOptions().CPlusPlus) {
-      const char *CXXExprs[] = { 
-        "delete", "new", "operator", "throw", "typeid" 
+      const char *CXXExprs[] = {
+        "delete", "new", "operator", "throw", "typeid"
       };
       const unsigned NumCXXExprs = sizeof(CXXExprs) / sizeof(CXXExprs[0]);
       for (unsigned I = 0; I != NumCXXExprs; ++I)
         Consumer.addKeywordResult(Context, CXXExprs[I]);
-      
+
       if (isa<CXXMethodDecl>(CurContext) &&
           cast<CXXMethodDecl>(CurContext)->isInstance())
         Consumer.addKeywordResult(Context, "this");
-      
+
       if (getLangOptions().CPlusPlus0x) {
         Consumer.addKeywordResult(Context, "alignof");
         Consumer.addKeywordResult(Context, "nullptr");
       }
     }
   }
-  
+
   if (WantRemainingKeywords) {
     if (getCurFunctionOrMethodDecl() || getCurBlock()) {
       // Statements.
@@ -2981,18 +3206,18 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
       const unsigned NumCStmts = sizeof(CStmts) / sizeof(CStmts[0]);
       for (unsigned I = 0; I != NumCStmts; ++I)
         Consumer.addKeywordResult(Context, CStmts[I]);
-      
+
       if (getLangOptions().CPlusPlus) {
         Consumer.addKeywordResult(Context, "catch");
         Consumer.addKeywordResult(Context, "try");
       }
-      
+
       if (S && S->getBreakParent())
         Consumer.addKeywordResult(Context, "break");
-      
+
       if (S && S->getContinueParent())
         Consumer.addKeywordResult(Context, "continue");
-      
+
       if (!getCurFunction()->SwitchStack.empty()) {
         Consumer.addKeywordResult(Context, "case");
         Consumer.addKeywordResult(Context, "default");
@@ -3013,7 +3238,7 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
         Consumer.addKeywordResult(Context, "virtual");
       }
     }
-        
+
     if (getLangOptions().CPlusPlus) {
       Consumer.addKeywordResult(Context, "using");
 
@@ -3021,122 +3246,134 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
         Consumer.addKeywordResult(Context, "static_assert");
     }
   }
-  
+
   // If we haven't found anything, we're done.
-  if (Consumer.empty())
+  if (Consumer.empty()) {
+    // If this was an unqualified lookup, note that no correction was found.
+    if (IsUnqualifiedLookup)
+      (void)UnqualifiedTyposCorrected[Typo];
+
     return DeclarationName();
+  }
 
-  // Only allow a single, closest name in the result set (it's okay to
-  // have overloads of that name, though).
-  DeclarationName BestName;
-  NamedDecl *BestIvarOrPropertyDecl = 0;
-  bool FoundIvarOrPropertyDecl = false;
-  
-  // Check all of the declaration results to find the best name so far.
-  for (TypoCorrectionConsumer::iterator I = Consumer.begin(), 
+  // Make sure that the user typed at least 3 characters for each correction
+  // made. Otherwise, we don't even both looking at the results.
+
+  // We also suppress exact matches; those should be handled by a
+  // different mechanism (e.g., one that introduces qualification in
+  // C++).
+  unsigned ED = Consumer.getBestEditDistance();
+  if (ED > 0 && Typo->getName().size() / ED < 3) {
+    // If this was an unqualified lookup, note that no correction was found.
+    if (IsUnqualifiedLookup)
+      (void)UnqualifiedTyposCorrected[Typo];
+
+    return DeclarationName();
+  }
+
+  // Weed out any names that could not be found by name lookup.
+  bool LastLookupWasAccepted = false;
+  for (TypoCorrectionConsumer::iterator I = Consumer.begin(),
                                      IEnd = Consumer.end();
-       I != IEnd; ++I) {
-    if (!BestName)
-      BestName = (*I)->getDeclName();
-    else if (BestName != (*I)->getDeclName())
-      return DeclarationName();
+       I != IEnd; /* Increment in loop. */) {
+    // Keywords are always found.
+    if (I->second) {
+      ++I;
+      continue;
+    }
+
+    // Perform name lookup on this name.
+    IdentifierInfo *Name = &Context.Idents.get(I->getKey());
+    LookupPotentialTypoResult(*this, Res, Name, S, SS, MemberContext,
+                              EnteringContext, CTC);
 
-    // \brief Keep track of either an Objective-C ivar or a property, but not
-    // both.
-    if (isa<ObjCIvarDecl>(*I) || isa<ObjCPropertyDecl>(*I)) {
-      if (FoundIvarOrPropertyDecl)
-        BestIvarOrPropertyDecl = 0;
-      else {
-        BestIvarOrPropertyDecl = *I;
-        FoundIvarOrPropertyDecl = true;
+    switch (Res.getResultKind()) {
+    case LookupResult::NotFound:
+    case LookupResult::NotFoundInCurrentInstantiation:
+    case LookupResult::Ambiguous:
+      // We didn't find this name in our scope, or didn't like what we found;
+      // ignore it.
+      Res.suppressDiagnostics();
+      {
+        TypoCorrectionConsumer::iterator Next = I;
+        ++Next;
+        Consumer.erase(I);
+        I = Next;
       }
+      LastLookupWasAccepted = false;
+      break;
+
+    case LookupResult::Found:
+    case LookupResult::FoundOverloaded:
+    case LookupResult::FoundUnresolvedValue:
+      ++I;
+      LastLookupWasAccepted = true;
+      break;
+    }
+
+    if (Res.isAmbiguous()) {
+      // We don't deal with ambiguities.
+      Res.suppressDiagnostics();
+      Res.clear();
+      return DeclarationName();
     }
   }
 
-  // Now check all of the keyword results to find the best name. 
-  switch (Consumer.keyword_size()) {
-    case 0:
-      // No keywords matched.
-      break;
-      
-    case 1:
-      // If we already have a name
-      if (!BestName) {
-        // We did not have anything previously, 
-        BestName = *Consumer.keyword_begin();
-      } else if (BestName.getAsIdentifierInfo() == *Consumer.keyword_begin()) {
-        // We have a declaration with the same name as a context-sensitive
-        // keyword. The keyword takes precedence.
-        BestIvarOrPropertyDecl = 0;
-        FoundIvarOrPropertyDecl = false;
-        Consumer.clear_decls();
-      } else if (CTC == CTC_ObjCMessageReceiver &&
-                 (*Consumer.keyword_begin())->isStr("super")) {
-        // In an Objective-C message send, give the "super" keyword a slight
-        // edge over entities not in function or method scope.
-        for (TypoCorrectionConsumer::iterator I = Consumer.begin(), 
-                                           IEnd = Consumer.end();
-             I != IEnd; ++I) {
-          if ((*I)->getDeclName() == BestName) {
-            if ((*I)->getDeclContext()->isFunctionOrMethod())
-              return DeclarationName();
-          }
-        }
-        
-        // Everything found was outside a function or method; the 'super'
-        // keyword takes precedence.
-        BestIvarOrPropertyDecl = 0;
-        FoundIvarOrPropertyDecl = false;
-        Consumer.clear_decls();        
-        BestName = *Consumer.keyword_begin();
-      } else {
-        // Name collision; we will not correct typos.
+  // If only a single name remains, return that result.
+  if (Consumer.size() == 1) {
+    IdentifierInfo *Name = &Context.Idents.get(Consumer.begin()->getKey());
+    if (Consumer.begin()->second) {
+      Res.suppressDiagnostics();
+      Res.clear();
+
+      // Don't correct to a keyword that's the same as the typo; the keyword
+      // wasn't actually in scope.
+      if (ED == 0) {
+        Res.setLookupName(Typo);
         return DeclarationName();
       }
-      break;
-      
-    default:
-      // Name collision; we will not correct typos.
-      return DeclarationName();
-  }
-  
-  // BestName is the closest viable name to what the user
-  // typed. However, to make sure that we don't pick something that's
-  // way off, make sure that the user typed at least 3 characters for
-  // each correction.
-  unsigned ED = Consumer.getBestEditDistance();
-  if (ED == 0 || !BestName.getAsIdentifierInfo() ||
-      (BestName.getAsIdentifierInfo()->getName().size() / ED) < 3)
-    return DeclarationName();
 
-  // Perform name lookup again with the name we chose, and declare
-  // success if we found something that was not ambiguous.
-  Res.clear();
-  Res.setLookupName(BestName);
-
-  // If we found an ivar or property, add that result; no further
-  // lookup is required.
-  if (BestIvarOrPropertyDecl)
-    Res.addDecl(BestIvarOrPropertyDecl);  
-  // If we're looking into the context of a member, perform qualified
-  // name lookup on the best name.
-  else if (!Consumer.keyword_empty()) {
-    // The best match was a keyword. Return it.
-    return BestName;
-  } else if (MemberContext)
-    LookupQualifiedName(Res, MemberContext);
-  // Perform lookup as if we had just parsed the best name.
-  else
-    LookupParsedName(Res, S, SS, /*AllowBuiltinCreation=*/false, 
-                     EnteringContext);
+    } else if (!LastLookupWasAccepted) {
+      // Perform name lookup on this name.
+      LookupPotentialTypoResult(*this, Res, Name, S, SS, MemberContext,
+                                EnteringContext, CTC);
+    }
 
-  if (Res.isAmbiguous()) {
+    // Record the correction for unqualified lookup.
+    if (IsUnqualifiedLookup)
+      UnqualifiedTyposCorrected[Typo]
+        = std::make_pair(Name->getName(), Consumer.begin()->second);
+
+    return &Context.Idents.get(Consumer.begin()->getKey());
+  }
+  else if (Consumer.size() > 1 && CTC == CTC_ObjCMessageReceiver
+           && Consumer["super"]) {
+    // Prefix 'super' when we're completing in a message-receiver
+    // context.
     Res.suppressDiagnostics();
-    return DeclarationName();
+    Res.clear();
+
+    // Don't correct to a keyword that's the same as the typo; the keyword
+    // wasn't actually in scope.
+    if (ED == 0) {
+      Res.setLookupName(Typo);
+      return DeclarationName();
+    }
+
+    // Record the correction for unqualified lookup.
+    if (IsUnqualifiedLookup)
+      UnqualifiedTyposCorrected[Typo]
+        = std::make_pair("super", Consumer.begin()->second);
+
+    return &Context.Idents.get("super");
   }
 
-  if (Res.getResultKind() != LookupResult::NotFound)
-    return BestName;
-  
+  Res.suppressDiagnostics();
+  Res.setLookupName(Typo);
+  Res.clear();
+  // Record the correction for unqualified lookup.
+  if (IsUnqualifiedLookup)
+    (void)UnqualifiedTyposCorrected[Typo];
+
   return DeclarationName();
 }
diff --git a/lib/Sema/SemaObjCProperty.cpp b/lib/Sema/SemaObjCProperty.cpp
index 7181d58f7fb0..b086ca751406 100644
--- a/lib/Sema/SemaObjCProperty.cpp
+++ b/lib/Sema/SemaObjCProperty.cpp
@@ -31,7 +31,8 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
                           Selector SetterSel,
                           Decl *ClassCategory,
                           bool *isOverridingProperty,
-                          tok::ObjCKeywordKind MethodImplKind) {
+                          tok::ObjCKeywordKind MethodImplKind,
+                          DeclContext *lexicalDC) {
   unsigned Attributes = ODS.getPropertyAttributes();
   bool isReadWrite = ((Attributes & ObjCDeclSpec::DQ_PR_readwrite) ||
                       // default is readwrite!
@@ -70,6 +71,9 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
                                  GetterSel, SetterSel,
                                  isAssign, isReadWrite,
                                  Attributes, TSI, MethodImplKind);
+  if (lexicalDC)
+    Res->setLexicalDeclContext(lexicalDC);
+
   // Validate the attributes on the @property.
   CheckObjCPropertyAttributes(Res, AtLoc, Attributes);
   return Res;
@@ -89,16 +93,25 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
   // Diagnose if this property is already in continuation class.
   DeclContext *DC = cast<DeclContext>(CDecl);
   IdentifierInfo *PropertyId = FD.D.getIdentifier();
-
-  if (ObjCPropertyDecl *prevDecl =
-        ObjCPropertyDecl::findPropertyDecl(DC, PropertyId)) {
-    Diag(AtLoc, diag::err_duplicate_property);
-    Diag(prevDecl->getLocation(), diag::note_property_declare);
-    return 0;
-  }
-
+  ObjCInterfaceDecl *CCPrimary = CDecl->getClassInterface();
+  
+  if (CCPrimary)
+    // Check for duplicate declaration of this property in current and
+    // other class extensions.
+    for (const ObjCCategoryDecl *ClsExtDecl = 
+         CCPrimary->getFirstClassExtension();
+         ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension()) {
+      if (ObjCPropertyDecl *prevDecl =
+          ObjCPropertyDecl::findPropertyDecl(ClsExtDecl, PropertyId)) {
+        Diag(AtLoc, diag::err_duplicate_property);
+        Diag(prevDecl->getLocation(), diag::note_property_declare);
+        return 0;
+      }
+    }
+  
   // Create a new ObjCPropertyDecl with the DeclContext being
   // the class extension.
+  // FIXME. We should really be using CreatePropertyDecl for this.
   ObjCPropertyDecl *PDecl =
     ObjCPropertyDecl::Create(Context, DC, FD.D.getIdentifierLoc(),
                              PropertyId, AtLoc, T);
@@ -106,12 +119,13 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_readonly);
   if (Attributes & ObjCDeclSpec::DQ_PR_readwrite)
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_readwrite);
-
+  // Set setter/getter selector name. Needed later.
+  PDecl->setGetterName(GetterSel);
+  PDecl->setSetterName(SetterSel);
   DC->addDecl(PDecl);
 
   // We need to look in the @interface to see if the @property was
   // already declared.
-  ObjCInterfaceDecl *CCPrimary = CDecl->getClassInterface();
   if (!CCPrimary) {
     Diag(CDecl->getLocation(), diag::err_continuation_class);
     *isOverridingProperty = true;
@@ -137,9 +151,9 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
     // A case of continuation class adding a new property in the class. This
     // is not what it was meant for. However, gcc supports it and so should we.
     // Make sure setter/getters are declared here.
-    ProcessPropertyDecl(PDecl, CCPrimary);
+    ProcessPropertyDecl(PDecl, CCPrimary, /* redeclaredProperty = */ 0,
+                        /* lexicalDC = */ CDecl);
     return PDecl;
-
   }
 
   // The property 'PIDecl's readonly attribute will be over-ridden
@@ -172,7 +186,8 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
                       PIDecl->getGetterName(),
                       PIDecl->getSetterName(),
                       CCPrimary, isOverridingProperty,
-                      MethodImplKind);
+                      MethodImplKind,
+                      /* lexicalDC = */ CDecl);
       PIDecl = cast<ObjCPropertyDecl>(ProtocolPtrTy);
     }
     PIDecl->makeitReadWriteAttribute();
@@ -182,13 +197,22 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
       PIDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_copy);
     PIDecl->setSetterName(SetterSel);
   } else {
-    Diag(AtLoc, diag::err_use_continuation_class)
+    // Tailor the diagnostics for the common case where a readwrite
+    // property is declared both in the @interface and the continuation.
+    // This is a common error where the user often intended the original
+    // declaration to be readonly.
+    unsigned diag =
+      (Attributes & ObjCDeclSpec::DQ_PR_readwrite) &&
+      (PIkind & ObjCPropertyDecl::OBJC_PR_readwrite)
+      ? diag::err_use_continuation_class_redeclaration_readwrite
+      : diag::err_use_continuation_class;
+    Diag(AtLoc, diag)
       << CCPrimary->getDeclName();
     Diag(PIDecl->getLocation(), diag::note_property_declare);
   }
   *isOverridingProperty = true;
   // Make sure setter decl is synthesized, and added to primary class's list.
-  ProcessPropertyDecl(PIDecl, CCPrimary);
+  ProcessPropertyDecl(PIDecl, CCPrimary, PDecl, CDecl);
   return 0;
 }
 
@@ -275,6 +299,8 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
 
   if (Attributes & ObjCDeclSpec::DQ_PR_nonatomic)
     PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_nonatomic);
+  else if (Attributes & ObjCDeclSpec::DQ_PR_atomic)
+    PDecl->setPropertyAttributes(ObjCPropertyDecl::OBJC_PR_atomic);
 
   PDecl->setPropertyAttributesAsWritten(PDecl->getPropertyAttributes());
   
@@ -297,7 +323,8 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                   bool Synthesize,
                                   Decl *ClassCatImpDecl,
                                   IdentifierInfo *PropertyId,
-                                  IdentifierInfo *PropertyIvar) {
+                                  IdentifierInfo *PropertyIvar,
+                                  SourceLocation PropertyIvarLoc) {
   ObjCContainerDecl *ClassImpDecl =
     cast_or_null<ObjCContainerDecl>(ClassCatImpDecl);
   // Make sure we have a context for the property implementation declaration.
@@ -324,6 +351,16 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       Diag(PropertyLoc, diag::error_bad_property_decl) << IDecl->getDeclName();
       return 0;
     }
+    unsigned PIkind = property->getPropertyAttributesAsWritten();
+    if ((PIkind & (ObjCPropertyDecl::OBJC_PR_atomic |
+                   ObjCPropertyDecl::OBJC_PR_nonatomic) ) == 0) {
+      if (AtLoc.isValid())
+        Diag(AtLoc, diag::warn_implicit_atomic_property);
+      else
+        Diag(IC->getLocation(), diag::warn_auto_implicit_atomic_property);
+      Diag(property->getLocation(), diag::note_property_declare);
+    }
+    
     if (const ObjCCategoryDecl *CD =
         dyn_cast<ObjCCategoryDecl>(property->getDeclContext())) {
       if (!CD->IsClassExtension()) {
@@ -373,7 +410,7 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     if (!Ivar) {
       Ivar = ObjCIvarDecl::Create(Context, ClassImpDecl, PropertyLoc,
                                   PropertyIvar, PropType, /*Dinfo=*/0,
-                                  ObjCIvarDecl::Protected,
+                                  ObjCIvarDecl::Private,
                                   (Expr *)0, true);
       ClassImpDecl->addDecl(Ivar);
       IDecl->makeDeclVisibleInContext(Ivar, false);
@@ -403,8 +440,13 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
           Context.canAssignObjCInterfaces(
                                   PropType->getAs<ObjCObjectPointerType>(),
                                   IvarType->getAs<ObjCObjectPointerType>());
-      else 
-        compat = (CheckAssignmentConstraints(PropType, IvarType) == Compatible);
+      else {
+        SourceLocation Loc = PropertyIvarLoc;
+        if (Loc.isInvalid())
+          Loc = PropertyLoc;
+        compat = (CheckAssignmentConstraints(Loc, PropType, IvarType)
+                    == Compatible);
+      }
       if (!compat) {
         Diag(PropertyLoc, diag::error_property_ivar_type)
           << property->getDeclName() << PropType
@@ -452,17 +494,18 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
                                (Synthesize ?
                                 ObjCPropertyImplDecl::Synthesize
                                 : ObjCPropertyImplDecl::Dynamic),
-                               Ivar);
+                               Ivar, PropertyIvarLoc);
   if (ObjCMethodDecl *getterMethod = property->getGetterMethodDecl()) {
     getterMethod->createImplicitParams(Context, IDecl);
-    if (getLangOptions().CPlusPlus && Synthesize) {
+    if (getLangOptions().CPlusPlus && Synthesize &&
+        Ivar->getType()->isRecordType()) {
       // For Objective-C++, need to synthesize the AST for the IVAR object to be
       // returned by the getter as it must conform to C++'s copy-return rules.
       // FIXME. Eventually we want to do this for Objective-C as well.
       ImplicitParamDecl *SelfDecl = getterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = 
-        new (Context) DeclRefExpr(SelfDecl,SelfDecl->getType(),
-                                  SourceLocation());
+        new (Context) DeclRefExpr(SelfDecl, SelfDecl->getType(),
+                                  VK_RValue, SourceLocation());
       Expr *IvarRefExpr =
         new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), AtLoc,
                                       SelfExpr, true, true);
@@ -476,27 +519,28 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       if (!Res.isInvalid()) {
         Expr *ResExpr = Res.takeAs<Expr>();
         if (ResExpr)
-          ResExpr = MaybeCreateCXXExprWithTemporaries(ResExpr);
+          ResExpr = MaybeCreateExprWithCleanups(ResExpr);
         PIDecl->setGetterCXXConstructor(ResExpr);
       }
     }
   }
   if (ObjCMethodDecl *setterMethod = property->getSetterMethodDecl()) {
     setterMethod->createImplicitParams(Context, IDecl);
-    if (getLangOptions().CPlusPlus && Synthesize) {
+    if (getLangOptions().CPlusPlus && Synthesize
+        && Ivar->getType()->isRecordType()) {
       // FIXME. Eventually we want to do this for Objective-C as well.
       ImplicitParamDecl *SelfDecl = setterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = 
-        new (Context) DeclRefExpr(SelfDecl,SelfDecl->getType(),
-                                  SourceLocation());
+        new (Context) DeclRefExpr(SelfDecl, SelfDecl->getType(),
+                                  VK_RValue, SourceLocation());
       Expr *lhs =
         new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), AtLoc,
                                       SelfExpr, true, true);
       ObjCMethodDecl::param_iterator P = setterMethod->param_begin();
       ParmVarDecl *Param = (*P);
-      Expr *rhs = new (Context) DeclRefExpr(Param,Param->getType(),
-                                            SourceLocation());
-      ExprResult Res = BuildBinOp(S, SourceLocation(), 
+      Expr *rhs = new (Context) DeclRefExpr(Param, Param->getType(),
+                                            VK_LValue, SourceLocation());
+      ExprResult Res = BuildBinOp(S, lhs->getLocEnd(), 
                                   BO_Assign, lhs, rhs);
       PIDecl->setSetterCXXAssignment(Res.takeAs<Expr>());
     }
@@ -519,11 +563,12 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       return 0;
     }
     IC->addPropertyImplementation(PIDecl);
-    if (getLangOptions().ObjCNonFragileABI2) {
+    if (getLangOptions().ObjCDefaultSynthProperties &&
+        getLangOptions().ObjCNonFragileABI2) {
       // Diagnose if an ivar was lazily synthesdized due to a previous
       // use and if 1) property is @dynamic or 2) property is synthesized
       // but it requires an ivar of different name.
-      ObjCInterfaceDecl *ClassDeclared;
+      ObjCInterfaceDecl *ClassDeclared=0;
       ObjCIvarDecl *Ivar = 0;
       if (!Synthesize)
         Ivar = IDecl->lookupInstanceVariable(PropertyId, ClassDeclared);
@@ -622,7 +667,7 @@ bool Sema::DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *property,
       GetterMethod->getResultType() != property->getType()) {
     AssignConvertType result = Incompatible;
     if (property->getType()->isObjCObjectPointerType())
-      result = CheckAssignmentConstraints(GetterMethod->getResultType(),
+      result = CheckAssignmentConstraints(Loc, GetterMethod->getResultType(),
                                           property->getType());
     if (result != Compatible) {
       Diag(Loc, diag::warn_accessor_property_type_mismatch)
@@ -966,10 +1011,16 @@ void Sema::DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
         continue;
     }
 
-    ActOnPropertyImplDecl(S, IMPDecl->getLocation(), IMPDecl->getLocation(),
-                          true, IMPDecl,
-                          Prop->getIdentifier(), Prop->getIdentifier());
-  }    
+
+    // We use invalid SourceLocations for the synthesized ivars since they
+    // aren't really synthesized at a particular location; they just exist.
+    // Saying that they are located at the @implementation isn't really going
+    // to help users.
+    ActOnPropertyImplDecl(S, SourceLocation(), SourceLocation(),
+                          true,IMPDecl,
+                          Prop->getIdentifier(), Prop->getIdentifier(),
+                          SourceLocation());
+  }
 }
 
 void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
@@ -1030,7 +1081,32 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
        E = IDecl->prop_end();
        I != E; ++I) {
     ObjCPropertyDecl *Property = (*I);
+    ObjCMethodDecl *GetterMethod = 0;
+    ObjCMethodDecl *SetterMethod = 0;
+    bool LookedUpGetterSetter = false;
+
     unsigned Attributes = Property->getPropertyAttributes();
+    unsigned AttributesAsWrittern = Property->getPropertyAttributesAsWritten();
+
+    if (!(AttributesAsWrittern & ObjCPropertyDecl::OBJC_PR_atomic) &&
+        !(AttributesAsWrittern & ObjCPropertyDecl::OBJC_PR_nonatomic)) {
+      GetterMethod = IMPDecl->getInstanceMethod(Property->getGetterName());
+      SetterMethod = IMPDecl->getInstanceMethod(Property->getSetterName());
+      LookedUpGetterSetter = true;
+      if (GetterMethod) {
+        Diag(GetterMethod->getLocation(),
+             diag::warn_default_atomic_custom_getter_setter)
+          << Property->getIdentifier() << 0;
+        Diag(Property->getLocation(), diag::note_property_declare);
+      }
+      if (SetterMethod) {
+        Diag(SetterMethod->getLocation(),
+             diag::warn_default_atomic_custom_getter_setter)
+          << Property->getIdentifier() << 1;
+        Diag(Property->getLocation(), diag::note_property_declare);
+      }
+    }
+
     // We only care about readwrite atomic property.
     if ((Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) ||
         !(Attributes & ObjCPropertyDecl::OBJC_PR_readwrite))
@@ -1039,10 +1115,11 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
          = IMPDecl->FindPropertyImplDecl(Property->getIdentifier())) {
       if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
         continue;
-      ObjCMethodDecl *GetterMethod =
-        IMPDecl->getInstanceMethod(Property->getGetterName());
-      ObjCMethodDecl *SetterMethod =
-        IMPDecl->getInstanceMethod(Property->getSetterName());
+      if (!LookedUpGetterSetter) {
+        GetterMethod = IMPDecl->getInstanceMethod(Property->getGetterName());
+        SetterMethod = IMPDecl->getInstanceMethod(Property->getSetterName());
+        LookedUpGetterSetter = true;
+      }
       if ((GetterMethod && !SetterMethod) || (!GetterMethod && SetterMethod)) {
         SourceLocation MethodLoc =
           (GetterMethod ? GetterMethod->getLocation()
@@ -1055,13 +1132,27 @@ Sema::AtomicPropertySetterGetterRules (ObjCImplDecl* IMPDecl,
   }
 }
 
+/// AddPropertyAttrs - Propagates attributes from a property to the
+/// implicitly-declared getter or setter for that property.
+static void AddPropertyAttrs(Sema &S, ObjCMethodDecl *PropertyMethod,
+                             ObjCPropertyDecl *Property) {
+  // Should we just clone all attributes over?
+  if (DeprecatedAttr *A = Property->getAttr<DeprecatedAttr>())
+    PropertyMethod->addAttr(A->clone(S.Context));
+  if (UnavailableAttr *A = Property->getAttr<UnavailableAttr>())
+    PropertyMethod->addAttr(A->clone(S.Context));
+}
+
 /// ProcessPropertyDecl - Make sure that any user-defined setter/getter methods
 /// have the property type and issue diagnostics if they don't.
 /// Also synthesize a getter/setter method if none exist (and update the
 /// appropriate lookup tables. FIXME: Should reconsider if adding synthesized
 /// methods is the "right" thing to do.
 void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
-                               ObjCContainerDecl *CD) {
+                               ObjCContainerDecl *CD,
+                               ObjCPropertyDecl *redeclaredProperty,
+                               ObjCContainerDecl *lexicalDC) {
+
   ObjCMethodDecl *GetterMethod, *SetterMethod;
 
   GetterMethod = CD->getInstanceMethod(property->getGetterName());
@@ -1096,8 +1187,12 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
     // No instance method of same name as property getter name was found.
     // Declare a getter method and add it to the list of methods
     // for this class.
-    GetterMethod = ObjCMethodDecl::Create(Context, property->getLocation(),
-                             property->getLocation(), property->getGetterName(),
+    SourceLocation Loc = redeclaredProperty ? 
+      redeclaredProperty->getLocation() :
+      property->getLocation();
+
+    GetterMethod = ObjCMethodDecl::Create(Context, Loc, Loc,
+                             property->getGetterName(),
                              property->getType(), 0, CD, true, false, true, 
                              false,
                              (property->getPropertyImplementation() ==
@@ -1105,11 +1200,13 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
                              ObjCMethodDecl::Optional :
                              ObjCMethodDecl::Required);
     CD->addDecl(GetterMethod);
+
+    AddPropertyAttrs(*this, GetterMethod, property);
+
     // FIXME: Eventually this shouldn't be needed, as the lexical context
     // and the real context should be the same.
-    if (DeclContext *lexicalDC = property->getLexicalDeclContext())
+    if (lexicalDC)
       GetterMethod->setLexicalDeclContext(lexicalDC);
-
   } else
     // A user declared getter will be synthesize when @synthesize of
     // the property with the same name is seen in the @implementation
@@ -1123,19 +1220,22 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
       // No instance method of same name as property setter name was found.
       // Declare a setter method and add it to the list of methods
       // for this class.
-      SetterMethod = ObjCMethodDecl::Create(Context, property->getLocation(),
-                               property->getLocation(),
-                               property->getSetterName(),
-                               Context.VoidTy, 0, CD, true, false, true,
-                               false,
+      SourceLocation Loc = redeclaredProperty ? 
+        redeclaredProperty->getLocation() :
+        property->getLocation();
+
+      SetterMethod =
+        ObjCMethodDecl::Create(Context, Loc, Loc,
+                               property->getSetterName(), Context.VoidTy, 0,
+                               CD, true, false, true, false,
                                (property->getPropertyImplementation() ==
                                 ObjCPropertyDecl::Optional) ?
-                               ObjCMethodDecl::Optional :
-                               ObjCMethodDecl::Required);
+                                ObjCMethodDecl::Optional :
+                                ObjCMethodDecl::Required);
+
       // Invent the arguments for the setter. We don't bother making a
       // nice name for the argument.
-      ParmVarDecl *Argument = ParmVarDecl::Create(Context, SetterMethod,
-                                                  property->getLocation(),
+      ParmVarDecl *Argument = ParmVarDecl::Create(Context, SetterMethod, Loc,
                                                   property->getIdentifier(),
                                                   property->getType(),
                                                   /*TInfo=*/0,
@@ -1143,10 +1243,13 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
                                                   SC_None,
                                                   0);
       SetterMethod->setMethodParams(Context, &Argument, 1, 1);
+
+      AddPropertyAttrs(*this, SetterMethod, property);
+
       CD->addDecl(SetterMethod);
       // FIXME: Eventually this shouldn't be needed, as the lexical context
       // and the real context should be the same.
-      if (DeclContext *lexicalDC = property->getLexicalDeclContext())
+      if (lexicalDC)
         SetterMethod->setLexicalDeclContext(lexicalDC);
     } else
       // A user declared setter will be synthesize when @synthesize of
@@ -1253,6 +1356,7 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
   }
 
   if (!(Attributes & ObjCDeclSpec::DQ_PR_copy)
+      &&!(Attributes & ObjCDeclSpec::DQ_PR_readonly)
       && getLangOptions().getGCMode() == LangOptions::GCOnly
       && PropertyTy->isBlockPointerType())
     Diag(Loc, diag::warn_objc_property_copy_missing_on_block);
diff --git a/lib/Sema/SemaOverload.cpp b/lib/Sema/SemaOverload.cpp
index 11b4bb3b92c6..6fb789cfd8a1 100644
--- a/lib/Sema/SemaOverload.cpp
+++ b/lib/Sema/SemaOverload.cpp
@@ -23,8 +23,10 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/PartialDiagnostic.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include <algorithm>
@@ -32,9 +34,20 @@
 namespace clang {
 using namespace sema;
 
+/// A convenience routine for creating a decayed reference to a
+/// function.
+static Expr *
+CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn,
+                      SourceLocation Loc = SourceLocation()) {
+  Expr *E = new (S.Context) DeclRefExpr(Fn, Fn->getType(), VK_LValue, Loc);
+  S.DefaultFunctionArrayConversion(E);
+  return E;
+}
+
 static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
                                  bool InOverloadResolution,
-                                 StandardConversionSequence &SCS);
+                                 StandardConversionSequence &SCS,
+                                 bool CStyle);
 static OverloadingResult
 IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
                         UserDefinedConversionSequence& User,
@@ -158,7 +171,10 @@ void StandardConversionSequence::setAsIdentityConversion() {
   DeprecatedStringLiteralToCharPtr = false;
   ReferenceBinding = false;
   DirectBinding = false;
-  RRefBinding = false;
+  IsLvalueReference = true;
+  BindsToFunctionLvalue = false;
+  BindsToRvalue = false;
+  BindsImplicitObjectArgumentWithoutRefQualifier = false;
   CopyConstructor = 0;
 }
 
@@ -189,6 +205,7 @@ bool StandardConversionSequence::isPointerConversionToBool() const {
       (getFromType()->isPointerType() ||
        getFromType()->isObjCObjectPointerType() ||
        getFromType()->isBlockPointerType() ||
+       getFromType()->isNullPtrType() ||
        First == ICK_Array_To_Pointer || First == ICK_Function_To_Pointer))
     return true;
 
@@ -323,7 +340,7 @@ namespace {
     TemplateArgument SecondArg;
   };
 }
-  
+
 /// \brief Convert from Sema's representation of template deduction information
 /// to the form used in overload-candidate information.
 OverloadCandidate::DeductionFailureInfo
@@ -339,12 +356,12 @@ static MakeDeductionFailureInfo(ASTContext &Context,
   case Sema::TDK_TooManyArguments:
   case Sema::TDK_TooFewArguments:
     break;
-      
+
   case Sema::TDK_Incomplete:
   case Sema::TDK_InvalidExplicitArguments:
     Result.Data = Info.Param.getOpaqueValue();
     break;
-      
+
   case Sema::TDK_Inconsistent:
   case Sema::TDK_Underqualified: {
     // FIXME: Should allocate from normal heap so that we can free this later.
@@ -355,16 +372,16 @@ static MakeDeductionFailureInfo(ASTContext &Context,
     Result.Data = Saved;
     break;
   }
-      
+
   case Sema::TDK_SubstitutionFailure:
     Result.Data = Info.take();
     break;
-      
+
   case Sema::TDK_NonDeducedMismatch:
   case Sema::TDK_FailedOverloadResolution:
-    break;  
+    break;
   }
-  
+
   return Result;
 }
 
@@ -377,7 +394,7 @@ void OverloadCandidate::DeductionFailureInfo::Destroy() {
   case Sema::TDK_TooFewArguments:
   case Sema::TDK_InvalidExplicitArguments:
     break;
-      
+
   case Sema::TDK_Inconsistent:
   case Sema::TDK_Underqualified:
     // FIXME: Destroy the data?
@@ -388,15 +405,15 @@ void OverloadCandidate::DeductionFailureInfo::Destroy() {
     // FIXME: Destroy the template arugment list?
     Data = 0;
     break;
-      
+
   // Unhandled
   case Sema::TDK_NonDeducedMismatch:
   case Sema::TDK_FailedOverloadResolution:
     break;
   }
 }
-  
-TemplateParameter 
+
+TemplateParameter
 OverloadCandidate::DeductionFailureInfo::getTemplateParameter() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
   case Sema::TDK_Success:
@@ -405,24 +422,24 @@ OverloadCandidate::DeductionFailureInfo::getTemplateParameter() {
   case Sema::TDK_TooFewArguments:
   case Sema::TDK_SubstitutionFailure:
     return TemplateParameter();
-    
+
   case Sema::TDK_Incomplete:
   case Sema::TDK_InvalidExplicitArguments:
-    return TemplateParameter::getFromOpaqueValue(Data);    
+    return TemplateParameter::getFromOpaqueValue(Data);
 
   case Sema::TDK_Inconsistent:
   case Sema::TDK_Underqualified:
     return static_cast<DFIParamWithArguments*>(Data)->Param;
-      
+
   // Unhandled
   case Sema::TDK_NonDeducedMismatch:
   case Sema::TDK_FailedOverloadResolution:
     break;
   }
-  
+
   return TemplateParameter();
 }
- 
+
 TemplateArgumentList *
 OverloadCandidate::DeductionFailureInfo::getTemplateArgumentList() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
@@ -438,7 +455,7 @@ OverloadCandidate::DeductionFailureInfo::getTemplateArgumentList() {
 
     case Sema::TDK_SubstitutionFailure:
       return static_cast<TemplateArgumentList*>(Data);
-      
+
     // Unhandled
     case Sema::TDK_NonDeducedMismatch:
     case Sema::TDK_FailedOverloadResolution:
@@ -461,16 +478,16 @@ const TemplateArgument *OverloadCandidate::DeductionFailureInfo::getFirstArg() {
 
   case Sema::TDK_Inconsistent:
   case Sema::TDK_Underqualified:
-    return &static_cast<DFIParamWithArguments*>(Data)->FirstArg;      
+    return &static_cast<DFIParamWithArguments*>(Data)->FirstArg;
 
   // Unhandled
   case Sema::TDK_NonDeducedMismatch:
   case Sema::TDK_FailedOverloadResolution:
     break;
   }
-  
+
   return 0;
-}    
+}
 
 const TemplateArgument *
 OverloadCandidate::DeductionFailureInfo::getSecondArg() {
@@ -493,7 +510,7 @@ OverloadCandidate::DeductionFailureInfo::getSecondArg() {
   case Sema::TDK_FailedOverloadResolution:
     break;
   }
-  
+
   return 0;
 }
 
@@ -501,7 +518,7 @@ void OverloadCandidateSet::clear() {
   inherited::clear();
   Functions.clear();
 }
-  
+
 // IsOverload - Determine whether the given New declaration is an
 // overload of the declarations in Old. This routine returns false if
 // New and Old cannot be overloaded, e.g., if New has the same
@@ -582,10 +599,12 @@ Sema::CheckOverload(Scope *S, FunctionDecl *New, const LookupResult &Old,
         Match = *I;
         return Ovl_Match;
       }
-    } else if (isa<UsingDecl>(OldD) || isa<TagDecl>(OldD)) {
+    } else if (isa<UsingDecl>(OldD)) {
       // We can overload with these, which can show up when doing
       // redeclaration checks for UsingDecls.
       assert(Old.getLookupKind() == LookupUsingDeclName);
+    } else if (isa<TagDecl>(OldD)) {
+      // We can always overload with tags by hiding them.
     } else if (isa<UnresolvedUsingValueDecl>(OldD)) {
       // Optimistically assume that an unresolved using decl will
       // overload; if it doesn't, we'll have to diagnose during
@@ -632,8 +651,8 @@ bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
       isa<FunctionNoProtoType>(NewQType.getTypePtr()))
     return false;
 
-  FunctionProtoType* OldType = cast<FunctionProtoType>(OldQType);
-  FunctionProtoType* NewType = cast<FunctionProtoType>(NewQType);
+  const FunctionProtoType* OldType = cast<FunctionProtoType>(OldQType);
+  const FunctionProtoType* NewType = cast<FunctionProtoType>(NewQType);
 
   // The signature of a function includes the types of its
   // parameters (C++ 1.3.10), which includes the presence or absence
@@ -664,7 +683,7 @@ bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
     return true;
 
   // If the function is a class member, its signature includes the
-  // cv-qualifiers (if any) on the function itself.
+  // cv-qualifiers (if any) and ref-qualifier (if any) on the function itself.
   //
   // As part of this, also check whether one of the member functions
   // is static, in which case they are not overloads (C++
@@ -675,9 +694,26 @@ bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
   CXXMethodDecl* NewMethod = dyn_cast<CXXMethodDecl>(New);
   if (OldMethod && NewMethod &&
       !OldMethod->isStatic() && !NewMethod->isStatic() &&
-      OldMethod->getTypeQualifiers() != NewMethod->getTypeQualifiers())
+      (OldMethod->getTypeQualifiers() != NewMethod->getTypeQualifiers() ||
+       OldMethod->getRefQualifier() != NewMethod->getRefQualifier())) {
+    if (!UseUsingDeclRules &&
+        OldMethod->getRefQualifier() != NewMethod->getRefQualifier() &&
+        (OldMethod->getRefQualifier() == RQ_None ||
+         NewMethod->getRefQualifier() == RQ_None)) {
+      // C++0x [over.load]p2:
+      //   - Member function declarations with the same name and the same
+      //     parameter-type-list as well as member function template
+      //     declarations with the same name, the same parameter-type-list, and
+      //     the same template parameter lists cannot be overloaded if any of
+      //     them, but not all, have a ref-qualifier (8.3.5).
+      Diag(NewMethod->getLocation(), diag::err_ref_qualifier_overload)
+        << NewMethod->getRefQualifier() << OldMethod->getRefQualifier();
+      Diag(OldMethod->getLocation(), diag::note_previous_declaration);
+    }
+
     return true;
-  
+  }
+
   // The signatures match; this is not an overload.
   return false;
 }
@@ -708,11 +744,12 @@ bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
 static ImplicitConversionSequence
 TryImplicitConversion(Sema &S, Expr *From, QualType ToType,
                       bool SuppressUserConversions,
-                      bool AllowExplicit, 
-                      bool InOverloadResolution) {
+                      bool AllowExplicit,
+                      bool InOverloadResolution,
+                      bool CStyle) {
   ImplicitConversionSequence ICS;
   if (IsStandardConversion(S, From, ToType, InOverloadResolution,
-                           ICS.Standard)) {
+                           ICS.Standard, CStyle)) {
     ICS.setStandard();
     return ICS;
   }
@@ -737,20 +774,20 @@ TryImplicitConversion(Sema &S, Expr *From, QualType ToType,
     ICS.Standard.setAsIdentityConversion();
     ICS.Standard.setFromType(FromType);
     ICS.Standard.setAllToTypes(ToType);
-    
+
     // We don't actually check at this point whether there is a valid
     // copy/move constructor, since overloading just assumes that it
     // exists. When we actually perform initialization, we'll find the
     // appropriate constructor to copy the returned object, if needed.
     ICS.Standard.CopyConstructor = 0;
-    
+
     // Determine whether this is considered a derived-to-base conversion.
     if (!S.Context.hasSameUnqualifiedType(FromType, ToType))
       ICS.Standard.Second = ICK_Derived_To_Base;
-    
+
     return ICS;
   }
-  
+
   if (SuppressUserConversions) {
     // We're not in the case above, so there is no conversion that
     // we can perform.
@@ -823,12 +860,14 @@ bool Sema::TryImplicitConversion(InitializationSequence &Sequence,
                                  Expr *Initializer,
                                  bool SuppressUserConversions,
                                  bool AllowExplicitConversions,
-                                 bool InOverloadResolution) {
+                                 bool InOverloadResolution,
+                                 bool CStyle) {
   ImplicitConversionSequence ICS
     = clang::TryImplicitConversion(*this, Initializer, Entity.getType(),
                                    SuppressUserConversions,
-                                   AllowExplicitConversions, 
-                                   InOverloadResolution);
+                                   AllowExplicitConversions,
+                                   InOverloadResolution,
+                                   CStyle);
   if (ICS.isBad()) return true;
 
   // Perform the actual conversion.
@@ -856,34 +895,66 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   ICS = clang::TryImplicitConversion(*this, From, ToType,
                                      /*SuppressUserConversions=*/false,
                                      AllowExplicit,
-                                     /*InOverloadResolution=*/false);
+                                     /*InOverloadResolution=*/false,
+                                     /*CStyle=*/false);
   return PerformImplicitConversion(From, ToType, ICS, Action);
 }
-  
-/// \brief Determine whether the conversion from FromType to ToType is a valid 
+
+/// \brief Determine whether the conversion from FromType to ToType is a valid
 /// conversion that strips "noreturn" off the nested function type.
-static bool IsNoReturnConversion(ASTContext &Context, QualType FromType, 
+static bool IsNoReturnConversion(ASTContext &Context, QualType FromType,
                                  QualType ToType, QualType &ResultTy) {
   if (Context.hasSameUnqualifiedType(FromType, ToType))
     return false;
-  
-  // Strip the noreturn off the type we're converting from; noreturn can
-  // safely be removed.
-  FromType = Context.getNoReturnType(FromType, false);
-  if (!Context.hasSameUnqualifiedType(FromType, ToType))
-    return false;
 
-  ResultTy = FromType;
+  // Permit the conversion F(t __attribute__((noreturn))) -> F(t)
+  // where F adds one of the following at most once:
+  //   - a pointer
+  //   - a member pointer
+  //   - a block pointer
+  CanQualType CanTo = Context.getCanonicalType(ToType);
+  CanQualType CanFrom = Context.getCanonicalType(FromType);
+  Type::TypeClass TyClass = CanTo->getTypeClass();
+  if (TyClass != CanFrom->getTypeClass()) return false;
+  if (TyClass != Type::FunctionProto && TyClass != Type::FunctionNoProto) {
+    if (TyClass == Type::Pointer) {
+      CanTo = CanTo.getAs<PointerType>()->getPointeeType();
+      CanFrom = CanFrom.getAs<PointerType>()->getPointeeType();
+    } else if (TyClass == Type::BlockPointer) {
+      CanTo = CanTo.getAs<BlockPointerType>()->getPointeeType();
+      CanFrom = CanFrom.getAs<BlockPointerType>()->getPointeeType();
+    } else if (TyClass == Type::MemberPointer) {
+      CanTo = CanTo.getAs<MemberPointerType>()->getPointeeType();
+      CanFrom = CanFrom.getAs<MemberPointerType>()->getPointeeType();
+    } else {
+      return false;
+    }
+
+    TyClass = CanTo->getTypeClass();
+    if (TyClass != CanFrom->getTypeClass()) return false;
+    if (TyClass != Type::FunctionProto && TyClass != Type::FunctionNoProto)
+      return false;
+  }
+
+  const FunctionType *FromFn = cast<FunctionType>(CanFrom);
+  FunctionType::ExtInfo EInfo = FromFn->getExtInfo();
+  if (!EInfo.getNoReturn()) return false;
+
+  FromFn = Context.adjustFunctionType(FromFn, EInfo.withNoReturn(false));
+  assert(QualType(FromFn, 0).isCanonical());
+  if (QualType(FromFn, 0) != CanTo) return false;
+
+  ResultTy = ToType;
   return true;
 }
- 
+
 /// \brief Determine whether the conversion from FromType to ToType is a valid
 /// vector conversion.
 ///
 /// \param ICK Will be set to the vector conversion kind, if this is a vector
 /// conversion.
-static bool IsVectorConversion(ASTContext &Context, QualType FromType, 
-                               QualType ToType, ImplicitConversionKind &ICK) {  
+static bool IsVectorConversion(ASTContext &Context, QualType FromType,
+                               QualType ToType, ImplicitConversionKind &ICK) {
   // We need at least one of these types to be a vector type to have a vector
   // conversion.
   if (!ToType->isVectorType() && !FromType->isVectorType())
@@ -899,7 +970,7 @@ static bool IsVectorConversion(ASTContext &Context, QualType FromType,
     // identity conversion.
     if (FromType->isExtVectorType())
       return false;
-   
+
     // Vector splat from any arithmetic type to a vector.
     if (FromType->isArithmeticType()) {
       ICK = ICK_Vector_Splat;
@@ -922,7 +993,7 @@ static bool IsVectorConversion(ASTContext &Context, QualType FromType,
 
   return false;
 }
-  
+
 /// IsStandardConversion - Determines whether there is a standard
 /// conversion sequence (C++ [conv], C++ [over.ics.scs]) from the
 /// expression From to the type ToType. Standard conversion sequences
@@ -933,9 +1004,10 @@ static bool IsVectorConversion(ASTContext &Context, QualType FromType,
 /// routine will return false and the value of SCS is unspecified.
 static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
                                  bool InOverloadResolution,
-                                 StandardConversionSequence &SCS) {
+                                 StandardConversionSequence &SCS,
+                                 bool CStyle) {
   QualType FromType = From->getType();
-  
+
   // Standard conversions (C++ [conv])
   SCS.setAsIdentityConversion();
   SCS.DeprecatedStringLiteralToCharPtr = false;
@@ -959,38 +1031,53 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
   if (FromType == S.Context.OverloadTy) {
     DeclAccessPair AccessPair;
     if (FunctionDecl *Fn
-          = S.ResolveAddressOfOverloadedFunction(From, ToType, false, 
+          = S.ResolveAddressOfOverloadedFunction(From, ToType, false,
                                                  AccessPair)) {
       // We were able to resolve the address of the overloaded function,
       // so we can convert to the type of that function.
       FromType = Fn->getType();
+
+      // we can sometimes resolve &foo<int> regardless of ToType, so check
+      // if the type matches (identity) or we are converting to bool
+      if (!S.Context.hasSameUnqualifiedType(
+                      S.ExtractUnqualifiedFunctionType(ToType), FromType)) {
+        QualType resultTy;
+        // if the function type matches except for [[noreturn]], it's ok
+        if (!IsNoReturnConversion(S.Context, FromType, 
+              S.ExtractUnqualifiedFunctionType(ToType), resultTy))
+          // otherwise, only a boolean conversion is standard   
+          if (!ToType->isBooleanType()) 
+            return false; 
+
+      }
+        
       if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Fn)) {
         if (!Method->isStatic()) {
-          Type *ClassType 
+          const Type *ClassType
             = S.Context.getTypeDeclType(Method->getParent()).getTypePtr();
           FromType = S.Context.getMemberPointerType(FromType, ClassType);
         }
       }
-      
+
       // If the "from" expression takes the address of the overloaded
       // function, update the type of the resulting expression accordingly.
       if (FromType->getAs<FunctionType>())
         if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(From->IgnoreParens()))
           if (UnOp->getOpcode() == UO_AddrOf)
             FromType = S.Context.getPointerType(FromType);
- 
+
       // Check that we've computed the proper type after overload resolution.
       assert(S.Context.hasSameType(FromType,
             S.FixOverloadedFunctionReference(From, AccessPair, Fn)->getType()));
     } else {
       return false;
     }
-  } 
+  }
   // Lvalue-to-rvalue conversion (C++ 4.1):
   //   An lvalue (3.10) of a non-function, non-array type T can be
   //   converted to an rvalue.
-  Expr::isLvalueResult argIsLvalue = From->isLvalue(S.Context);
-  if (argIsLvalue == Expr::LV_Valid &&
+  bool argIsLValue = From->isLValue();
+  if (argIsLValue &&
       !FromType->isFunctionType() && !FromType->isArrayType() &&
       S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) {
     SCS.First = ICK_Lvalue_To_Rvalue;
@@ -1022,7 +1109,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
       SCS.setAllToTypes(FromType);
       return true;
     }
-  } else if (FromType->isFunctionType() && argIsLvalue == Expr::LV_Valid) {
+  } else if (FromType->isFunctionType() && argIsLValue) {
     // Function-to-pointer conversion (C++ 4.3).
     SCS.First = ICK_Function_To_Pointer;
 
@@ -1060,17 +1147,26 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // Complex promotion (Clang extension)
     SCS.Second = ICK_Complex_Promotion;
     FromType = ToType.getUnqualifiedType();
-  } else if (FromType->isIntegralOrEnumerationType() &&
+  } else if (ToType->isBooleanType() &&
+             (FromType->isArithmeticType() ||
+              FromType->isAnyPointerType() ||
+              FromType->isBlockPointerType() ||
+              FromType->isMemberPointerType() ||
+              FromType->isNullPtrType())) {
+    // Boolean conversions (C++ 4.12).
+    SCS.Second = ICK_Boolean_Conversion;
+    FromType = S.Context.BoolTy;
+  } else if (FromType->isIntegralOrUnscopedEnumerationType() &&
              ToType->isIntegralType(S.Context)) {
     // Integral conversions (C++ 4.7).
     SCS.Second = ICK_Integral_Conversion;
     FromType = ToType.getUnqualifiedType();
-  } else if (FromType->isComplexType() && ToType->isComplexType()) {
+  } else if (FromType->isAnyComplexType() && ToType->isComplexType()) {
     // Complex conversions (C99 6.3.1.6)
     SCS.Second = ICK_Complex_Conversion;
     FromType = ToType.getUnqualifiedType();
-  } else if ((FromType->isComplexType() && ToType->isArithmeticType()) ||
-             (ToType->isComplexType() && FromType->isArithmeticType())) {
+  } else if ((FromType->isAnyComplexType() && ToType->isArithmeticType()) ||
+             (ToType->isAnyComplexType() && FromType->isArithmeticType())) {
     // Complex-real conversions (C99 6.3.1.7)
     SCS.Second = ICK_Complex_Real;
     FromType = ToType.getUnqualifiedType();
@@ -1078,32 +1174,24 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     // Floating point conversions (C++ 4.8).
     SCS.Second = ICK_Floating_Conversion;
     FromType = ToType.getUnqualifiedType();
-  } else if ((FromType->isRealFloatingType() && 
-              ToType->isIntegralType(S.Context) && !ToType->isBooleanType()) ||
-             (FromType->isIntegralOrEnumerationType() &&
+  } else if ((FromType->isRealFloatingType() &&
+              ToType->isIntegralType(S.Context)) ||
+             (FromType->isIntegralOrUnscopedEnumerationType() &&
               ToType->isRealFloatingType())) {
     // Floating-integral conversions (C++ 4.9).
     SCS.Second = ICK_Floating_Integral;
     FromType = ToType.getUnqualifiedType();
+  } else if (S.IsBlockPointerConversion(FromType, ToType, FromType)) {
+               SCS.Second = ICK_Block_Pointer_Conversion;
   } else if (S.IsPointerConversion(From, FromType, ToType, InOverloadResolution,
                                    FromType, IncompatibleObjC)) {
     // Pointer conversions (C++ 4.10).
     SCS.Second = ICK_Pointer_Conversion;
     SCS.IncompatibleObjC = IncompatibleObjC;
-  } else if (S.IsMemberPointerConversion(From, FromType, ToType, 
+  } else if (S.IsMemberPointerConversion(From, FromType, ToType,
                                          InOverloadResolution, FromType)) {
     // Pointer to member conversions (4.11).
     SCS.Second = ICK_Pointer_Member;
-  } else if (ToType->isBooleanType() &&
-             (FromType->isArithmeticType() ||
-              FromType->isEnumeralType() ||
-              FromType->isAnyPointerType() ||
-              FromType->isBlockPointerType() ||
-              FromType->isMemberPointerType() ||
-              FromType->isNullPtrType())) {
-    // Boolean conversions (C++ 4.12).
-    SCS.Second = ICK_Boolean_Conversion;
-    FromType = S.Context.BoolTy;
   } else if (IsVectorConversion(S.Context, FromType, ToType, SecondICK)) {
     SCS.Second = SecondICK;
     FromType = ToType.getUnqualifiedType();
@@ -1124,7 +1212,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
   QualType CanonFrom;
   QualType CanonTo;
   // The third conversion can be a qualification conversion (C++ 4p1).
-  if (S.IsQualificationConversion(FromType, ToType)) {
+  if (S.IsQualificationConversion(FromType, ToType, CStyle)) {
     SCS.Third = ICK_Qualification;
     FromType = ToType;
     CanonFrom = S.Context.getCanonicalType(FromType);
@@ -1139,7 +1227,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
     //   a conversion. [...]
     CanonFrom = S.Context.getCanonicalType(FromType);
     CanonTo = S.Context.getCanonicalType(ToType);
-    if (CanonFrom.getLocalUnqualifiedType() 
+    if (CanonFrom.getLocalUnqualifiedType()
                                        == CanonTo.getLocalUnqualifiedType() &&
         (CanonFrom.getLocalCVRQualifiers() != CanonTo.getLocalCVRQualifiers()
          || CanonFrom.getObjCGCAttr() != CanonTo.getObjCGCAttr())) {
@@ -1187,23 +1275,47 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
     return To->getKind() == BuiltinType::UInt;
   }
 
-  // An rvalue of type wchar_t (3.9.1) or an enumeration type (7.2)
-  // can be converted to an rvalue of the first of the following types
-  // that can represent all the values of its underlying type: int,
-  // unsigned int, long, or unsigned long (C++ 4.5p2).
+  // C++0x [conv.prom]p3:
+  //   A prvalue of an unscoped enumeration type whose underlying type is not
+  //   fixed (7.2) can be converted to an rvalue a prvalue of the first of the
+  //   following types that can represent all the values of the enumeration
+  //   (i.e., the values in the range bmin to bmax as described in 7.2): int,
+  //   unsigned int, long int, unsigned long int, long long int, or unsigned
+  //   long long int. If none of the types in that list can represent all the
+  //   values of the enumeration, an rvalue a prvalue of an unscoped enumeration
+  //   type can be converted to an rvalue a prvalue of the extended integer type
+  //   with lowest integer conversion rank (4.13) greater than the rank of long
+  //   long in which all the values of the enumeration can be represented. If
+  //   there are two such extended types, the signed one is chosen.
+  if (const EnumType *FromEnumType = FromType->getAs<EnumType>()) {
+    // C++0x 7.2p9: Note that this implicit enum to int conversion is not
+    // provided for a scoped enumeration.
+    if (FromEnumType->getDecl()->isScoped())
+      return false;
 
-  // We pre-calculate the promotion type for enum types.
-  if (const EnumType *FromEnumType = FromType->getAs<EnumType>())
-    if (ToType->isIntegerType())
+    // We have already pre-calculated the promotion type, so this is trivial.
+    if (ToType->isIntegerType() &&
+        !RequireCompleteType(From->getLocStart(), FromType, PDiag()))
       return Context.hasSameUnqualifiedType(ToType,
                                 FromEnumType->getDecl()->getPromotionType());
+  }
 
-  if (FromType->isWideCharType() && ToType->isIntegerType()) {
+  // C++0x [conv.prom]p2:
+  //   A prvalue of type char16_t, char32_t, or wchar_t (3.9.1) can be converted
+  //   to an rvalue a prvalue of the first of the following types that can
+  //   represent all the values of its underlying type: int, unsigned int,
+  //   long int, unsigned long int, long long int, or unsigned long long int.
+  //   If none of the types in that list can represent all the values of its
+  //   underlying type, an rvalue a prvalue of type char16_t, char32_t,
+  //   or wchar_t can be converted to an rvalue a prvalue of its underlying
+  //   type.
+  if (FromType->isAnyCharacterType() && !FromType->isCharType() &&
+      ToType->isIntegerType()) {
     // Determine whether the type we're converting from is signed or
     // unsigned.
     bool FromIsSigned;
     uint64_t FromSize = Context.getTypeSize(FromType);
-    
+
     // FIXME: Is wchar_t signed or unsigned? We assume it's signed for now.
     FromIsSigned = true;
 
@@ -1321,10 +1433,19 @@ bool Sema::IsComplexPromotion(QualType FromType, QualType ToType) {
 /// if non-empty, will be a pointer to ToType that may or may not have
 /// the right set of qualifiers on its pointee.
 static QualType
-BuildSimilarlyQualifiedPointerType(const PointerType *FromPtr,
+BuildSimilarlyQualifiedPointerType(const Type *FromPtr,
                                    QualType ToPointee, QualType ToType,
                                    ASTContext &Context) {
-  QualType CanonFromPointee = Context.getCanonicalType(FromPtr->getPointeeType());
+  assert((FromPtr->getTypeClass() == Type::Pointer ||
+          FromPtr->getTypeClass() == Type::ObjCObjectPointer) &&
+         "Invalid similarly-qualified pointer type");
+
+  /// \brief Conversions to 'id' subsume cv-qualifier conversions.
+  if (ToType->isObjCIdType() || ToType->isObjCQualifiedIdType())
+    return ToType.getUnqualifiedType();
+
+  QualType CanonFromPointee
+    = Context.getCanonicalType(FromPtr->getPointeeType());
   QualType CanonToPointee = Context.getCanonicalType(ToPointee);
   Qualifiers Quals = CanonFromPointee.getQualifiers();
 
@@ -1336,34 +1457,20 @@ BuildSimilarlyQualifiedPointerType(const PointerType *FromPtr,
 
     // Build a pointer to ToPointee. It has the right qualifiers
     // already.
+    if (isa<ObjCObjectPointerType>(ToType))
+      return Context.getObjCObjectPointerType(ToPointee);
     return Context.getPointerType(ToPointee);
   }
 
   // Just build a canonical type that has the right qualifiers.
-  return Context.getPointerType(
-         Context.getQualifiedType(CanonToPointee.getLocalUnqualifiedType(), 
-                                  Quals));
-}
+  QualType QualifiedCanonToPointee
+    = Context.getQualifiedType(CanonToPointee.getLocalUnqualifiedType(), Quals);
 
-/// BuildSimilarlyQualifiedObjCObjectPointerType - In a pointer conversion from
-/// the FromType, which is an objective-c pointer, to ToType, which may or may
-/// not have the right set of qualifiers.
-static QualType
-BuildSimilarlyQualifiedObjCObjectPointerType(QualType FromType,
-                                             QualType ToType,
-                                             ASTContext &Context) {
-  QualType CanonFromType = Context.getCanonicalType(FromType);
-  QualType CanonToType = Context.getCanonicalType(ToType);
-  Qualifiers Quals = CanonFromType.getQualifiers();
-    
-  // Exact qualifier match -> return the pointer type we're converting to.
-  if (CanonToType.getLocalQualifiers() == Quals)
-    return ToType;
-  
-  // Just build a canonical type that has the right qualifiers.
-  return Context.getQualifiedType(CanonToType.getLocalUnqualifiedType(), Quals);
+  if (isa<ObjCObjectPointerType>(ToType))
+    return Context.getObjCObjectPointerType(QualifiedCanonToPointee);
+  return Context.getPointerType(QualifiedCanonToPointee);
 }
-  
+
 static bool isNullPointerConstantForConversion(Expr *Expr,
                                                bool InOverloadResolution,
                                                ASTContext &Context) {
@@ -1399,7 +1506,8 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
                                QualType& ConvertedType,
                                bool &IncompatibleObjC) {
   IncompatibleObjC = false;
-  if (isObjCPointerConversion(FromType, ToType, ConvertedType, IncompatibleObjC))
+  if (isObjCPointerConversion(FromType, ToType, ConvertedType,
+                              IncompatibleObjC))
     return true;
 
   // Conversion from a null pointer constant to any Objective-C pointer type.
@@ -1441,14 +1549,15 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
     return true;
   }
 
-  // Beyond this point, both types need to be pointers 
+  // Beyond this point, both types need to be pointers
   // , including objective-c pointers.
   QualType ToPointeeType = ToTypePtr->getPointeeType();
   if (FromType->isObjCObjectPointerType() && ToPointeeType->isVoidType()) {
-    ConvertedType = BuildSimilarlyQualifiedObjCObjectPointerType(FromType,
+    ConvertedType = BuildSimilarlyQualifiedPointerType(
+                                      FromType->getAs<ObjCObjectPointerType>(),
+                                                       ToPointeeType,
                                                        ToType, Context);
     return true;
-    
   }
   const PointerType *FromTypePtr = FromType->getAs<PointerType>();
   if (!FromTypePtr)
@@ -1456,7 +1565,7 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
 
   QualType FromPointeeType = FromTypePtr->getPointeeType();
 
-  // If the unqualified pointee types are the same, this can't be a 
+  // If the unqualified pointee types are the same, this can't be a
   // pointer conversion, so don't do all of the work below.
   if (Context.hasSameUnqualifiedType(FromPointeeType, ToPointeeType))
     return false;
@@ -1517,13 +1626,20 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
                                    bool &IncompatibleObjC) {
   if (!getLangOptions().ObjC1)
     return false;
- 
+
   // First, we handle all conversions on ObjC object pointer types.
-  const ObjCObjectPointerType* ToObjCPtr = ToType->getAs<ObjCObjectPointerType>();
+  const ObjCObjectPointerType* ToObjCPtr =
+    ToType->getAs<ObjCObjectPointerType>();
   const ObjCObjectPointerType *FromObjCPtr =
     FromType->getAs<ObjCObjectPointerType>();
 
   if (ToObjCPtr && FromObjCPtr) {
+    // If the pointee types are the same (ignoring qualifications),
+    // then this is not a pointer conversion.
+    if (Context.hasSameUnqualifiedType(ToObjCPtr->getPointeeType(),
+                                       FromObjCPtr->getPointeeType()))
+      return false;
+
     // Objective C++: We're able to convert between "id" or "Class" and a
     // pointer to any interface (in both directions).
     if (ToObjCPtr->isObjCBuiltinType() && FromObjCPtr->isObjCBuiltinType()) {
@@ -1547,7 +1663,9 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
           !ToObjCPtr->getPointeeType().isAtLeastAsQualifiedAs(
                                                 FromObjCPtr->getPointeeType()))
         return false;
-      ConvertedType = ToType;
+      ConvertedType = BuildSimilarlyQualifiedPointerType(FromObjCPtr,
+                                                   ToObjCPtr->getPointeeType(),
+                                                         ToType, Context);
       return true;
     }
 
@@ -1556,7 +1674,9 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
       // interfaces, which is permitted. However, we're going to
       // complain about it.
       IncompatibleObjC = true;
-      ConvertedType = FromType;
+      ConvertedType = BuildSimilarlyQualifiedPointerType(FromObjCPtr,
+                                                   ToObjCPtr->getPointeeType(),
+                                                         ToType, Context);
       return true;
     }
   }
@@ -1564,7 +1684,7 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
   QualType ToPointeeType;
   if (const PointerType *ToCPtr = ToType->getAs<PointerType>())
     ToPointeeType = ToCPtr->getPointeeType();
-  else if (const BlockPointerType *ToBlockPtr = 
+  else if (const BlockPointerType *ToBlockPtr =
             ToType->getAs<BlockPointerType>()) {
     // Objective C++: We're able to convert from a pointer to any object
     // to a block pointer type.
@@ -1574,9 +1694,9 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
     }
     ToPointeeType = ToBlockPtr->getPointeeType();
   }
-  else if (FromType->getAs<BlockPointerType>() && 
+  else if (FromType->getAs<BlockPointerType>() &&
            ToObjCPtr && ToObjCPtr->isObjCBuiltinType()) {
-    // Objective C++: We're able to convert from a block pointer type to a 
+    // Objective C++: We're able to convert from a block pointer type to a
     // pointer to any object.
     ConvertedType = ToType;
     return true;
@@ -1587,7 +1707,8 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
   QualType FromPointeeType;
   if (const PointerType *FromCPtr = FromType->getAs<PointerType>())
     FromPointeeType = FromCPtr->getPointeeType();
-  else if (const BlockPointerType *FromBlockPtr = FromType->getAs<BlockPointerType>())
+  else if (const BlockPointerType *FromBlockPtr =
+           FromType->getAs<BlockPointerType>())
     FromPointeeType = FromBlockPtr->getPointeeType();
   else
     return false;
@@ -1599,7 +1720,7 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
                               IncompatibleObjC)) {
     // We always complain about this conversion.
     IncompatibleObjC = true;
-    ConvertedType = ToType;
+    ConvertedType = Context.getPointerType(ConvertedType);
     return true;
   }
   // Allow conversion of pointee being objective-c pointer to another one;
@@ -1608,10 +1729,10 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
       ToPointeeType->getAs<ObjCObjectPointerType>() &&
       isObjCPointerConversion(FromPointeeType, ToPointeeType, ConvertedType,
                               IncompatibleObjC)) {
-    ConvertedType = ToType;
+    ConvertedType = Context.getPointerType(ConvertedType);
     return true;
   }
-  
+
   // If we have pointers to functions or blocks, check whether the only
   // differences in the argument and result types are in Objective-C
   // pointer conversions. If so, we permit the conversion (but
@@ -1677,17 +1798,102 @@ bool Sema::isObjCPointerConversion(QualType FromType, QualType ToType,
 
   return false;
 }
- 
+
+bool Sema::IsBlockPointerConversion(QualType FromType, QualType ToType,
+                                    QualType& ConvertedType) {
+  QualType ToPointeeType;
+  if (const BlockPointerType *ToBlockPtr =
+        ToType->getAs<BlockPointerType>())
+    ToPointeeType = ToBlockPtr->getPointeeType();
+  else
+    return false;
+  
+  QualType FromPointeeType;
+  if (const BlockPointerType *FromBlockPtr =
+      FromType->getAs<BlockPointerType>())
+    FromPointeeType = FromBlockPtr->getPointeeType();
+  else
+    return false;
+  // We have pointer to blocks, check whether the only
+  // differences in the argument and result types are in Objective-C
+  // pointer conversions. If so, we permit the conversion.
+  
+  const FunctionProtoType *FromFunctionType
+    = FromPointeeType->getAs<FunctionProtoType>();
+  const FunctionProtoType *ToFunctionType
+    = ToPointeeType->getAs<FunctionProtoType>();
+  
+  if (!FromFunctionType || !ToFunctionType)
+    return false;
+
+  if (Context.hasSameType(FromPointeeType, ToPointeeType))
+    return true;
+    
+  // Perform the quick checks that will tell us whether these
+  // function types are obviously different.
+  if (FromFunctionType->getNumArgs() != ToFunctionType->getNumArgs() ||
+      FromFunctionType->isVariadic() != ToFunctionType->isVariadic())
+    return false;
+    
+  FunctionType::ExtInfo FromEInfo = FromFunctionType->getExtInfo();
+  FunctionType::ExtInfo ToEInfo = ToFunctionType->getExtInfo();
+  if (FromEInfo != ToEInfo)
+    return false;
+
+  bool IncompatibleObjC = false;
+  if (Context.hasSameType(FromFunctionType->getResultType(), 
+                          ToFunctionType->getResultType())) {
+    // Okay, the types match exactly. Nothing to do.
+  } else {
+    QualType RHS = FromFunctionType->getResultType();
+    QualType LHS = ToFunctionType->getResultType();
+    if ((!getLangOptions().CPlusPlus || !RHS->isRecordType()) &&
+        !RHS.hasQualifiers() && LHS.hasQualifiers())
+       LHS = LHS.getUnqualifiedType();
+
+     if (Context.hasSameType(RHS,LHS)) {
+       // OK exact match.
+     } else if (isObjCPointerConversion(RHS, LHS,
+                                        ConvertedType, IncompatibleObjC)) {
+     if (IncompatibleObjC)
+       return false;
+     // Okay, we have an Objective-C pointer conversion.
+     }
+     else
+       return false;
+   }
+    
+   // Check argument types.
+   for (unsigned ArgIdx = 0, NumArgs = FromFunctionType->getNumArgs();
+        ArgIdx != NumArgs; ++ArgIdx) {
+     IncompatibleObjC = false;
+     QualType FromArgType = FromFunctionType->getArgType(ArgIdx);
+     QualType ToArgType = ToFunctionType->getArgType(ArgIdx);
+     if (Context.hasSameType(FromArgType, ToArgType)) {
+       // Okay, the types match exactly. Nothing to do.
+     } else if (isObjCPointerConversion(ToArgType, FromArgType,
+                                        ConvertedType, IncompatibleObjC)) {
+       if (IncompatibleObjC)
+         return false;
+       // Okay, we have an Objective-C pointer conversion.
+     } else
+       // Argument types are too different. Abort.
+       return false;
+   }
+   ConvertedType = ToType;
+   return true;
+}
+
 /// FunctionArgTypesAreEqual - This routine checks two function proto types
 /// for equlity of their argument types. Caller has already checked that
 /// they have same number of arguments. This routine assumes that Objective-C
 /// pointer types which only differ in their protocol qualifiers are equal.
-bool Sema::FunctionArgTypesAreEqual(FunctionProtoType*  OldType, 
-                            FunctionProtoType*  NewType){
+bool Sema::FunctionArgTypesAreEqual(const FunctionProtoType *OldType,
+                                    const FunctionProtoType *NewType) {
   if (!getLangOptions().ObjC1)
     return std::equal(OldType->arg_type_begin(), OldType->arg_type_end(),
                       NewType->arg_type_begin());
-  
+
   for (FunctionProtoType::arg_type_iterator O = OldType->arg_type_begin(),
        N = NewType->arg_type_begin(),
        E = OldType->arg_type_end(); O && (O != E); ++O, ++N) {
@@ -1704,12 +1910,12 @@ bool Sema::FunctionArgTypesAreEqual(FunctionProtoType*  OldType,
       }
       else if (const ObjCObjectPointerType *PTTo =
                  ToType->getAs<ObjCObjectPointerType>()) {
-        if (const ObjCObjectPointerType *PTFr = 
+        if (const ObjCObjectPointerType *PTFr =
               FromType->getAs<ObjCObjectPointerType>())
           if (PTTo->getInterfaceDecl() == PTFr->getInterfaceDecl())
             continue;
       }
-      return false;  
+      return false;
     }
   }
   return true;
@@ -1726,10 +1932,13 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
                                   CXXCastPath& BasePath,
                                   bool IgnoreBaseAccess) {
   QualType FromType = From->getType();
+  bool IsCStyleOrFunctionalCast = IgnoreBaseAccess;
+
+  Kind = CK_BitCast;
 
   if (CXXBoolLiteralExpr* LitBool
                           = dyn_cast<CXXBoolLiteralExpr>(From->IgnoreParens()))
-    if (LitBool->getValue() == false)
+    if (!IsCStyleOrFunctionalCast && LitBool->getValue() == false)
       Diag(LitBool->getExprLoc(), diag::warn_init_pointer_from_false)
         << ToType;
 
@@ -1747,13 +1956,13 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
                                          From->getSourceRange(), &BasePath,
                                          IgnoreBaseAccess))
           return true;
-        
+
         // The conversion was successful.
         Kind = CK_DerivedToBase;
       }
     }
   if (const ObjCObjectPointerType *FromPtrType =
-        FromType->getAs<ObjCObjectPointerType>())
+        FromType->getAs<ObjCObjectPointerType>()) {
     if (const ObjCObjectPointerType *ToPtrType =
           ToType->getAs<ObjCObjectPointerType>()) {
       // Objective-C++ conversions are always okay.
@@ -1761,8 +1970,14 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
       // Objective-C++ implicit conversions.
       if (FromPtrType->isObjCBuiltinType() || ToPtrType->isObjCBuiltinType())
         return false;
-
+    }
   }
+
+  // We shouldn't fall into this case unless it's valid for other
+  // reasons.
+  if (From->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull))
+    Kind = CK_NullToPointer;
+
   return false;
 }
 
@@ -1772,7 +1987,7 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
 /// If so, returns true and places the converted type (that might differ from
 /// ToType in its cv-qualifiers at some level) into ConvertedType.
 bool Sema::IsMemberPointerConversion(Expr *From, QualType FromType,
-                                     QualType ToType, 
+                                     QualType ToType,
                                      bool InOverloadResolution,
                                      QualType &ConvertedType) {
   const MemberPointerType *ToTypePtr = ToType->getAs<MemberPointerType>();
@@ -1796,9 +2011,10 @@ bool Sema::IsMemberPointerConversion(Expr *From, QualType FromType,
   // where D is derived from B (C++ 4.11p2).
   QualType FromClass(FromTypePtr->getClass(), 0);
   QualType ToClass(ToTypePtr->getClass(), 0);
-  // FIXME: What happens when these are dependent? Is this function even called?
 
-  if (IsDerivedFrom(ToClass, FromClass)) {
+  if (!Context.hasSameUnqualifiedType(FromClass, ToClass) &&
+      !RequireCompleteType(From->getLocStart(), ToClass, PDiag()) &&
+      IsDerivedFrom(ToClass, FromClass)) {
     ConvertedType = Context.getMemberPointerType(FromTypePtr->getPointeeType(),
                                                  ToClass.getTypePtr());
     return true;
@@ -1806,7 +2022,7 @@ bool Sema::IsMemberPointerConversion(Expr *From, QualType FromType,
 
   return false;
 }
-  
+
 /// CheckMemberPointerConversion - Check the member pointer conversion from the
 /// expression From to the type ToType. This routine checks for ambiguous or
 /// virtual or inaccessible base-to-derived member pointer conversions
@@ -1821,7 +2037,7 @@ bool Sema::CheckMemberPointerConversion(Expr *From, QualType ToType,
   const MemberPointerType *FromPtrType = FromType->getAs<MemberPointerType>();
   if (!FromPtrType) {
     // This must be a null pointer to member pointer conversion
-    assert(From->isNullPointerConstant(Context, 
+    assert(From->isNullPointerConstant(Context,
                                        Expr::NPC_ValueDependentIsNull) &&
            "Expr must be null pointer constant!");
     Kind = CK_NullToMemberPointer;
@@ -1876,7 +2092,8 @@ bool Sema::CheckMemberPointerConversion(Expr *From, QualType ToType,
 /// an rvalue of type FromType to ToType is a qualification conversion
 /// (C++ 4.4).
 bool
-Sema::IsQualificationConversion(QualType FromType, QualType ToType) {
+Sema::IsQualificationConversion(QualType FromType, QualType ToType,
+                                bool CStyle) {
   FromType = Context.getCanonicalType(FromType);
   ToType = Context.getCanonicalType(ToType);
 
@@ -1901,12 +2118,12 @@ Sema::IsQualificationConversion(QualType FromType, QualType ToType) {
 
     //   -- for every j > 0, if const is in cv 1,j then const is in cv
     //      2,j, and similarly for volatile.
-    if (!ToType.isAtLeastAsQualifiedAs(FromType))
+    if (!CStyle && !ToType.isAtLeastAsQualifiedAs(FromType))
       return false;
 
     //   -- if the cv 1,j and cv 2,j are different, then const is in
     //      every cv for 0 < k < j.
-    if (FromType.getCVRQualifiers() != ToType.getCVRQualifiers()
+    if (!CStyle && FromType.getCVRQualifiers() != ToType.getCVRQualifiers()
         && !PreviousToQualsIncludeConst)
       return false;
 
@@ -1977,13 +2194,13 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
             = cast<CXXConstructorDecl>(ConstructorTmpl->getTemplatedDecl());
         else
           Constructor = cast<CXXConstructorDecl>(D);
-        
+
         if (!Constructor->isInvalidDecl() &&
             Constructor->isConvertingConstructor(AllowExplicit)) {
           if (ConstructorTmpl)
             S.AddTemplateOverloadCandidate(ConstructorTmpl, FoundDecl,
                                            /*ExplicitArgs*/ 0,
-                                           &From, 1, CandidateSet, 
+                                           &From, 1, CandidateSet,
                                            /*SuppressUserConversions=*/
                                              !ConstructorsOnly);
           else
@@ -2039,7 +2256,7 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
   }
 
   OverloadCandidateSet::iterator Best;
-  switch (CandidateSet.BestViableFunction(S, From->getLocStart(), Best)) {
+  switch (CandidateSet.BestViableFunction(S, From->getLocStart(), Best, true)) {
   case OR_Success:
     // Record the standard conversion we used and the conversion function.
     if (CXXConstructorDecl *Constructor
@@ -2058,6 +2275,7 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
         User.EllipsisConversion = false;
       }
       User.ConversionFunction = Constructor;
+      User.FoundConversionFunction = Best->FoundDecl.getDecl();
       User.After.setAsIdentityConversion();
       User.After.setFromType(ThisType->getAs<PointerType>()->getPointeeType());
       User.After.setAllToTypes(ToType);
@@ -2072,6 +2290,7 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
       //   implicit object parameter of the conversion function.
       User.Before = Best->Conversions[0].Standard;
       User.ConversionFunction = Conversion;
+      User.FoundConversionFunction = Best->FoundDecl.getDecl();
       User.EllipsisConversion = false;
 
       // C++ [over.ics.user]p2:
@@ -2095,19 +2314,19 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
   case OR_Deleted:
     // No conversion here! We're done.
     return OR_Deleted;
-    
+
   case OR_Ambiguous:
     return OR_Ambiguous;
   }
 
   return OR_No_Viable_Function;
 }
-  
+
 bool
 Sema::DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType) {
   ImplicitConversionSequence ICS;
   OverloadCandidateSet CandidateSet(From->getExprLoc());
-  OverloadingResult OvResult = 
+  OverloadingResult OvResult =
     IsUserDefinedConversion(*this, From, ToType, ICS.UserDefined,
                             CandidateSet, false);
   if (OvResult == OR_Ambiguous)
@@ -2121,7 +2340,7 @@ Sema::DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType) {
   else
     return false;
   CandidateSet.NoteCandidates(*this, OCD_AllCandidates, &From, 1);
-  return true;  
+  return true;
 }
 
 /// CompareImplicitConversionSequences - Compare two implicit
@@ -2182,12 +2401,12 @@ static bool hasSimilarType(ASTContext &Context, QualType T1, QualType T2) {
   while (Context.UnwrapSimilarPointerTypes(T1, T2)) {
     Qualifiers Quals;
     T1 = Context.getUnqualifiedArrayType(T1, Quals);
-    T2 = Context.getUnqualifiedArrayType(T2, Quals);    
+    T2 = Context.getUnqualifiedArrayType(T2, Quals);
   }
-  
+
   return Context.hasSameUnqualifiedType(T1, T2);
 }
-  
+
 // Per 13.3.3.2p3, compare the given standard conversion sequences to
 // determine if one is a proper subset of the other.
 static ImplicitConversionSequence::CompareKind
@@ -2197,7 +2416,7 @@ compareStandardConversionSubsets(ASTContext &Context,
   ImplicitConversionSequence::CompareKind Result
     = ImplicitConversionSequence::Indistinguishable;
 
-  // the identity conversion sequence is considered to be a subsequence of 
+  // the identity conversion sequence is considered to be a subsequence of
   // any non-identity conversion sequence
   if (SCS1.ReferenceBinding == SCS2.ReferenceBinding) {
     if (SCS1.isIdentityConversion() && !SCS2.isIdentityConversion())
@@ -2205,7 +2424,7 @@ compareStandardConversionSubsets(ASTContext &Context,
     else if (!SCS1.isIdentityConversion() && SCS2.isIdentityConversion())
       return ImplicitConversionSequence::Worse;
   }
-    
+
   if (SCS1.Second != SCS2.Second) {
     if (SCS1.Second == ICK_Identity)
       Result = ImplicitConversionSequence::Better;
@@ -2230,10 +2449,37 @@ compareStandardConversionSubsets(ASTContext &Context,
     return Result == ImplicitConversionSequence::Better
              ? ImplicitConversionSequence::Indistinguishable
              : ImplicitConversionSequence::Worse;
-       
+
   return ImplicitConversionSequence::Indistinguishable;
 }
 
+/// \brief Determine whether one of the given reference bindings is better
+/// than the other based on what kind of bindings they are.
+static bool isBetterReferenceBindingKind(const StandardConversionSequence &SCS1,
+                                       const StandardConversionSequence &SCS2) {
+  // C++0x [over.ics.rank]p3b4:
+  //   -- S1 and S2 are reference bindings (8.5.3) and neither refers to an
+  //      implicit object parameter of a non-static member function declared
+  //      without a ref-qualifier, and *either* S1 binds an rvalue reference
+  //      to an rvalue and S2 binds an lvalue reference *or S1 binds an
+  //      lvalue reference to a function lvalue and S2 binds an rvalue
+  //      reference*.
+  //
+  // FIXME: Rvalue references. We're going rogue with the above edits,
+  // because the semantics in the current C++0x working paper (N3225 at the
+  // time of this writing) break the standard definition of std::forward
+  // and std::reference_wrapper when dealing with references to functions.
+  // Proposed wording changes submitted to CWG for consideration.
+  if (SCS1.BindsImplicitObjectArgumentWithoutRefQualifier ||
+      SCS2.BindsImplicitObjectArgumentWithoutRefQualifier)
+    return false;
+
+  return (!SCS1.IsLvalueReference && SCS1.BindsToRvalue &&
+          SCS2.IsLvalueReference) ||
+         (SCS1.IsLvalueReference && SCS1.BindsToFunctionLvalue &&
+          !SCS2.IsLvalueReference);
+}
+
 /// CompareStandardConversionSequences - Compare two standard
 /// conversion sequences to determine whether one is better than the
 /// other or if they are indistinguishable (C++ 13.3.3.2p3).
@@ -2339,17 +2585,11 @@ CompareStandardConversionSequences(Sema &S,
     return QualCK;
 
   if (SCS1.ReferenceBinding && SCS2.ReferenceBinding) {
-    // C++0x [over.ics.rank]p3b4:
-    //   -- S1 and S2 are reference bindings (8.5.3) and neither refers to an
-    //      implicit object parameter of a non-static member function declared
-    //      without a ref-qualifier, and S1 binds an rvalue reference to an
-    //      rvalue and S2 binds an lvalue reference.
-    // FIXME: We don't know if we're dealing with the implicit object parameter,
-    // or if the member function in this case has a ref qualifier.
-    // (Of course, we don't have ref qualifiers yet.)
-    if (SCS1.RRefBinding != SCS2.RRefBinding)
-      return SCS1.RRefBinding ? ImplicitConversionSequence::Better
-                              : ImplicitConversionSequence::Worse;
+    // Check for a better reference binding based on the kind of bindings.
+    if (isBetterReferenceBindingKind(SCS1, SCS2))
+      return ImplicitConversionSequence::Better;
+    else if (isBetterReferenceBindingKind(SCS2, SCS1))
+      return ImplicitConversionSequence::Worse;
 
     // C++ [over.ics.rank]p3b4:
     //   -- S1 and S2 are reference bindings (8.5.3), and the types to
@@ -2365,8 +2605,8 @@ CompareStandardConversionSequences(Sema &S,
     QualType UnqualT1 = S.Context.getUnqualifiedArrayType(T1, T1Quals);
     QualType UnqualT2 = S.Context.getUnqualifiedArrayType(T2, T2Quals);
     if (UnqualT1 == UnqualT2) {
-      // If the type is an array type, promote the element qualifiers to the type
-      // for comparison.
+      // If the type is an array type, promote the element qualifiers to the
+      // type for comparison.
       if (isa<ArrayType>(T1) && T1Quals)
         T1 = S.Context.getQualifiedType(UnqualT1, T1Quals);
       if (isa<ArrayType>(T2) && T2Quals)
@@ -2513,9 +2753,6 @@ CompareDerivedToBaseConversions(Sema &S,
   //   If class B is derived directly or indirectly from class A and
   //   class C is derived directly or indirectly from B,
   //
-  // For Objective-C, we let A, B, and C also be Objective-C
-  // interfaces.
-
   // Compare based on pointer conversions.
   if (SCS1.Second == ICK_Pointer_Conversion &&
       SCS2.Second == ICK_Pointer_Conversion &&
@@ -2531,24 +2768,12 @@ CompareDerivedToBaseConversions(Sema &S,
     QualType ToPointee2
       = ToType2->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
 
-    const ObjCObjectType* FromIface1 = FromPointee1->getAs<ObjCObjectType>();
-    const ObjCObjectType* FromIface2 = FromPointee2->getAs<ObjCObjectType>();
-    const ObjCObjectType* ToIface1 = ToPointee1->getAs<ObjCObjectType>();
-    const ObjCObjectType* ToIface2 = ToPointee2->getAs<ObjCObjectType>();
-
     //   -- conversion of C* to B* is better than conversion of C* to A*,
     if (FromPointee1 == FromPointee2 && ToPointee1 != ToPointee2) {
       if (S.IsDerivedFrom(ToPointee1, ToPointee2))
         return ImplicitConversionSequence::Better;
       else if (S.IsDerivedFrom(ToPointee2, ToPointee1))
         return ImplicitConversionSequence::Worse;
-
-      if (ToIface1 && ToIface2) {
-        if (S.Context.canAssignObjCInterfaces(ToIface2, ToIface1))
-          return ImplicitConversionSequence::Better;
-        else if (S.Context.canAssignObjCInterfaces(ToIface1, ToIface2))
-          return ImplicitConversionSequence::Worse;
-      }
     }
 
     //   -- conversion of B* to A* is better than conversion of C* to A*,
@@ -2557,27 +2782,90 @@ CompareDerivedToBaseConversions(Sema &S,
         return ImplicitConversionSequence::Better;
       else if (S.IsDerivedFrom(FromPointee1, FromPointee2))
         return ImplicitConversionSequence::Worse;
+    }
+  } else if (SCS1.Second == ICK_Pointer_Conversion &&
+             SCS2.Second == ICK_Pointer_Conversion) {
+    const ObjCObjectPointerType *FromPtr1
+      = FromType1->getAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *FromPtr2
+      = FromType2->getAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *ToPtr1
+      = ToType1->getAs<ObjCObjectPointerType>();
+    const ObjCObjectPointerType *ToPtr2
+      = ToType2->getAs<ObjCObjectPointerType>();
+    
+    if (FromPtr1 && FromPtr2 && ToPtr1 && ToPtr2) {
+      // Apply the same conversion ranking rules for Objective-C pointer types
+      // that we do for C++ pointers to class types. However, we employ the
+      // Objective-C pseudo-subtyping relationship used for assignment of
+      // Objective-C pointer types.
+      bool FromAssignLeft
+        = S.Context.canAssignObjCInterfaces(FromPtr1, FromPtr2);
+      bool FromAssignRight
+        = S.Context.canAssignObjCInterfaces(FromPtr2, FromPtr1);
+      bool ToAssignLeft
+        = S.Context.canAssignObjCInterfaces(ToPtr1, ToPtr2);
+      bool ToAssignRight
+        = S.Context.canAssignObjCInterfaces(ToPtr2, ToPtr1);
+      
+      // A conversion to an a non-id object pointer type or qualified 'id' 
+      // type is better than a conversion to 'id'.
+      if (ToPtr1->isObjCIdType() &&
+          (ToPtr2->isObjCQualifiedIdType() || ToPtr2->getInterfaceDecl()))
+        return ImplicitConversionSequence::Worse;
+      if (ToPtr2->isObjCIdType() &&
+          (ToPtr1->isObjCQualifiedIdType() || ToPtr1->getInterfaceDecl()))
+        return ImplicitConversionSequence::Better;
+      
+      // A conversion to a non-id object pointer type is better than a 
+      // conversion to a qualified 'id' type 
+      if (ToPtr1->isObjCQualifiedIdType() && ToPtr2->getInterfaceDecl())
+        return ImplicitConversionSequence::Worse;
+      if (ToPtr2->isObjCQualifiedIdType() && ToPtr1->getInterfaceDecl())
+        return ImplicitConversionSequence::Better;
+  
+      // A conversion to an a non-Class object pointer type or qualified 'Class' 
+      // type is better than a conversion to 'Class'.
+      if (ToPtr1->isObjCClassType() &&
+          (ToPtr2->isObjCQualifiedClassType() || ToPtr2->getInterfaceDecl()))
+        return ImplicitConversionSequence::Worse;
+      if (ToPtr2->isObjCClassType() &&
+          (ToPtr1->isObjCQualifiedClassType() || ToPtr1->getInterfaceDecl()))
+        return ImplicitConversionSequence::Better;
+      
+      // A conversion to a non-Class object pointer type is better than a 
+      // conversion to a qualified 'Class' type.
+      if (ToPtr1->isObjCQualifiedClassType() && ToPtr2->getInterfaceDecl())
+        return ImplicitConversionSequence::Worse;
+      if (ToPtr2->isObjCQualifiedClassType() && ToPtr1->getInterfaceDecl())
+        return ImplicitConversionSequence::Better;
 
-      if (FromIface1 && FromIface2) {
-        if (S.Context.canAssignObjCInterfaces(FromIface1, FromIface2))
-          return ImplicitConversionSequence::Better;
-        else if (S.Context.canAssignObjCInterfaces(FromIface2, FromIface1))
-          return ImplicitConversionSequence::Worse;
-      }
+      //   -- "conversion of C* to B* is better than conversion of C* to A*,"
+      if (S.Context.hasSameType(FromType1, FromType2) && 
+          !FromPtr1->isObjCIdType() && !FromPtr1->isObjCClassType() &&
+          (ToAssignLeft != ToAssignRight))
+        return ToAssignLeft? ImplicitConversionSequence::Worse
+                           : ImplicitConversionSequence::Better;
+
+      //   -- "conversion of B* to A* is better than conversion of C* to A*,"
+      if (S.Context.hasSameUnqualifiedType(ToType1, ToType2) &&
+          (FromAssignLeft != FromAssignRight))
+        return FromAssignLeft? ImplicitConversionSequence::Better
+        : ImplicitConversionSequence::Worse;
     }
   }
-
+  
   // Ranking of member-pointer types.
   if (SCS1.Second == ICK_Pointer_Member && SCS2.Second == ICK_Pointer_Member &&
       FromType1->isMemberPointerType() && FromType2->isMemberPointerType() &&
       ToType1->isMemberPointerType() && ToType2->isMemberPointerType()) {
-    const MemberPointerType * FromMemPointer1 = 
+    const MemberPointerType * FromMemPointer1 =
                                         FromType1->getAs<MemberPointerType>();
-    const MemberPointerType * ToMemPointer1 = 
+    const MemberPointerType * ToMemPointer1 =
                                           ToType1->getAs<MemberPointerType>();
-    const MemberPointerType * FromMemPointer2 = 
+    const MemberPointerType * FromMemPointer2 =
                                           FromType2->getAs<MemberPointerType>();
-    const MemberPointerType * ToMemPointer2 = 
+    const MemberPointerType * ToMemPointer2 =
                                           ToType2->getAs<MemberPointerType>();
     const Type *FromPointeeType1 = FromMemPointer1->getClass();
     const Type *ToPointeeType1 = ToMemPointer1->getClass();
@@ -2602,7 +2890,7 @@ CompareDerivedToBaseConversions(Sema &S,
         return ImplicitConversionSequence::Worse;
     }
   }
-  
+
   if (SCS1.Second == ICK_Derived_To_Base) {
     //   -- conversion of C to B is better than conversion of C to A,
     //   -- binding of an expression of type C to a reference of type
@@ -2708,10 +2996,6 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
   CXXRecordDecl *T2RecordDecl
     = dyn_cast<CXXRecordDecl>(T2->getAs<RecordType>()->getDecl());
 
-  QualType ToType
-    = AllowRvalues? DeclType->getAs<ReferenceType>()->getPointeeType()
-                  : DeclType;
-
   OverloadCandidateSet CandidateSet(DeclLoc);
   const UnresolvedSetImpl *Conversions
     = T2RecordDecl->getVisibleConversionFunctions();
@@ -2730,20 +3014,22 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
     else
       Conv = cast<CXXConversionDecl>(D);
 
-    // If this is an explicit conversion, and we're not allowed to consider 
+    // If this is an explicit conversion, and we're not allowed to consider
     // explicit conversions, skip it.
     if (!AllowExplicit && Conv->isExplicit())
       continue;
-    
+
     if (AllowRvalues) {
       bool DerivedToBase = false;
       bool ObjCConversion = false;
       if (!ConvTemplate &&
-          S.CompareReferenceRelationship(DeclLoc,
-                                         Conv->getConversionType().getNonReferenceType().getUnqualifiedType(),
-                                         DeclType.getNonReferenceType().getUnqualifiedType(),
-                                         DerivedToBase, ObjCConversion)
-            == Sema::Ref_Incompatible)
+          S.CompareReferenceRelationship(
+            DeclLoc,
+            Conv->getConversionType().getNonReferenceType()
+              .getUnqualifiedType(),
+            DeclType.getNonReferenceType().getUnqualifiedType(),
+            DerivedToBase, ObjCConversion) ==
+          Sema::Ref_Incompatible)
         continue;
     } else {
       // If the conversion function doesn't return a reference type,
@@ -2757,17 +3043,17 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
            !RefType->getPointeeType()->isFunctionType()))
         continue;
     }
-    
+
     if (ConvTemplate)
       S.AddTemplateConversionCandidate(ConvTemplate, I.getPair(), ActingDC,
-                                       Init, ToType, CandidateSet);
+                                       Init, DeclType, CandidateSet);
     else
       S.AddConversionCandidate(Conv, I.getPair(), ActingDC, Init,
-                               ToType, CandidateSet);
+                               DeclType, CandidateSet);
   }
 
   OverloadCandidateSet::iterator Best;
-  switch (CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(S, DeclLoc, Best, true)) {
   case OR_Success:
     // C++ [over.ics.ref]p1:
     //
@@ -2786,6 +3072,7 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
     ICS.UserDefined.Before = Best->Conversions[0].Standard;
     ICS.UserDefined.After = Best->FinalConversion;
     ICS.UserDefined.ConversionFunction = Best->Function;
+    ICS.UserDefined.FoundConversionFunction = Best->FoundDecl.getDecl();
     ICS.UserDefined.EllipsisConversion = false;
     assert(ICS.UserDefined.After.ReferenceBinding &&
            ICS.UserDefined.After.DirectBinding &&
@@ -2806,7 +3093,7 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
     // conversion; continue with other checks.
     return false;
   }
-  
+
   return false;
 }
 
@@ -2851,9 +3138,7 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   //   of type "cv2 T2" as follows:
 
   //     -- If reference is an lvalue reference and the initializer expression
-  // The next bullet point (T1 is a function) is pretty much equivalent to this
-  // one, so it's handled here.
-  if (!isRValRef || T1->isFunctionType()) {
+  if (!isRValRef) {
     //     -- is an lvalue (but is not a bit-field), and "cv1 T1" is
     //        reference-compatible with "cv2 T2," or
     //
@@ -2879,7 +3164,10 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
       ICS.Standard.setToType(2, T1);
       ICS.Standard.ReferenceBinding = true;
       ICS.Standard.DirectBinding = true;
-      ICS.Standard.RRefBinding = isRValRef;
+      ICS.Standard.IsLvalueReference = !isRValRef;
+      ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType();
+      ICS.Standard.BindsToRvalue = false;
+      ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
       ICS.Standard.CopyConstructor = 0;
 
       // Nothing more to do: the inaccessibility/ambiguity check for
@@ -2897,7 +3185,7 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
     //          conversion functions (13.3.1.6) and choosing the best
     //          one through overload resolution (13.3)),
     if (!SuppressUserConversions && T2->isRecordType() &&
-        !S.RequireCompleteType(DeclLoc, T2, 0) && 
+        !S.RequireCompleteType(DeclLoc, T2, 0) &&
         RefRelationship == Sema::Ref_Incompatible) {
       if (FindConversionForRefInit(S, ICS, DeclType, DeclLoc,
                                    Init, T2, /*AllowRvalues=*/false,
@@ -2908,9 +3196,8 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
 
   //     -- Otherwise, the reference shall be an lvalue reference to a
   //        non-volatile const type (i.e., cv1 shall be const), or the reference
-  //        shall be an rvalue reference and the initializer expression shall be
-  //        an rvalue or have a function type.
-  // 
+  //        shall be an rvalue reference.
+  //
   // We actually handle one oddity of C++ [over.ics.ref] at this
   // point, which is that, due to p2 (which short-circuits reference
   // binding by only attempting a simple conversion for non-direct
@@ -2920,70 +3207,70 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   // qualifier.
   // This is also the point where rvalue references and lvalue inits no longer
   // go together.
-  if ((!isRValRef && !T1.isConstQualified()) ||
-      (isRValRef && InitCategory.isLValue()))
+  if (!isRValRef && !T1.isConstQualified())
     return ICS;
 
-  //       -- If T1 is a function type, then
-  //          -- if T2 is the same type as T1, the reference is bound to the
-  //             initializer expression lvalue;
-  //          -- if T2 is a class type and the initializer expression can be
-  //             implicitly converted to an lvalue of type T1 [...], the
-  //             reference is bound to the function lvalue that is the result
-  //             of the conversion;
-  // This is the same as for the lvalue case above, so it was handled there.
-  //          -- otherwise, the program is ill-formed.
-  // This is the one difference to the lvalue case.
-  if (T1->isFunctionType())
+  //       -- If the initializer expression
+  //
+  //            -- is an xvalue, class prvalue, array prvalue or function
+  //               lvalue and "cv1T1" is reference-compatible with "cv2 T2", or
+  if (RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification &&
+      (InitCategory.isXValue() ||
+      (InitCategory.isPRValue() && (T2->isRecordType() || T2->isArrayType())) ||
+      (InitCategory.isLValue() && T2->isFunctionType()))) {
+    ICS.setStandard();
+    ICS.Standard.First = ICK_Identity;
+    ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base
+                      : ObjCConversion? ICK_Compatible_Conversion
+                      : ICK_Identity;
+    ICS.Standard.Third = ICK_Identity;
+    ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
+    ICS.Standard.setToType(0, T2);
+    ICS.Standard.setToType(1, T1);
+    ICS.Standard.setToType(2, T1);
+    ICS.Standard.ReferenceBinding = true;
+    // In C++0x, this is always a direct binding. In C++98/03, it's a direct
+    // binding unless we're binding to a class prvalue.
+    // Note: Although xvalues wouldn't normally show up in C++98/03 code, we
+    // allow the use of rvalue references in C++98/03 for the benefit of
+    // standard library implementors; therefore, we need the xvalue check here.
+    ICS.Standard.DirectBinding =
+      S.getLangOptions().CPlusPlus0x ||
+      (InitCategory.isPRValue() && !T2->isRecordType());
+    ICS.Standard.IsLvalueReference = !isRValRef;
+    ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType();
+    ICS.Standard.BindsToRvalue = InitCategory.isRValue();
+    ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
+    ICS.Standard.CopyConstructor = 0;
     return ICS;
+  }
 
-  //       -- Otherwise, if T2 is a class type and
-  //          -- the initializer expression is an rvalue and "cv1 T1"
-  //             is reference-compatible with "cv2 T2," or
-  //
-  //          -- T1 is not reference-related to T2 and the initializer
-  //             expression can be implicitly converted to an rvalue
-  //             of type "cv3 T3" (this conversion is selected by
-  //             enumerating the applicable conversion functions
-  //             (13.3.1.6) and choosing the best one through overload
-  //             resolution (13.3)),
+  //            -- has a class type (i.e., T2 is a class type), where T1 is not
+  //               reference-related to T2, and can be implicitly converted to
+  //               an xvalue, class prvalue, or function lvalue of type
+  //               "cv3 T3", where "cv1 T1" is reference-compatible with
+  //               "cv3 T3",
   //
-  //          then the reference is bound to the initializer
-  //          expression rvalue in the first case and to the object
-  //          that is the result of the conversion in the second case
-  //          (or, in either case, to the appropriate base class
-  //          subobject of the object).
-  if (T2->isRecordType()) {
-    // First case: "cv1 T1" is reference-compatible with "cv2 T2". This is a
-    // direct binding in C++0x but not in C++03.
-    if (InitCategory.isRValue() && 
-        RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification) {
-      ICS.setStandard();
-      ICS.Standard.First = ICK_Identity;
-      ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base 
-                          : ObjCConversion? ICK_Compatible_Conversion
-                          : ICK_Identity;
-      ICS.Standard.Third = ICK_Identity;
-      ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
-      ICS.Standard.setToType(0, T2);
-      ICS.Standard.setToType(1, T1);
-      ICS.Standard.setToType(2, T1);
-      ICS.Standard.ReferenceBinding = true;
-      ICS.Standard.DirectBinding = S.getLangOptions().CPlusPlus0x;
-      ICS.Standard.RRefBinding = isRValRef;
-      ICS.Standard.CopyConstructor = 0;
-      return ICS;
-    }
-    
-    // Second case: not reference-related.
-    if (RefRelationship == Sema::Ref_Incompatible &&
-        !S.RequireCompleteType(DeclLoc, T2, 0) && 
-        FindConversionForRefInit(S, ICS, DeclType, DeclLoc,
-                                 Init, T2, /*AllowRvalues=*/true,
-                                 AllowExplicit))
-      return ICS;
+  //          then the reference is bound to the value of the initializer
+  //          expression in the first case and to the result of the conversion
+  //          in the second case (or, in either case, to an appropriate base
+  //          class subobject).
+  if (!SuppressUserConversions && RefRelationship == Sema::Ref_Incompatible &&
+      T2->isRecordType() && !S.RequireCompleteType(DeclLoc, T2, 0) &&
+      FindConversionForRefInit(S, ICS, DeclType, DeclLoc,
+                               Init, T2, /*AllowRvalues=*/true,
+                               AllowExplicit)) {
+    // In the second case, if the reference is an rvalue reference
+    // and the second standard conversion sequence of the
+    // user-defined conversion sequence includes an lvalue-to-rvalue
+    // conversion, the program is ill-formed.
+    if (ICS.isUserDefined() && isRValRef &&
+        ICS.UserDefined.After.First == ICK_Lvalue_To_Rvalue)
+      ICS.setBad(BadConversionSequence::no_conversion, Init, DeclType);
+
+    return ICS;
   }
-  
+
   //       -- Otherwise, a temporary of type "cv1 T1" is created and
   //          initialized from the initializer expression using the
   //          rules for a non-reference copy initialization (8.5). The
@@ -3008,6 +3295,12 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
       (T1->isRecordType() || T2->isRecordType()))
     return ICS;
 
+  // If T1 is reference-related to T2 and the reference is an rvalue
+  // reference, the initializer expression shall not be an lvalue.
+  if (RefRelationship >= Sema::Ref_Related &&
+      isRValRef && Init->Classify(S.Context).isLValue())
+    return ICS;
+
   // C++ [over.ics.ref]p2:
   //   When a parameter of reference type is not bound directly to
   //   an argument expression, the conversion sequence is the one
@@ -3020,16 +3313,24 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   //   and does not constitute a conversion.
   ICS = TryImplicitConversion(S, Init, T1, SuppressUserConversions,
                               /*AllowExplicit=*/false,
-                              /*InOverloadResolution=*/false);
+                              /*InOverloadResolution=*/false,
+                              /*CStyle=*/false);
 
   // Of course, that's still a reference binding.
   if (ICS.isStandard()) {
     ICS.Standard.ReferenceBinding = true;
-    ICS.Standard.RRefBinding = isRValRef;
+    ICS.Standard.IsLvalueReference = !isRValRef;
+    ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType();
+    ICS.Standard.BindsToRvalue = true;
+    ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
   } else if (ICS.isUserDefined()) {
     ICS.UserDefined.After.ReferenceBinding = true;
-    ICS.UserDefined.After.RRefBinding = isRValRef;
+    ICS.Standard.IsLvalueReference = !isRValRef;
+    ICS.Standard.BindsToFunctionLvalue = T2->isFunctionType();
+    ICS.Standard.BindsToRvalue = true;
+    ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier = false;
   }
+
   return ICS;
 }
 
@@ -3041,7 +3342,7 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
 /// do not permit any user-defined conversion sequences.
 static ImplicitConversionSequence
 TryCopyInitialization(Sema &S, Expr *From, QualType ToType,
-                      bool SuppressUserConversions, 
+                      bool SuppressUserConversions,
                       bool InOverloadResolution) {
   if (ToType->isReferenceType())
     return TryReferenceInit(S, From, ToType,
@@ -3052,7 +3353,8 @@ TryCopyInitialization(Sema &S, Expr *From, QualType ToType,
   return TryImplicitConversion(S, From, ToType,
                                SuppressUserConversions,
                                /*AllowExplicit=*/false,
-                               InOverloadResolution);
+                               InOverloadResolution,
+                               /*CStyle=*/false);
 }
 
 /// TryObjectArgumentInitialization - Try to initialize the object
@@ -3060,6 +3362,7 @@ TryCopyInitialization(Sema &S, Expr *From, QualType ToType,
 /// expression @p From.
 static ImplicitConversionSequence
 TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
+                                Expr::Classification FromClassification,
                                 CXXMethodDecl *Method,
                                 CXXRecordDecl *ActingContext) {
   QualType ClassType = S.Context.getTypeDeclType(ActingContext);
@@ -3075,24 +3378,37 @@ TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
 
   // We need to have an object of class type.
   QualType FromType = OrigFromType;
-  if (const PointerType *PT = FromType->getAs<PointerType>())
+  if (const PointerType *PT = FromType->getAs<PointerType>()) {
     FromType = PT->getPointeeType();
 
+    // When we had a pointer, it's implicitly dereferenced, so we
+    // better have an lvalue.
+    assert(FromClassification.isLValue());
+  }
+
   assert(FromType->isRecordType());
 
-  // The implicit object parameter is has the type "reference to cv X",
-  // where X is the class of which the function is a member
-  // (C++ [over.match.funcs]p4). However, when finding an implicit
-  // conversion sequence for the argument, we are not allowed to
-  // create temporaries or perform user-defined conversions
+  // C++0x [over.match.funcs]p4:
+  //   For non-static member functions, the type of the implicit object
+  //   parameter is
+  //
+  //     - "lvalue reference to cv X" for functions declared without a
+  //        ref-qualifier or with the & ref-qualifier
+  //     - "rvalue reference to cv X" for functions declared with the &&
+  //        ref-qualifier
+  //
+  // where X is the class of which the function is a member and cv is the
+  // cv-qualification on the member function declaration.
+  //
+  // However, when finding an implicit conversion sequence for the argument, we
+  // are not allowed to create temporaries or perform user-defined conversions
   // (C++ [over.match.funcs]p5). We perform a simplified version of
   // reference binding here, that allows class rvalues to bind to
   // non-constant references.
 
-  // First check the qualifiers. We don't care about lvalue-vs-rvalue
-  // with the implicit object parameter (C++ [over.match.funcs]p5).
+  // First check the qualifiers.
   QualType FromTypeCanon = S.Context.getCanonicalType(FromType);
-  if (ImplicitParamType.getCVRQualifiers() 
+  if (ImplicitParamType.getCVRQualifiers()
                                     != FromTypeCanon.getLocalCVRQualifiers() &&
       !ImplicitParamType.isAtLeastAsQualifiedAs(FromTypeCanon)) {
     ICS.setBad(BadConversionSequence::bad_qualifiers,
@@ -3114,6 +3430,31 @@ TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
     return ICS;
   }
 
+  // Check the ref-qualifier.
+  switch (Method->getRefQualifier()) {
+  case RQ_None:
+    // Do nothing; we don't care about lvalueness or rvalueness.
+    break;
+
+  case RQ_LValue:
+    if (!FromClassification.isLValue() && Quals != Qualifiers::Const) {
+      // non-const lvalue reference cannot bind to an rvalue
+      ICS.setBad(BadConversionSequence::lvalue_ref_to_rvalue, FromType,
+                 ImplicitParamType);
+      return ICS;
+    }
+    break;
+
+  case RQ_RValue:
+    if (!FromClassification.isRValue()) {
+      // rvalue reference cannot bind to an lvalue
+      ICS.setBad(BadConversionSequence::rvalue_ref_to_lvalue, FromType,
+                 ImplicitParamType);
+      return ICS;
+    }
+    break;
+  }
+
   // Success. Mark this as a reference binding.
   ICS.setStandard();
   ICS.Standard.setAsIdentityConversion();
@@ -3122,7 +3463,11 @@ TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
   ICS.Standard.setAllToTypes(ImplicitParamType);
   ICS.Standard.ReferenceBinding = true;
   ICS.Standard.DirectBinding = true;
-  ICS.Standard.RRefBinding = false;
+  ICS.Standard.IsLvalueReference = Method->getRefQualifier() != RQ_RValue;
+  ICS.Standard.BindsToFunctionLvalue = false;
+  ICS.Standard.BindsToRvalue = FromClassification.isRValue();
+  ICS.Standard.BindsImplicitObjectArgumentWithoutRefQualifier
+    = (Method->getRefQualifier() == RQ_None);
   return ICS;
 }
 
@@ -3130,31 +3475,50 @@ TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
 /// the implicit object parameter for the given Method with the given
 /// expression.
 bool
-Sema::PerformObjectArgumentInitialization(Expr *&From, 
-                                          NestedNameSpecifier *Qualifier, 
+Sema::PerformObjectArgumentInitialization(Expr *&From,
+                                          NestedNameSpecifier *Qualifier,
                                           NamedDecl *FoundDecl,
                                           CXXMethodDecl *Method) {
   QualType FromRecordType, DestType;
   QualType ImplicitParamRecordType  =
     Method->getThisType(Context)->getAs<PointerType>()->getPointeeType();
 
+  Expr::Classification FromClassification;
   if (const PointerType *PT = From->getType()->getAs<PointerType>()) {
     FromRecordType = PT->getPointeeType();
     DestType = Method->getThisType(Context);
+    FromClassification = Expr::Classification::makeSimpleLValue();
   } else {
     FromRecordType = From->getType();
     DestType = ImplicitParamRecordType;
+    FromClassification = From->Classify(Context);
   }
 
   // Note that we always use the true parent context when performing
   // the actual argument initialization.
   ImplicitConversionSequence ICS
-    = TryObjectArgumentInitialization(*this, From->getType(), Method,
-                                      Method->getParent());
-  if (ICS.isBad())
+    = TryObjectArgumentInitialization(*this, From->getType(), FromClassification,
+                                      Method, Method->getParent());
+  if (ICS.isBad()) {
+    if (ICS.Bad.Kind == BadConversionSequence::bad_qualifiers) {
+      Qualifiers FromQs = FromRecordType.getQualifiers();
+      Qualifiers ToQs = DestType.getQualifiers();
+      unsigned CVR = FromQs.getCVRQualifiers() & ~ToQs.getCVRQualifiers();
+      if (CVR) {
+        Diag(From->getSourceRange().getBegin(),
+             diag::err_member_function_call_bad_cvr)
+          << Method->getDeclName() << FromRecordType << (CVR - 1)
+          << From->getSourceRange();
+        Diag(Method->getLocation(), diag::note_previous_decl)
+          << Method->getDeclName();
+        return true;
+      }
+    }
+
     return Diag(From->getSourceRange().getBegin(),
                 diag::err_implicit_object_parameter_init)
        << ImplicitParamRecordType << FromRecordType << From->getSourceRange();
+  }
 
   if (ICS.Standard.Second == ICK_Derived_To_Base)
     return PerformObjectMemberConversion(From, Qualifier, FoundDecl, Method);
@@ -3174,7 +3538,8 @@ TryContextuallyConvertToBool(Sema &S, Expr *From) {
                                // FIXME: Are these flags correct?
                                /*SuppressUserConversions=*/false,
                                /*AllowExplicit=*/true,
-                               /*InOverloadResolution=*/false);
+                               /*InOverloadResolution=*/false,
+                               /*CStyle=*/false);
 }
 
 /// PerformContextuallyConvertToBool - Perform a contextual conversion
@@ -3183,14 +3548,14 @@ bool Sema::PerformContextuallyConvertToBool(Expr *&From) {
   ImplicitConversionSequence ICS = TryContextuallyConvertToBool(*this, From);
   if (!ICS.isBad())
     return PerformImplicitConversion(From, Context.BoolTy, ICS, AA_Converting);
-  
+
   if (!DiagnoseMultipleUserDefinedConversion(From, Context.BoolTy))
     return  Diag(From->getSourceRange().getBegin(),
                  diag::err_typecheck_bool_condition)
                   << From->getType() << From->getSourceRange();
   return true;
 }
-  
+
 /// TryContextuallyConvertToObjCId - Attempt to contextually convert the
 /// expression From to 'id'.
 static ImplicitConversionSequence
@@ -3200,7 +3565,8 @@ TryContextuallyConvertToObjCId(Sema &S, Expr *From) {
                                // FIXME: Are these flags correct?
                                /*SuppressUserConversions=*/false,
                                /*AllowExplicit=*/true,
-                               /*InOverloadResolution=*/false);
+                               /*InOverloadResolution=*/false,
+                               /*CStyle=*/false);
 }
 
 /// PerformContextuallyConvertToObjCId - Perform a contextual conversion
@@ -3213,7 +3579,7 @@ bool Sema::PerformContextuallyConvertToObjCId(Expr *&From) {
   return true;
 }
 
-/// \brief Attempt to convert the given expression to an integral or 
+/// \brief Attempt to convert the given expression to an integral or
 /// enumeration type.
 ///
 /// This routine will attempt to convert an expression of class type to an
@@ -3241,7 +3607,7 @@ bool Sema::PerformContextuallyConvertToObjCId(Expr *&From) {
 /// \param AmbigDiag The diagnostic to be emitted if there is more than one
 /// conversion function that could convert to integral or enumeration type.
 ///
-/// \param AmbigNote The note to be emitted with \p AmbigDiag for each 
+/// \param AmbigNote The note to be emitted with \p AmbigDiag for each
 /// usable conversion function.
 ///
 /// \param ConvDiag The diagnostic to be emitted if we are calling a conversion
@@ -3249,7 +3615,7 @@ bool Sema::PerformContextuallyConvertToObjCId(Expr *&From) {
 ///
 /// \returns The expression, converted to an integral or enumeration type if
 /// successful.
-ExprResult 
+ExprResult
 Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
                                          const PartialDiagnostic &NotIntDiag,
                                        const PartialDiagnostic &IncompleteDiag,
@@ -3261,7 +3627,7 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
   // We can't perform any more checking for type-dependent expressions.
   if (From->isTypeDependent())
     return Owned(From);
-  
+
   // If the expression already has integral or enumeration type, we're golden.
   QualType T = From->getType();
   if (T->isIntegralOrEnumerationType())
@@ -3269,7 +3635,7 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
 
   // FIXME: Check for missing '()' if T is a function type?
 
-  // If we don't have a class type in C++, there's no way we can get an 
+  // If we don't have a class type in C++, there's no way we can get an
   // expression of integral or enumeration type.
   const RecordType *RecordTy = T->getAs<RecordType>();
   if (!RecordTy || !getLangOptions().CPlusPlus) {
@@ -3277,20 +3643,20 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
       << T << From->getSourceRange();
     return Owned(From);
   }
-    
+
   // We must have a complete class type.
   if (RequireCompleteType(Loc, T, IncompleteDiag))
     return Owned(From);
-  
+
   // Look for a conversion to an integral or enumeration type.
   UnresolvedSet<4> ViableConversions;
   UnresolvedSet<4> ExplicitConversions;
   const UnresolvedSetImpl *Conversions
     = cast<CXXRecordDecl>(RecordTy->getDecl())->getVisibleConversionFunctions();
-  
+
   for (UnresolvedSetImpl::iterator I = Conversions->begin(),
-                                   E = Conversions->end(); 
-       I != E; 
+                                   E = Conversions->end();
+       I != E;
        ++I) {
     if (CXXConversionDecl *Conversion
           = dyn_cast<CXXConversionDecl>((*I)->getUnderlyingDecl()))
@@ -3302,21 +3668,21 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
           ViableConversions.addDecl(I.getDecl(), I.getAccess());
       }
   }
-    
+
   switch (ViableConversions.size()) {
   case 0:
     if (ExplicitConversions.size() == 1) {
       DeclAccessPair Found = ExplicitConversions[0];
       CXXConversionDecl *Conversion
         = cast<CXXConversionDecl>(Found->getUnderlyingDecl());
-      
+
       // The user probably meant to invoke the given explicit
       // conversion; use it.
       QualType ConvTy
         = Conversion->getConversionType().getNonReferenceType();
       std::string TypeStr;
       ConvTy.getAsStringInternal(TypeStr, Context.PrintingPolicy);
-      
+
       Diag(Loc, ExplicitConvDiag)
         << T << ConvTy
         << FixItHint::CreateInsertion(From->getLocStart(),
@@ -3325,41 +3691,49 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
                                       ")");
       Diag(Conversion->getLocation(), ExplicitConvNote)
         << ConvTy->isEnumeralType() << ConvTy;
-      
-      // If we aren't in a SFINAE context, build a call to the 
+
+      // If we aren't in a SFINAE context, build a call to the
       // explicit conversion function.
       if (isSFINAEContext())
         return ExprError();
-      
+
       CheckMemberOperatorAccess(From->getExprLoc(), From, 0, Found);
-      From = BuildCXXMemberCallExpr(From, Found, Conversion);
+      ExprResult Result = BuildCXXMemberCallExpr(From, Found, Conversion);
+      if (Result.isInvalid())
+        return ExprError();
+
+      From = Result.get();
     }
-    
+
     // We'll complain below about a non-integral condition type.
     break;
-      
+
   case 1: {
     // Apply this conversion.
     DeclAccessPair Found = ViableConversions[0];
     CheckMemberOperatorAccess(From->getExprLoc(), From, 0, Found);
-    
+
     CXXConversionDecl *Conversion
       = cast<CXXConversionDecl>(Found->getUnderlyingDecl());
     QualType ConvTy
-      = Conversion->getConversionType().getNonReferenceType();    
+      = Conversion->getConversionType().getNonReferenceType();
     if (ConvDiag.getDiagID()) {
       if (isSFINAEContext())
         return ExprError();
-      
+
       Diag(Loc, ConvDiag)
         << T << ConvTy->isEnumeralType() << ConvTy << From->getSourceRange();
     }
-    
-    From = BuildCXXMemberCallExpr(From, Found,
+
+    ExprResult Result = BuildCXXMemberCallExpr(From, Found,
                           cast<CXXConversionDecl>(Found->getUnderlyingDecl()));
+    if (Result.isInvalid())
+      return ExprError();
+
+    From = Result.get();
     break;
   }
-    
+
   default:
     Diag(Loc, AmbigDiag)
       << T << From->getSourceRange();
@@ -3372,7 +3746,7 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
     }
     return Owned(From);
   }
-  
+
   if (!From->getType()->isIntegralOrEnumerationType())
     Diag(Loc, NotIntDiag)
       << From->getType() << From->getSourceRange();
@@ -3411,7 +3785,8 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
       // object argument (C++ [over.call.func]p3), and the acting context
       // is irrelevant.
       AddMethodCandidate(Method, FoundDecl, Method->getParent(),
-                         QualType(), Args, NumArgs, CandidateSet,
+                         QualType(), Expr::Classification::makeSimpleLValue(),
+                         Args, NumArgs, CandidateSet,
                          SuppressUserConversions);
       return;
     }
@@ -3430,13 +3805,13 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
     //   A member function template is never instantiated to perform the copy
     //   of a class object to an object of its class type.
     QualType ClassType = Context.getTypeDeclType(Constructor->getParent());
-    if (NumArgs == 1 && 
-        Constructor->isCopyConstructorLikeSpecialization() &&
+    if (NumArgs == 1 &&
+        Constructor->isSpecializationCopyingObject() &&
         (Context.hasSameUnqualifiedType(ClassType, Args[0]->getType()) ||
          IsDerivedFrom(Args[0]->getType(), ClassType)))
       return;
   }
-  
+
   // Add this candidate
   CandidateSet.push_back(OverloadCandidate());
   OverloadCandidate& Candidate = CandidateSet.back();
@@ -3445,13 +3820,14 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
   Candidate.Viable = true;
   Candidate.IsSurrogate = false;
   Candidate.IgnoreObjectArgument = false;
+  Candidate.ExplicitCallArguments = NumArgs;
 
   unsigned NumArgsInProto = Proto->getNumArgs();
 
   // (C++ 13.3.2p2): A candidate function having fewer than m
   // parameters is viable only if it has an ellipsis in its parameter
   // list (8.3.5).
-  if ((NumArgs + (PartialOverloading && NumArgs)) > NumArgsInProto && 
+  if ((NumArgs + (PartialOverloading && NumArgs)) > NumArgsInProto &&
       !Proto->isVariadic()) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_too_many_arguments;
@@ -3483,7 +3859,7 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
       QualType ParamType = Proto->getArgType(ArgIdx);
       Candidate.Conversions[ArgIdx]
         = TryCopyInitialization(*this, Args[ArgIdx], ParamType,
-                                SuppressUserConversions, 
+                                SuppressUserConversions,
                                 /*InOverloadResolution=*/true);
       if (Candidate.Conversions[ArgIdx].isBad()) {
         Candidate.Viable = false;
@@ -3511,7 +3887,8 @@ void Sema::AddFunctionCandidates(const UnresolvedSetImpl &Fns,
       if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic())
         AddMethodCandidate(cast<CXXMethodDecl>(FD), F.getPair(),
                            cast<CXXMethodDecl>(FD)->getParent(),
-                           Args[0]->getType(), Args + 1, NumArgs - 1, 
+                           Args[0]->getType(), Args[0]->Classify(Context),
+                           Args + 1, NumArgs - 1,
                            CandidateSet, SuppressUserConversions);
       else
         AddOverloadCandidate(FD, F.getPair(), Args, NumArgs, CandidateSet,
@@ -3523,7 +3900,9 @@ void Sema::AddFunctionCandidates(const UnresolvedSetImpl &Fns,
         AddMethodTemplateCandidate(FunTmpl, F.getPair(),
                               cast<CXXRecordDecl>(FunTmpl->getDeclContext()),
                                    /*FIXME: explicit args */ 0,
-                                   Args[0]->getType(), Args + 1, NumArgs - 1,
+                                   Args[0]->getType(),
+                                   Args[0]->Classify(Context),
+                                   Args + 1, NumArgs - 1,
                                    CandidateSet,
                                    SuppressUserConversions);
       else
@@ -3539,6 +3918,7 @@ void Sema::AddFunctionCandidates(const UnresolvedSetImpl &Fns,
 /// method) as a method candidate to the given overload set.
 void Sema::AddMethodCandidate(DeclAccessPair FoundDecl,
                               QualType ObjectType,
+                              Expr::Classification ObjectClassification,
                               Expr **Args, unsigned NumArgs,
                               OverloadCandidateSet& CandidateSet,
                               bool SuppressUserConversions) {
@@ -3547,18 +3927,18 @@ void Sema::AddMethodCandidate(DeclAccessPair FoundDecl,
 
   if (isa<UsingShadowDecl>(Decl))
     Decl = cast<UsingShadowDecl>(Decl)->getTargetDecl();
-  
+
   if (FunctionTemplateDecl *TD = dyn_cast<FunctionTemplateDecl>(Decl)) {
     assert(isa<CXXMethodDecl>(TD->getTemplatedDecl()) &&
            "Expected a member function template");
     AddMethodTemplateCandidate(TD, FoundDecl, ActingContext,
                                /*ExplicitArgs*/ 0,
-                               ObjectType, Args, NumArgs,
+                               ObjectType, ObjectClassification, Args, NumArgs,
                                CandidateSet,
                                SuppressUserConversions);
   } else {
     AddMethodCandidate(cast<CXXMethodDecl>(Decl), FoundDecl, ActingContext,
-                       ObjectType, Args, NumArgs,
+                       ObjectType, ObjectClassification, Args, NumArgs,
                        CandidateSet, SuppressUserConversions);
   }
 }
@@ -3573,6 +3953,7 @@ void Sema::AddMethodCandidate(DeclAccessPair FoundDecl,
 void
 Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
                          CXXRecordDecl *ActingContext, QualType ObjectType,
+                         Expr::Classification ObjectClassification,
                          Expr **Args, unsigned NumArgs,
                          OverloadCandidateSet& CandidateSet,
                          bool SuppressUserConversions) {
@@ -3595,6 +3976,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
   Candidate.Function = Method;
   Candidate.IsSurrogate = false;
   Candidate.IgnoreObjectArgument = false;
+  Candidate.ExplicitCallArguments = NumArgs;
 
   unsigned NumArgsInProto = Proto->getNumArgs();
 
@@ -3630,8 +4012,8 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
     // Determine the implicit conversion sequence for the object
     // parameter.
     Candidate.Conversions[0]
-      = TryObjectArgumentInitialization(*this, ObjectType, Method,
-                                        ActingContext);
+      = TryObjectArgumentInitialization(*this, ObjectType, ObjectClassification,
+                                        Method, ActingContext);
     if (Candidate.Conversions[0].isBad()) {
       Candidate.Viable = false;
       Candidate.FailureKind = ovl_fail_bad_conversion;
@@ -3650,7 +4032,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
       QualType ParamType = Proto->getArgType(ArgIdx);
       Candidate.Conversions[ArgIdx + 1]
         = TryCopyInitialization(*this, Args[ArgIdx], ParamType,
-                                SuppressUserConversions, 
+                                SuppressUserConversions,
                                 /*InOverloadResolution=*/true);
       if (Candidate.Conversions[ArgIdx + 1].isBad()) {
         Candidate.Viable = false;
@@ -3665,7 +4047,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
     }
   }
 }
-  
+
 /// \brief Add a C++ member function template as a candidate to the candidate
 /// set, using template argument deduction to produce an appropriate member
 /// function template specialization.
@@ -3675,6 +4057,7 @@ Sema::AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl,
                                  CXXRecordDecl *ActingContext,
                         const TemplateArgumentListInfo *ExplicitTemplateArgs,
                                  QualType ObjectType,
+                                 Expr::Classification ObjectClassification,
                                  Expr **Args, unsigned NumArgs,
                                  OverloadCandidateSet& CandidateSet,
                                  bool SuppressUserConversions) {
@@ -3703,7 +4086,8 @@ Sema::AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl,
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.IsSurrogate = false;
     Candidate.IgnoreObjectArgument = false;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result, 
+    Candidate.ExplicitCallArguments = NumArgs;
+    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
                                                           Info);
     return;
   }
@@ -3714,8 +4098,8 @@ Sema::AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl,
   assert(isa<CXXMethodDecl>(Specialization) &&
          "Specialization is not a member function?");
   AddMethodCandidate(cast<CXXMethodDecl>(Specialization), FoundDecl,
-                     ActingContext, ObjectType, Args, NumArgs,
-                     CandidateSet, SuppressUserConversions);
+                     ActingContext, ObjectType, ObjectClassification,
+                     Args, NumArgs, CandidateSet, SuppressUserConversions);
 }
 
 /// \brief Add a C++ function template specialization as a candidate
@@ -3753,7 +4137,8 @@ Sema::AddTemplateOverloadCandidate(FunctionTemplateDecl *FunctionTemplate,
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.IsSurrogate = false;
     Candidate.IgnoreObjectArgument = false;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result, 
+    Candidate.ExplicitCallArguments = NumArgs;
+    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
                                                           Info);
     return;
   }
@@ -3798,10 +4183,11 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   Candidate.FinalConversion.setAllToTypes(ToType);
   Candidate.Viable = true;
   Candidate.Conversions.resize(1);
+  Candidate.ExplicitCallArguments = 1;
 
   // C++ [over.match.funcs]p4:
-  //   For conversion functions, the function is considered to be a member of 
-  //   the class of the implicit implied object argument for the purpose of 
+  //   For conversion functions, the function is considered to be a member of
+  //   the class of the implicit implied object argument for the purpose of
   //   defining the type of the implicit object parameter.
   //
   // Determine the implicit conversion sequence for the implicit
@@ -3811,18 +4197,19 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
     ImplicitParamType = FromPtrType->getPointeeType();
   CXXRecordDecl *ConversionContext
     = cast<CXXRecordDecl>(ImplicitParamType->getAs<RecordType>()->getDecl());
-  
+
   Candidate.Conversions[0]
-    = TryObjectArgumentInitialization(*this, From->getType(), Conversion,
-                                      ConversionContext);
-  
+    = TryObjectArgumentInitialization(*this, From->getType(),
+                                      From->Classify(Context),
+                                      Conversion, ConversionContext);
+
   if (Candidate.Conversions[0].isBad()) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_conversion;
     return;
   }
 
-  // We won't go through a user-define type conversion function to convert a 
+  // We won't go through a user-define type conversion function to convert a
   // derived to base as such conversions are given Conversion Rank. They only
   // go through a copy constructor. 13.3.3.1.2-p4 [over.ics.user]
   QualType FromCanon
@@ -3833,7 +4220,7 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
     Candidate.FailureKind = ovl_fail_trivial_conversion;
     return;
   }
-  
+
   // To determine what the conversion from the result of calling the
   // conversion function to the type we're eventually trying to
   // convert to (ToType), we need to synthesize a call to the
@@ -3843,17 +4230,26 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   // call on the stack and we don't need its arguments to be
   // well-formed.
   DeclRefExpr ConversionRef(Conversion, Conversion->getType(),
-                            From->getLocStart());
+                            VK_LValue, From->getLocStart());
   ImplicitCastExpr ConversionFn(ImplicitCastExpr::OnStack,
                                 Context.getPointerType(Conversion->getType()),
                                 CK_FunctionToPointerDecay,
                                 &ConversionRef, VK_RValue);
 
+  QualType CallResultType
+    = Conversion->getConversionType().getNonLValueExprType(Context);
+  if (RequireCompleteType(From->getLocStart(), CallResultType, 0)) {
+    Candidate.Viable = false;
+    Candidate.FailureKind = ovl_fail_bad_final_conversion;
+    return;
+  }
+
+  ExprValueKind VK = Expr::getValueKindForType(Conversion->getConversionType());
+
   // Note that it is safe to allocate CallExpr on the stack here because
   // there are 0 arguments (i.e., nothing is allocated using ASTContext's
   // allocator).
-  CallExpr Call(Context, &ConversionFn, 0, 0,
-                Conversion->getConversionType().getNonLValueExprType(Context),
+  CallExpr Call(Context, &ConversionFn, 0, 0, CallResultType, VK,
                 From->getLocStart());
   ImplicitConversionSequence ICS =
     TryCopyInitialization(*this, &Call, ToType,
@@ -3863,17 +4259,27 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   switch (ICS.getKind()) {
   case ImplicitConversionSequence::StandardConversion:
     Candidate.FinalConversion = ICS.Standard;
-      
+
     // C++ [over.ics.user]p3:
     //   If the user-defined conversion is specified by a specialization of a
-    //   conversion function template, the second standard conversion sequence 
+    //   conversion function template, the second standard conversion sequence
     //   shall have exact match rank.
     if (Conversion->getPrimaryTemplate() &&
         GetConversionRank(ICS.Standard.Second) != ICR_Exact_Match) {
       Candidate.Viable = false;
       Candidate.FailureKind = ovl_fail_final_conversion_not_exact;
     }
-      
+
+    // C++0x [dcl.init.ref]p5:
+    //    In the second case, if the reference is an rvalue reference and
+    //    the second standard conversion sequence of the user-defined
+    //    conversion sequence includes an lvalue-to-rvalue conversion, the
+    //    program is ill-formed.
+    if (ToType->isRValueReferenceType() &&
+        ICS.Standard.First == ICK_Lvalue_To_Rvalue) {
+      Candidate.Viable = false;
+      Candidate.FailureKind = ovl_fail_bad_final_conversion;
+    }
     break;
 
   case ImplicitConversionSequence::BadConversion:
@@ -3917,7 +4323,8 @@ Sema::AddTemplateConversionCandidate(FunctionTemplateDecl *FunctionTemplate,
     Candidate.FailureKind = ovl_fail_bad_deduction;
     Candidate.IsSurrogate = false;
     Candidate.IgnoreObjectArgument = false;
-    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result, 
+    Candidate.ExplicitCallArguments = 1;
+    Candidate.DeductionFailure = MakeDeductionFailureInfo(Context, Result,
                                                           Info);
     return;
   }
@@ -3938,7 +4345,7 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
                                  DeclAccessPair FoundDecl,
                                  CXXRecordDecl *ActingContext,
                                  const FunctionProtoType *Proto,
-                                 QualType ObjectType,
+                                 Expr *Object,
                                  Expr **Args, unsigned NumArgs,
                                  OverloadCandidateSet& CandidateSet) {
   if (!CandidateSet.isNewCandidate(Conversion))
@@ -3956,12 +4363,14 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
   Candidate.IsSurrogate = true;
   Candidate.IgnoreObjectArgument = false;
   Candidate.Conversions.resize(NumArgs + 1);
+  Candidate.ExplicitCallArguments = NumArgs;
 
   // Determine the implicit conversion sequence for the implicit
   // object parameter.
   ImplicitConversionSequence ObjectInit
-    = TryObjectArgumentInitialization(*this, ObjectType, Conversion,
-                                      ActingContext);
+    = TryObjectArgumentInitialization(*this, Object->getType(),
+                                      Object->Classify(Context),
+                                      Conversion, ActingContext);
   if (ObjectInit.isBad()) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_conversion;
@@ -3976,6 +4385,8 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
   Candidate.Conversions[0].UserDefined.Before = ObjectInit.Standard;
   Candidate.Conversions[0].UserDefined.EllipsisConversion = false;
   Candidate.Conversions[0].UserDefined.ConversionFunction = Conversion;
+  Candidate.Conversions[0].UserDefined.FoundConversionFunction
+    = FoundDecl.getDecl();
   Candidate.Conversions[0].UserDefined.After
     = Candidate.Conversions[0].UserDefined.Before;
   Candidate.Conversions[0].UserDefined.After.setAsIdentityConversion();
@@ -4071,7 +4482,8 @@ void Sema::AddMemberOperatorCandidates(OverloadedOperatorKind Op,
          Oper != OperEnd;
          ++Oper)
       AddMethodCandidate(Oper.getPair(), Args[0]->getType(),
-                         Args + 1, NumArgs - 1, CandidateSet,
+                         Args[0]->Classify(Context), Args + 1, NumArgs - 1,
+                         CandidateSet,
                          /* SuppressUserConversions = */ false);
   }
 }
@@ -4107,6 +4519,7 @@ void Sema::AddBuiltinCandidate(QualType ResultTy, QualType *ParamTys,
   // arguments.
   Candidate.Viable = true;
   Candidate.Conversions.resize(NumArgs);
+  Candidate.ExplicitCallArguments = NumArgs;
   for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
     // C++ [over.match.oper]p4:
     //   For the built-in assignment operators, conversions of the
@@ -4159,10 +4572,17 @@ class BuiltinCandidateTypeSet  {
   /// used in the built-in candidates.
   TypeSet EnumerationTypes;
 
-  /// \brief The set of vector types that will be used in the built-in 
+  /// \brief The set of vector types that will be used in the built-in
   /// candidates.
   TypeSet VectorTypes;
-  
+
+  /// \brief A flag indicating non-record types are viable candidates
+  bool HasNonRecordTypes;
+
+  /// \brief A flag indicating whether either arithmetic or enumeration types
+  /// were present in the candidate set.
+  bool HasArithmeticOrEnumeralTypes;
+
   /// Sema - The semantic analysis instance where we are building the
   /// candidate type set.
   Sema &SemaRef;
@@ -4179,9 +4599,12 @@ public:
   typedef TypeSet::iterator iterator;
 
   BuiltinCandidateTypeSet(Sema &SemaRef)
-    : SemaRef(SemaRef), Context(SemaRef.Context) { }
+    : HasNonRecordTypes(false),
+      HasArithmeticOrEnumeralTypes(false),
+      SemaRef(SemaRef),
+      Context(SemaRef.Context) { }
 
-  void AddTypesConvertedFrom(QualType Ty, 
+  void AddTypesConvertedFrom(QualType Ty,
                              SourceLocation Loc,
                              bool AllowUserConversions,
                              bool AllowExplicitConversions,
@@ -4204,9 +4627,12 @@ public:
 
   /// enumeration_end - Past the last enumeration type found;
   iterator enumeration_end() { return EnumerationTypes.end(); }
-  
+
   iterator vector_begin() { return VectorTypes.begin(); }
   iterator vector_end() { return VectorTypes.end(); }
+
+  bool hasNonRecordTypes() { return HasNonRecordTypes; }
+  bool hasArithmeticOrEnumeralTypes() { return HasArithmeticOrEnumeralTypes; }
 };
 
 /// AddPointerWithMoreQualifiedTypeVariants - Add the pointer type @p Ty to
@@ -4225,7 +4651,7 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
   // Insert this type.
   if (!PointerTypes.insert(Ty))
     return false;
-    
+
   QualType PointeeTy;
   const PointerType *PointerTy = Ty->getAs<PointerType>();
   bool buildObjCPtr = false;
@@ -4239,7 +4665,7 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
   }
   else
     PointeeTy = PointerTy->getPointeeType();
-  
+
   // Don't add qualified variants of arrays. For one, they're not allowed
   // (the qualifier would sink to the element type), and for another, the
   // only overload situation where it matters is subscript or pointer +- int,
@@ -4251,7 +4677,7 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
     BaseCVR = Array->getElementType().getCVRQualifiers();
   bool hasVolatile = VisibleQuals.hasVolatile();
   bool hasRestrict = VisibleQuals.hasRestrict();
-  
+
   // Iterate through all strict supersets of BaseCVR.
   for (unsigned CVR = BaseCVR+1; CVR <= Qualifiers::CVRMask; ++CVR) {
     if ((CVR | BaseCVR) != CVR) continue;
@@ -4302,9 +4728,10 @@ BuiltinCandidateTypeSet::AddMemberPointerWithMoreQualifiedTypeVariants(
   unsigned BaseCVR = PointeeTy.getCVRQualifiers();
   for (unsigned CVR = BaseCVR+1; CVR <= Qualifiers::CVRMask; ++CVR) {
     if ((CVR | BaseCVR) != CVR) continue;
-    
+
     QualType QPointeeTy = Context.getCVRQualifiedType(PointeeTy, CVR);
-    MemberPointerTypes.insert(Context.getMemberPointerType(QPointeeTy, ClassTy));
+    MemberPointerTypes.insert(
+      Context.getMemberPointerType(QPointeeTy, ClassTy));
   }
 
   return true;
@@ -4332,12 +4759,21 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
   if (const ReferenceType *RefTy = Ty->getAs<ReferenceType>())
     Ty = RefTy->getPointeeType();
 
-  // We don't care about qualifiers on the type.
-  Ty = Ty.getLocalUnqualifiedType();
-
   // If we're dealing with an array type, decay to the pointer.
   if (Ty->isArrayType())
     Ty = SemaRef.Context.getArrayDecayedType(Ty);
+
+  // Otherwise, we don't care about qualifiers on the type.
+  Ty = Ty.getLocalUnqualifiedType();
+
+  // Flag if we ever add a non-record type.
+  const RecordType *TyRec = Ty->getAs<RecordType>();
+  HasNonRecordTypes = HasNonRecordTypes || !TyRec;
+
+  // Flag if we encounter an arithmetic type.
+  HasArithmeticOrEnumeralTypes =
+    HasArithmeticOrEnumeralTypes || Ty->isArithmeticType();
+
   if (Ty->isObjCIdType() || Ty->isObjCClassType())
     PointerTypes.insert(Ty);
   else if (Ty->getAs<PointerType>() || Ty->getAs<ObjCObjectPointerType>()) {
@@ -4350,35 +4786,36 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
     if (!AddMemberPointerWithMoreQualifiedTypeVariants(Ty))
       return;
   } else if (Ty->isEnumeralType()) {
+    HasArithmeticOrEnumeralTypes = true;
     EnumerationTypes.insert(Ty);
   } else if (Ty->isVectorType()) {
+    // We treat vector types as arithmetic types in many contexts as an
+    // extension.
+    HasArithmeticOrEnumeralTypes = true;
     VectorTypes.insert(Ty);
-  } else if (AllowUserConversions) {
-    if (const RecordType *TyRec = Ty->getAs<RecordType>()) {
-      if (SemaRef.RequireCompleteType(Loc, Ty, 0)) {
-        // No conversion functions in incomplete types.
-        return;
-      }
+  } else if (AllowUserConversions && TyRec) {
+    // No conversion functions in incomplete types.
+    if (SemaRef.RequireCompleteType(Loc, Ty, 0))
+      return;
 
-      CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(TyRec->getDecl());
-      const UnresolvedSetImpl *Conversions
-        = ClassDecl->getVisibleConversionFunctions();
-      for (UnresolvedSetImpl::iterator I = Conversions->begin(),
-             E = Conversions->end(); I != E; ++I) {
-        NamedDecl *D = I.getDecl();
-        if (isa<UsingShadowDecl>(D))
-          D = cast<UsingShadowDecl>(D)->getTargetDecl();
+    CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(TyRec->getDecl());
+    const UnresolvedSetImpl *Conversions
+      = ClassDecl->getVisibleConversionFunctions();
+    for (UnresolvedSetImpl::iterator I = Conversions->begin(),
+           E = Conversions->end(); I != E; ++I) {
+      NamedDecl *D = I.getDecl();
+      if (isa<UsingShadowDecl>(D))
+        D = cast<UsingShadowDecl>(D)->getTargetDecl();
 
-        // Skip conversion function templates; they don't tell us anything
-        // about which builtin types we can convert to.
-        if (isa<FunctionTemplateDecl>(D))
-          continue;
+      // Skip conversion function templates; they don't tell us anything
+      // about which builtin types we can convert to.
+      if (isa<FunctionTemplateDecl>(D))
+        continue;
 
-        CXXConversionDecl *Conv = cast<CXXConversionDecl>(D);
-        if (AllowExplicitConversions || !Conv->isExplicit()) {
-          AddTypesConvertedFrom(Conv->getConversionType(), Loc, false, false, 
-                                VisibleQuals);
-        }
+      CXXConversionDecl *Conv = cast<CXXConversionDecl>(D);
+      if (AllowExplicitConversions || !Conv->isExplicit()) {
+        AddTypesConvertedFrom(Conv->getConversionType(), Loc, false, false,
+                              VisibleQuals);
       }
     }
   }
@@ -4426,14 +4863,14 @@ static  Qualifiers CollectVRQualifiers(ASTContext &Context, Expr* ArgExpr) {
       VRQuals.addRestrict();
       return VRQuals;
     }
-    
+
     CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(TyRec->getDecl());
     if (!ClassDecl->hasDefinition())
       return VRQuals;
 
     const UnresolvedSetImpl *Conversions =
       ClassDecl->getVisibleConversionFunctions();
-    
+
     for (UnresolvedSetImpl::iterator I = Conversions->begin(),
            E = Conversions->end(); I != E; ++I) {
       NamedDecl *D = I.getDecl();
@@ -4449,7 +4886,7 @@ static  Qualifiers CollectVRQualifiers(ASTContext &Context, Expr* ArgExpr) {
         while (!done) {
           if (const PointerType *ResTypePtr = CanTy->getAs<PointerType>())
             CanTy = ResTypePtr->getPointeeType();
-          else if (const MemberPointerType *ResTypeMPtr = 
+          else if (const MemberPointerType *ResTypeMPtr =
                 CanTy->getAs<MemberPointerType>())
             CanTy = ResTypeMPtr->getPointeeType();
           else
@@ -4465,765 +4902,1174 @@ static  Qualifiers CollectVRQualifiers(ASTContext &Context, Expr* ArgExpr) {
     }
     return VRQuals;
 }
-  
-/// AddBuiltinOperatorCandidates - Add the appropriate built-in
-/// operator overloads to the candidate set (C++ [over.built]), based
-/// on the operator @p Op and the arguments given. For example, if the
-/// operator is a binary '+', this routine might add "int
-/// operator+(int, int)" to cover integer addition.
-void
-Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
-                                   SourceLocation OpLoc,
-                                   Expr **Args, unsigned NumArgs,
-                                   OverloadCandidateSet& CandidateSet) {
-  // The set of "promoted arithmetic types", which are the arithmetic
-  // types are that preserved by promotion (C++ [over.built]p2). Note
-  // that the first few of these types are the promoted integral
-  // types; these types need to be first.
-  // FIXME: What about complex?
-  const unsigned FirstIntegralType = 0;
-  const unsigned LastIntegralType = 13;
-  const unsigned FirstPromotedIntegralType = 7,
-                 LastPromotedIntegralType = 13;
-  const unsigned FirstPromotedArithmeticType = 7,
-                 LastPromotedArithmeticType = 16;
-  const unsigned NumArithmeticTypes = 16;
-  QualType ArithmeticTypes[NumArithmeticTypes] = {
-    Context.BoolTy, Context.CharTy, Context.WCharTy,
-// FIXME:   Context.Char16Ty, Context.Char32Ty,
-    Context.SignedCharTy, Context.ShortTy,
-    Context.UnsignedCharTy, Context.UnsignedShortTy,
-    Context.IntTy, Context.LongTy, Context.LongLongTy,
-    Context.UnsignedIntTy, Context.UnsignedLongTy, Context.UnsignedLongLongTy,
-    Context.FloatTy, Context.DoubleTy, Context.LongDoubleTy
-  };
-  assert(ArithmeticTypes[FirstPromotedIntegralType] == Context.IntTy &&
-         "Invalid first promoted integral type");
-  assert(ArithmeticTypes[LastPromotedIntegralType - 1] 
-           == Context.UnsignedLongLongTy &&
-         "Invalid last promoted integral type");
-  assert(ArithmeticTypes[FirstPromotedArithmeticType] == Context.IntTy &&
-         "Invalid first promoted arithmetic type");
-  assert(ArithmeticTypes[LastPromotedArithmeticType - 1] 
-            == Context.LongDoubleTy &&
-         "Invalid last promoted arithmetic type");
-         
-  // Find all of the types that the arguments can convert to, but only
-  // if the operator we're looking at has built-in operator candidates
-  // that make use of these types.
-  Qualifiers VisibleTypeConversionsQuals;
-  VisibleTypeConversionsQuals.addConst();
-  for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
-    VisibleTypeConversionsQuals += CollectVRQualifiers(Context, Args[ArgIdx]);
-  
-  BuiltinCandidateTypeSet CandidateTypes(*this);
-  for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
-    CandidateTypes.AddTypesConvertedFrom(Args[ArgIdx]->getType(),
-                                         OpLoc,
-                                         true,
-                                         (Op == OO_Exclaim ||
-                                          Op == OO_AmpAmp ||
-                                          Op == OO_PipePipe),
-                                         VisibleTypeConversionsQuals);
-
-  bool isComparison = false;
-  switch (Op) {
-  case OO_None:
-  case NUM_OVERLOADED_OPERATORS:
-    assert(false && "Expected an overloaded operator");
-    break;
 
-  case OO_Star: // '*' is either unary or binary
-    if (NumArgs == 1)
-      goto UnaryStar;
-    else
-      goto BinaryStar;
-    break;
+namespace {
 
-  case OO_Plus: // '+' is either unary or binary
-    if (NumArgs == 1)
-      goto UnaryPlus;
-    else
-      goto BinaryPlus;
-    break;
+/// \brief Helper class to manage the addition of builtin operator overload
+/// candidates. It provides shared state and utility methods used throughout
+/// the process, as well as a helper method to add each group of builtin
+/// operator overloads from the standard to a candidate set.
+class BuiltinOperatorOverloadBuilder {
+  // Common instance state available to all overload candidate addition methods.
+  Sema &S;
+  Expr **Args;
+  unsigned NumArgs;
+  Qualifiers VisibleTypeConversionsQuals;
+  bool HasArithmeticOrEnumeralCandidateType;
+  llvm::SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes;
+  OverloadCandidateSet &CandidateSet;
+
+  // Define some constants used to index and iterate over the arithemetic types
+  // provided via the getArithmeticType() method below.
+  // The "promoted arithmetic types" are the arithmetic
+  // types are that preserved by promotion (C++ [over.built]p2).
+  static const unsigned FirstIntegralType = 3;
+  static const unsigned LastIntegralType = 18;
+  static const unsigned FirstPromotedIntegralType = 3,
+                        LastPromotedIntegralType = 9;
+  static const unsigned FirstPromotedArithmeticType = 0,
+                        LastPromotedArithmeticType = 9;
+  static const unsigned NumArithmeticTypes = 18;
+
+  /// \brief Get the canonical type for a given arithmetic type index.
+  CanQualType getArithmeticType(unsigned index) {
+    assert(index < NumArithmeticTypes);
+    static CanQualType ASTContext::* const
+      ArithmeticTypes[NumArithmeticTypes] = {
+      // Start of promoted types.
+      &ASTContext::FloatTy,
+      &ASTContext::DoubleTy,
+      &ASTContext::LongDoubleTy,
+
+      // Start of integral types.
+      &ASTContext::IntTy,
+      &ASTContext::LongTy,
+      &ASTContext::LongLongTy,
+      &ASTContext::UnsignedIntTy,
+      &ASTContext::UnsignedLongTy,
+      &ASTContext::UnsignedLongLongTy,
+      // End of promoted types.
+
+      &ASTContext::BoolTy,
+      &ASTContext::CharTy,
+      &ASTContext::WCharTy,
+      &ASTContext::Char16Ty,
+      &ASTContext::Char32Ty,
+      &ASTContext::SignedCharTy,
+      &ASTContext::ShortTy,
+      &ASTContext::UnsignedCharTy,
+      &ASTContext::UnsignedShortTy,
+      // End of integral types.
+      // FIXME: What about complex?
+    };
+    return S.Context.*ArithmeticTypes[index];
+  }
+
+  /// \brief Gets the canonical type resulting from the usual arithemetic
+  /// converions for the given arithmetic types.
+  CanQualType getUsualArithmeticConversions(unsigned L, unsigned R) {
+    // Accelerator table for performing the usual arithmetic conversions.
+    // The rules are basically:
+    //   - if either is floating-point, use the wider floating-point
+    //   - if same signedness, use the higher rank
+    //   - if same size, use unsigned of the higher rank
+    //   - use the larger type
+    // These rules, together with the axiom that higher ranks are
+    // never smaller, are sufficient to precompute all of these results
+    // *except* when dealing with signed types of higher rank.
+    // (we could precompute SLL x UI for all known platforms, but it's
+    // better not to make any assumptions).
+    enum PromotedType {
+                  Flt,  Dbl, LDbl,   SI,   SL,  SLL,   UI,   UL,  ULL, Dep=-1
+    };
+    static PromotedType ConversionsTable[LastPromotedArithmeticType]
+                                        [LastPromotedArithmeticType] = {
+      /* Flt*/ {  Flt,  Dbl, LDbl,  Flt,  Flt,  Flt,  Flt,  Flt,  Flt },
+      /* Dbl*/ {  Dbl,  Dbl, LDbl,  Dbl,  Dbl,  Dbl,  Dbl,  Dbl,  Dbl },
+      /*LDbl*/ { LDbl, LDbl, LDbl, LDbl, LDbl, LDbl, LDbl, LDbl, LDbl },
+      /*  SI*/ {  Flt,  Dbl, LDbl,   SI,   SL,  SLL,   UI,   UL,  ULL },
+      /*  SL*/ {  Flt,  Dbl, LDbl,   SL,   SL,  SLL,  Dep,   UL,  ULL },
+      /* SLL*/ {  Flt,  Dbl, LDbl,  SLL,  SLL,  SLL,  Dep,  Dep,  ULL },
+      /*  UI*/ {  Flt,  Dbl, LDbl,   UI,  Dep,  Dep,   UI,   UL,  ULL },
+      /*  UL*/ {  Flt,  Dbl, LDbl,   UL,   UL,  Dep,   UL,   UL,  ULL },
+      /* ULL*/ {  Flt,  Dbl, LDbl,  ULL,  ULL,  ULL,  ULL,  ULL,  ULL },
+    };
 
-  case OO_Minus: // '-' is either unary or binary
-    if (NumArgs == 1)
-      goto UnaryMinus;
-    else
-      goto BinaryMinus;
-    break;
+    assert(L < LastPromotedArithmeticType);
+    assert(R < LastPromotedArithmeticType);
+    int Idx = ConversionsTable[L][R];
+
+    // Fast path: the table gives us a concrete answer.
+    if (Idx != Dep) return getArithmeticType(Idx);
+
+    // Slow path: we need to compare widths.
+    // An invariant is that the signed type has higher rank.
+    CanQualType LT = getArithmeticType(L),
+                RT = getArithmeticType(R);
+    unsigned LW = S.Context.getIntWidth(LT),
+             RW = S.Context.getIntWidth(RT);
+
+    // If they're different widths, use the signed type.
+    if (LW > RW) return LT;
+    else if (LW < RW) return RT;
+
+    // Otherwise, use the unsigned type of the signed type's rank.
+    if (L == SL || R == SL) return S.Context.UnsignedLongTy;
+    assert(L == SLL || R == SLL);
+    return S.Context.UnsignedLongLongTy;
+  }
+
+  /// \brief Helper method to factor out the common pattern of adding overloads
+  /// for '++' and '--' builtin operators.
+  void addPlusPlusMinusMinusStyleOverloads(QualType CandidateTy,
+                                           bool HasVolatile) {
+    QualType ParamTypes[2] = {
+      S.Context.getLValueReferenceType(CandidateTy),
+      S.Context.IntTy
+    };
 
-  case OO_Amp: // '&' is either unary or binary
+    // Non-volatile version.
     if (NumArgs == 1)
-      goto UnaryAmp;
+      S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
     else
-      goto BinaryAmp;
+      S.AddBuiltinCandidate(CandidateTy, ParamTypes, Args, 2, CandidateSet);
+
+    // Use a heuristic to reduce number of builtin candidates in the set:
+    // add volatile version only if there are conversions to a volatile type.
+    if (HasVolatile) {
+      ParamTypes[0] =
+        S.Context.getLValueReferenceType(
+          S.Context.getVolatileType(CandidateTy));
+      if (NumArgs == 1)
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
+      else
+        S.AddBuiltinCandidate(CandidateTy, ParamTypes, Args, 2, CandidateSet);
+    }
+  }
+
+public:
+  BuiltinOperatorOverloadBuilder(
+    Sema &S, Expr **Args, unsigned NumArgs,
+    Qualifiers VisibleTypeConversionsQuals,
+    bool HasArithmeticOrEnumeralCandidateType,
+    llvm::SmallVectorImpl<BuiltinCandidateTypeSet> &CandidateTypes,
+    OverloadCandidateSet &CandidateSet)
+    : S(S), Args(Args), NumArgs(NumArgs),
+      VisibleTypeConversionsQuals(VisibleTypeConversionsQuals),
+      HasArithmeticOrEnumeralCandidateType(
+        HasArithmeticOrEnumeralCandidateType),
+      CandidateTypes(CandidateTypes),
+      CandidateSet(CandidateSet) {
+    // Validate some of our static helper constants in debug builds.
+    assert(getArithmeticType(FirstPromotedIntegralType) == S.Context.IntTy &&
+           "Invalid first promoted integral type");
+    assert(getArithmeticType(LastPromotedIntegralType - 1)
+             == S.Context.UnsignedLongLongTy &&
+           "Invalid last promoted integral type");
+    assert(getArithmeticType(FirstPromotedArithmeticType)
+             == S.Context.FloatTy &&
+           "Invalid first promoted arithmetic type");
+    assert(getArithmeticType(LastPromotedArithmeticType - 1)
+             == S.Context.UnsignedLongLongTy &&
+           "Invalid last promoted arithmetic type");
+  }
+
+  // C++ [over.built]p3:
+  //
+  //   For every pair (T, VQ), where T is an arithmetic type, and VQ
+  //   is either volatile or empty, there exist candidate operator
+  //   functions of the form
+  //
+  //       VQ T&      operator++(VQ T&);
+  //       T          operator++(VQ T&, int);
+  //
+  // C++ [over.built]p4:
+  //
+  //   For every pair (T, VQ), where T is an arithmetic type other
+  //   than bool, and VQ is either volatile or empty, there exist
+  //   candidate operator functions of the form
+  //
+  //       VQ T&      operator--(VQ T&);
+  //       T          operator--(VQ T&, int);
+  void addPlusPlusMinusMinusArithmeticOverloads(OverloadedOperatorKind Op) {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  case OO_PlusPlus:
-  case OO_MinusMinus:
-    // C++ [over.built]p3:
-    //
-    //   For every pair (T, VQ), where T is an arithmetic type, and VQ
-    //   is either volatile or empty, there exist candidate operator
-    //   functions of the form
-    //
-    //       VQ T&      operator++(VQ T&);
-    //       T          operator++(VQ T&, int);
-    //
-    // C++ [over.built]p4:
-    //
-    //   For every pair (T, VQ), where T is an arithmetic type other
-    //   than bool, and VQ is either volatile or empty, there exist
-    //   candidate operator functions of the form
-    //
-    //       VQ T&      operator--(VQ T&);
-    //       T          operator--(VQ T&, int);
     for (unsigned Arith = (Op == OO_PlusPlus? 0 : 1);
          Arith < NumArithmeticTypes; ++Arith) {
-      QualType ArithTy = ArithmeticTypes[Arith];
-      QualType ParamTypes[2]
-        = { Context.getLValueReferenceType(ArithTy), Context.IntTy };
-
-      // Non-volatile version.
-      if (NumArgs == 1)
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
-      else
-        AddBuiltinCandidate(ArithTy, ParamTypes, Args, 2, CandidateSet);
-      // heuristic to reduce number of builtin candidates in the set.
-      // Add volatile version only if there are conversions to a volatile type.
-      if (VisibleTypeConversionsQuals.hasVolatile()) {
-        // Volatile version
-        ParamTypes[0]
-          = Context.getLValueReferenceType(Context.getVolatileType(ArithTy));
-        if (NumArgs == 1)
-          AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
-        else
-          AddBuiltinCandidate(ArithTy, ParamTypes, Args, 2, CandidateSet);
-      }
+      addPlusPlusMinusMinusStyleOverloads(
+        getArithmeticType(Arith),
+        VisibleTypeConversionsQuals.hasVolatile());
     }
+  }
 
-    // C++ [over.built]p5:
-    //
-    //   For every pair (T, VQ), where T is a cv-qualified or
-    //   cv-unqualified object type, and VQ is either volatile or
-    //   empty, there exist candidate operator functions of the form
-    //
-    //       T*VQ&      operator++(T*VQ&);
-    //       T*VQ&      operator--(T*VQ&);
-    //       T*         operator++(T*VQ&, int);
-    //       T*         operator--(T*VQ&, int);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
+  // C++ [over.built]p5:
+  //
+  //   For every pair (T, VQ), where T is a cv-qualified or
+  //   cv-unqualified object type, and VQ is either volatile or
+  //   empty, there exist candidate operator functions of the form
+  //
+  //       T*VQ&      operator++(T*VQ&);
+  //       T*VQ&      operator--(T*VQ&);
+  //       T*         operator++(T*VQ&, int);
+  //       T*         operator--(T*VQ&, int);
+  void addPlusPlusMinusMinusPointerOverloads() {
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
       // Skip pointer types that aren't pointers to object types.
-      if (!(*Ptr)->getPointeeType()->isIncompleteOrObjectType())
+      if (!(*Ptr)->getPointeeType()->isObjectType())
         continue;
 
-      QualType ParamTypes[2] = {
-        Context.getLValueReferenceType(*Ptr), Context.IntTy
-      };
-
-      // Without volatile
-      if (NumArgs == 1)
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
-      else
-        AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
-
-      if (!Context.getCanonicalType(*Ptr).isVolatileQualified() &&
-          VisibleTypeConversionsQuals.hasVolatile()) {
-        // With volatile
-        ParamTypes[0]
-          = Context.getLValueReferenceType(Context.getVolatileType(*Ptr));
-        if (NumArgs == 1)
-          AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 1, CandidateSet);
-        else
-          AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
-      }
+      addPlusPlusMinusMinusStyleOverloads(*Ptr,
+        (!S.Context.getCanonicalType(*Ptr).isVolatileQualified() &&
+         VisibleTypeConversionsQuals.hasVolatile()));
     }
-    break;
+  }
 
-  UnaryStar:
-    // C++ [over.built]p6:
-    //   For every cv-qualified or cv-unqualified object type T, there
-    //   exist candidate operator functions of the form
-    //
-    //       T&         operator*(T*);
-    //
-    // C++ [over.built]p7:
-    //   For every function type T, there exist candidate operator
-    //   functions of the form
-    //       T&         operator*(T*);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
+  // C++ [over.built]p6:
+  //   For every cv-qualified or cv-unqualified object type T, there
+  //   exist candidate operator functions of the form
+  //
+  //       T&         operator*(T*);
+  //
+  // C++ [over.built]p7:
+  //   For every function type T that does not have cv-qualifiers or a
+  //   ref-qualifier, there exist candidate operator functions of the form
+  //       T&         operator*(T*);
+  void addUnaryStarPointerOverloads() {
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
       QualType ParamTy = *Ptr;
       QualType PointeeTy = ParamTy->getPointeeType();
-      AddBuiltinCandidate(Context.getLValueReferenceType(PointeeTy),
-                          &ParamTy, Args, 1, CandidateSet);
-    }
-    break;
+      if (!PointeeTy->isObjectType() && !PointeeTy->isFunctionType())
+        continue;
 
-  UnaryPlus:
-    // C++ [over.built]p8:
-    //   For every type T, there exist candidate operator functions of
-    //   the form
-    //
-    //       T*         operator+(T*);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-      QualType ParamTy = *Ptr;
-      AddBuiltinCandidate(ParamTy, &ParamTy, Args, 1, CandidateSet);
+      if (const FunctionProtoType *Proto =PointeeTy->getAs<FunctionProtoType>())
+        if (Proto->getTypeQuals() || Proto->getRefQualifier())
+          continue;
+
+      S.AddBuiltinCandidate(S.Context.getLValueReferenceType(PointeeTy),
+                            &ParamTy, Args, 1, CandidateSet);
     }
+  }
 
-    // Fall through
+  // C++ [over.built]p9:
+  //  For every promoted arithmetic type T, there exist candidate
+  //  operator functions of the form
+  //
+  //       T         operator+(T);
+  //       T         operator-(T);
+  void addUnaryPlusOrMinusArithmeticOverloads() {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  UnaryMinus:
-    // C++ [over.built]p9:
-    //  For every promoted arithmetic type T, there exist candidate
-    //  operator functions of the form
-    //
-    //       T         operator+(T);
-    //       T         operator-(T);
     for (unsigned Arith = FirstPromotedArithmeticType;
          Arith < LastPromotedArithmeticType; ++Arith) {
-      QualType ArithTy = ArithmeticTypes[Arith];
-      AddBuiltinCandidate(ArithTy, &ArithTy, Args, 1, CandidateSet);
+      QualType ArithTy = getArithmeticType(Arith);
+      S.AddBuiltinCandidate(ArithTy, &ArithTy, Args, 1, CandidateSet);
     }
-      
+
     // Extension: We also add these operators for vector types.
-    for (BuiltinCandidateTypeSet::iterator Vec = CandidateTypes.vector_begin(),
-                                        VecEnd = CandidateTypes.vector_end(); 
+    for (BuiltinCandidateTypeSet::iterator
+              Vec = CandidateTypes[0].vector_begin(),
+           VecEnd = CandidateTypes[0].vector_end();
          Vec != VecEnd; ++Vec) {
       QualType VecTy = *Vec;
-      AddBuiltinCandidate(VecTy, &VecTy, Args, 1, CandidateSet);
+      S.AddBuiltinCandidate(VecTy, &VecTy, Args, 1, CandidateSet);
     }
-    break;
+  }
+
+  // C++ [over.built]p8:
+  //   For every type T, there exist candidate operator functions of
+  //   the form
+  //
+  //       T*         operator+(T*);
+  void addUnaryPlusPointerOverloads() {
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
+      QualType ParamTy = *Ptr;
+      S.AddBuiltinCandidate(ParamTy, &ParamTy, Args, 1, CandidateSet);
+    }
+  }
+
+  // C++ [over.built]p10:
+  //   For every promoted integral type T, there exist candidate
+  //   operator functions of the form
+  //
+  //        T         operator~(T);
+  void addUnaryTildePromotedIntegralOverloads() {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  case OO_Tilde:
-    // C++ [over.built]p10:
-    //   For every promoted integral type T, there exist candidate
-    //   operator functions of the form
-    //
-    //        T         operator~(T);
     for (unsigned Int = FirstPromotedIntegralType;
          Int < LastPromotedIntegralType; ++Int) {
-      QualType IntTy = ArithmeticTypes[Int];
-      AddBuiltinCandidate(IntTy, &IntTy, Args, 1, CandidateSet);
+      QualType IntTy = getArithmeticType(Int);
+      S.AddBuiltinCandidate(IntTy, &IntTy, Args, 1, CandidateSet);
     }
-      
+
     // Extension: We also add this operator for vector types.
-    for (BuiltinCandidateTypeSet::iterator Vec = CandidateTypes.vector_begin(),
-                                        VecEnd = CandidateTypes.vector_end(); 
+    for (BuiltinCandidateTypeSet::iterator
+              Vec = CandidateTypes[0].vector_begin(),
+           VecEnd = CandidateTypes[0].vector_end();
          Vec != VecEnd; ++Vec) {
       QualType VecTy = *Vec;
-      AddBuiltinCandidate(VecTy, &VecTy, Args, 1, CandidateSet);
-    }      
-    break;
+      S.AddBuiltinCandidate(VecTy, &VecTy, Args, 1, CandidateSet);
+    }
+  }
 
-  case OO_New:
-  case OO_Delete:
-  case OO_Array_New:
-  case OO_Array_Delete:
-  case OO_Call:
-    assert(false && "Special operators don't use AddBuiltinOperatorCandidates");
-    break;
+  // C++ [over.match.oper]p16:
+  //   For every pointer to member type T, there exist candidate operator
+  //   functions of the form
+  //
+  //        bool operator==(T,T);
+  //        bool operator!=(T,T);
+  void addEqualEqualOrNotEqualMemberPointerOverloads() {
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
+
+    for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
+      for (BuiltinCandidateTypeSet::iterator
+                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
+             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
+           MemPtr != MemPtrEnd;
+           ++MemPtr) {
+        // Don't add the same builtin candidate twice.
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)))
+          continue;
 
-  case OO_Comma:
-  UnaryAmp:
-  case OO_Arrow:
-    // C++ [over.match.oper]p3:
-    //   -- For the operator ',', the unary operator '&', or the
-    //      operator '->', the built-in candidates set is empty.
-    break;
+        QualType ParamTypes[2] = { *MemPtr, *MemPtr };
+        S.AddBuiltinCandidate(S.Context.BoolTy, ParamTypes, Args, 2,
+                              CandidateSet);
+      }
+    }
+  }
 
-  case OO_EqualEqual:
-  case OO_ExclaimEqual:
-    // C++ [over.match.oper]p16:
-    //   For every pointer to member type T, there exist candidate operator
-    //   functions of the form
+  // C++ [over.built]p15:
+  //
+  //   For every pointer or enumeration type T, there exist
+  //   candidate operator functions of the form
+  //
+  //        bool       operator<(T, T);
+  //        bool       operator>(T, T);
+  //        bool       operator<=(T, T);
+  //        bool       operator>=(T, T);
+  //        bool       operator==(T, T);
+  //        bool       operator!=(T, T);
+  void addRelationalPointerOrEnumeralOverloads() {
+    // C++ [over.built]p1:
+    //   If there is a user-written candidate with the same name and parameter
+    //   types as a built-in candidate operator function, the built-in operator
+    //   function is hidden and is not included in the set of candidate
+    //   functions.
     //
-    //        bool operator==(T,T);
-    //        bool operator!=(T,T);
-    for (BuiltinCandidateTypeSet::iterator
-           MemPtr = CandidateTypes.member_pointer_begin(),
-           MemPtrEnd = CandidateTypes.member_pointer_end();
-         MemPtr != MemPtrEnd;
-         ++MemPtr) {
-      QualType ParamTypes[2] = { *MemPtr, *MemPtr };
-      AddBuiltinCandidate(Context.BoolTy, ParamTypes, Args, 2, CandidateSet);
-    }
+    // The text is actually in a note, but if we don't implement it then we end
+    // up with ambiguities when the user provides an overloaded operator for
+    // an enumeration type. Note that only enumeration types have this problem,
+    // so we track which enumeration types we've seen operators for. Also, the
+    // only other overloaded operator with enumeration argumenst, operator=,
+    // cannot be overloaded for enumeration types, so this is the only place
+    // where we must suppress candidates like this.
+    llvm::DenseSet<std::pair<CanQualType, CanQualType> >
+      UserDefinedBinaryOperators;
+
+    for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
+      if (CandidateTypes[ArgIdx].enumeration_begin() !=
+          CandidateTypes[ArgIdx].enumeration_end()) {
+        for (OverloadCandidateSet::iterator C = CandidateSet.begin(),
+                                         CEnd = CandidateSet.end();
+             C != CEnd; ++C) {
+          if (!C->Viable || !C->Function || C->Function->getNumParams() != 2)
+            continue;
 
-    // Fall through
+          QualType FirstParamType =
+            C->Function->getParamDecl(0)->getType().getUnqualifiedType();
+          QualType SecondParamType =
+            C->Function->getParamDecl(1)->getType().getUnqualifiedType();
 
-  case OO_Less:
-  case OO_Greater:
-  case OO_LessEqual:
-  case OO_GreaterEqual:
-    // C++ [over.built]p15:
-    //
-    //   For every pointer or enumeration type T, there exist
-    //   candidate operator functions of the form
-    //
-    //        bool       operator<(T, T);
-    //        bool       operator>(T, T);
-    //        bool       operator<=(T, T);
-    //        bool       operator>=(T, T);
-    //        bool       operator==(T, T);
-    //        bool       operator!=(T, T);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-      QualType ParamTypes[2] = { *Ptr, *Ptr };
-      AddBuiltinCandidate(Context.BoolTy, ParamTypes, Args, 2, CandidateSet);
-    }
-    for (BuiltinCandidateTypeSet::iterator Enum
-           = CandidateTypes.enumeration_begin();
-         Enum != CandidateTypes.enumeration_end(); ++Enum) {
-      QualType ParamTypes[2] = { *Enum, *Enum };
-      AddBuiltinCandidate(Context.BoolTy, ParamTypes, Args, 2, CandidateSet);
+          // Skip if either parameter isn't of enumeral type.
+          if (!FirstParamType->isEnumeralType() ||
+              !SecondParamType->isEnumeralType())
+            continue;
+
+          // Add this operator to the set of known user-defined operators.
+          UserDefinedBinaryOperators.insert(
+            std::make_pair(S.Context.getCanonicalType(FirstParamType),
+                           S.Context.getCanonicalType(SecondParamType)));
+        }
+      }
     }
 
-    // Fall through.
-    isComparison = true;
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
-  BinaryPlus:
-  BinaryMinus:
-    if (!isComparison) {
-      // We didn't fall through, so we must have OO_Plus or OO_Minus.
+    for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
+      for (BuiltinCandidateTypeSet::iterator
+                Ptr = CandidateTypes[ArgIdx].pointer_begin(),
+             PtrEnd = CandidateTypes[ArgIdx].pointer_end();
+           Ptr != PtrEnd; ++Ptr) {
+        // Don't add the same builtin candidate twice.
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)))
+          continue;
 
-      // C++ [over.built]p13:
-      //
-      //   For every cv-qualified or cv-unqualified object type T
-      //   there exist candidate operator functions of the form
-      //
-      //      T*         operator+(T*, ptrdiff_t);
-      //      T&         operator[](T*, ptrdiff_t);    [BELOW]
-      //      T*         operator-(T*, ptrdiff_t);
-      //      T*         operator+(ptrdiff_t, T*);
-      //      T&         operator[](ptrdiff_t, T*);    [BELOW]
-      //
-      // C++ [over.built]p14:
-      //
-      //   For every T, where T is a pointer to object type, there
-      //   exist candidate operator functions of the form
-      //
-      //      ptrdiff_t  operator-(T, T);
-      for (BuiltinCandidateTypeSet::iterator Ptr
-             = CandidateTypes.pointer_begin();
-           Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-        QualType ParamTypes[2] = { *Ptr, Context.getPointerDiffType() };
+        QualType ParamTypes[2] = { *Ptr, *Ptr };
+        S.AddBuiltinCandidate(S.Context.BoolTy, ParamTypes, Args, 2,
+                              CandidateSet);
+      }
+      for (BuiltinCandidateTypeSet::iterator
+                Enum = CandidateTypes[ArgIdx].enumeration_begin(),
+             EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
+           Enum != EnumEnd; ++Enum) {
+        CanQualType CanonType = S.Context.getCanonicalType(*Enum);
+
+        // Don't add the same builtin candidate twice, or if a user defined
+        // candidate exists.
+        if (!AddedTypes.insert(CanonType) ||
+            UserDefinedBinaryOperators.count(std::make_pair(CanonType,
+                                                            CanonType)))
+          continue;
 
-        // operator+(T*, ptrdiff_t) or operator-(T*, ptrdiff_t)
-        AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
+        QualType ParamTypes[2] = { *Enum, *Enum };
+        S.AddBuiltinCandidate(S.Context.BoolTy, ParamTypes, Args, 2,
+                              CandidateSet);
+      }
+    }
+  }
 
-        if (Op == OO_Plus) {
+  // C++ [over.built]p13:
+  //
+  //   For every cv-qualified or cv-unqualified object type T
+  //   there exist candidate operator functions of the form
+  //
+  //      T*         operator+(T*, ptrdiff_t);
+  //      T&         operator[](T*, ptrdiff_t);    [BELOW]
+  //      T*         operator-(T*, ptrdiff_t);
+  //      T*         operator+(ptrdiff_t, T*);
+  //      T&         operator[](ptrdiff_t, T*);    [BELOW]
+  //
+  // C++ [over.built]p14:
+  //
+  //   For every T, where T is a pointer to object type, there
+  //   exist candidate operator functions of the form
+  //
+  //      ptrdiff_t  operator-(T, T);
+  void addBinaryPlusOrMinusPointerOverloads(OverloadedOperatorKind Op) {
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
+
+    for (int Arg = 0; Arg < 2; ++Arg) {
+      QualType AsymetricParamTypes[2] = {
+        S.Context.getPointerDiffType(),
+        S.Context.getPointerDiffType(),
+      };
+      for (BuiltinCandidateTypeSet::iterator
+                Ptr = CandidateTypes[Arg].pointer_begin(),
+             PtrEnd = CandidateTypes[Arg].pointer_end();
+           Ptr != PtrEnd; ++Ptr) {
+        QualType PointeeTy = (*Ptr)->getPointeeType();
+        if (!PointeeTy->isObjectType())
+          continue;
+
+        AsymetricParamTypes[Arg] = *Ptr;
+        if (Arg == 0 || Op == OO_Plus) {
+          // operator+(T*, ptrdiff_t) or operator-(T*, ptrdiff_t)
           // T* operator+(ptrdiff_t, T*);
-          ParamTypes[0] = ParamTypes[1];
-          ParamTypes[1] = *Ptr;
-          AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
-        } else {
+          S.AddBuiltinCandidate(*Ptr, AsymetricParamTypes, Args, 2,
+                                CandidateSet);
+        }
+        if (Op == OO_Minus) {
           // ptrdiff_t operator-(T, T);
-          ParamTypes[1] = *Ptr;
-          AddBuiltinCandidate(Context.getPointerDiffType(), ParamTypes,
-                              Args, 2, CandidateSet);
+          if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)))
+            continue;
+
+          QualType ParamTypes[2] = { *Ptr, *Ptr };
+          S.AddBuiltinCandidate(S.Context.getPointerDiffType(), ParamTypes,
+                                Args, 2, CandidateSet);
         }
       }
     }
-    // Fall through
+  }
+
+  // C++ [over.built]p12:
+  //
+  //   For every pair of promoted arithmetic types L and R, there
+  //   exist candidate operator functions of the form
+  //
+  //        LR         operator*(L, R);
+  //        LR         operator/(L, R);
+  //        LR         operator+(L, R);
+  //        LR         operator-(L, R);
+  //        bool       operator<(L, R);
+  //        bool       operator>(L, R);
+  //        bool       operator<=(L, R);
+  //        bool       operator>=(L, R);
+  //        bool       operator==(L, R);
+  //        bool       operator!=(L, R);
+  //
+  //   where LR is the result of the usual arithmetic conversions
+  //   between types L and R.
+  //
+  // C++ [over.built]p24:
+  //
+  //   For every pair of promoted arithmetic types L and R, there exist
+  //   candidate operator functions of the form
+  //
+  //        LR       operator?(bool, L, R);
+  //
+  //   where LR is the result of the usual arithmetic conversions
+  //   between types L and R.
+  // Our candidates ignore the first parameter.
+  void addGenericBinaryArithmeticOverloads(bool isComparison) {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  case OO_Slash:
-  BinaryStar:
-  Conditional:
-    // C++ [over.built]p12:
-    //
-    //   For every pair of promoted arithmetic types L and R, there
-    //   exist candidate operator functions of the form
-    //
-    //        LR         operator*(L, R);
-    //        LR         operator/(L, R);
-    //        LR         operator+(L, R);
-    //        LR         operator-(L, R);
-    //        bool       operator<(L, R);
-    //        bool       operator>(L, R);
-    //        bool       operator<=(L, R);
-    //        bool       operator>=(L, R);
-    //        bool       operator==(L, R);
-    //        bool       operator!=(L, R);
-    //
-    //   where LR is the result of the usual arithmetic conversions
-    //   between types L and R.
-    //
-    // C++ [over.built]p24:
-    //
-    //   For every pair of promoted arithmetic types L and R, there exist
-    //   candidate operator functions of the form
-    //
-    //        LR       operator?(bool, L, R);
-    //
-    //   where LR is the result of the usual arithmetic conversions
-    //   between types L and R.
-    // Our candidates ignore the first parameter.
     for (unsigned Left = FirstPromotedArithmeticType;
          Left < LastPromotedArithmeticType; ++Left) {
       for (unsigned Right = FirstPromotedArithmeticType;
            Right < LastPromotedArithmeticType; ++Right) {
-        QualType LandR[2] = { ArithmeticTypes[Left], ArithmeticTypes[Right] };
-        QualType Result
-          = isComparison
-          ? Context.BoolTy
-          : Context.UsualArithmeticConversionsType(LandR[0], LandR[1]);
-        AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
+        QualType LandR[2] = { getArithmeticType(Left),
+                              getArithmeticType(Right) };
+        QualType Result =
+          isComparison ? S.Context.BoolTy
+                       : getUsualArithmeticConversions(Left, Right);
+        S.AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
       }
     }
 
     // Extension: Add the binary operators ==, !=, <, <=, >=, >, *, /, and the
     // conditional operator for vector types.
-    for (BuiltinCandidateTypeSet::iterator Vec1 = CandidateTypes.vector_begin(),
-         Vec1End = CandidateTypes.vector_end(); 
-         Vec1 != Vec1End; ++Vec1)
-      for (BuiltinCandidateTypeSet::iterator 
-           Vec2 = CandidateTypes.vector_begin(),
-           Vec2End = CandidateTypes.vector_end(); 
+    for (BuiltinCandidateTypeSet::iterator
+              Vec1 = CandidateTypes[0].vector_begin(),
+           Vec1End = CandidateTypes[0].vector_end();
+         Vec1 != Vec1End; ++Vec1) {
+      for (BuiltinCandidateTypeSet::iterator
+                Vec2 = CandidateTypes[1].vector_begin(),
+             Vec2End = CandidateTypes[1].vector_end();
            Vec2 != Vec2End; ++Vec2) {
         QualType LandR[2] = { *Vec1, *Vec2 };
-        QualType Result;
-        if (isComparison)
-          Result = Context.BoolTy;
-        else {
+        QualType Result = S.Context.BoolTy;
+        if (!isComparison) {
           if ((*Vec1)->isExtVectorType() || !(*Vec2)->isExtVectorType())
             Result = *Vec1;
           else
             Result = *Vec2;
         }
-        
-        AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
+
+        S.AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
       }
-      
-    break;
+    }
+  }
+
+  // C++ [over.built]p17:
+  //
+  //   For every pair of promoted integral types L and R, there
+  //   exist candidate operator functions of the form
+  //
+  //      LR         operator%(L, R);
+  //      LR         operator&(L, R);
+  //      LR         operator^(L, R);
+  //      LR         operator|(L, R);
+  //      L          operator<<(L, R);
+  //      L          operator>>(L, R);
+  //
+  //   where LR is the result of the usual arithmetic conversions
+  //   between types L and R.
+  void addBinaryBitwiseArithmeticOverloads(OverloadedOperatorKind Op) {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  case OO_Percent:
-  BinaryAmp:
-  case OO_Caret:
-  case OO_Pipe:
-  case OO_LessLess:
-  case OO_GreaterGreater:
-    // C++ [over.built]p17:
-    //
-    //   For every pair of promoted integral types L and R, there
-    //   exist candidate operator functions of the form
-    //
-    //      LR         operator%(L, R);
-    //      LR         operator&(L, R);
-    //      LR         operator^(L, R);
-    //      LR         operator|(L, R);
-    //      L          operator<<(L, R);
-    //      L          operator>>(L, R);
-    //
-    //   where LR is the result of the usual arithmetic conversions
-    //   between types L and R.
     for (unsigned Left = FirstPromotedIntegralType;
          Left < LastPromotedIntegralType; ++Left) {
       for (unsigned Right = FirstPromotedIntegralType;
            Right < LastPromotedIntegralType; ++Right) {
-        QualType LandR[2] = { ArithmeticTypes[Left], ArithmeticTypes[Right] };
+        QualType LandR[2] = { getArithmeticType(Left),
+                              getArithmeticType(Right) };
         QualType Result = (Op == OO_LessLess || Op == OO_GreaterGreater)
             ? LandR[0]
-            : Context.UsualArithmeticConversionsType(LandR[0], LandR[1]);
-        AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
+            : getUsualArithmeticConversions(Left, Right);
+        S.AddBuiltinCandidate(Result, LandR, Args, 2, CandidateSet);
       }
     }
-    break;
+  }
 
-  case OO_Equal:
-    // C++ [over.built]p20:
-    //
-    //   For every pair (T, VQ), where T is an enumeration or
-    //   pointer to member type and VQ is either volatile or
-    //   empty, there exist candidate operator functions of the form
-    //
-    //        VQ T&      operator=(VQ T&, T);
-    for (BuiltinCandidateTypeSet::iterator
-           Enum = CandidateTypes.enumeration_begin(),
-           EnumEnd = CandidateTypes.enumeration_end();
-         Enum != EnumEnd; ++Enum)
-      AddBuiltinAssignmentOperatorCandidates(*this, *Enum, Args, 2,
-                                             CandidateSet);
-    for (BuiltinCandidateTypeSet::iterator
-           MemPtr = CandidateTypes.member_pointer_begin(),
-         MemPtrEnd = CandidateTypes.member_pointer_end();
-         MemPtr != MemPtrEnd; ++MemPtr)
-      AddBuiltinAssignmentOperatorCandidates(*this, *MemPtr, Args, 2,
-                                             CandidateSet);
-      
-    // Fall through.
+  // C++ [over.built]p20:
+  //
+  //   For every pair (T, VQ), where T is an enumeration or
+  //   pointer to member type and VQ is either volatile or
+  //   empty, there exist candidate operator functions of the form
+  //
+  //        VQ T&      operator=(VQ T&, T);
+  void addAssignmentMemberPointerOrEnumeralOverloads() {
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
+
+    for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
+      for (BuiltinCandidateTypeSet::iterator
+                Enum = CandidateTypes[ArgIdx].enumeration_begin(),
+             EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
+           Enum != EnumEnd; ++Enum) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*Enum)))
+          continue;
 
-  case OO_PlusEqual:
-  case OO_MinusEqual:
-    // C++ [over.built]p19:
-    //
-    //   For every pair (T, VQ), where T is any type and VQ is either
-    //   volatile or empty, there exist candidate operator functions
-    //   of the form
-    //
-    //        T*VQ&      operator=(T*VQ&, T*);
-    //
-    // C++ [over.built]p21:
-    //
-    //   For every pair (T, VQ), where T is a cv-qualified or
-    //   cv-unqualified object type and VQ is either volatile or
-    //   empty, there exist candidate operator functions of the form
-    //
-    //        T*VQ&      operator+=(T*VQ&, ptrdiff_t);
-    //        T*VQ&      operator-=(T*VQ&, ptrdiff_t);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-      QualType ParamTypes[2];
-      ParamTypes[1] = (Op == OO_Equal)? *Ptr : Context.getPointerDiffType();
+        AddBuiltinAssignmentOperatorCandidates(S, *Enum, Args, 2,
+                                               CandidateSet);
+      }
+
+      for (BuiltinCandidateTypeSet::iterator
+                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
+             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
+           MemPtr != MemPtrEnd; ++MemPtr) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)))
+          continue;
+
+        AddBuiltinAssignmentOperatorCandidates(S, *MemPtr, Args, 2,
+                                               CandidateSet);
+      }
+    }
+  }
+
+  // C++ [over.built]p19:
+  //
+  //   For every pair (T, VQ), where T is any type and VQ is either
+  //   volatile or empty, there exist candidate operator functions
+  //   of the form
+  //
+  //        T*VQ&      operator=(T*VQ&, T*);
+  //
+  // C++ [over.built]p21:
+  //
+  //   For every pair (T, VQ), where T is a cv-qualified or
+  //   cv-unqualified object type and VQ is either volatile or
+  //   empty, there exist candidate operator functions of the form
+  //
+  //        T*VQ&      operator+=(T*VQ&, ptrdiff_t);
+  //        T*VQ&      operator-=(T*VQ&, ptrdiff_t);
+  void addAssignmentPointerOverloads(bool isEqualOp) {
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
+
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
+      // If this is operator=, keep track of the builtin candidates we added.
+      if (isEqualOp)
+        AddedTypes.insert(S.Context.getCanonicalType(*Ptr));
+      else if (!(*Ptr)->getPointeeType()->isObjectType())
+        continue;
 
       // non-volatile version
-      ParamTypes[0] = Context.getLValueReferenceType(*Ptr);
-      AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                          /*IsAssigmentOperator=*/Op == OO_Equal);
+      QualType ParamTypes[2] = {
+        S.Context.getLValueReferenceType(*Ptr),
+        isEqualOp ? *Ptr : S.Context.getPointerDiffType(),
+      };
+      S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
+                            /*IsAssigmentOperator=*/ isEqualOp);
 
-      if (!Context.getCanonicalType(*Ptr).isVolatileQualified() &&
+      if (!S.Context.getCanonicalType(*Ptr).isVolatileQualified() &&
           VisibleTypeConversionsQuals.hasVolatile()) {
         // volatile version
-        ParamTypes[0]
-          = Context.getLValueReferenceType(Context.getVolatileType(*Ptr));
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                            /*IsAssigmentOperator=*/Op == OO_Equal);
+        ParamTypes[0] =
+          S.Context.getLValueReferenceType(S.Context.getVolatileType(*Ptr));
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
+                              /*IsAssigmentOperator=*/isEqualOp);
       }
     }
-    // Fall through.
 
-  case OO_StarEqual:
-  case OO_SlashEqual:
-    // C++ [over.built]p18:
-    //
-    //   For every triple (L, VQ, R), where L is an arithmetic type,
-    //   VQ is either volatile or empty, and R is a promoted
-    //   arithmetic type, there exist candidate operator functions of
-    //   the form
-    //
-    //        VQ L&      operator=(VQ L&, R);
-    //        VQ L&      operator*=(VQ L&, R);
-    //        VQ L&      operator/=(VQ L&, R);
-    //        VQ L&      operator+=(VQ L&, R);
-    //        VQ L&      operator-=(VQ L&, R);
+    if (isEqualOp) {
+      for (BuiltinCandidateTypeSet::iterator
+                Ptr = CandidateTypes[1].pointer_begin(),
+             PtrEnd = CandidateTypes[1].pointer_end();
+           Ptr != PtrEnd; ++Ptr) {
+        // Make sure we don't add the same candidate twice.
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)))
+          continue;
+
+        QualType ParamTypes[2] = {
+          S.Context.getLValueReferenceType(*Ptr),
+          *Ptr,
+        };
+
+        // non-volatile version
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
+                              /*IsAssigmentOperator=*/true);
+
+        if (!S.Context.getCanonicalType(*Ptr).isVolatileQualified() &&
+            VisibleTypeConversionsQuals.hasVolatile()) {
+          // volatile version
+          ParamTypes[0] =
+            S.Context.getLValueReferenceType(S.Context.getVolatileType(*Ptr));
+          S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2,
+                                CandidateSet, /*IsAssigmentOperator=*/true);
+        }
+      }
+    }
+  }
+
+  // C++ [over.built]p18:
+  //
+  //   For every triple (L, VQ, R), where L is an arithmetic type,
+  //   VQ is either volatile or empty, and R is a promoted
+  //   arithmetic type, there exist candidate operator functions of
+  //   the form
+  //
+  //        VQ L&      operator=(VQ L&, R);
+  //        VQ L&      operator*=(VQ L&, R);
+  //        VQ L&      operator/=(VQ L&, R);
+  //        VQ L&      operator+=(VQ L&, R);
+  //        VQ L&      operator-=(VQ L&, R);
+  void addAssignmentArithmeticOverloads(bool isEqualOp) {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
+
     for (unsigned Left = 0; Left < NumArithmeticTypes; ++Left) {
       for (unsigned Right = FirstPromotedArithmeticType;
            Right < LastPromotedArithmeticType; ++Right) {
         QualType ParamTypes[2];
-        ParamTypes[1] = ArithmeticTypes[Right];
+        ParamTypes[1] = getArithmeticType(Right);
 
         // Add this built-in operator as a candidate (VQ is empty).
-        ParamTypes[0] = Context.getLValueReferenceType(ArithmeticTypes[Left]);
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                            /*IsAssigmentOperator=*/Op == OO_Equal);
+        ParamTypes[0] =
+          S.Context.getLValueReferenceType(getArithmeticType(Left));
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
+                              /*IsAssigmentOperator=*/isEqualOp);
 
         // Add this built-in operator as a candidate (VQ is 'volatile').
         if (VisibleTypeConversionsQuals.hasVolatile()) {
-          ParamTypes[0] = Context.getVolatileType(ArithmeticTypes[Left]);
-          ParamTypes[0] = Context.getLValueReferenceType(ParamTypes[0]);
-          AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                              /*IsAssigmentOperator=*/Op == OO_Equal);
+          ParamTypes[0] =
+            S.Context.getVolatileType(getArithmeticType(Left));
+          ParamTypes[0] = S.Context.getLValueReferenceType(ParamTypes[0]);
+          S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2,
+                                CandidateSet,
+                                /*IsAssigmentOperator=*/isEqualOp);
         }
       }
     }
-      
+
     // Extension: Add the binary operators =, +=, -=, *=, /= for vector types.
-    for (BuiltinCandidateTypeSet::iterator Vec1 = CandidateTypes.vector_begin(),
-                                        Vec1End = CandidateTypes.vector_end(); 
-         Vec1 != Vec1End; ++Vec1)
-      for (BuiltinCandidateTypeSet::iterator 
-                Vec2 = CandidateTypes.vector_begin(),
-             Vec2End = CandidateTypes.vector_end(); 
+    for (BuiltinCandidateTypeSet::iterator
+              Vec1 = CandidateTypes[0].vector_begin(),
+           Vec1End = CandidateTypes[0].vector_end();
+         Vec1 != Vec1End; ++Vec1) {
+      for (BuiltinCandidateTypeSet::iterator
+                Vec2 = CandidateTypes[1].vector_begin(),
+             Vec2End = CandidateTypes[1].vector_end();
            Vec2 != Vec2End; ++Vec2) {
         QualType ParamTypes[2];
         ParamTypes[1] = *Vec2;
         // Add this built-in operator as a candidate (VQ is empty).
-        ParamTypes[0] = Context.getLValueReferenceType(*Vec1);
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                            /*IsAssigmentOperator=*/Op == OO_Equal);
-        
+        ParamTypes[0] = S.Context.getLValueReferenceType(*Vec1);
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
+                              /*IsAssigmentOperator=*/isEqualOp);
+
         // Add this built-in operator as a candidate (VQ is 'volatile').
         if (VisibleTypeConversionsQuals.hasVolatile()) {
-          ParamTypes[0] = Context.getVolatileType(*Vec1);
-          ParamTypes[0] = Context.getLValueReferenceType(ParamTypes[0]);
-          AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet,
-                              /*IsAssigmentOperator=*/Op == OO_Equal);
+          ParamTypes[0] = S.Context.getVolatileType(*Vec1);
+          ParamTypes[0] = S.Context.getLValueReferenceType(ParamTypes[0]);
+          S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2,
+                                CandidateSet,
+                                /*IsAssigmentOperator=*/isEqualOp);
         }
       }
-    break;
+    }
+  }
+
+  // C++ [over.built]p22:
+  //
+  //   For every triple (L, VQ, R), where L is an integral type, VQ
+  //   is either volatile or empty, and R is a promoted integral
+  //   type, there exist candidate operator functions of the form
+  //
+  //        VQ L&       operator%=(VQ L&, R);
+  //        VQ L&       operator<<=(VQ L&, R);
+  //        VQ L&       operator>>=(VQ L&, R);
+  //        VQ L&       operator&=(VQ L&, R);
+  //        VQ L&       operator^=(VQ L&, R);
+  //        VQ L&       operator|=(VQ L&, R);
+  void addAssignmentIntegralOverloads() {
+    if (!HasArithmeticOrEnumeralCandidateType)
+      return;
 
-  case OO_PercentEqual:
-  case OO_LessLessEqual:
-  case OO_GreaterGreaterEqual:
-  case OO_AmpEqual:
-  case OO_CaretEqual:
-  case OO_PipeEqual:
-    // C++ [over.built]p22:
-    //
-    //   For every triple (L, VQ, R), where L is an integral type, VQ
-    //   is either volatile or empty, and R is a promoted integral
-    //   type, there exist candidate operator functions of the form
-    //
-    //        VQ L&       operator%=(VQ L&, R);
-    //        VQ L&       operator<<=(VQ L&, R);
-    //        VQ L&       operator>>=(VQ L&, R);
-    //        VQ L&       operator&=(VQ L&, R);
-    //        VQ L&       operator^=(VQ L&, R);
-    //        VQ L&       operator|=(VQ L&, R);
     for (unsigned Left = FirstIntegralType; Left < LastIntegralType; ++Left) {
       for (unsigned Right = FirstPromotedIntegralType;
            Right < LastPromotedIntegralType; ++Right) {
         QualType ParamTypes[2];
-        ParamTypes[1] = ArithmeticTypes[Right];
+        ParamTypes[1] = getArithmeticType(Right);
 
         // Add this built-in operator as a candidate (VQ is empty).
-        ParamTypes[0] = Context.getLValueReferenceType(ArithmeticTypes[Left]);
-        AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet);
+        ParamTypes[0] =
+          S.Context.getLValueReferenceType(getArithmeticType(Left));
+        S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet);
         if (VisibleTypeConversionsQuals.hasVolatile()) {
           // Add this built-in operator as a candidate (VQ is 'volatile').
-          ParamTypes[0] = ArithmeticTypes[Left];
-          ParamTypes[0] = Context.getVolatileType(ParamTypes[0]);
-          ParamTypes[0] = Context.getLValueReferenceType(ParamTypes[0]);
-          AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2, CandidateSet);
+          ParamTypes[0] = getArithmeticType(Left);
+          ParamTypes[0] = S.Context.getVolatileType(ParamTypes[0]);
+          ParamTypes[0] = S.Context.getLValueReferenceType(ParamTypes[0]);
+          S.AddBuiltinCandidate(ParamTypes[0], ParamTypes, Args, 2,
+                                CandidateSet);
         }
       }
     }
-    break;
-
-  case OO_Exclaim: {
-    // C++ [over.operator]p23:
-    //
-    //   There also exist candidate operator functions of the form
-    //
-    //        bool        operator!(bool);
-    //        bool        operator&&(bool, bool);     [BELOW]
-    //        bool        operator||(bool, bool);     [BELOW]
-    QualType ParamTy = Context.BoolTy;
-    AddBuiltinCandidate(ParamTy, &ParamTy, Args, 1, CandidateSet,
-                        /*IsAssignmentOperator=*/false,
-                        /*NumContextualBoolArguments=*/1);
-    break;
-  }
-
-  case OO_AmpAmp:
-  case OO_PipePipe: {
-    // C++ [over.operator]p23:
-    //
-    //   There also exist candidate operator functions of the form
-    //
-    //        bool        operator!(bool);            [ABOVE]
-    //        bool        operator&&(bool, bool);
-    //        bool        operator||(bool, bool);
-    QualType ParamTypes[2] = { Context.BoolTy, Context.BoolTy };
-    AddBuiltinCandidate(Context.BoolTy, ParamTypes, Args, 2, CandidateSet,
-                        /*IsAssignmentOperator=*/false,
-                        /*NumContextualBoolArguments=*/2);
-    break;
   }
 
-  case OO_Subscript:
-    // C++ [over.built]p13:
-    //
-    //   For every cv-qualified or cv-unqualified object type T there
-    //   exist candidate operator functions of the form
-    //
-    //        T*         operator+(T*, ptrdiff_t);     [ABOVE]
-    //        T&         operator[](T*, ptrdiff_t);
-    //        T*         operator-(T*, ptrdiff_t);     [ABOVE]
-    //        T*         operator+(ptrdiff_t, T*);     [ABOVE]
-    //        T&         operator[](ptrdiff_t, T*);
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
-         Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-      QualType ParamTypes[2] = { *Ptr, Context.getPointerDiffType() };
+  // C++ [over.operator]p23:
+  //
+  //   There also exist candidate operator functions of the form
+  //
+  //        bool        operator!(bool);
+  //        bool        operator&&(bool, bool);
+  //        bool        operator||(bool, bool);
+  void addExclaimOverload() {
+    QualType ParamTy = S.Context.BoolTy;
+    S.AddBuiltinCandidate(ParamTy, &ParamTy, Args, 1, CandidateSet,
+                          /*IsAssignmentOperator=*/false,
+                          /*NumContextualBoolArguments=*/1);
+  }
+  void addAmpAmpOrPipePipeOverload() {
+    QualType ParamTypes[2] = { S.Context.BoolTy, S.Context.BoolTy };
+    S.AddBuiltinCandidate(S.Context.BoolTy, ParamTypes, Args, 2, CandidateSet,
+                          /*IsAssignmentOperator=*/false,
+                          /*NumContextualBoolArguments=*/2);
+  }
+
+  // C++ [over.built]p13:
+  //
+  //   For every cv-qualified or cv-unqualified object type T there
+  //   exist candidate operator functions of the form
+  //
+  //        T*         operator+(T*, ptrdiff_t);     [ABOVE]
+  //        T&         operator[](T*, ptrdiff_t);
+  //        T*         operator-(T*, ptrdiff_t);     [ABOVE]
+  //        T*         operator+(ptrdiff_t, T*);     [ABOVE]
+  //        T&         operator[](ptrdiff_t, T*);
+  void addSubscriptOverloads() {
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
+      QualType ParamTypes[2] = { *Ptr, S.Context.getPointerDiffType() };
       QualType PointeeType = (*Ptr)->getPointeeType();
-      QualType ResultTy = Context.getLValueReferenceType(PointeeType);
+      if (!PointeeType->isObjectType())
+        continue;
+
+      QualType ResultTy = S.Context.getLValueReferenceType(PointeeType);
 
       // T& operator[](T*, ptrdiff_t)
-      AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
+      S.AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
+    }
 
-      // T& operator[](ptrdiff_t, T*);
-      ParamTypes[0] = ParamTypes[1];
-      ParamTypes[1] = *Ptr;
-      AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
-    }      
-    break;
+    for (BuiltinCandidateTypeSet::iterator
+              Ptr = CandidateTypes[1].pointer_begin(),
+           PtrEnd = CandidateTypes[1].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
+      QualType ParamTypes[2] = { S.Context.getPointerDiffType(), *Ptr };
+      QualType PointeeType = (*Ptr)->getPointeeType();
+      if (!PointeeType->isObjectType())
+        continue;
 
-  case OO_ArrowStar:
-    // C++ [over.built]p11:
-    //    For every quintuple (C1, C2, T, CV1, CV2), where C2 is a class type, 
-    //    C1 is the same type as C2 or is a derived class of C2, T is an object 
-    //    type or a function type, and CV1 and CV2 are cv-qualifier-seqs, 
-    //    there exist candidate operator functions of the form 
-    //    CV12 T& operator->*(CV1 C1*, CV2 T C2::*); 
-    //    where CV12 is the union of CV1 and CV2.
-    {
-      for (BuiltinCandidateTypeSet::iterator Ptr = 
-             CandidateTypes.pointer_begin();
-           Ptr != CandidateTypes.pointer_end(); ++Ptr) {
-        QualType C1Ty = (*Ptr);
-        QualType C1;
-        QualifierCollector Q1;
-        C1 = QualType(Q1.strip(C1Ty->getPointeeType()), 0);
-        if (!isa<RecordType>(C1))
+      QualType ResultTy = S.Context.getLValueReferenceType(PointeeType);
+
+      // T& operator[](ptrdiff_t, T*)
+      S.AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
+    }
+  }
+
+  // C++ [over.built]p11:
+  //    For every quintuple (C1, C2, T, CV1, CV2), where C2 is a class type,
+  //    C1 is the same type as C2 or is a derived class of C2, T is an object
+  //    type or a function type, and CV1 and CV2 are cv-qualifier-seqs,
+  //    there exist candidate operator functions of the form
+  //
+  //      CV12 T& operator->*(CV1 C1*, CV2 T C2::*);
+  //
+  //    where CV12 is the union of CV1 and CV2.
+  void addArrowStarOverloads() {
+    for (BuiltinCandidateTypeSet::iterator
+             Ptr = CandidateTypes[0].pointer_begin(),
+           PtrEnd = CandidateTypes[0].pointer_end();
+         Ptr != PtrEnd; ++Ptr) {
+      QualType C1Ty = (*Ptr);
+      QualType C1;
+      QualifierCollector Q1;
+      C1 = QualType(Q1.strip(C1Ty->getPointeeType()), 0);
+      if (!isa<RecordType>(C1))
+        continue;
+      // heuristic to reduce number of builtin candidates in the set.
+      // Add volatile/restrict version only if there are conversions to a
+      // volatile/restrict type.
+      if (!VisibleTypeConversionsQuals.hasVolatile() && Q1.hasVolatile())
+        continue;
+      if (!VisibleTypeConversionsQuals.hasRestrict() && Q1.hasRestrict())
+        continue;
+      for (BuiltinCandidateTypeSet::iterator
+                MemPtr = CandidateTypes[1].member_pointer_begin(),
+             MemPtrEnd = CandidateTypes[1].member_pointer_end();
+           MemPtr != MemPtrEnd; ++MemPtr) {
+        const MemberPointerType *mptr = cast<MemberPointerType>(*MemPtr);
+        QualType C2 = QualType(mptr->getClass(), 0);
+        C2 = C2.getUnqualifiedType();
+        if (C1 != C2 && !S.IsDerivedFrom(C1, C2))
+          break;
+        QualType ParamTypes[2] = { *Ptr, *MemPtr };
+        // build CV12 T&
+        QualType T = mptr->getPointeeType();
+        if (!VisibleTypeConversionsQuals.hasVolatile() &&
+            T.isVolatileQualified())
+          continue;
+        if (!VisibleTypeConversionsQuals.hasRestrict() &&
+            T.isRestrictQualified())
           continue;
-        // heuristic to reduce number of builtin candidates in the set.
-        // Add volatile/restrict version only if there are conversions to a
-        // volatile/restrict type.
-        if (!VisibleTypeConversionsQuals.hasVolatile() && Q1.hasVolatile())
+        T = Q1.apply(S.Context, T);
+        QualType ResultTy = S.Context.getLValueReferenceType(T);
+        S.AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
+      }
+    }
+  }
+
+  // Note that we don't consider the first argument, since it has been
+  // contextually converted to bool long ago. The candidates below are
+  // therefore added as binary.
+  //
+  // C++ [over.built]p25:
+  //   For every type T, where T is a pointer, pointer-to-member, or scoped
+  //   enumeration type, there exist candidate operator functions of the form
+  //
+  //        T        operator?(bool, T, T);
+  //
+  void addConditionalOperatorOverloads() {
+    /// Set of (canonical) types that we've already handled.
+    llvm::SmallPtrSet<QualType, 8> AddedTypes;
+
+    for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
+      for (BuiltinCandidateTypeSet::iterator
+                Ptr = CandidateTypes[ArgIdx].pointer_begin(),
+             PtrEnd = CandidateTypes[ArgIdx].pointer_end();
+           Ptr != PtrEnd; ++Ptr) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)))
           continue;
-        if (!VisibleTypeConversionsQuals.hasRestrict() && Q1.hasRestrict())
+
+        QualType ParamTypes[2] = { *Ptr, *Ptr };
+        S.AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
+      }
+
+      for (BuiltinCandidateTypeSet::iterator
+                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
+             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
+           MemPtr != MemPtrEnd; ++MemPtr) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)))
           continue;
+
+        QualType ParamTypes[2] = { *MemPtr, *MemPtr };
+        S.AddBuiltinCandidate(*MemPtr, ParamTypes, Args, 2, CandidateSet);
+      }
+
+      if (S.getLangOptions().CPlusPlus0x) {
         for (BuiltinCandidateTypeSet::iterator
-             MemPtr = CandidateTypes.member_pointer_begin(),
-             MemPtrEnd = CandidateTypes.member_pointer_end();
-             MemPtr != MemPtrEnd; ++MemPtr) {
-          const MemberPointerType *mptr = cast<MemberPointerType>(*MemPtr);
-          QualType C2 = QualType(mptr->getClass(), 0);
-          C2 = C2.getUnqualifiedType();
-          if (C1 != C2 && !IsDerivedFrom(C1, C2))
-            break;
-          QualType ParamTypes[2] = { *Ptr, *MemPtr };
-          // build CV12 T&
-          QualType T = mptr->getPointeeType();
-          if (!VisibleTypeConversionsQuals.hasVolatile() && 
-              T.isVolatileQualified())
+                  Enum = CandidateTypes[ArgIdx].enumeration_begin(),
+               EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
+             Enum != EnumEnd; ++Enum) {
+          if (!(*Enum)->getAs<EnumType>()->getDecl()->isScoped())
             continue;
-          if (!VisibleTypeConversionsQuals.hasRestrict() && 
-              T.isRestrictQualified())
+
+          if (!AddedTypes.insert(S.Context.getCanonicalType(*Enum)))
             continue;
-          T = Q1.apply(T);
-          QualType ResultTy = Context.getLValueReferenceType(T);
-          AddBuiltinCandidate(ResultTy, ParamTypes, Args, 2, CandidateSet);
+
+          QualType ParamTypes[2] = { *Enum, *Enum };
+          S.AddBuiltinCandidate(*Enum, ParamTypes, Args, 2, CandidateSet);
         }
       }
     }
+  }
+};
+
+} // end anonymous namespace
+
+/// AddBuiltinOperatorCandidates - Add the appropriate built-in
+/// operator overloads to the candidate set (C++ [over.built]), based
+/// on the operator @p Op and the arguments given. For example, if the
+/// operator is a binary '+', this routine might add "int
+/// operator+(int, int)" to cover integer addition.
+void
+Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
+                                   SourceLocation OpLoc,
+                                   Expr **Args, unsigned NumArgs,
+                                   OverloadCandidateSet& CandidateSet) {
+  // Find all of the types that the arguments can convert to, but only
+  // if the operator we're looking at has built-in operator candidates
+  // that make use of these types. Also record whether we encounter non-record
+  // candidate types or either arithmetic or enumeral candidate types.
+  Qualifiers VisibleTypeConversionsQuals;
+  VisibleTypeConversionsQuals.addConst();
+  for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
+    VisibleTypeConversionsQuals += CollectVRQualifiers(Context, Args[ArgIdx]);
+
+  bool HasNonRecordCandidateType = false;
+  bool HasArithmeticOrEnumeralCandidateType = false;
+  llvm::SmallVector<BuiltinCandidateTypeSet, 2> CandidateTypes;
+  for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx) {
+    CandidateTypes.push_back(BuiltinCandidateTypeSet(*this));
+    CandidateTypes[ArgIdx].AddTypesConvertedFrom(Args[ArgIdx]->getType(),
+                                                 OpLoc,
+                                                 true,
+                                                 (Op == OO_Exclaim ||
+                                                  Op == OO_AmpAmp ||
+                                                  Op == OO_PipePipe),
+                                                 VisibleTypeConversionsQuals);
+    HasNonRecordCandidateType = HasNonRecordCandidateType ||
+        CandidateTypes[ArgIdx].hasNonRecordTypes();
+    HasArithmeticOrEnumeralCandidateType =
+        HasArithmeticOrEnumeralCandidateType ||
+        CandidateTypes[ArgIdx].hasArithmeticOrEnumeralTypes();
+  }
+
+  // Exit early when no non-record types have been added to the candidate set
+  // for any of the arguments to the operator.
+  if (!HasNonRecordCandidateType)
+    return;
+
+  // Setup an object to manage the common state for building overloads.
+  BuiltinOperatorOverloadBuilder OpBuilder(*this, Args, NumArgs,
+                                           VisibleTypeConversionsQuals,
+                                           HasArithmeticOrEnumeralCandidateType,
+                                           CandidateTypes, CandidateSet);
+
+  // Dispatch over the operation to add in only those overloads which apply.
+  switch (Op) {
+  case OO_None:
+  case NUM_OVERLOADED_OPERATORS:
+    assert(false && "Expected an overloaded operator");
     break;
 
-  case OO_Conditional:
-    // Note that we don't consider the first argument, since it has been
-    // contextually converted to bool long ago. The candidates below are
-    // therefore added as binary.
-    //
-    // C++ [over.built]p24:
-    //   For every type T, where T is a pointer or pointer-to-member type,
-    //   there exist candidate operator functions of the form
-    //
-    //        T        operator?(bool, T, T);
-    //
-    for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin(),
-         E = CandidateTypes.pointer_end(); Ptr != E; ++Ptr) {
-      QualType ParamTypes[2] = { *Ptr, *Ptr };
-      AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
-    }
-    for (BuiltinCandidateTypeSet::iterator Ptr =
-           CandidateTypes.member_pointer_begin(),
-         E = CandidateTypes.member_pointer_end(); Ptr != E; ++Ptr) {
-      QualType ParamTypes[2] = { *Ptr, *Ptr };
-      AddBuiltinCandidate(*Ptr, ParamTypes, Args, 2, CandidateSet);
+  case OO_New:
+  case OO_Delete:
+  case OO_Array_New:
+  case OO_Array_Delete:
+  case OO_Call:
+    assert(false && "Special operators don't use AddBuiltinOperatorCandidates");
+    break;
+
+  case OO_Comma:
+  case OO_Arrow:
+    // C++ [over.match.oper]p3:
+    //   -- For the operator ',', the unary operator '&', or the
+    //      operator '->', the built-in candidates set is empty.
+    break;
+
+  case OO_Plus: // '+' is either unary or binary
+    if (NumArgs == 1)
+      OpBuilder.addUnaryPlusPointerOverloads();
+    // Fall through.
+
+  case OO_Minus: // '-' is either unary or binary
+    if (NumArgs == 1) {
+      OpBuilder.addUnaryPlusOrMinusArithmeticOverloads();
+    } else {
+      OpBuilder.addBinaryPlusOrMinusPointerOverloads(Op);
+      OpBuilder.addGenericBinaryArithmeticOverloads(/*isComparison=*/false);
     }
-    goto Conditional;
+    break;
+
+  case OO_Star: // '*' is either unary or binary
+    if (NumArgs == 1)
+      OpBuilder.addUnaryStarPointerOverloads();
+    else
+      OpBuilder.addGenericBinaryArithmeticOverloads(/*isComparison=*/false);
+    break;
+
+  case OO_Slash:
+    OpBuilder.addGenericBinaryArithmeticOverloads(/*isComparison=*/false);
+    break;
+
+  case OO_PlusPlus:
+  case OO_MinusMinus:
+    OpBuilder.addPlusPlusMinusMinusArithmeticOverloads(Op);
+    OpBuilder.addPlusPlusMinusMinusPointerOverloads();
+    break;
+
+  case OO_EqualEqual:
+  case OO_ExclaimEqual:
+    OpBuilder.addEqualEqualOrNotEqualMemberPointerOverloads();
+    // Fall through.
+
+  case OO_Less:
+  case OO_Greater:
+  case OO_LessEqual:
+  case OO_GreaterEqual:
+    OpBuilder.addRelationalPointerOrEnumeralOverloads();
+    OpBuilder.addGenericBinaryArithmeticOverloads(/*isComparison=*/true);
+    break;
+
+  case OO_Percent:
+  case OO_Caret:
+  case OO_Pipe:
+  case OO_LessLess:
+  case OO_GreaterGreater:
+    OpBuilder.addBinaryBitwiseArithmeticOverloads(Op);
+    break;
+
+  case OO_Amp: // '&' is either unary or binary
+    if (NumArgs == 1)
+      // C++ [over.match.oper]p3:
+      //   -- For the operator ',', the unary operator '&', or the
+      //      operator '->', the built-in candidates set is empty.
+      break;
+
+    OpBuilder.addBinaryBitwiseArithmeticOverloads(Op);
+    break;
+
+  case OO_Tilde:
+    OpBuilder.addUnaryTildePromotedIntegralOverloads();
+    break;
+
+  case OO_Equal:
+    OpBuilder.addAssignmentMemberPointerOrEnumeralOverloads();
+    // Fall through.
+
+  case OO_PlusEqual:
+  case OO_MinusEqual:
+    OpBuilder.addAssignmentPointerOverloads(Op == OO_Equal);
+    // Fall through.
+
+  case OO_StarEqual:
+  case OO_SlashEqual:
+    OpBuilder.addAssignmentArithmeticOverloads(Op == OO_Equal);
+    break;
+
+  case OO_PercentEqual:
+  case OO_LessLessEqual:
+  case OO_GreaterGreaterEqual:
+  case OO_AmpEqual:
+  case OO_CaretEqual:
+  case OO_PipeEqual:
+    OpBuilder.addAssignmentIntegralOverloads();
+    break;
+
+  case OO_Exclaim:
+    OpBuilder.addExclaimOverload();
+    break;
+
+  case OO_AmpAmp:
+  case OO_PipePipe:
+    OpBuilder.addAmpAmpOrPipePipeOverload();
+    break;
+
+  case OO_Subscript:
+    OpBuilder.addSubscriptOverloads();
+    break;
+
+  case OO_ArrowStar:
+    OpBuilder.addArrowStarOverloads();
+    break;
+
+  case OO_Conditional:
+    OpBuilder.addConditionalOperatorOverloads();
+    OpBuilder.addGenericBinaryArithmeticOverloads(/*isComparison=*/false);
+    break;
   }
 }
 
@@ -5270,7 +6116,7 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
     if (FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
       if (ExplicitTemplateArgs)
         continue;
-      
+
       AddOverloadCandidate(FD, FoundDecl, Args, NumArgs, CandidateSet,
                            false, PartialOverloading);
     } else
@@ -5284,9 +6130,10 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
 /// candidate is a better candidate than the second (C++ 13.3.3p1).
 bool
 isBetterOverloadCandidate(Sema &S,
-                          const OverloadCandidate& Cand1,
-                          const OverloadCandidate& Cand2,
-                          SourceLocation Loc) {
+                          const OverloadCandidate &Cand1,
+                          const OverloadCandidate &Cand2,
+                          SourceLocation Loc,
+                          bool UserDefinedConversion) {
   // Define viable functions to be better candidates than non-viable
   // functions.
   if (!Cand2.Viable)
@@ -5346,14 +6193,16 @@ isBetterOverloadCandidate(Sema &S,
   //      according to the partial ordering rules described in 14.5.5.2, or,
   //      if not that,
   if (Cand1.Function && Cand1.Function->getPrimaryTemplate() &&
-      Cand2.Function && Cand2.Function->getPrimaryTemplate())
+      Cand2.Function && Cand2.Function->getPrimaryTemplate()) {
     if (FunctionTemplateDecl *BetterTemplate
           = S.getMoreSpecializedTemplate(Cand1.Function->getPrimaryTemplate(),
                                          Cand2.Function->getPrimaryTemplate(),
                                          Loc,
-                       isa<CXXConversionDecl>(Cand1.Function)? TPOC_Conversion 
-                                                             : TPOC_Call))
+                       isa<CXXConversionDecl>(Cand1.Function)? TPOC_Conversion
+                                                             : TPOC_Call,
+                                         Cand1.ExplicitCallArguments))
       return BetterTemplate == Cand1.Function->getPrimaryTemplate();
+  }
 
   //   -- the context is an initialization by user-defined conversion
   //      (see 8.5, 13.3.1.5) and the standard conversion sequence
@@ -5361,7 +6210,7 @@ isBetterOverloadCandidate(Sema &S,
   //      the type of the entity being initialized) is a better
   //      conversion sequence than the standard conversion sequence
   //      from the return type of F2 to the destination type.
-  if (Cand1.Function && Cand2.Function &&
+  if (UserDefinedConversion && Cand1.Function && Cand2.Function &&
       isa<CXXConversionDecl>(Cand1.Function) &&
       isa<CXXConversionDecl>(Cand2.Function)) {
     switch (CompareStandardConversionSequences(S,
@@ -5398,12 +6247,14 @@ isBetterOverloadCandidate(Sema &S,
 /// \returns The result of overload resolution.
 OverloadingResult
 OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
-                                         iterator& Best) {
+                                         iterator &Best,
+                                         bool UserDefinedConversion) {
   // Find the best viable function.
   Best = end();
   for (iterator Cand = begin(); Cand != end(); ++Cand) {
     if (Cand->Viable)
-      if (Best == end() || isBetterOverloadCandidate(S, *Cand, *Best, Loc))
+      if (Best == end() || isBetterOverloadCandidate(S, *Cand, *Best, Loc,
+                                                     UserDefinedConversion))
         Best = Cand;
   }
 
@@ -5416,7 +6267,8 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   for (iterator Cand = begin(); Cand != end(); ++Cand) {
     if (Cand->Viable &&
         Cand != Best &&
-        !isBetterOverloadCandidate(S, *Best, *Cand, Loc)) {
+        !isBetterOverloadCandidate(S, *Best, *Cand, Loc,
+                                   UserDefinedConversion)) {
       Best = end();
       return OR_Ambiguous;
     }
@@ -5436,6 +6288,7 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   //   placement new (5.3.4), as well as non-default initialization (8.5).
   if (Best->Function)
     S.MarkDeclarationReferenced(Loc, Best->Function);
+
   return OR_Success;
 }
 
@@ -5450,7 +6303,8 @@ enum OverloadCandidateKind {
   oc_constructor_template,
   oc_implicit_default_constructor,
   oc_implicit_copy_constructor,
-  oc_implicit_copy_assignment
+  oc_implicit_copy_assignment,
+  oc_implicit_inherited_constructor
 };
 
 OverloadCandidateKind ClassifyOverloadCandidate(Sema &S,
@@ -5468,6 +6322,9 @@ OverloadCandidateKind ClassifyOverloadCandidate(Sema &S,
     if (!Ctor->isImplicit())
       return isTemplate ? oc_constructor_template : oc_constructor;
 
+    if (Ctor->getInheritedConstructor())
+      return oc_implicit_inherited_constructor;
+
     return Ctor->isCopyConstructor() ? oc_implicit_copy_constructor
                                      : oc_implicit_default_constructor;
   }
@@ -5478,7 +6335,7 @@ OverloadCandidateKind ClassifyOverloadCandidate(Sema &S,
     if (!Meth->isImplicit())
       return isTemplate ? oc_method_template : oc_method;
 
-    assert(Meth->isCopyAssignment()
+    assert(Meth->isCopyAssignmentOperator()
            && "implicit method is not copy assignment operator?");
     return oc_implicit_copy_assignment;
   }
@@ -5486,6 +6343,16 @@ OverloadCandidateKind ClassifyOverloadCandidate(Sema &S,
   return isTemplate ? oc_function_template : oc_function;
 }
 
+void MaybeEmitInheritedConstructorNote(Sema &S, FunctionDecl *Fn) {
+  const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(Fn);
+  if (!Ctor) return;
+
+  Ctor = Ctor->getInheritedConstructor();
+  if (!Ctor) return;
+
+  S.Diag(Ctor->getLocation(), diag::note_ovl_candidate_inherited_constructor);
+}
+
 } // end anonymous namespace
 
 // Notes the location of an overload candidate.
@@ -5494,6 +6361,28 @@ void Sema::NoteOverloadCandidate(FunctionDecl *Fn) {
   OverloadCandidateKind K = ClassifyOverloadCandidate(*this, Fn, FnDesc);
   Diag(Fn->getLocation(), diag::note_ovl_candidate)
     << (unsigned) K << FnDesc;
+  MaybeEmitInheritedConstructorNote(*this, Fn);
+}
+
+//Notes the location of all overload candidates designated through 
+// OverloadedExpr
+void Sema::NoteAllOverloadCandidates(Expr* OverloadedExpr) {
+  assert(OverloadedExpr->getType() == Context.OverloadTy);
+
+  OverloadExpr::FindResult Ovl = OverloadExpr::find(OverloadedExpr);
+  OverloadExpr *OvlExpr = Ovl.Expression;
+
+  for (UnresolvedSetIterator I = OvlExpr->decls_begin(),
+                            IEnd = OvlExpr->decls_end(); 
+       I != IEnd; ++I) {
+    if (FunctionTemplateDecl *FunTmpl = 
+                dyn_cast<FunctionTemplateDecl>((*I)->getUnderlyingDecl()) ) {
+      NoteOverloadCandidate(FunTmpl->getTemplatedDecl());   
+    } else if (FunctionDecl *Fun 
+                      = dyn_cast<FunctionDecl>((*I)->getUnderlyingDecl()) ) {
+      NoteOverloadCandidate(Fun);
+    }
+  }
 }
 
 /// Diagnoses an ambiguous conversion.  The partial diagnostic is the
@@ -5548,6 +6437,7 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
       << (unsigned) FnKind << FnDesc
       << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
       << ToTy << Name << I+1;
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
@@ -5582,6 +6472,7 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
         << FromTy
         << FromQs.getAddressSpace() << ToQs.getAddressSpace()
         << (unsigned) isObjectArgument << I+1;
+      MaybeEmitInheritedConstructorNote(S, Fn);
       return;
     }
 
@@ -5599,6 +6490,7 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
         << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
         << FromTy << (CVR - 1) << I+1;
     }
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
@@ -5613,6 +6505,7 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
       << (unsigned) FnKind << FnDesc
       << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
       << FromTy << ToTy << (unsigned) isObjectArgument << I+1;
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
@@ -5624,7 +6517,7 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
                                                FromPtrTy->getPointeeType()) &&
           !FromPtrTy->getPointeeType()->isIncompleteType() &&
           !ToPtrTy->getPointeeType()->isIncompleteType() &&
-          S.IsDerivedFrom(ToPtrTy->getPointeeType(), 
+          S.IsDerivedFrom(ToPtrTy->getPointeeType(),
                           FromPtrTy->getPointeeType()))
         BaseToDerivedConversion = 1;
     }
@@ -5645,22 +6538,24 @@ void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand, unsigned I) {
           S.IsDerivedFrom(ToRefTy->getPointeeType(), FromTy))
         BaseToDerivedConversion = 3;
     }
-    
+
   if (BaseToDerivedConversion) {
-    S.Diag(Fn->getLocation(), 
+    S.Diag(Fn->getLocation(),
            diag::note_ovl_candidate_bad_base_to_derived_conv)
       << (unsigned) FnKind << FnDesc
       << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
       << (BaseToDerivedConversion - 1)
-      << FromTy << ToTy << I+1;    
+      << FromTy << ToTy << I+1;
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
-      
+
   // TODO: specialize more based on the kind of mismatch
   S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_conv)
     << (unsigned) FnKind << FnDesc
     << (FromExpr ? FromExpr->getSourceRange() : SourceRange())
     << FromTy << ToTy << (unsigned) isObjectArgument << I+1;
+  MaybeEmitInheritedConstructorNote(S, Fn);
 }
 
 void DiagnoseArityMismatch(Sema &S, OverloadCandidate *Cand,
@@ -5671,15 +6566,15 @@ void DiagnoseArityMismatch(Sema &S, OverloadCandidate *Cand,
   const FunctionProtoType *FnTy = Fn->getType()->getAs<FunctionProtoType>();
 
   unsigned MinParams = Fn->getMinRequiredArguments();
-  
+
   // at least / at most / exactly
-  // FIXME: variadic templates "at most" should account for parameter packs
   unsigned mode, modeCount;
   if (NumFormalArgs < MinParams) {
     assert((Cand->FailureKind == ovl_fail_too_few_arguments) ||
            (Cand->FailureKind == ovl_fail_bad_deduction &&
             Cand->DeductionFailure.Result == Sema::TDK_TooFewArguments));
-    if (MinParams != FnTy->getNumArgs() || FnTy->isVariadic())
+    if (MinParams != FnTy->getNumArgs() ||
+        FnTy->isVariadic() || FnTy->isTemplateVariadic())
       mode = 0; // "at least"
     else
       mode = 2; // "exactly"
@@ -5699,8 +6594,9 @@ void DiagnoseArityMismatch(Sema &S, OverloadCandidate *Cand,
   OverloadCandidateKind FnKind = ClassifyOverloadCandidate(S, Fn, Description);
 
   S.Diag(Fn->getLocation(), diag::note_ovl_candidate_arity)
-    << (unsigned) FnKind << (Fn->getDescribedFunctionTemplate() != 0) << mode 
+    << (unsigned) FnKind << (Fn->getDescribedFunctionTemplate() != 0) << mode
     << modeCount << NumFormalArgs;
+  MaybeEmitInheritedConstructorNote(S, Fn);
 }
 
 /// Diagnose a failed template-argument deduction.
@@ -5721,6 +6617,7 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
     assert(ParamD && "no parameter found for incomplete deduction result");
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_incomplete_deduction)
       << ParamD->getDeclName();
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
@@ -5732,9 +6629,9 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
 
     // Param will have been canonicalized, but it should just be a
     // qualified version of ParamD, so move the qualifiers to that.
-    QualifierCollector Qs(S.Context);
+    QualifierCollector Qs;
     Qs.strip(Param);
-    QualType NonCanonParam = Qs.apply(TParam->getTypeForDecl());
+    QualType NonCanonParam = Qs.apply(S.Context, TParam->getTypeForDecl());
     assert(S.Context.hasSameType(Param, NonCanonParam));
 
     // Arg has also been canonicalized, but there's nothing we can do
@@ -5745,11 +6642,12 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
 
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_underqualified)
       << ParamD->getDeclName() << Arg << NonCanonParam;
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
   case Sema::TDK_Inconsistent: {
-    assert(ParamD && "no parameter found for inconsistent deduction result");    
+    assert(ParamD && "no parameter found for inconsistent deduction result");
     int which = 0;
     if (isa<TemplateTypeParmDecl>(ParamD))
       which = 0;
@@ -5758,18 +6656,19 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
     else {
       which = 2;
     }
-    
+
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_inconsistent_deduction)
-      << which << ParamD->getDeclName() 
+      << which << ParamD->getDeclName()
       << *Cand->DeductionFailure.getFirstArg()
       << *Cand->DeductionFailure.getSecondArg();
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
   case Sema::TDK_InvalidExplicitArguments:
-    assert(ParamD && "no parameter found for invalid explicit arguments");    
+    assert(ParamD && "no parameter found for invalid explicit arguments");
     if (ParamD->getDeclName())
-      S.Diag(Fn->getLocation(), 
+      S.Diag(Fn->getLocation(),
              diag::note_ovl_candidate_explicit_arg_mismatch_named)
         << ParamD->getDeclName();
     else {
@@ -5781,12 +6680,13 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
         index = NTTP->getIndex();
       else
         index = cast<TemplateTemplateParmDecl>(ParamD)->getIndex();
-      S.Diag(Fn->getLocation(), 
+      S.Diag(Fn->getLocation(),
              diag::note_ovl_candidate_explicit_arg_mismatch_unnamed)
         << (index + 1);
     }
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
-      
+
   case Sema::TDK_TooManyArguments:
   case Sema::TDK_TooFewArguments:
     DiagnoseArityMismatch(S, Cand, NumArgs);
@@ -5794,6 +6694,7 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
 
   case Sema::TDK_InstantiationDepth:
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_instantiation_depth);
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
 
   case Sema::TDK_SubstitutionFailure: {
@@ -5805,14 +6706,16 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
                                                     *Args);
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_substitution_failure)
       << ArgString;
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
-      
+
   // TODO: diagnose these individually, then kill off
   // note_ovl_candidate_bad_deduction, which is uselessly vague.
   case Sema::TDK_NonDeducedMismatch:
   case Sema::TDK_FailedOverloadResolution:
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_bad_deduction);
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 }
@@ -5841,6 +6744,7 @@ void NoteFunctionCandidate(Sema &S, OverloadCandidate *Cand,
 
     S.Diag(Fn->getLocation(), diag::note_ovl_candidate_deleted)
       << FnKind << FnDesc << Fn->isDeleted();
+    MaybeEmitInheritedConstructorNote(S, Fn);
     return;
   }
 
@@ -5868,7 +6772,7 @@ void NoteFunctionCandidate(Sema &S, OverloadCandidate *Cand,
     for (unsigned N = Cand->Conversions.size(); I != N; ++I)
       if (Cand->Conversions[I].isBad())
         return DiagnoseBadConversion(S, Cand, I);
-    
+
     // FIXME: this currently happens when we're called from SemaInit
     // when user-conversion overload fails.  Figure out how to handle
     // those conditions and diagnose them well.
@@ -5907,6 +6811,7 @@ void NoteSurrogateCandidate(Sema &S, OverloadCandidate *Cand) {
 
   S.Diag(Cand->Surrogate->getLocation(), diag::note_ovl_surrogate_cand)
     << FnType;
+  MaybeEmitInheritedConstructorNote(S, Cand->Surrogate);
 }
 
 void NoteBuiltinOperatorCandidate(Sema &S,
@@ -6083,7 +6988,7 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
       Cand->Conversions[ConvIdx]
         = TryCopyInitialization(S, Args[ConvIdx],
                                 Cand->BuiltinTypes.ParamTypes[ConvIdx],
-                                SuppressUserConversions, 
+                                SuppressUserConversions,
                                 /*InOverloadResolution*/ true);
     return;
   }
@@ -6094,7 +6999,7 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
     if (ArgIdx < NumArgsInProto)
       Cand->Conversions[ConvIdx]
         = TryCopyInitialization(S, Args[ArgIdx], Proto->getArgType(ArgIdx),
-                                SuppressUserConversions, 
+                                SuppressUserConversions,
                                 /*InOverloadResolution=*/true);
     else
       Cand->Conversions[ConvIdx].setEllipsis();
@@ -6129,7 +7034,7 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
 
   std::sort(Cands.begin(), Cands.end(),
             CompareOverloadCandidatesForDisplay(S));
-  
+
   bool ReportedAmbiguousConversions = false;
 
   llvm::SmallVectorImpl<OverloadCandidate*>::iterator I, E;
@@ -6180,186 +7085,207 @@ static bool CheckUnresolvedAccess(Sema &S, OverloadExpr *E, DeclAccessPair D) {
   return S.CheckUnresolvedMemberAccess(cast<UnresolvedMemberExpr>(E), D);
 }
 
-/// ResolveAddressOfOverloadedFunction - Try to resolve the address of
-/// an overloaded function (C++ [over.over]), where @p From is an
-/// expression with overloaded function type and @p ToType is the type
-/// we're trying to resolve to. For example:
-///
-/// @code
-/// int f(double);
-/// int f(int);
-///
-/// int (*pfd)(double) = f; // selects f(double)
-/// @endcode
-///
-/// This routine returns the resulting FunctionDecl if it could be
-/// resolved, and NULL otherwise. When @p Complain is true, this
-/// routine will emit diagnostics if there is an error.
-FunctionDecl *
-Sema::ResolveAddressOfOverloadedFunction(Expr *From, QualType ToType,
-                                         bool Complain,
-                                         DeclAccessPair &FoundResult) {
-  QualType FunctionType = ToType;
-  bool IsMember = false;
-  if (const PointerType *ToTypePtr = ToType->getAs<PointerType>())
-    FunctionType = ToTypePtr->getPointeeType();
-  else if (const ReferenceType *ToTypeRef = ToType->getAs<ReferenceType>())
-    FunctionType = ToTypeRef->getPointeeType();
-  else if (const MemberPointerType *MemTypePtr =
-                    ToType->getAs<MemberPointerType>()) {
-    FunctionType = MemTypePtr->getPointeeType();
-    IsMember = true;
-  }
 
-  // C++ [over.over]p1:
-  //   [...] [Note: any redundant set of parentheses surrounding the
-  //   overloaded function name is ignored (5.1). ]
-  // C++ [over.over]p1:
-  //   [...] The overloaded function name can be preceded by the &
-  //   operator.
-  // However, remember whether the expression has member-pointer form:
-  // C++ [expr.unary.op]p4:
-  //     A pointer to member is only formed when an explicit & is used
-  //     and its operand is a qualified-id not enclosed in
-  //     parentheses.
-  OverloadExpr::FindResult Ovl = OverloadExpr::find(From);
-  OverloadExpr *OvlExpr = Ovl.Expression;
-  
-  // We expect a pointer or reference to function, or a function pointer.
-  FunctionType = Context.getCanonicalType(FunctionType).getUnqualifiedType();
-  if (!FunctionType->isFunctionType()) {
-    if (Complain)
-      Diag(From->getLocStart(), diag::err_addr_ovl_not_func_ptrref)
-        << OvlExpr->getName() << ToType;
-    
-    return 0;
-  }
 
-  // If the overload expression doesn't have the form of a pointer to
-  // member, don't try to convert it to a pointer-to-member type.
-  if (IsMember && !Ovl.HasFormOfMemberPointer) {
-    if (!Complain) return 0;
 
-    // TODO: Should we condition this on whether any functions might
-    // have matched, or is it more appropriate to do that in callers?
-    // TODO: a fixit wouldn't hurt.
-    Diag(OvlExpr->getNameLoc(), diag::err_addr_ovl_no_qualifier)
-      << ToType << OvlExpr->getSourceRange();
-    return 0;
-  }
+// [PossiblyAFunctionType]  -->   [Return]
+// NonFunctionType --> NonFunctionType
+// R (A) --> R(A)
+// R (*)(A) --> R (A)
+// R (&)(A) --> R (A)
+// R (S::*)(A) --> R (A)
+QualType Sema::ExtractUnqualifiedFunctionType(QualType PossiblyAFunctionType) {
+  QualType Ret = PossiblyAFunctionType;
+  if (const PointerType *ToTypePtr = 
+    PossiblyAFunctionType->getAs<PointerType>())
+    Ret = ToTypePtr->getPointeeType();
+  else if (const ReferenceType *ToTypeRef = 
+    PossiblyAFunctionType->getAs<ReferenceType>())
+    Ret = ToTypeRef->getPointeeType();
+  else if (const MemberPointerType *MemTypePtr =
+    PossiblyAFunctionType->getAs<MemberPointerType>()) 
+    Ret = MemTypePtr->getPointeeType();   
+  Ret = 
+    Context.getCanonicalType(Ret).getUnqualifiedType();
+  return Ret;
+}
 
-  TemplateArgumentListInfo ETABuffer, *ExplicitTemplateArgs = 0;
-  if (OvlExpr->hasExplicitTemplateArgs()) {
-    OvlExpr->getExplicitTemplateArgs().copyInto(ETABuffer);
-    ExplicitTemplateArgs = &ETABuffer;
-  }
+// A helper class to help with address of function resolution
+// - allows us to avoid passing around all those ugly parameters
+class AddressOfFunctionResolver 
+{
+  Sema& S;
+  Expr* SourceExpr;
+  const QualType& TargetType; 
+  QualType TargetFunctionType; // Extracted function type from target type 
+   
+  bool Complain;
+  //DeclAccessPair& ResultFunctionAccessPair;
+  ASTContext& Context;
 
-  assert(From->getType() == Context.OverloadTy);
+  bool TargetTypeIsNonStaticMemberFunction;
+  bool FoundNonTemplateFunction;
 
-  // Look through all of the overloaded functions, searching for one
-  // whose type matches exactly.
+  OverloadExpr::FindResult OvlExprInfo; 
+  OverloadExpr *OvlExpr;
+  TemplateArgumentListInfo OvlExplicitTemplateArgs;
   llvm::SmallVector<std::pair<DeclAccessPair, FunctionDecl*>, 4> Matches;
-  llvm::SmallVector<FunctionDecl *, 4> NonMatches;
-
-  bool FoundNonTemplateFunction = false;
-  for (UnresolvedSetIterator I = OvlExpr->decls_begin(),
-         E = OvlExpr->decls_end(); I != E; ++I) {
-    // Look through any using declarations to find the underlying function.
-    NamedDecl *Fn = (*I)->getUnderlyingDecl();
-
-    // C++ [over.over]p3:
-    //   Non-member functions and static member functions match
-    //   targets of type "pointer-to-function" or "reference-to-function."
-    //   Nonstatic member functions match targets of
-    //   type "pointer-to-member-function."
-    // Note that according to DR 247, the containing class does not matter.
-
-    if (FunctionTemplateDecl *FunctionTemplate
-          = dyn_cast<FunctionTemplateDecl>(Fn)) {
-      if (CXXMethodDecl *Method
-            = dyn_cast<CXXMethodDecl>(FunctionTemplate->getTemplatedDecl())) {
-        // Skip non-static function templates when converting to pointer, and
-        // static when converting to member pointer.
-        if (Method->isStatic() == IsMember)
-          continue;
-      } else if (IsMember)
-        continue;
 
-      // C++ [over.over]p2:
-      //   If the name is a function template, template argument deduction is
-      //   done (14.8.2.2), and if the argument deduction succeeds, the
-      //   resulting template argument list is used to generate a single
-      //   function template specialization, which is added to the set of
-      //   overloaded functions considered.
-      // FIXME: We don't really want to build the specialization here, do we?
-      FunctionDecl *Specialization = 0;
-      TemplateDeductionInfo Info(Context, OvlExpr->getNameLoc());
-      if (TemplateDeductionResult Result
-            = DeduceTemplateArguments(FunctionTemplate, ExplicitTemplateArgs,
-                                      FunctionType, Specialization, Info)) {
-        // FIXME: make a note of the failed deduction for diagnostics.
-        (void)Result;
-      } else {
-        // FIXME: If the match isn't exact, shouldn't we just drop this as
-        // a candidate? Find a testcase before changing the code.
-        assert(FunctionType
-                 == Context.getCanonicalType(Specialization->getType()));
-        Matches.push_back(std::make_pair(I.getPair(),
-                    cast<FunctionDecl>(Specialization->getCanonicalDecl())));
+public:
+  AddressOfFunctionResolver(Sema &S, Expr* SourceExpr, 
+                            const QualType& TargetType, bool Complain)
+    : S(S), SourceExpr(SourceExpr), TargetType(TargetType), 
+      Complain(Complain), Context(S.getASTContext()), 
+      TargetTypeIsNonStaticMemberFunction(
+                                    !!TargetType->getAs<MemberPointerType>()),
+      FoundNonTemplateFunction(false),
+      OvlExprInfo(OverloadExpr::find(SourceExpr)),
+      OvlExpr(OvlExprInfo.Expression)
+  {
+    ExtractUnqualifiedFunctionTypeFromTargetType();
+    
+    if (!TargetFunctionType->isFunctionType()) {        
+      if (OvlExpr->hasExplicitTemplateArgs()) {
+        DeclAccessPair dap;
+        if( FunctionDecl* Fn = S.ResolveSingleFunctionTemplateSpecialization(
+                                            OvlExpr, false, &dap) ) {
+          Matches.push_back(std::make_pair(dap,Fn));
+        }
+      }
+      return;
+    }
+    
+    if (OvlExpr->hasExplicitTemplateArgs())
+      OvlExpr->getExplicitTemplateArgs().copyInto(OvlExplicitTemplateArgs);
+
+    if (FindAllFunctionsThatMatchTargetTypeExactly()) {
+      // C++ [over.over]p4:
+      //   If more than one function is selected, [...]
+      if (Matches.size() > 1) {
+        if (FoundNonTemplateFunction)
+          EliminateAllTemplateMatches();
+        else
+          EliminateAllExceptMostSpecializedTemplate();
       }
-
-      continue;
     }
+  }
+  
+private:
+  bool isTargetTypeAFunction() const {
+    return TargetFunctionType->isFunctionType();
+  }
+
+  // [ToType]     [Return]
 
+  // R (*)(A) --> R (A), IsNonStaticMemberFunction = false
+  // R (&)(A) --> R (A), IsNonStaticMemberFunction = false
+  // R (S::*)(A) --> R (A), IsNonStaticMemberFunction = true
+  void inline ExtractUnqualifiedFunctionTypeFromTargetType() {
+    TargetFunctionType = S.ExtractUnqualifiedFunctionType(TargetType);
+  }
+
+  // return true if any matching specializations were found
+  bool AddMatchingTemplateFunction(FunctionTemplateDecl* FunctionTemplate, 
+                                   const DeclAccessPair& CurAccessFunPair) {
+    if (CXXMethodDecl *Method
+              = dyn_cast<CXXMethodDecl>(FunctionTemplate->getTemplatedDecl())) {
+      // Skip non-static function templates when converting to pointer, and
+      // static when converting to member pointer.
+      if (Method->isStatic() == TargetTypeIsNonStaticMemberFunction)
+        return false;
+    } 
+    else if (TargetTypeIsNonStaticMemberFunction)
+      return false;
+
+    // C++ [over.over]p2:
+    //   If the name is a function template, template argument deduction is
+    //   done (14.8.2.2), and if the argument deduction succeeds, the
+    //   resulting template argument list is used to generate a single
+    //   function template specialization, which is added to the set of
+    //   overloaded functions considered.
+    FunctionDecl *Specialization = 0;
+    TemplateDeductionInfo Info(Context, OvlExpr->getNameLoc());
+    if (Sema::TemplateDeductionResult Result
+          = S.DeduceTemplateArguments(FunctionTemplate, 
+                                      &OvlExplicitTemplateArgs,
+                                      TargetFunctionType, Specialization, 
+                                      Info)) {
+      // FIXME: make a note of the failed deduction for diagnostics.
+      (void)Result;
+      return false;
+    } 
+    
+    // Template argument deduction ensures that we have an exact match.
+    // This function template specicalization works.
+    Specialization = cast<FunctionDecl>(Specialization->getCanonicalDecl());
+    assert(TargetFunctionType
+                      == Context.getCanonicalType(Specialization->getType()));
+    Matches.push_back(std::make_pair(CurAccessFunPair, Specialization));
+    return true;
+  }
+  
+  bool AddMatchingNonTemplateFunction(NamedDecl* Fn, 
+                                      const DeclAccessPair& CurAccessFunPair) {
     if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Fn)) {
       // Skip non-static functions when converting to pointer, and static
       // when converting to member pointer.
-      if (Method->isStatic() == IsMember)
-        continue;
-      
-      // If we have explicit template arguments, skip non-templates.
-      if (OvlExpr->hasExplicitTemplateArgs())
-        continue;
-    } else if (IsMember)
-      continue;
+      if (Method->isStatic() == TargetTypeIsNonStaticMemberFunction)
+        return false;
+    } 
+    else if (TargetTypeIsNonStaticMemberFunction)
+      return false;
 
     if (FunctionDecl *FunDecl = dyn_cast<FunctionDecl>(Fn)) {
       QualType ResultTy;
-      if (Context.hasSameUnqualifiedType(FunctionType, FunDecl->getType()) ||
-          IsNoReturnConversion(Context, FunDecl->getType(), FunctionType, 
+      if (Context.hasSameUnqualifiedType(TargetFunctionType, 
+                                         FunDecl->getType()) ||
+          IsNoReturnConversion(Context, FunDecl->getType(), TargetFunctionType, 
                                ResultTy)) {
-        Matches.push_back(std::make_pair(I.getPair(),
-                           cast<FunctionDecl>(FunDecl->getCanonicalDecl())));
+        Matches.push_back(std::make_pair(CurAccessFunPair,
+          cast<FunctionDecl>(FunDecl->getCanonicalDecl())));
         FoundNonTemplateFunction = true;
+        return true;
       }
     }
+    
+    return false;
   }
+  
+  bool FindAllFunctionsThatMatchTargetTypeExactly() {
+    bool Ret = false;
+    
+    // If the overload expression doesn't have the form of a pointer to
+    // member, don't try to convert it to a pointer-to-member type.
+    if (IsInvalidFormOfPointerToMemberFunction())
+      return false;
 
-  // If there were 0 or 1 matches, we're done.
-  if (Matches.empty()) {
-    if (Complain) {
-      Diag(From->getLocStart(), diag::err_addr_ovl_no_viable)
-        << OvlExpr->getName() << FunctionType;
-      for (UnresolvedSetIterator I = OvlExpr->decls_begin(),
-                                 E = OvlExpr->decls_end(); 
-           I != E; ++I)
-        if (FunctionDecl *F = dyn_cast<FunctionDecl>((*I)->getUnderlyingDecl()))
-          NoteOverloadCandidate(F);
+    for (UnresolvedSetIterator I = OvlExpr->decls_begin(),
+                               E = OvlExpr->decls_end(); 
+         I != E; ++I) {
+      // Look through any using declarations to find the underlying function.
+      NamedDecl *Fn = (*I)->getUnderlyingDecl();
+
+      // C++ [over.over]p3:
+      //   Non-member functions and static member functions match
+      //   targets of type "pointer-to-function" or "reference-to-function."
+      //   Nonstatic member functions match targets of
+      //   type "pointer-to-member-function."
+      // Note that according to DR 247, the containing class does not matter.
+      if (FunctionTemplateDecl *FunctionTemplate
+                                        = dyn_cast<FunctionTemplateDecl>(Fn)) {
+        if (AddMatchingTemplateFunction(FunctionTemplate, I.getPair()))
+          Ret = true;
+      }
+      // If we have explicit template arguments supplied, skip non-templates.
+      else if (!OvlExpr->hasExplicitTemplateArgs() &&
+               AddMatchingNonTemplateFunction(Fn, I.getPair()))
+        Ret = true;
     }
-    
-    return 0;
-  } else if (Matches.size() == 1) {
-    FunctionDecl *Result = Matches[0].second;
-    FoundResult = Matches[0].first;
-    MarkDeclarationReferenced(From->getLocStart(), Result);
-    if (Complain)
-      CheckAddressOfMemberAccess(OvlExpr, Matches[0].first);
-    return Result;
+    assert(Ret || Matches.empty());
+    return Ret;
   }
 
-  // C++ [over.over]p4:
-  //   If more than one function is selected, [...]
-  if (!FoundNonTemplateFunction) {
+  void EliminateAllExceptMostSpecializedTemplate() {
     //   [...] and any given function template specialization F1 is
     //   eliminated if the set contains a second function template
     //   specialization whose function template is more specialized
@@ -6374,84 +7300,159 @@ Sema::ResolveAddressOfOverloadedFunction(Expr *From, QualType ToType,
     UnresolvedSet<4> MatchesCopy; // TODO: avoid!
     for (unsigned I = 0, E = Matches.size(); I != E; ++I)
       MatchesCopy.addDecl(Matches[I].second, Matches[I].first.getAccess());
-    
+
     UnresolvedSetIterator Result =
-        getMostSpecialized(MatchesCopy.begin(), MatchesCopy.end(),
-                           TPOC_Other, From->getLocStart(),
-                           PDiag(),
-                           PDiag(diag::err_addr_ovl_ambiguous)
-                               << Matches[0].second->getDeclName(),
-                           PDiag(diag::note_ovl_candidate)
-                               << (unsigned) oc_function_template);
-    assert(Result != MatchesCopy.end() && "no most-specialized template");
-    MarkDeclarationReferenced(From->getLocStart(), *Result);
-    FoundResult = Matches[Result - MatchesCopy.begin()].first;
-    if (Complain) {
-      CheckUnresolvedAccess(*this, OvlExpr, FoundResult);
-      DiagnoseUseOfDecl(FoundResult, OvlExpr->getNameLoc());
-    }
-    return cast<FunctionDecl>(*Result);
-  }
-
-  //   [...] any function template specializations in the set are
-  //   eliminated if the set also contains a non-template function, [...]
-  for (unsigned I = 0, N = Matches.size(); I != N; ) {
-    if (Matches[I].second->getPrimaryTemplate() == 0)
-      ++I;
-    else {
-      Matches[I] = Matches[--N];
-      Matches.set_size(N);
+      S.getMostSpecialized(MatchesCopy.begin(), MatchesCopy.end(),
+                           TPOC_Other, 0, SourceExpr->getLocStart(),
+                           S.PDiag(),
+                           S.PDiag(diag::err_addr_ovl_ambiguous)
+                             << Matches[0].second->getDeclName(),
+                           S.PDiag(diag::note_ovl_candidate)
+                             << (unsigned) oc_function_template,
+                           Complain);
+
+    if (Result != MatchesCopy.end()) {
+      // Make it the first and only element
+      Matches[0].first = Matches[Result - MatchesCopy.begin()].first;
+      Matches[0].second = cast<FunctionDecl>(*Result);
+      Matches.resize(1);
+    }
+  }
+
+  void EliminateAllTemplateMatches() {
+    //   [...] any function template specializations in the set are
+    //   eliminated if the set also contains a non-template function, [...]
+    for (unsigned I = 0, N = Matches.size(); I != N; ) {
+      if (Matches[I].second->getPrimaryTemplate() == 0)
+        ++I;
+      else {
+        Matches[I] = Matches[--N];
+        Matches.set_size(N);
+      }
     }
   }
+
+public:
+  void ComplainNoMatchesFound() const {
+    assert(Matches.empty());
+    S.Diag(OvlExpr->getLocStart(), diag::err_addr_ovl_no_viable)
+        << OvlExpr->getName() << TargetFunctionType
+        << OvlExpr->getSourceRange();
+    S.NoteAllOverloadCandidates(OvlExpr);
+  } 
   
-  // [...] After such eliminations, if any, there shall remain exactly one
-  // selected function.
-  if (Matches.size() == 1) {
-    MarkDeclarationReferenced(From->getLocStart(), Matches[0].second);
-    FoundResult = Matches[0].first;
-    if (Complain) {
-      CheckUnresolvedAccess(*this, OvlExpr, Matches[0].first);
-      DiagnoseUseOfDecl(Matches[0].first, OvlExpr->getNameLoc());
-    }
-    return cast<FunctionDecl>(Matches[0].second);
-  }
-
-  // FIXME: We should probably return the same thing that BestViableFunction
-  // returns (even if we issue the diagnostics here).
-  Diag(From->getLocStart(), diag::err_addr_ovl_ambiguous)
-    << Matches[0].second->getDeclName();
-  for (unsigned I = 0, E = Matches.size(); I != E; ++I)
-    NoteOverloadCandidate(Matches[I].second);
-  return 0;
+  bool IsInvalidFormOfPointerToMemberFunction() const {
+    return TargetTypeIsNonStaticMemberFunction &&
+      !OvlExprInfo.HasFormOfMemberPointer;
+  }
+  
+  void ComplainIsInvalidFormOfPointerToMemberFunction() const {
+      // TODO: Should we condition this on whether any functions might
+      // have matched, or is it more appropriate to do that in callers?
+      // TODO: a fixit wouldn't hurt.
+      S.Diag(OvlExpr->getNameLoc(), diag::err_addr_ovl_no_qualifier)
+        << TargetType << OvlExpr->getSourceRange();
+  }
+  
+  void ComplainOfInvalidConversion() const {
+    S.Diag(OvlExpr->getLocStart(), diag::err_addr_ovl_not_func_ptrref)
+      << OvlExpr->getName() << TargetType;
+  }
+
+  void ComplainMultipleMatchesFound() const {
+    assert(Matches.size() > 1);
+    S.Diag(OvlExpr->getLocStart(), diag::err_addr_ovl_ambiguous)
+      << OvlExpr->getName()
+      << OvlExpr->getSourceRange();
+    S.NoteAllOverloadCandidates(OvlExpr);
+  }
+  
+  int getNumMatches() const { return Matches.size(); }
+  
+  FunctionDecl* getMatchingFunctionDecl() const {
+    if (Matches.size() != 1) return 0;
+    return Matches[0].second;
+  }
+  
+  const DeclAccessPair* getMatchingFunctionAccessPair() const {
+    if (Matches.size() != 1) return 0;
+    return &Matches[0].first;
+  }
+};
+  
+/// ResolveAddressOfOverloadedFunction - Try to resolve the address of
+/// an overloaded function (C++ [over.over]), where @p From is an
+/// expression with overloaded function type and @p ToType is the type
+/// we're trying to resolve to. For example:
+///
+/// @code
+/// int f(double);
+/// int f(int);
+///
+/// int (*pfd)(double) = f; // selects f(double)
+/// @endcode
+///
+/// This routine returns the resulting FunctionDecl if it could be
+/// resolved, and NULL otherwise. When @p Complain is true, this
+/// routine will emit diagnostics if there is an error.
+FunctionDecl *
+Sema::ResolveAddressOfOverloadedFunction(Expr *AddressOfExpr, QualType TargetType,
+                                    bool Complain,
+                                    DeclAccessPair &FoundResult) {
+
+  assert(AddressOfExpr->getType() == Context.OverloadTy);
+  
+  AddressOfFunctionResolver Resolver(*this, AddressOfExpr, TargetType, Complain);
+  int NumMatches = Resolver.getNumMatches();
+  FunctionDecl* Fn = 0;
+  if ( NumMatches == 0 && Complain) {
+    if (Resolver.IsInvalidFormOfPointerToMemberFunction())
+      Resolver.ComplainIsInvalidFormOfPointerToMemberFunction();
+    else
+      Resolver.ComplainNoMatchesFound();
+  }
+  else if (NumMatches > 1 && Complain)
+    Resolver.ComplainMultipleMatchesFound();
+  else if (NumMatches == 1) {
+    Fn = Resolver.getMatchingFunctionDecl();
+    assert(Fn);
+    FoundResult = *Resolver.getMatchingFunctionAccessPair();
+    MarkDeclarationReferenced(AddressOfExpr->getLocStart(), Fn);
+    if (Complain)
+      CheckAddressOfMemberAccess(AddressOfExpr, FoundResult);
+  }
+  
+  return Fn;
 }
 
-/// \brief Given an expression that refers to an overloaded function, try to 
+/// \brief Given an expression that refers to an overloaded function, try to
 /// resolve that overloaded function expression down to a single function.
 ///
 /// This routine can only resolve template-ids that refer to a single function
 /// template, where that template-id refers to a single template whose template
-/// arguments are either provided by the template-id or have defaults, 
+/// arguments are either provided by the template-id or have defaults,
 /// as described in C++0x [temp.arg.explicit]p3.
-FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From) {
+FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From, 
+                                                                bool Complain,
+                                                  DeclAccessPair* FoundResult) {
   // C++ [over.over]p1:
   //   [...] [Note: any redundant set of parentheses surrounding the
   //   overloaded function name is ignored (5.1). ]
   // C++ [over.over]p1:
   //   [...] The overloaded function name can be preceded by the &
   //   operator.
-
   if (From->getType() != Context.OverloadTy)
     return 0;
 
   OverloadExpr *OvlExpr = OverloadExpr::find(From).Expression;
-  
+
   // If we didn't actually find any template-ids, we're done.
   if (!OvlExpr->hasExplicitTemplateArgs())
     return 0;
 
   TemplateArgumentListInfo ExplicitTemplateArgs;
   OvlExpr->getExplicitTemplateArgs().copyInto(ExplicitTemplateArgs);
-  
+
   // Look through all of the overloaded functions, searching for one
   // whose type matches exactly.
   FunctionDecl *Matched = 0;
@@ -6459,13 +7460,13 @@ FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From) {
          E = OvlExpr->decls_end(); I != E; ++I) {
     // C++0x [temp.arg.explicit]p3:
     //   [...] In contexts where deduction is done and fails, or in contexts
-    //   where deduction is not done, if a template argument list is 
-    //   specified and it, along with any default template arguments, 
-    //   identifies a single function template specialization, then the 
+    //   where deduction is not done, if a template argument list is
+    //   specified and it, along with any default template arguments,
+    //   identifies a single function template specialization, then the
     //   template-id is an lvalue for the function template specialization.
     FunctionTemplateDecl *FunctionTemplate
       = cast<FunctionTemplateDecl>((*I)->getUnderlyingDecl());
-    
+
     // C++ [over.over]p2:
     //   If the name is a function template, template argument deduction is
     //   done (14.8.2.2), and if the argument deduction succeeds, the
@@ -6480,18 +7481,28 @@ FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From) {
       // FIXME: make a note of the failed deduction for diagnostics.
       (void)Result;
       continue;
-    } 
-    
+    }
+
     // Multiple matches; we can't resolve to a single declaration.
-    if (Matched)
+    if (Matched) {
+      if (FoundResult) 
+          *FoundResult = DeclAccessPair();
+      
+      if (Complain) {
+        Diag(From->getLocStart(), diag::err_addr_ovl_ambiguous)
+          << OvlExpr->getName();
+        NoteAllOverloadCandidates(OvlExpr);
+      }
       return 0;
-
-    Matched = Specialization;
+    }   
+    
+    if ((Matched = Specialization) && FoundResult) 
+      *FoundResult = I.getPair();    
   }
 
   return Matched;
 }
-    
+
 /// \brief Add a single candidate to the overload set.
 static void AddOverloadedCallCandidate(Sema &S,
                                        DeclAccessPair FoundDecl,
@@ -6522,7 +7533,7 @@ static void AddOverloadedCallCandidate(Sema &S,
 
   // do nothing?
 }
-  
+
 /// \brief Add the overload candidates named by callee and/or found by argument
 /// dependent lookup to the given overload set.
 void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
@@ -6570,7 +7581,7 @@ void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
   for (UnresolvedLookupExpr::decls_iterator I = ULE->decls_begin(),
          E = ULE->decls_end(); I != E; ++I)
     AddOverloadedCallCandidate(*this, I.getPair(), ExplicitTemplateArgs,
-                               Args, NumArgs, CandidateSet, 
+                               Args, NumArgs, CandidateSet,
                                PartialOverloading);
 
   if (ULE->requiresADL())
@@ -6578,7 +7589,7 @@ void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
                                          Args, NumArgs,
                                          ExplicitTemplateArgs,
                                          CandidateSet,
-                                         PartialOverloading);  
+                                         PartialOverloading);
 }
 
 /// Attempts to recover from a call where no functions were found.
@@ -6589,7 +7600,6 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                       UnresolvedLookupExpr *ULE,
                       SourceLocation LParenLoc,
                       Expr **Args, unsigned NumArgs,
-                      SourceLocation *CommaLocs,
                       SourceLocation RParenLoc) {
 
   CXXScopeSpec SS;
@@ -6616,7 +7626,8 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
   // casts and such from the call, we don't really care.
   ExprResult NewFn = ExprError();
   if ((*R.begin())->isCXXClassMember())
-    NewFn = SemaRef.BuildPossibleImplicitMemberExpr(SS, R, ExplicitTemplateArgs);
+    NewFn = SemaRef.BuildPossibleImplicitMemberExpr(SS, R,
+                                                    ExplicitTemplateArgs);
   else if (ExplicitTemplateArgs)
     NewFn = SemaRef.BuildTemplateIdExpr(SS, R, false, *ExplicitTemplateArgs);
   else
@@ -6629,8 +7640,7 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
   // an expression with non-empty lookup results, which should never
   // end up here.
   return SemaRef.ActOnCallExpr(/*Scope*/ 0, NewFn.take(), LParenLoc,
-                               MultiExprArg(Args, NumArgs),
-                               CommaLocs, RParenLoc);
+                               MultiExprArg(Args, NumArgs), RParenLoc);
 }
 
 /// ResolveOverloadedCallFn - Given the call expression that calls Fn
@@ -6644,8 +7654,8 @@ ExprResult
 Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
                               SourceLocation LParenLoc,
                               Expr **Args, unsigned NumArgs,
-                              SourceLocation *CommaLocs,
-                              SourceLocation RParenLoc) {
+                              SourceLocation RParenLoc,
+                              Expr *ExecConfig) {
 #ifndef NDEBUG
   if (ULE->requiresADL()) {
     // To do ADL, we must have found an unqualified name.
@@ -6658,7 +7668,7 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
         (F = dyn_cast<FunctionDecl>(*ULE->decls_begin())) &&
         F->getBuiltinID() && F->isImplicit())
       assert(0 && "performing ADL for builtin");
-      
+
     // We don't perform ADL in C.
     assert(getLangOptions().CPlusPlus && "ADL enabled in C");
   }
@@ -6675,16 +7685,18 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
   // bailout out if it fails.
   if (CandidateSet.empty())
     return BuildRecoveryCallExpr(*this, S, Fn, ULE, LParenLoc, Args, NumArgs,
-                                 CommaLocs, RParenLoc);
+                                 RParenLoc);
 
   OverloadCandidateSet::iterator Best;
   switch (CandidateSet.BestViableFunction(*this, Fn->getLocStart(), Best)) {
   case OR_Success: {
     FunctionDecl *FDecl = Best->Function;
     CheckUnresolvedLookupAccess(ULE, Best->FoundDecl);
-    DiagnoseUseOfDecl(Best->FoundDecl, ULE->getNameLoc());
+    DiagnoseUseOfDecl(FDecl? FDecl : Best->FoundDecl.getDecl(),
+                      ULE->getNameLoc());
     Fn = FixOverloadedFunctionReference(Fn, Best->FoundDecl, FDecl);
-    return BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs, RParenLoc);
+    return BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs, RParenLoc,
+                                 ExecConfig);
   }
 
   case OR_No_Viable_Function:
@@ -6746,6 +7758,9 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
   // TODO: provide better source location info.
   DeclarationNameInfo OpNameInfo(OpName, OpLoc);
 
+  if (Input->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Input);
+
   Expr *Args[2] = { Input, 0 };
   unsigned NumArgs = 1;
 
@@ -6762,19 +7777,21 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
   if (Input->isTypeDependent()) {
     if (Fns.empty())
       return Owned(new (Context) UnaryOperator(Input,
-                                               Opc, 
+                                               Opc,
                                                Context.DependentTy,
+                                               VK_RValue, OK_Ordinary,
                                                OpLoc));
-    
+
     CXXRecordDecl *NamingClass = 0; // because lookup ignores member operators
     UnresolvedLookupExpr *Fn
-      = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
+      = UnresolvedLookupExpr::Create(Context, NamingClass,
                                      0, SourceRange(), OpNameInfo,
                                      /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
     return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
                                                    &Args[0], NumArgs,
                                                    Context.DependentTy,
+                                                   VK_RValue,
                                                    OpLoc));
   }
 
@@ -6818,8 +7835,9 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
         // Convert the arguments.
         ExprResult InputInit
           = PerformCopyInitialization(InitializedEntity::InitializeParameter(
+                                                      Context,
                                                       FnDecl->getParamDecl(0)),
-                                      SourceLocation(), 
+                                      SourceLocation(),
                                       Input);
         if (InputInit.isInvalid())
           return ExprError();
@@ -6828,20 +7846,20 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
 
       DiagnoseUseOfDecl(Best->FoundDecl, OpLoc);
 
-      // Determine the result type
-      QualType ResultTy = FnDecl->getCallResultType();
+      // Determine the result type.
+      QualType ResultTy = FnDecl->getResultType();
+      ExprValueKind VK = Expr::getValueKindForType(ResultTy);
+      ResultTy = ResultTy.getNonLValueExprType(Context);
 
       // Build the actual expression node.
-      Expr *FnExpr = new (Context) DeclRefExpr(FnDecl, FnDecl->getType(),
-                                               SourceLocation());
-      UsualUnaryConversions(FnExpr);
+      Expr *FnExpr = CreateFunctionRefExpr(*this, FnDecl);
 
       Args[0] = Input;
       CallExpr *TheCall =
         new (Context) CXXOperatorCallExpr(Context, Op, FnExpr,
-                                          Args, NumArgs, ResultTy, OpLoc);
+                                          Args, NumArgs, ResultTy, VK, OpLoc);
 
-      if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall, 
+      if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall,
                               FnDecl))
         return ExprError();
 
@@ -6864,8 +7882,9 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
       break;
 
     case OR_Ambiguous:
-      Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
+      Diag(OpLoc,  diag::err_ovl_ambiguous_oper_unary)
           << UnaryOperator::getOpcodeStr(Opc)
+          << Input->getType()
           << Input->getSourceRange();
       CandidateSet.NoteCandidates(*this, OCD_ViableCandidates,
                                   Args, NumArgs,
@@ -6920,14 +7939,18 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
   // expression.
   if (Args[0]->isTypeDependent() || Args[1]->isTypeDependent()) {
     if (Fns.empty()) {
-      // If there are no functions to store, just build a dependent 
+      // If there are no functions to store, just build a dependent
       // BinaryOperator or CompoundAssignment.
       if (Opc <= BO_Assign || Opc > BO_OrAssign)
         return Owned(new (Context) BinaryOperator(Args[0], Args[1], Opc,
-                                                  Context.DependentTy, OpLoc));
-      
+                                                  Context.DependentTy,
+                                                  VK_RValue, OK_Ordinary,
+                                                  OpLoc));
+
       return Owned(new (Context) CompoundAssignOperator(Args[0], Args[1], Opc,
                                                         Context.DependentTy,
+                                                        VK_LValue,
+                                                        OK_Ordinary,
                                                         Context.DependentTy,
                                                         Context.DependentTy,
                                                         OpLoc));
@@ -6938,20 +7961,48 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
     // TODO: provide better source location info in DNLoc component.
     DeclarationNameInfo OpNameInfo(OpName, OpLoc);
     UnresolvedLookupExpr *Fn
-      = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
-                                     0, SourceRange(), OpNameInfo,
-                                     /*ADL*/ true, IsOverloaded(Fns),
+      = UnresolvedLookupExpr::Create(Context, NamingClass, 0, SourceRange(),
+                                     OpNameInfo, /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
     return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
                                                    Args, 2,
                                                    Context.DependentTy,
+                                                   VK_RValue,
                                                    OpLoc));
   }
 
-  // If this is the .* operator, which is not overloadable, just
-  // create a built-in binary operator.
-  if (Opc == BO_PtrMemD)
-    return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
+  // Always do property rvalue conversions on the RHS.
+  if (Args[1]->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Args[1]);
+
+  // The LHS is more complicated.
+  if (Args[0]->getObjectKind() == OK_ObjCProperty) {
+
+    // There's a tension for assignment operators between primitive
+    // property assignment and the overloaded operators.
+    if (BinaryOperator::isAssignmentOp(Opc)) {
+      const ObjCPropertyRefExpr *PRE = LHS->getObjCProperty();
+
+      // Is the property "logically" settable?
+      bool Settable = (PRE->isExplicitProperty() ||
+                       PRE->getImplicitPropertySetter());
+
+      // To avoid gratuitously inventing semantics, use the primitive
+      // unless it isn't.  Thoughts in case we ever really care:
+      // - If the property isn't logically settable, we have to
+      //   load and hope.
+      // - If the property is settable and this is simple assignment,
+      //   we really should use the primitive.
+      // - If the property is settable, then we could try overloading
+      //   on a generic lvalue of the appropriate type;  if it works
+      //   out to a builtin candidate, we would do that same operation
+      //   on the property, and otherwise just error.
+      if (Settable)
+        return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
+    }
+
+    ConvertPropertyForRValue(Args[0]);
+  }
 
   // If this is the assignment operator, we only perform overload resolution
   // if the left-hand side is a class or enumeration type. This is actually
@@ -6962,6 +8013,11 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
   if (Opc == BO_Assign && !Args[0]->getType()->isOverloadableType())
     return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
 
+  // If this is the .* operator, which is not overloadable, just
+  // create a built-in binary operator.
+  if (Opc == BO_PtrMemD)
+    return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
+
   // Build an empty overload set.
   OverloadCandidateSet CandidateSet(OpLoc);
 
@@ -6996,37 +8052,33 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           // Best->Access is only meaningful for class members.
           CheckMemberOperatorAccess(OpLoc, Args[0], Args[1], Best->FoundDecl);
 
-          ExprResult Arg1
-            = PerformCopyInitialization(
-                                        InitializedEntity::InitializeParameter(
-                                                        FnDecl->getParamDecl(0)),
-                                        SourceLocation(),
-                                        Owned(Args[1]));
+          ExprResult Arg1 =
+            PerformCopyInitialization(
+              InitializedEntity::InitializeParameter(Context,
+                                                     FnDecl->getParamDecl(0)),
+              SourceLocation(), Owned(Args[1]));
           if (Arg1.isInvalid())
             return ExprError();
 
-          if (PerformObjectArgumentInitialization(Args[0], /*Qualifier=*/0, 
+          if (PerformObjectArgumentInitialization(Args[0], /*Qualifier=*/0,
                                                   Best->FoundDecl, Method))
             return ExprError();
 
           Args[1] = RHS = Arg1.takeAs<Expr>();
         } else {
           // Convert the arguments.
-          ExprResult Arg0
-            = PerformCopyInitialization(
-                                        InitializedEntity::InitializeParameter(
-                                                        FnDecl->getParamDecl(0)),
-                                        SourceLocation(),
-                                        Owned(Args[0]));
+          ExprResult Arg0 = PerformCopyInitialization(
+            InitializedEntity::InitializeParameter(Context,
+                                                   FnDecl->getParamDecl(0)),
+            SourceLocation(), Owned(Args[0]));
           if (Arg0.isInvalid())
             return ExprError();
 
-          ExprResult Arg1
-            = PerformCopyInitialization(
-                                        InitializedEntity::InitializeParameter(
-                                                        FnDecl->getParamDecl(1)),
-                                        SourceLocation(),
-                                        Owned(Args[1]));
+          ExprResult Arg1 =
+            PerformCopyInitialization(
+              InitializedEntity::InitializeParameter(Context,
+                                                     FnDecl->getParamDecl(1)),
+              SourceLocation(), Owned(Args[1]));
           if (Arg1.isInvalid())
             return ExprError();
           Args[0] = LHS = Arg0.takeAs<Expr>();
@@ -7035,21 +8087,19 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
         DiagnoseUseOfDecl(Best->FoundDecl, OpLoc);
 
-        // Determine the result type
-        QualType ResultTy
-          = FnDecl->getType()->getAs<FunctionType>()
-                                                ->getCallResultType(Context);
+        // Determine the result type.
+        QualType ResultTy = FnDecl->getResultType();
+        ExprValueKind VK = Expr::getValueKindForType(ResultTy);
+        ResultTy = ResultTy.getNonLValueExprType(Context);
 
         // Build the actual expression node.
-        Expr *FnExpr = new (Context) DeclRefExpr(FnDecl, FnDecl->getType(),
-                                                 OpLoc);
-        UsualUnaryConversions(FnExpr);
+        Expr *FnExpr = CreateFunctionRefExpr(*this, FnDecl, OpLoc);
 
         CXXOperatorCallExpr *TheCall =
           new (Context) CXXOperatorCallExpr(Context, Op, FnExpr,
-                                            Args, 2, ResultTy, OpLoc);
-        
-        if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall, 
+                                            Args, 2, ResultTy, VK, OpLoc);
+
+        if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall,
                                 FnDecl))
           return ExprError();
 
@@ -7076,11 +8126,11 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       if (Opc == BO_Comma)
         break;
 
-      // For class as left operand for assignment or compound assigment operator
-      // do not fall through to handling in built-in, but report that no overloaded
-      // assignment operator found
+      // For class as left operand for assignment or compound assigment
+      // operator do not fall through to handling in built-in, but report that
+      // no overloaded assignment operator found
       ExprResult Result = ExprError();
-      if (Args[0]->getType()->isRecordType() && 
+      if (Args[0]->getType()->isRecordType() &&
           Opc >= BO_Assign && Opc <= BO_OrAssign) {
         Diag(OpLoc,  diag::err_ovl_no_viable_oper)
              << BinaryOperator::getOpcodeStr(Opc)
@@ -7090,7 +8140,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         // produce an error. Then, show the non-viable candidates.
         Result = CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
       }
-      assert(Result.isInvalid() && 
+      assert(Result.isInvalid() &&
              "C++ binary operator overloading is missing candidates!");
       if (Result.isInvalid())
         CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2,
@@ -7099,8 +8149,9 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
     }
 
     case OR_Ambiguous:
-      Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
+      Diag(OpLoc,  diag::err_ovl_ambiguous_oper_binary)
           << BinaryOperator::getOpcodeStr(Opc)
+          << Args[0]->getType() << Args[1]->getType()
           << Args[0]->getSourceRange() << Args[1]->getSourceRange();
       CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, 2,
                                   BinaryOperator::getOpcodeStr(Opc), OpLoc);
@@ -7136,7 +8187,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
     DeclarationNameInfo OpNameInfo(OpName, LLoc);
     OpNameInfo.setCXXOperatorNameRange(SourceRange(LLoc, RLoc));
     UnresolvedLookupExpr *Fn
-      = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
+      = UnresolvedLookupExpr::Create(Context, NamingClass,
                                      0, SourceRange(), OpNameInfo,
                                      /*ADL*/ true, /*Overloaded*/ false,
                                      UnresolvedSetIterator(),
@@ -7146,9 +8197,15 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
     return Owned(new (Context) CXXOperatorCallExpr(Context, OO_Subscript, Fn,
                                                    Args, 2,
                                                    Context.DependentTy,
+                                                   VK_RValue,
                                                    RLoc));
   }
 
+  if (Args[0]->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Args[0]);
+  if (Args[1]->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Args[1]);
+
   // Build an empty overload set.
   OverloadCandidateSet CandidateSet(LLoc);
 
@@ -7176,15 +8233,16 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 
         // Convert the arguments.
         CXXMethodDecl *Method = cast<CXXMethodDecl>(FnDecl);
-        if (PerformObjectArgumentInitialization(Args[0], /*Qualifier=*/0, 
+        if (PerformObjectArgumentInitialization(Args[0], /*Qualifier=*/0,
                                                 Best->FoundDecl, Method))
           return ExprError();
 
         // Convert the arguments.
         ExprResult InputInit
           = PerformCopyInitialization(InitializedEntity::InitializeParameter(
+                                                      Context,
                                                       FnDecl->getParamDecl(0)),
-                                      SourceLocation(), 
+                                      SourceLocation(),
                                       Owned(Args[1]));
         if (InputInit.isInvalid())
           return ExprError();
@@ -7192,19 +8250,17 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
         Args[1] = InputInit.takeAs<Expr>();
 
         // Determine the result type
-        QualType ResultTy
-          = FnDecl->getType()->getAs<FunctionType>()
-                                                  ->getCallResultType(Context);
+        QualType ResultTy = FnDecl->getResultType();
+        ExprValueKind VK = Expr::getValueKindForType(ResultTy);
+        ResultTy = ResultTy.getNonLValueExprType(Context);
 
         // Build the actual expression node.
-        Expr *FnExpr = new (Context) DeclRefExpr(FnDecl, FnDecl->getType(),
-                                                 LLoc);
-        UsualUnaryConversions(FnExpr);
+        Expr *FnExpr = CreateFunctionRefExpr(*this, FnDecl, LLoc);
 
         CXXOperatorCallExpr *TheCall =
           new (Context) CXXOperatorCallExpr(Context, OO_Subscript,
                                             FnExpr, Args, 2,
-                                            ResultTy, RLoc);
+                                            ResultTy, VK, RLoc);
 
         if (CheckCallReturnType(FnDecl->getResultType(), LLoc, TheCall,
                                 FnDecl))
@@ -7240,8 +8296,10 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
     }
 
     case OR_Ambiguous:
-      Diag(LLoc,  diag::err_ovl_ambiguous_oper)
-          << "[]" << Args[0]->getSourceRange() << Args[1]->getSourceRange();
+      Diag(LLoc,  diag::err_ovl_ambiguous_oper_binary)
+          << "[]"
+          << Args[0]->getType() << Args[1]->getType()
+          << Args[0]->getSourceRange() << Args[1]->getSourceRange();
       CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, 2,
                                   "[]", LLoc);
       return ExprError();
@@ -7269,12 +8327,11 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 ExprResult
 Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
                                 SourceLocation LParenLoc, Expr **Args,
-                                unsigned NumArgs, SourceLocation *CommaLocs,
-                                SourceLocation RParenLoc) {
+                                unsigned NumArgs, SourceLocation RParenLoc) {
   // Dig out the member expression. This holds both the object
   // argument and the member function we're referring to.
   Expr *NakedMemExpr = MemExprE->IgnoreParens();
-  
+
   MemberExpr *MemExpr;
   CXXMethodDecl *Method = 0;
   DeclAccessPair FoundDecl = DeclAccessPair::make(0, AS_public);
@@ -7287,8 +8344,11 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   } else {
     UnresolvedMemberExpr *UnresExpr = cast<UnresolvedMemberExpr>(NakedMemExpr);
     Qualifier = UnresExpr->getQualifier();
-    
+
     QualType ObjectType = UnresExpr->getBaseType();
+    Expr::Classification ObjectClassification
+      = UnresExpr->isArrow()? Expr::Classification::makeSimpleLValue()
+                            : UnresExpr->getBase()->Classify(Context);
 
     // Add overload candidates
     OverloadCandidateSet CandidateSet(UnresExpr->getMemberLoc());
@@ -7308,20 +8368,26 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       if (isa<UsingShadowDecl>(Func))
         Func = cast<UsingShadowDecl>(Func)->getTargetDecl();
 
-      if ((Method = dyn_cast<CXXMethodDecl>(Func))) {
+
+      // Microsoft supports direct constructor calls.
+      if (getLangOptions().Microsoft && isa<CXXConstructorDecl>(Func)) {
+        AddOverloadCandidate(cast<CXXConstructorDecl>(Func), I.getPair(), Args, NumArgs,
+                             CandidateSet);
+      } else if ((Method = dyn_cast<CXXMethodDecl>(Func))) {
         // If explicit template arguments were provided, we can't call a
         // non-template member function.
         if (TemplateArgs)
           continue;
-        
+
         AddMethodCandidate(Method, I.getPair(), ActingDC, ObjectType,
-                           Args, NumArgs,
-                           CandidateSet, /*SuppressUserConversions=*/false);
+                           ObjectClassification,
+                           Args, NumArgs, CandidateSet,
+                           /*SuppressUserConversions=*/false);
       } else {
         AddMethodTemplateCandidate(cast<FunctionTemplateDecl>(Func),
                                    I.getPair(), ActingDC, TemplateArgs,
-                                   ObjectType, Args, NumArgs,
-                                   CandidateSet,
+                                   ObjectType,  ObjectClassification,
+                                   Args, NumArgs, CandidateSet,
                                    /*SuppressUsedConversions=*/false);
       }
     }
@@ -7330,7 +8396,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
     OverloadCandidateSet::iterator Best;
     switch (CandidateSet.BestViableFunction(*this, UnresExpr->getLocStart(),
-                               Best)) {
+                                            Best)) {
     case OR_Success:
       Method = cast<CXXMethodDecl>(Best->Function);
       FoundDecl = Best->FoundDecl;
@@ -7374,17 +8440,20 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     MemExpr = cast<MemberExpr>(MemExprE->IgnoreParens());
   }
 
+  QualType ResultType = Method->getResultType();
+  ExprValueKind VK = Expr::getValueKindForType(ResultType);
+  ResultType = ResultType.getNonLValueExprType(Context);
+
   assert(Method && "Member call to something that isn't a method?");
-  CXXMemberCallExpr *TheCall = 
+  CXXMemberCallExpr *TheCall =
     new (Context) CXXMemberCallExpr(Context, MemExprE, Args, NumArgs,
-                                    Method->getCallResultType(),
-                                    RParenLoc);
+                                    ResultType, VK, RParenLoc);
 
   // Check for a valid return type.
-  if (CheckCallReturnType(Method->getResultType(), MemExpr->getMemberLoc(), 
+  if (CheckCallReturnType(Method->getResultType(), MemExpr->getMemberLoc(),
                           TheCall, Method))
     return ExprError();
-  
+
   // Convert the object argument (for a non-static member function call).
   // We only need to do this if there was actually an overload; otherwise
   // it was done at lookup.
@@ -7396,7 +8465,8 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   MemExpr->setBase(ObjectArg);
 
   // Convert the rest of the arguments
-  const FunctionProtoType *Proto = Method->getType()->getAs<FunctionProtoType>();
+  const FunctionProtoType *Proto =
+    Method->getType()->getAs<FunctionProtoType>();
   if (ConvertArgumentsForCall(TheCall, MemExpr, Method, Proto, Args, NumArgs,
                               RParenLoc))
     return ExprError();
@@ -7415,8 +8485,10 @@ ExprResult
 Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
                                    SourceLocation LParenLoc,
                                    Expr **Args, unsigned NumArgs,
-                                   SourceLocation *CommaLocs,
                                    SourceLocation RParenLoc) {
+  if (Object->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Object);
+
   assert(Object->getType()->isRecordType() && "Requires object type argument");
   const RecordType *Record = Object->getType()->getAs<RecordType>();
 
@@ -7430,11 +8502,11 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   OverloadCandidateSet CandidateSet(LParenLoc);
   DeclarationName OpName = Context.DeclarationNames.getCXXOperatorName(OO_Call);
 
-  if (RequireCompleteType(LParenLoc, Object->getType(), 
+  if (RequireCompleteType(LParenLoc, Object->getType(),
                           PDiag(diag::err_incomplete_object_call)
                           << Object->getSourceRange()))
     return true;
-  
+
   LookupResult R(*this, OpName, LParenLoc, LookupOrdinaryName);
   LookupQualifiedName(R, Record->getDecl());
   R.suppressDiagnostics();
@@ -7442,10 +8514,10 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   for (LookupResult::iterator Oper = R.begin(), OperEnd = R.end();
        Oper != OperEnd; ++Oper) {
     AddMethodCandidate(Oper.getPair(), Object->getType(),
-                       Args, NumArgs, CandidateSet,
+                       Object->Classify(Context), Args, NumArgs, CandidateSet,
                        /*SuppressUserConversions=*/ false);
   }
-  
+
   // C++ [over.call.object]p2:
   //   In addition, for each conversion function declared in T of the
   //   form
@@ -7471,7 +8543,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
     CXXRecordDecl *ActingContext = cast<CXXRecordDecl>(D->getDeclContext());
     if (isa<UsingShadowDecl>(D))
       D = cast<UsingShadowDecl>(D)->getTargetDecl();
-    
+
     // Skip over templated conversion functions; they aren't
     // surrogates.
     if (isa<FunctionTemplateDecl>(D))
@@ -7487,8 +8559,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 
     if (const FunctionProtoType *Proto = ConvType->getAs<FunctionProtoType>())
       AddSurrogateCandidate(Conv, I.getPair(), ActingContext, Proto,
-                            Object->getType(), Args, NumArgs,
-                            CandidateSet);
+                            Object, Args, NumArgs, CandidateSet);
   }
 
   // Perform overload resolution.
@@ -7544,14 +8615,15 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
     // We selected one of the surrogate functions that converts the
     // object parameter to a function pointer. Perform the conversion
     // on the object argument, then let ActOnCallExpr finish the job.
-    
+
     // Create an implicit member expr to refer to the conversion operator.
     // and then call it.
-    CXXMemberCallExpr *CE = BuildCXXMemberCallExpr(Object, Best->FoundDecl,
-                                                   Conv);
-      
-    return ActOnCallExpr(S, CE, LParenLoc, MultiExprArg(Args, NumArgs),
-                         CommaLocs, RParenLoc);
+    ExprResult Call = BuildCXXMemberCallExpr(Object, Best->FoundDecl, Conv);
+    if (Call.isInvalid())
+      return ExprError();
+
+    return ActOnCallExpr(S, Call.get(), LParenLoc, MultiExprArg(Args, NumArgs),
+                         RParenLoc);
   }
 
   CheckMemberOperatorAccess(LParenLoc, Object, 0, Best->FoundDecl);
@@ -7561,7 +8633,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   // that calls this method, using Object for the implicit object
   // parameter and passing along the remaining arguments.
   CXXMethodDecl *Method = cast<CXXMethodDecl>(Best->Function);
-  const FunctionProtoType *Proto = Method->getType()->getAs<FunctionProtoType>();
+  const FunctionProtoType *Proto =
+    Method->getType()->getAs<FunctionProtoType>();
 
   unsigned NumArgsInProto = Proto->getNumArgs();
   unsigned NumArgsToCheck = NumArgs;
@@ -7580,23 +8653,24 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
     MethodArgs[ArgIdx + 1] = Args[ArgIdx];
 
-  Expr *NewFn = new (Context) DeclRefExpr(Method, Method->getType(),
-                                          SourceLocation());
-  UsualUnaryConversions(NewFn);
+  Expr *NewFn = CreateFunctionRefExpr(*this, Method);
 
   // Once we've built TheCall, all of the expressions are properly
   // owned.
-  QualType ResultTy = Method->getCallResultType();
+  QualType ResultTy = Method->getResultType();
+  ExprValueKind VK = Expr::getValueKindForType(ResultTy);
+  ResultTy = ResultTy.getNonLValueExprType(Context);
+
   CXXOperatorCallExpr *TheCall =
     new (Context) CXXOperatorCallExpr(Context, OO_Call, NewFn,
                                       MethodArgs, NumArgs + 1,
-                                      ResultTy, RParenLoc);
+                                      ResultTy, VK, RParenLoc);
   delete [] MethodArgs;
 
-  if (CheckCallReturnType(Method->getResultType(), LParenLoc, TheCall, 
+  if (CheckCallReturnType(Method->getResultType(), LParenLoc, TheCall,
                           Method))
     return true;
-  
+
   // We may have default arguments. If so, we need to allocate more
   // slots in the call for them.
   if (NumArgs < NumArgsInProto)
@@ -7607,7 +8681,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   bool IsError = false;
 
   // Initialize the implicit object parameter.
-  IsError |= PerformObjectArgumentInitialization(Object, /*Qualifier=*/0, 
+  IsError |= PerformObjectArgumentInitialization(Object, /*Qualifier=*/0,
                                                  Best->FoundDecl, Method);
   TheCall->setArg(0, Object);
 
@@ -7622,9 +8696,10 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 
       ExprResult InputInit
         = PerformCopyInitialization(InitializedEntity::InitializeParameter(
+                                                    Context,
                                                     Method->getParamDecl(i)),
                                     SourceLocation(), Arg);
-      
+
       IsError |= InputInit.isInvalid();
       Arg = InputInit.takeAs<Expr>();
     } else {
@@ -7634,7 +8709,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
         IsError = true;
         break;
       }
-      
+
       Arg = DefArg.takeAs<Expr>();
     }
 
@@ -7664,7 +8739,11 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 /// @c Member is the name of the member we're trying to find.
 ExprResult
 Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
-  assert(Base->getType()->isRecordType() && "left-hand side must have class type");
+  assert(Base->getType()->isRecordType() &&
+         "left-hand side must have class type");
+
+  if (Base->getObjectKind() == OK_ObjCProperty)
+    ConvertPropertyForRValue(Base);
 
   SourceLocation Loc = Base->getExprLoc();
 
@@ -7674,7 +8753,8 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
   //   for a class object x of type T if T::operator->() exists and if
   //   the operator is selected as the best match function by the
   //   overload resolution mechanism (13.3).
-  DeclarationName OpName = Context.DeclarationNames.getCXXOperatorName(OO_Arrow);
+  DeclarationName OpName =
+    Context.DeclarationNames.getCXXOperatorName(OO_Arrow);
   OverloadCandidateSet CandidateSet(Loc);
   const RecordType *BaseRecord = Base->getType()->getAs<RecordType>();
 
@@ -7689,8 +8769,8 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
 
   for (LookupResult::iterator Oper = R.begin(), OperEnd = R.end();
        Oper != OperEnd; ++Oper) {
-    AddMethodCandidate(Oper.getPair(), Base->getType(), 0, 0, CandidateSet,
-                       /*SuppressUserConversions=*/false);
+    AddMethodCandidate(Oper.getPair(), Base->getType(), Base->Classify(Context),
+                       0, 0, CandidateSet, /*SuppressUserConversions=*/false);
   }
 
   // Perform overload resolution.
@@ -7711,8 +8791,8 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
     return ExprError();
 
   case OR_Ambiguous:
-    Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
-      << "->" << Base->getSourceRange();
+    Diag(OpLoc,  diag::err_ovl_ambiguous_oper_unary)
+      << "->" << Base->getType() << Base->getSourceRange();
     CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, &Base, 1);
     return ExprError();
 
@@ -7734,16 +8814,16 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
     return ExprError();
 
   // Build the operator call.
-  Expr *FnExpr = new (Context) DeclRefExpr(Method, Method->getType(),
-                                           SourceLocation());
-  UsualUnaryConversions(FnExpr);
-  
-  QualType ResultTy = Method->getCallResultType();
+  Expr *FnExpr = CreateFunctionRefExpr(*this, Method);
+
+  QualType ResultTy = Method->getResultType();
+  ExprValueKind VK = Expr::getValueKindForType(ResultTy);
+  ResultTy = ResultTy.getNonLValueExprType(Context);
   CXXOperatorCallExpr *TheCall =
-    new (Context) CXXOperatorCallExpr(Context, OO_Arrow, FnExpr, 
-                                      &Base, 1, ResultTy, OpLoc);
+    new (Context) CXXOperatorCallExpr(Context, OO_Arrow, FnExpr,
+                                      &Base, 1, ResultTy, VK, OpLoc);
 
-  if (CheckCallReturnType(Method->getResultType(), OpLoc, TheCall, 
+  if (CheckCallReturnType(Method->getResultType(), OpLoc, TheCall,
                           Method))
           return ExprError();
   return Owned(TheCall);
@@ -7760,27 +8840,27 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
     Expr *SubExpr = FixOverloadedFunctionReference(PE->getSubExpr(),
                                                    Found, Fn);
     if (SubExpr == PE->getSubExpr())
-      return PE->Retain();
-    
+      return PE;
+
     return new (Context) ParenExpr(PE->getLParen(), PE->getRParen(), SubExpr);
-  } 
-  
+  }
+
   if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
     Expr *SubExpr = FixOverloadedFunctionReference(ICE->getSubExpr(),
                                                    Found, Fn);
-    assert(Context.hasSameType(ICE->getSubExpr()->getType(), 
+    assert(Context.hasSameType(ICE->getSubExpr()->getType(),
                                SubExpr->getType()) &&
            "Implicit cast type cannot be determined from overload");
     assert(ICE->path_empty() && "fixing up hierarchy conversion?");
     if (SubExpr == ICE->getSubExpr())
-      return ICE->Retain();
-    
-    return ImplicitCastExpr::Create(Context, ICE->getType(), 
+      return ICE;
+
+    return ImplicitCastExpr::Create(Context, ICE->getType(),
                                     ICE->getCastKind(),
                                     SubExpr, 0,
                                     ICE->getValueKind());
-  } 
-  
+  }
+
   if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(E)) {
     assert(UnOp->getOpcode() == UO_AddrOf &&
            "Can only take the address of an overloaded function");
@@ -7795,7 +8875,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
         Expr *SubExpr = FixOverloadedFunctionReference(UnOp->getSubExpr(),
                                                        Found, Fn);
         if (SubExpr == UnOp->getSubExpr())
-          return UnOp->Retain();
+          return UnOp;
 
         assert(isa<DeclRefExpr>(SubExpr)
                && "fixed to something other than a decl ref");
@@ -7810,19 +8890,21 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
         QualType MemPtrType
           = Context.getMemberPointerType(Fn->getType(), ClassType.getTypePtr());
 
-        return new (Context) UnaryOperator(SubExpr, UO_AddrOf,
-                                           MemPtrType, UnOp->getOperatorLoc());
+        return new (Context) UnaryOperator(SubExpr, UO_AddrOf, MemPtrType,
+                                           VK_RValue, OK_Ordinary,
+                                           UnOp->getOperatorLoc());
       }
     }
     Expr *SubExpr = FixOverloadedFunctionReference(UnOp->getSubExpr(),
                                                    Found, Fn);
     if (SubExpr == UnOp->getSubExpr())
-      return UnOp->Retain();
-    
+      return UnOp;
+
     return new (Context) UnaryOperator(SubExpr, UO_AddrOf,
                                      Context.getPointerType(SubExpr->getType()),
+                                       VK_RValue, OK_Ordinary,
                                        UnOp->getOperatorLoc());
-  } 
+  }
 
   if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(E)) {
     // FIXME: avoid copy.
@@ -7838,6 +8920,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
                                Fn,
                                ULE->getNameLoc(),
                                Fn->getType(),
+                               VK_LValue,
                                TemplateArgs);
   }
 
@@ -7851,7 +8934,8 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
 
     Expr *Base;
 
-    // If we're filling in 
+    // If we're filling in a static method where we used to have an
+    // implicit member access, rewrite to a simple decl ref.
     if (MemExpr->isImplicitAccess()) {
       if (cast<CXXMethodDecl>(Fn)->isStatic()) {
         return DeclRefExpr::Create(Context,
@@ -7860,6 +8944,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
                                    Fn,
                                    MemExpr->getMemberLoc(),
                                    Fn->getType(),
+                                   VK_LValue,
                                    TemplateArgs);
       } else {
         SourceLocation Loc = MemExpr->getMemberLoc();
@@ -7870,24 +8955,27 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
                                          /*isImplicit=*/true);
       }
     } else
-      Base = MemExpr->getBase()->Retain();
+      Base = MemExpr->getBase();
 
     return MemberExpr::Create(Context, Base,
-                              MemExpr->isArrow(), 
-                              MemExpr->getQualifier(), 
+                              MemExpr->isArrow(),
+                              MemExpr->getQualifier(),
                               MemExpr->getQualifierRange(),
-                              Fn, 
+                              Fn,
                               Found,
                               MemExpr->getMemberNameInfo(),
                               TemplateArgs,
-                              Fn->getType());
+                              Fn->getType(),
+                              cast<CXXMethodDecl>(Fn)->isStatic()
+                                ? VK_LValue : VK_RValue,
+                              OK_Ordinary);
   }
-  
-  assert(false && "Invalid reference to overloaded function");
-  return E->Retain();
+
+  llvm_unreachable("Invalid reference to overloaded function");
+  return E;
 }
 
-ExprResult Sema::FixOverloadedFunctionReference(ExprResult E, 
+ExprResult Sema::FixOverloadedFunctionReference(ExprResult E,
                                                 DeclAccessPair Found,
                                                 FunctionDecl *Fn) {
   return Owned(FixOverloadedFunctionReference((Expr *)E.get(), Found, Fn));
diff --git a/lib/Sema/SemaStmt.cpp b/lib/Sema/SemaStmt.cpp
index 083e4dba85e3..e995e8f207df 100644
--- a/lib/Sema/SemaStmt.cpp
+++ b/lib/Sema/SemaStmt.cpp
@@ -32,7 +32,9 @@ using namespace sema;
 
 StmtResult Sema::ActOnExprStmt(FullExprArg expr) {
   Expr *E = expr.get();
-  assert(E && "ActOnExprStmt(): missing expression");
+  if (!E) // FIXME: FullExprArg has no error state?
+    return StmtError();
+
   // C99 6.8.3p2: The expression in an expression statement is evaluated as a
   // void expression for its side effects.  Conversion to void allows any
   // operand, even incomplete types.
@@ -42,13 +44,12 @@ StmtResult Sema::ActOnExprStmt(FullExprArg expr) {
 }
 
 
-StmtResult Sema::ActOnNullStmt(SourceLocation SemiLoc) {
-  return Owned(new (Context) NullStmt(SemiLoc));
+StmtResult Sema::ActOnNullStmt(SourceLocation SemiLoc, bool LeadingEmptyMacro) {
+  return Owned(new (Context) NullStmt(SemiLoc, LeadingEmptyMacro));
 }
 
-StmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg,
-                                           SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
+StmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg, SourceLocation StartLoc,
+                               SourceLocation EndLoc) {
   DeclGroupRef DG = dg.getAsVal<DeclGroupRef>();
 
   // If we have an invalid decl, just return an error.
@@ -59,7 +60,7 @@ StmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg,
 
 void Sema::ActOnForEachDeclStmt(DeclGroupPtrTy dg) {
   DeclGroupRef DG = dg.getAsVal<DeclGroupRef>();
-  
+
   // If we have an invalid decl, just return.
   if (DG.isNull() || !DG.isSingleDecl()) return;
   // suppress any potential 'unused variable' warning.
@@ -67,10 +68,19 @@ void Sema::ActOnForEachDeclStmt(DeclGroupPtrTy dg) {
 }
 
 void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
+  if (const LabelStmt *Label = dyn_cast_or_null<LabelStmt>(S))
+    return DiagnoseUnusedExprResult(Label->getSubStmt());
+
   const Expr *E = dyn_cast_or_null<Expr>(S);
   if (!E)
     return;
 
+  if (E->isBoundMemberFunction(Context)) {
+    Diag(E->getLocStart(), diag::err_invalid_use_of_bound_member_func)
+      << E->getSourceRange();
+    return;
+  }
+
   SourceLocation Loc;
   SourceRange R1, R2;
   if (!E->isUnusedResultAWarning(Loc, R1, R2, Context))
@@ -80,13 +90,10 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
   // we might want to make a more specific diagnostic.  Check for one of these
   // cases now.
   unsigned DiagID = diag::warn_unused_expr;
-  E = E->IgnoreParens();
-  if (isa<ObjCImplicitSetterGetterRefExpr>(E))
-    DiagID = diag::warn_unused_property_expr;
-  
-  if (const CXXExprWithTemporaries *Temps = dyn_cast<CXXExprWithTemporaries>(E))
+  if (const ExprWithCleanups *Temps = dyn_cast<ExprWithCleanups>(E))
     E = Temps->getSubExpr();
-      
+
+  E = E->IgnoreParenImpCasts();
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
     if (E->getType()->isVoidType())
       return;
@@ -106,14 +113,15 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
         Diag(Loc, diag::warn_unused_call) << R1 << R2 << "const";
         return;
       }
-    }        
-  }
-  else if (const ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
+    }
+  } else if (const ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
     const ObjCMethodDecl *MD = ME->getMethodDecl();
     if (MD && MD->getAttr<WarnUnusedResultAttr>()) {
       Diag(Loc, diag::warn_unused_call) << R1 << R2 << "warn_unused_result";
       return;
     }
+  } else if (isa<ObjCPropertyRefExpr>(E)) {
+    DiagID = diag::warn_unused_property_expr;
   } else if (const CXXFunctionalCastExpr *FC
                                        = dyn_cast<CXXFunctionalCastExpr>(E)) {
     if (isa<CXXConstructExpr>(FC->getSubExpr()) ||
@@ -223,30 +231,23 @@ Sema::ActOnDefaultStmt(SourceLocation DefaultLoc, SourceLocation ColonLoc,
 }
 
 StmtResult
-Sema::ActOnLabelStmt(SourceLocation IdentLoc, IdentifierInfo *II,
+Sema::ActOnLabelStmt(SourceLocation IdentLoc, LabelDecl *TheDecl,
                      SourceLocation ColonLoc, Stmt *SubStmt) {
-  // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[II];
-
-  // If not forward referenced or defined already, just create a new LabelStmt.
-  if (LabelDecl == 0)
-    return Owned(LabelDecl = new (Context) LabelStmt(IdentLoc, II, SubStmt));
-
-  assert(LabelDecl->getID() == II && "Label mismatch!");
-
-  // Otherwise, this label was either forward reference or multiply defined.  If
-  // multiply defined, reject it now.
-  if (LabelDecl->getSubStmt()) {
-    Diag(IdentLoc, diag::err_redefinition_of_label) << LabelDecl->getID();
-    Diag(LabelDecl->getIdentLoc(), diag::note_previous_definition);
+  
+  // If the label was multiply defined, reject it now.
+  if (TheDecl->getStmt()) {
+    Diag(IdentLoc, diag::err_redefinition_of_label) << TheDecl->getDeclName();
+    Diag(TheDecl->getLocation(), diag::note_previous_definition);
     return Owned(SubStmt);
   }
 
-  // Otherwise, this label was forward declared, and we just found its real
-  // definition.  Fill in the forward definition and return it.
-  LabelDecl->setIdentLoc(IdentLoc);
-  LabelDecl->setSubStmt(SubStmt);
-  return Owned(LabelDecl);
+  // Otherwise, things are good.  Fill in the declaration and return it.
+  TheDecl->setLocation(IdentLoc);
+  
+  LabelStmt *LS = new (Context) LabelStmt(IdentLoc, TheDecl, SubStmt);
+  TheDecl->setStmt(LS);
+  TheDecl->setLocation(IdentLoc);
+  return Owned(LS);
 }
 
 StmtResult
@@ -265,21 +266,29 @@ Sema::ActOnIfStmt(SourceLocation IfLoc, FullExprArg CondVal, Decl *CondVar,
   Expr *ConditionExpr = CondResult.takeAs<Expr>();
   if (!ConditionExpr)
     return StmtError();
-  
+
   DiagnoseUnusedExprResult(thenStmt);
 
   // Warn if the if block has a null body without an else value.
   // this helps prevent bugs due to typos, such as
   // if (condition);
   //   do_stuff();
+  //
   if (!elseStmt) {
     if (NullStmt* stmt = dyn_cast<NullStmt>(thenStmt))
-      Diag(stmt->getSemiLoc(), diag::warn_empty_if_body);
+      // But do not warn if the body is a macro that expands to nothing, e.g:
+      //
+      // #define CALL(x)
+      // if (condition)
+      //   CALL(0);
+      //
+      if (!stmt->hasLeadingEmptyMacro())
+        Diag(stmt->getSemiLoc(), diag::warn_empty_if_body);
   }
 
   DiagnoseUnusedExprResult(elseStmt);
 
-  return Owned(new (Context) IfStmt(Context, IfLoc, ConditionVar, ConditionExpr, 
+  return Owned(new (Context) IfStmt(Context, IfLoc, ConditionVar, ConditionExpr,
                                     thenStmt, ElseLoc, elseStmt));
 }
 
@@ -293,7 +302,7 @@ void Sema::ConvertIntegerToTypeWarnOnOverflow(llvm::APSInt &Val,
   // Perform a conversion to the promoted condition type if needed.
   if (NewWidth > Val.getBitWidth()) {
     // If this is an extension, just do it.
-    Val.extend(NewWidth);
+    Val = Val.extend(NewWidth);
     Val.setIsSigned(NewSign);
 
     // If the input was signed and negative and the output is
@@ -303,21 +312,21 @@ void Sema::ConvertIntegerToTypeWarnOnOverflow(llvm::APSInt &Val,
   } else if (NewWidth < Val.getBitWidth()) {
     // If this is a truncation, check for overflow.
     llvm::APSInt ConvVal(Val);
-    ConvVal.trunc(NewWidth);
+    ConvVal = ConvVal.trunc(NewWidth);
     ConvVal.setIsSigned(NewSign);
-    ConvVal.extend(Val.getBitWidth());
+    ConvVal = ConvVal.extend(Val.getBitWidth());
     ConvVal.setIsSigned(Val.isSigned());
     if (ConvVal != Val)
       Diag(Loc, DiagID) << Val.toString(10) << ConvVal.toString(10);
 
     // Regardless of whether a diagnostic was emitted, really do the
     // truncation.
-    Val.trunc(NewWidth);
+    Val = Val.trunc(NewWidth);
     Val.setIsSigned(NewSign);
   } else if (NewSign != Val.isSigned()) {
     // Convert the sign to match the sign of the condition.  This can cause
     // overflow as well: unsigned(INTMIN)
-    // We don't diagnose this overflow, because it is implementation-defined 
+    // We don't diagnose this overflow, because it is implementation-defined
     // behavior.
     // FIXME: Introduce a second, default-ignored warning for this case?
     llvm::APSInt OldVal(Val);
@@ -386,7 +395,7 @@ static QualType GetTypeBeforeIntegralPromotion(const Expr* expr) {
 }
 
 StmtResult
-Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond, 
+Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond,
                              Decl *CondVar) {
   ExprResult CondResult;
 
@@ -396,15 +405,15 @@ Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond,
     CondResult = CheckConditionVariable(ConditionVar, SourceLocation(), false);
     if (CondResult.isInvalid())
       return StmtError();
-    
+
     Cond = CondResult.release();
   }
-  
+
   if (!Cond)
     return StmtError();
-  
+
   CondResult
-    = ConvertToIntegralOrEnumerationType(SwitchLoc, Cond, 
+    = ConvertToIntegralOrEnumerationType(SwitchLoc, Cond,
                           PDiag(diag::err_typecheck_statement_requires_integer),
                                    PDiag(diag::err_switch_incomplete_class_type)
                                      << Cond->getSourceRange(),
@@ -415,21 +424,30 @@ Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond,
                                          PDiag(0));
   if (CondResult.isInvalid()) return StmtError();
   Cond = CondResult.take();
-  
+
   if (!CondVar) {
-    CondResult = MaybeCreateCXXExprWithTemporaries(Cond);
+    CheckImplicitConversions(Cond, SwitchLoc);
+    CondResult = MaybeCreateExprWithCleanups(Cond);
     if (CondResult.isInvalid())
       return StmtError();
     Cond = CondResult.take();
   }
 
   getCurFunction()->setHasBranchIntoScope();
-    
+
   SwitchStmt *SS = new (Context) SwitchStmt(Context, ConditionVar, Cond);
   getCurFunction()->SwitchStack.push_back(SS);
   return Owned(SS);
 }
 
+static void AdjustAPSInt(llvm::APSInt &Val, unsigned BitWidth, bool IsSigned) {
+  if (Val.getBitWidth() < BitWidth)
+    Val = Val.extend(BitWidth);
+  else if (Val.getBitWidth() > BitWidth)
+    Val = Val.trunc(BitWidth);
+  Val.setIsSigned(IsSigned);
+}
+
 StmtResult
 Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
                             Stmt *BodyStmt) {
@@ -442,7 +460,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 
   if (SS->getCond() == 0)
     return StmtError();
-    
+
   Expr *CondExpr = SS->getCond();
   Expr *CondExprBeforePromotion = CondExpr;
   QualType CondTypeBeforePromotion =
@@ -462,7 +480,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
   // the pre-promotion type of the switch condition.
   if (!CondExpr->isTypeDependent()) {
     // We have already converted the expression to an integral or enumeration
-    // type, when we started the switch statement. If we don't have an 
+    // type, when we started the switch statement. If we don't have an
     // appropriate type now, just return an error.
     if (!CondType->isIntegralOrEnumerationType())
       return StmtError();
@@ -531,7 +549,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 
       // Convert the value to the same width/sign as the condition.
       ConvertIntegerToTypeWarnOnOverflow(LoVal, CondWidth, CondIsSigned,
-                                         CS->getLHS()->getLocStart(),
+                                         Lo->getLocStart(),
                                          diag::warn_case_value_overflow);
 
       // If the LHS is not the same type as the condition, insert an implicit
@@ -610,7 +628,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 
         // Convert the value to the same width/sign as the condition.
         ConvertIntegerToTypeWarnOnOverflow(HiVal, CondWidth, CondIsSigned,
-                                           CR->getRHS()->getLocStart(),
+                                           Hi->getLocStart(),
                                            diag::warn_case_value_overflow);
 
         // If the LHS is not the same type as the condition, insert an implicit
@@ -622,7 +640,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
         if (LoVal > HiVal) {
           Diag(CR->getLHS()->getLocStart(), diag::warn_case_empty_range)
             << SourceRange(CR->getLHS()->getLocStart(),
-                           CR->getRHS()->getLocEnd());
+                           Hi->getLocEnd());
           CaseRanges.erase(CaseRanges.begin()+i);
           --i, --e;
           continue;
@@ -696,14 +714,14 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
     }
 
     // Check to see if switch is over an Enum and handles all of its
-    // values.  We don't need to do this if there's a default
-    // statement or if we have a constant condition.
+    // values.  We only issue a warning if there is not 'default:', but
+    // we still do the analysis to preserve this information in the AST
+    // (which can be used by flow-based analyes).
     //
-    // TODO: we might want to check whether case values are out of the
-    // enum even if we don't want to check whether all cases are handled.
-    const EnumType* ET = CondTypeBeforePromotion->getAs<EnumType>();
+    const EnumType *ET = CondTypeBeforePromotion->getAs<EnumType>();
+
     // If switch has default case, then ignore it.
-    if (!CaseListIsErroneous && !TheDefaultStmt && !HasConstantCond && ET) {
+    if (!CaseListIsErroneous  && !HasConstantCond && ET) {
       const EnumDecl *ED = ET->getDecl();
       typedef llvm::SmallVector<std::pair<llvm::APSInt, EnumConstantDecl*>, 64> EnumValsTy;
       EnumValsTy EnumVals;
@@ -711,71 +729,105 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
       // Gather all enum values, set their type and sort them,
       // allowing easier comparison with CaseVals.
       for (EnumDecl::enumerator_iterator EDI = ED->enumerator_begin();
-             EDI != ED->enumerator_end(); EDI++) {
-        llvm::APSInt Val = (*EDI)->getInitVal();
-        if(Val.getBitWidth() < CondWidth)
-          Val.extend(CondWidth);
-        else if (Val.getBitWidth() > CondWidth)
-          Val.trunc(CondWidth);
-        Val.setIsSigned(CondIsSigned);
-        EnumVals.push_back(std::make_pair(Val, (*EDI)));
+           EDI != ED->enumerator_end(); ++EDI) {
+        llvm::APSInt Val = EDI->getInitVal();
+        AdjustAPSInt(Val, CondWidth, CondIsSigned);
+        EnumVals.push_back(std::make_pair(Val, *EDI));
       }
       std::stable_sort(EnumVals.begin(), EnumVals.end(), CmpEnumVals);
       EnumValsTy::iterator EIend =
         std::unique(EnumVals.begin(), EnumVals.end(), EqEnumVals);
-      // See which case values aren't in enum 
-      EnumValsTy::const_iterator EI = EnumVals.begin();
-      for (CaseValsTy::const_iterator CI = CaseVals.begin();
+
+      // See which case values aren't in enum.
+      // TODO: we might want to check whether case values are out of the
+      // enum even if we don't want to check whether all cases are handled.
+      if (!TheDefaultStmt) {
+        EnumValsTy::const_iterator EI = EnumVals.begin();
+        for (CaseValsTy::const_iterator CI = CaseVals.begin();
              CI != CaseVals.end(); CI++) {
-        while (EI != EIend && EI->first < CI->first)
-          EI++;
-        if (EI == EIend || EI->first > CI->first)
+          while (EI != EIend && EI->first < CI->first)
+            EI++;
+          if (EI == EIend || EI->first > CI->first)
             Diag(CI->second->getLHS()->getExprLoc(), diag::warn_not_in_enum)
               << ED->getDeclName();
-      }
-      // See which of case ranges aren't in enum
-      EI = EnumVals.begin();
-      for (CaseRangesTy::const_iterator RI = CaseRanges.begin();
-             RI != CaseRanges.end() && EI != EIend; RI++) {
-        while (EI != EIend && EI->first < RI->first)
-          EI++;
-        
-        if (EI == EIend || EI->first != RI->first) {
-          Diag(RI->second->getLHS()->getExprLoc(), diag::warn_not_in_enum)
-            << ED->getDeclName();
         }
+        // See which of case ranges aren't in enum
+        EI = EnumVals.begin();
+        for (CaseRangesTy::const_iterator RI = CaseRanges.begin();
+             RI != CaseRanges.end() && EI != EIend; RI++) {
+          while (EI != EIend && EI->first < RI->first)
+            EI++;
+
+          if (EI == EIend || EI->first != RI->first) {
+            Diag(RI->second->getLHS()->getExprLoc(), diag::warn_not_in_enum)
+              << ED->getDeclName();
+          }
 
-        llvm::APSInt Hi = RI->second->getRHS()->EvaluateAsInt(Context);
-        while (EI != EIend && EI->first < Hi)
-          EI++;
-        if (EI == EIend || EI->first != Hi)
-          Diag(RI->second->getRHS()->getExprLoc(), diag::warn_not_in_enum)
-            << ED->getDeclName();
+          llvm::APSInt Hi = RI->second->getRHS()->EvaluateAsInt(Context);
+          AdjustAPSInt(Hi, CondWidth, CondIsSigned);
+          while (EI != EIend && EI->first < Hi)
+            EI++;
+          if (EI == EIend || EI->first != Hi)
+            Diag(RI->second->getRHS()->getExprLoc(), diag::warn_not_in_enum)
+              << ED->getDeclName();
+        }
       }
-      //Check which enum vals aren't in switch
+
+      // Check which enum vals aren't in switch
       CaseValsTy::const_iterator CI = CaseVals.begin();
       CaseRangesTy::const_iterator RI = CaseRanges.begin();
-      EI = EnumVals.begin();
-      for (; EI != EIend; EI++) {
-        //Drop unneeded case values
+      bool hasCasesNotInSwitch = false;
+
+      llvm::SmallVector<DeclarationName,8> UnhandledNames;
+
+      for (EnumValsTy::const_iterator EI = EnumVals.begin(); EI != EIend; EI++){
+        // Drop unneeded case values
         llvm::APSInt CIVal;
         while (CI != CaseVals.end() && CI->first < EI->first)
           CI++;
-        
+
         if (CI != CaseVals.end() && CI->first == EI->first)
           continue;
 
-        //Drop unneeded case ranges
+        // Drop unneeded case ranges
         for (; RI != CaseRanges.end(); RI++) {
           llvm::APSInt Hi = RI->second->getRHS()->EvaluateAsInt(Context);
+          AdjustAPSInt(Hi, CondWidth, CondIsSigned);
           if (EI->first <= Hi)
             break;
         }
 
-        if (RI == CaseRanges.end() || EI->first < RI->first)
-          Diag(CondExpr->getExprLoc(), diag::warn_missing_cases)
-            << EI->second->getDeclName();
+        if (RI == CaseRanges.end() || EI->first < RI->first) {
+          hasCasesNotInSwitch = true;
+          if (!TheDefaultStmt)
+            UnhandledNames.push_back(EI->second->getDeclName());
+        }
+      }
+
+      // Produce a nice diagnostic if multiple values aren't handled.
+      switch (UnhandledNames.size()) {
+      case 0: break;
+      case 1:
+        Diag(CondExpr->getExprLoc(), diag::warn_missing_case1)
+          << UnhandledNames[0];
+        break;
+      case 2:
+        Diag(CondExpr->getExprLoc(), diag::warn_missing_case2)
+          << UnhandledNames[0] << UnhandledNames[1];
+        break;
+      case 3:
+        Diag(CondExpr->getExprLoc(), diag::warn_missing_case3)
+          << UnhandledNames[0] << UnhandledNames[1] << UnhandledNames[2];
+        break;
+      default:
+        Diag(CondExpr->getExprLoc(), diag::warn_missing_cases)
+          << (unsigned)UnhandledNames.size()
+          << UnhandledNames[0] << UnhandledNames[1] << UnhandledNames[2];
+        break;
       }
+
+      if (!hasCasesNotInSwitch)
+        SS->setAllEnumCasesCovered();
     }
   }
 
@@ -788,10 +840,10 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 }
 
 StmtResult
-Sema::ActOnWhileStmt(SourceLocation WhileLoc, FullExprArg Cond, 
+Sema::ActOnWhileStmt(SourceLocation WhileLoc, FullExprArg Cond,
                      Decl *CondVar, Stmt *Body) {
   ExprResult CondResult(Cond.release());
-  
+
   VarDecl *ConditionVar = 0;
   if (CondVar) {
     ConditionVar = cast<VarDecl>(CondVar);
@@ -802,7 +854,7 @@ Sema::ActOnWhileStmt(SourceLocation WhileLoc, FullExprArg Cond,
   Expr *ConditionExpr = CondResult.take();
   if (!ConditionExpr)
     return StmtError();
-  
+
   DiagnoseUnusedExprResult(Body);
 
   return Owned(new (Context) WhileStmt(Context, ConditionVar, ConditionExpr,
@@ -818,11 +870,12 @@ Sema::ActOnDoStmt(SourceLocation DoLoc, Stmt *Body,
   if (CheckBooleanCondition(Cond, DoLoc))
     return StmtError();
 
-  ExprResult CondResult = MaybeCreateCXXExprWithTemporaries(Cond);
+  CheckImplicitConversions(Cond, DoLoc);
+  ExprResult CondResult = MaybeCreateExprWithCleanups(Cond);
   if (CondResult.isInvalid())
     return StmtError();
   Cond = CondResult.take();
-  
+
   DiagnoseUnusedExprResult(Body);
 
   return Owned(new (Context) DoStmt(Body, Cond, DoLoc, WhileLoc, CondRParen));
@@ -841,7 +894,7 @@ Sema::ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
       for (DeclStmt::decl_iterator DI=DS->decl_begin(), DE=DS->decl_end();
            DI!=DE; ++DI) {
         VarDecl *VD = dyn_cast<VarDecl>(*DI);
-        if (VD && VD->isBlockVarDecl() && !VD->hasLocalStorage())
+        if (VD && VD->isLocalVarDecl() && !VD->hasLocalStorage())
           VD = 0;
         if (VD == 0)
           Diag((*DI)->getLocation(), diag::err_non_variable_decl_in_for);
@@ -858,19 +911,30 @@ Sema::ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
     if (SecondResult.isInvalid())
       return StmtError();
   }
-  
+
   Expr *Third  = third.release().takeAs<Expr>();
-  
+
   DiagnoseUnusedExprResult(First);
   DiagnoseUnusedExprResult(Third);
   DiagnoseUnusedExprResult(Body);
 
-  return Owned(new (Context) ForStmt(Context, First, 
-                                     SecondResult.take(), ConditionVar, 
-                                     Third, Body, ForLoc, LParenLoc, 
+  return Owned(new (Context) ForStmt(Context, First,
+                                     SecondResult.take(), ConditionVar,
+                                     Third, Body, ForLoc, LParenLoc,
                                      RParenLoc));
 }
 
+/// In an Objective C collection iteration statement:
+///   for (x in y)
+/// x can be an arbitrary l-value expression.  Bind it up as a
+/// full-expression.
+StmtResult Sema::ActOnForEachLValueExpr(Expr *E) {
+  CheckImplicitConversions(E);
+  ExprResult Result = MaybeCreateExprWithCleanups(E);
+  if (Result.isInvalid()) return StmtError();
+  return Owned(static_cast<Stmt*>(Result.get()));
+}
+
 StmtResult
 Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
                                  SourceLocation LParenLoc,
@@ -889,13 +953,12 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
       // declare identifiers for objects having storage class 'auto' or
       // 'register'.
       VarDecl *VD = cast<VarDecl>(D);
-      if (VD->isBlockVarDecl() && !VD->hasLocalStorage())
+      if (VD->isLocalVarDecl() && !VD->hasLocalStorage())
         return StmtError(Diag(VD->getLocation(),
                               diag::err_non_variable_decl_in_for));
     } else {
       Expr *FirstE = cast<Expr>(First);
-      if (!FirstE->isTypeDependent() &&
-          FirstE->isLvalue(Context) != Expr::LV_Valid)
+      if (!FirstE->isTypeDependent() && !FirstE->isLValue())
         return StmtError(Diag(First->getLocStart(),
                    diag::err_selector_element_not_lvalue)
           << First->getSourceRange());
@@ -917,7 +980,7 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
     else if (const ObjCObjectPointerType *OPT =
              SecondType->getAsObjCInterfacePointerType()) {
       llvm::SmallVector<IdentifierInfo *, 4> KeyIdents;
-      IdentifierInfo* selIdent = 
+      IdentifierInfo* selIdent =
         &Context.Idents.get("countByEnumeratingWithState");
       KeyIdents.push_back(selIdent);
       selIdent = &Context.Idents.get("objects");
@@ -926,7 +989,7 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
       KeyIdents.push_back(selIdent);
       Selector CSelector = Context.Selectors.getSelector(3, &KeyIdents[0]);
       if (ObjCInterfaceDecl *IDecl = OPT->getInterfaceDecl()) {
-        if (!IDecl->isForwardDecl() && 
+        if (!IDecl->isForwardDecl() &&
             !IDecl->lookupInstanceMethod(CSelector)) {
           // Must further look into private implementation methods.
           if (!LookupPrivateInstanceMethod(CSelector, IDecl))
@@ -940,19 +1003,12 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
                                                    ForLoc, RParenLoc));
 }
 
-StmtResult
-Sema::ActOnGotoStmt(SourceLocation GotoLoc, SourceLocation LabelLoc,
-                    IdentifierInfo *LabelII) {
-  // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[LabelII];
-
+StmtResult Sema::ActOnGotoStmt(SourceLocation GotoLoc,
+                               SourceLocation LabelLoc,
+                               LabelDecl *TheDecl) {
   getCurFunction()->setHasBranchIntoScope();
-
-  // If we haven't seen this label yet, create a forward reference.
-  if (LabelDecl == 0)
-    LabelDecl = new (Context) LabelStmt(LabelLoc, LabelII, 0);
-
-  return Owned(new (Context) GotoStmt(LabelDecl, GotoLoc, LabelLoc));
+  TheDecl->setUsed();
+  return Owned(new (Context) GotoStmt(TheDecl, GotoLoc, LabelLoc));
 }
 
 StmtResult
@@ -995,45 +1051,130 @@ Sema::ActOnBreakStmt(SourceLocation BreakLoc, Scope *CurScope) {
   return Owned(new (Context) BreakStmt(BreakLoc));
 }
 
-/// \brief Determine whether a return statement is a candidate for the named
-/// return value optimization (C++0x 12.8p34, bullet 1).
+/// \brief Determine whether the given expression is a candidate for
+/// copy elision in either a return statement or a throw expression.
 ///
-/// \param Ctx The context in which the return expression and type occur.
+/// \param ReturnType If we're determining the copy elision candidate for
+/// a return statement, this is the return type of the function. If we're
+/// determining the copy elision candidate for a throw expression, this will
+/// be a NULL type.
 ///
-/// \param RetType The return type of the function or block.
+/// \param E The expression being returned from the function or block, or
+/// being thrown.
 ///
-/// \param RetExpr The expression being returned from the function or block.
+/// \param AllowFunctionParameter
 ///
 /// \returns The NRVO candidate variable, if the return statement may use the
 /// NRVO, or NULL if there is no such candidate.
-static const VarDecl *getNRVOCandidate(ASTContext &Ctx, QualType RetType,
-                                       Expr *RetExpr) {
-  QualType ExprType = RetExpr->getType();
+const VarDecl *Sema::getCopyElisionCandidate(QualType ReturnType,
+                                             Expr *E,
+                                             bool AllowFunctionParameter) {
+  QualType ExprType = E->getType();
   // - in a return statement in a function with ...
   // ... a class return type ...
-  if (!RetType->isRecordType())
-    return 0;
-  // ... the same cv-unqualified type as the function return type ...
-  if (!Ctx.hasSameUnqualifiedType(RetType, ExprType))
-    return 0;
-  // ... the expression is the name of a non-volatile automatic object ...
-  // We ignore parentheses here.
-  // FIXME: Is this compliant? (Everyone else does it)
-  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(RetExpr->IgnoreParens());
+  if (!ReturnType.isNull()) {
+    if (!ReturnType->isRecordType())
+      return 0;
+    // ... the same cv-unqualified type as the function return type ...
+    if (!Context.hasSameUnqualifiedType(ReturnType, ExprType))
+      return 0;
+  }
+
+  // ... the expression is the name of a non-volatile automatic object
+  // (other than a function or catch-clause parameter)) ...
+  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E->IgnoreParens());
   if (!DR)
     return 0;
   const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
   if (!VD)
     return 0;
-  
-  if (VD->getKind() == Decl::Var && VD->hasLocalStorage() && 
+
+  if (VD->hasLocalStorage() && !VD->isExceptionVariable() &&
       !VD->getType()->isReferenceType() && !VD->hasAttr<BlocksAttr>() &&
-      !VD->getType().isVolatileQualified())
+      !VD->getType().isVolatileQualified() &&
+      ((VD->getKind() == Decl::Var) ||
+       (AllowFunctionParameter && VD->getKind() == Decl::ParmVar)))
     return VD;
-  
+
   return 0;
 }
 
+/// \brief Perform the initialization of a potentially-movable value, which
+/// is the result of return value.
+///
+/// This routine implements C++0x [class.copy]p33, which attempts to treat
+/// returned lvalues as rvalues in certain cases (to prefer move construction),
+/// then falls back to treating them as lvalues if that failed.
+ExprResult
+Sema::PerformMoveOrCopyInitialization(const InitializedEntity &Entity,
+                                      const VarDecl *NRVOCandidate,
+                                      QualType ResultType,
+                                      Expr *Value) {
+  // C++0x [class.copy]p33:
+  //   When the criteria for elision of a copy operation are met or would
+  //   be met save for the fact that the source object is a function
+  //   parameter, and the object to be copied is designated by an lvalue,
+  //   overload resolution to select the constructor for the copy is first
+  //   performed as if the object were designated by an rvalue.
+  ExprResult Res = ExprError();
+  if (NRVOCandidate || getCopyElisionCandidate(ResultType, Value, true)) {
+    ImplicitCastExpr AsRvalue(ImplicitCastExpr::OnStack,
+                              Value->getType(), CK_LValueToRValue,
+                              Value, VK_XValue);
+
+    Expr *InitExpr = &AsRvalue;
+    InitializationKind Kind
+      = InitializationKind::CreateCopy(Value->getLocStart(),
+                                       Value->getLocStart());
+    InitializationSequence Seq(*this, Entity, Kind, &InitExpr, 1);
+
+    //   [...] If overload resolution fails, or if the type of the first
+    //   parameter of the selected constructor is not an rvalue reference
+    //   to the object's type (possibly cv-qualified), overload resolution
+    //   is performed again, considering the object as an lvalue.
+    if (Seq.getKind() != InitializationSequence::FailedSequence) {
+      for (InitializationSequence::step_iterator Step = Seq.step_begin(),
+           StepEnd = Seq.step_end();
+           Step != StepEnd; ++Step) {
+        if (Step->Kind
+            != InitializationSequence::SK_ConstructorInitialization)
+          continue;
+
+        CXXConstructorDecl *Constructor
+        = cast<CXXConstructorDecl>(Step->Function.Function);
+
+        const RValueReferenceType *RRefType
+          = Constructor->getParamDecl(0)->getType()
+                                                 ->getAs<RValueReferenceType>();
+
+        // If we don't meet the criteria, break out now.
+        if (!RRefType ||
+            !Context.hasSameUnqualifiedType(RRefType->getPointeeType(),
+                            Context.getTypeDeclType(Constructor->getParent())))
+          break;
+
+        // Promote "AsRvalue" to the heap, since we now need this
+        // expression node to persist.
+        Value = ImplicitCastExpr::Create(Context, Value->getType(),
+                                         CK_LValueToRValue, Value, 0,
+                                         VK_XValue);
+
+        // Complete type-checking the initialization of the return type
+        // using the constructor we found.
+        Res = Seq.Perform(*this, Entity, Kind, MultiExprArg(&Value, 1));
+      }
+    }
+  }
+
+  // Either we didn't meet the criteria for treating an lvalue as an rvalue,
+  // above, or overload resolution failed. Either way, we need to try
+  // (again) now with the return value expression as written.
+  if (Res.isInvalid())
+    Res = PerformCopyInitialization(Entity, SourceLocation(), Value);
+
+  return Res;
+}
+
 /// ActOnBlockReturnStmt - Utility routine to figure out block's return type.
 ///
 StmtResult
@@ -1052,14 +1193,14 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
         // part of the implementation spec. and not the actual qualifier for
         // the variable.
         if (CDRE->isConstQualAdded())
-           CurBlock->ReturnType.removeConst();
+          CurBlock->ReturnType.removeLocalConst(); // FIXME: local???
       }
     } else
       CurBlock->ReturnType = Context.VoidTy;
   }
   QualType FnRetType = CurBlock->ReturnType;
 
-  if (CurBlock->TheDecl->hasAttr<NoReturnAttr>()) {
+  if (CurBlock->FunctionType->getAs<FunctionType>()->getNoReturnAttr()) {
     Diag(ReturnLoc, diag::err_noreturn_block_has_return_expr)
       << getCurFunctionOrMethodDecl()->getDeclName();
     return StmtError();
@@ -1079,7 +1220,7 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
     return StmtError(Diag(ReturnLoc, diag::err_block_return_missing_expr));
   } else {
     const VarDecl *NRVOCandidate = 0;
-    
+
     if (!FnRetType->isDependentType() && !RetValExp->isTypeDependent()) {
       // we have a non-void block with an expression, continue checking
 
@@ -1089,35 +1230,36 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
 
       // In C++ the return statement is handled via a copy initialization.
       // the C version of which boils down to CheckSingleAssignmentConstraints.
-      NRVOCandidate = getNRVOCandidate(Context, FnRetType, RetValExp);
-      ExprResult Res = PerformCopyInitialization(
-                               InitializedEntity::InitializeResult(ReturnLoc, 
-                                                                   FnRetType,
-                                                            NRVOCandidate != 0),
-                               SourceLocation(),
-                               Owned(RetValExp));
+      NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
+      InitializedEntity Entity = InitializedEntity::InitializeResult(ReturnLoc,
+                                                                     FnRetType,
+                                                           NRVOCandidate != 0);
+      ExprResult Res = PerformMoveOrCopyInitialization(Entity, NRVOCandidate,
+                                                       FnRetType, RetValExp);
       if (Res.isInvalid()) {
         // FIXME: Cleanup temporaries here, anyway?
         return StmtError();
       }
-      
-      if (RetValExp)
-        RetValExp = MaybeCreateCXXExprWithTemporaries(RetValExp);
+
+      if (RetValExp) {
+        CheckImplicitConversions(RetValExp, ReturnLoc);
+        RetValExp = MaybeCreateExprWithCleanups(RetValExp);
+      }
 
       RetValExp = Res.takeAs<Expr>();
-      if (RetValExp) 
+      if (RetValExp)
         CheckReturnStackAddr(RetValExp, FnRetType, ReturnLoc);
     }
-    
+
     Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, NRVOCandidate);
   }
 
-  // If we need to check for the named return value optimization, save the 
+  // If we need to check for the named return value optimization, save the
   // return statement in our scope for later processing.
   if (getLangOptions().CPlusPlus && FnRetType->isRecordType() &&
       !CurContext->isDependentContext())
     FunctionScopes.back()->Returns.push_back(Result);
-  
+
   return Owned(Result);
 }
 
@@ -1145,6 +1287,10 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
       unsigned D = diag::ext_return_has_expr;
       if (RetValExp->getType()->isVoidType())
         D = diag::ext_return_has_void_expr;
+      else {
+        IgnoredValueConversions(RetValExp);
+        ImpCastExprToType(RetValExp, Context.VoidTy, CK_ToVoid);
+      }
 
       // return (some void expression); is legal in C++.
       if (D != diag::ext_return_has_void_expr ||
@@ -1155,9 +1301,10 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
           << RetValExp->getSourceRange();
       }
 
-      RetValExp = MaybeCreateCXXExprWithTemporaries(RetValExp);
+      CheckImplicitConversions(RetValExp, ReturnLoc);
+      RetValExp = MaybeCreateExprWithCleanups(RetValExp);
     }
-    
+
     Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, 0);
   } else if (!RetValExp && !FnRetType->isDependentType()) {
     unsigned DiagID = diag::warn_return_missing_expr;  // C90 6.6.6.4p4
@@ -1180,34 +1327,35 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
 
       // In C++ the return statement is handled via a copy initialization.
       // the C version of which boils down to CheckSingleAssignmentConstraints.
-      NRVOCandidate = getNRVOCandidate(Context, FnRetType, RetValExp);
-      ExprResult Res = PerformCopyInitialization(
-                               InitializedEntity::InitializeResult(ReturnLoc, 
-                                                                   FnRetType,
-                                                            NRVOCandidate != 0),
-                               SourceLocation(),
-                               Owned(RetValExp));
+      NRVOCandidate = getCopyElisionCandidate(FnRetType, RetValExp, false);
+      InitializedEntity Entity = InitializedEntity::InitializeResult(ReturnLoc,
+                                                                     FnRetType,
+                                                                     NRVOCandidate != 0);
+      ExprResult Res = PerformMoveOrCopyInitialization(Entity, NRVOCandidate,
+                                                       FnRetType, RetValExp);
       if (Res.isInvalid()) {
         // FIXME: Cleanup temporaries here, anyway?
         return StmtError();
       }
 
       RetValExp = Res.takeAs<Expr>();
-      if (RetValExp) 
+      if (RetValExp)
         CheckReturnStackAddr(RetValExp, FnRetType, ReturnLoc);
     }
-    
-    if (RetValExp)
-      RetValExp = MaybeCreateCXXExprWithTemporaries(RetValExp);
+
+    if (RetValExp) {
+      CheckImplicitConversions(RetValExp, ReturnLoc);
+      RetValExp = MaybeCreateExprWithCleanups(RetValExp);
+    }
     Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, NRVOCandidate);
   }
-  
-  // If we need to check for the named return value optimization, save the 
+
+  // If we need to check for the named return value optimization, save the
   // return statement in our scope for later processing.
   if (getLangOptions().CPlusPlus && FnRetType->isRecordType() &&
       !CurContext->isDependentContext())
     FunctionScopes.back()->Returns.push_back(Result);
-  
+
   return Owned(Result);
 }
 
@@ -1222,14 +1370,14 @@ static bool CheckAsmLValue(const Expr *E, Sema &S) {
   // Type dependent expressions will be checked during instantiation.
   if (E->isTypeDependent())
     return false;
-  
-  if (E->isLvalue(S.Context) == Expr::LV_Valid)
+
+  if (E->isLValue())
     return false;  // Cool, this is an lvalue.
 
   // Okay, this is not an lvalue, but perhaps it is the result of a cast that we
   // are supposed to allow.
   const Expr *E2 = E->IgnoreParenNoopCasts(S.Context);
-  if (E != E2 && E2->isLvalue(S.Context) == Expr::LV_Valid) {
+  if (E != E2 && E2->isLValue()) {
     if (!S.getLangOptions().HeinousExtensions)
       S.Diag(E2->getLocStart(), diag::err_invalid_asm_cast_lvalue)
         << E->getSourceRange();
@@ -1358,8 +1506,8 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
   }
 
   AsmStmt *NS =
-    new (Context) AsmStmt(Context, AsmLoc, IsSimple, IsVolatile, MSAsm, 
-                          NumOutputs, NumInputs, Names, Constraints, Exprs, 
+    new (Context) AsmStmt(Context, AsmLoc, IsSimple, IsVolatile, MSAsm,
+                          NumOutputs, NumInputs, Names, Constraints, Exprs,
                           AsmString, NumClobbers, Clobbers, RParenLoc);
   // Validate the asm string, ensuring it makes sense given the operands we
   // have.
@@ -1393,7 +1541,7 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
     enum AsmDomain {
       AD_Int, AD_FP, AD_Other
     } InputDomain, OutputDomain;
-    
+
     if (InTy->isIntegerType() || InTy->isPointerType())
       InputDomain = AD_Int;
     else if (InTy->isRealFloatingType())
@@ -1407,19 +1555,19 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
       OutputDomain = AD_FP;
     else
       OutputDomain = AD_Other;
-    
+
     // They are ok if they are the same size and in the same domain.  This
     // allows tying things like:
     //   void* to int*
     //   void* to int            if they are the same size.
     //   double to long double   if they are the same size.
-    // 
+    //
     uint64_t OutSize = Context.getTypeSize(OutTy);
     uint64_t InSize = Context.getTypeSize(InTy);
     if (OutSize == InSize && InputDomain == OutputDomain &&
         InputDomain != AD_Other)
       continue;
-    
+
     // If the smaller input/output operand is not mentioned in the asm string,
     // then we can promote it and the asm string won't notice.  Check this
     // case now.
@@ -1471,7 +1619,7 @@ Sema::ActOnObjCAtCatchStmt(SourceLocation AtLoc,
   VarDecl *Var = cast_or_null<VarDecl>(Parm);
   if (Var && Var->isInvalidDecl())
     return StmtError();
-  
+
   return Owned(new (Context) ObjCAtCatchStmt(AtLoc, RParen, Var, Body));
 }
 
@@ -1481,8 +1629,11 @@ Sema::ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body) {
 }
 
 StmtResult
-Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try, 
+Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
                          MultiStmtArg CatchStmts, Stmt *Finally) {
+  if (!getLangOptions().ObjCExceptions)
+    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@try";
+
   getCurFunction()->setHasBranchProtectedScope();
   unsigned NumCatchStmts = CatchStmts.size();
   return Owned(ObjCAtTryStmt::Create(Context, AtLoc, Try,
@@ -1494,6 +1645,8 @@ Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
 StmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
                                                   Expr *Throw) {
   if (Throw) {
+    DefaultLvalueConversion(Throw);
+
     QualType ThrowType = Throw->getType();
     // Make sure the expression type is an ObjC pointer or "void *".
     if (!ThrowType->isDependentType() &&
@@ -1504,13 +1657,16 @@ StmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
                          << Throw->getType() << Throw->getSourceRange());
     }
   }
-  
+
   return Owned(new (Context) ObjCAtThrowStmt(AtLoc, Throw));
 }
 
 StmtResult
-Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw, 
+Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
                            Scope *CurScope) {
+  if (!getLangOptions().ObjCExceptions)
+    Diag(AtLoc, diag::err_objc_exceptions_disabled) << "@throw";
+
   if (!Throw) {
     // @throw without an expression designates a rethrow (which much occur
     // in the context of an @catch clause).
@@ -1519,8 +1675,8 @@ Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw,
       AtCatchParent = AtCatchParent->getParent();
     if (!AtCatchParent)
       return StmtError(Diag(AtLoc, diag::error_rethrow_used_outside_catch));
-  } 
-  
+  }
+
   return BuildObjCAtThrowStmt(AtLoc, Throw);
 }
 
@@ -1529,6 +1685,8 @@ Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SyncExpr,
                                   Stmt *SyncBody) {
   getCurFunction()->setHasBranchProtectedScope();
 
+  DefaultLvalueConversion(SyncExpr);
+
   // Make sure the expression type is an ObjC pointer or "void *".
   if (!SyncExpr->getType()->isDependentType() &&
       !SyncExpr->getType()->isObjCObjectPointerType()) {
@@ -1589,6 +1747,9 @@ public:
 StmtResult
 Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
                        MultiStmtArg RawHandlers) {
+  if (!getLangOptions().Exceptions)
+    Diag(TryLoc, diag::err_exceptions_disabled) << "try";
+
   unsigned NumHandlers = RawHandlers.size();
   assert(NumHandlers > 0 &&
          "The parser shouldn't call this if there are no handlers.");
diff --git a/lib/Sema/SemaTemplate.cpp b/lib/Sema/SemaTemplate.cpp
index 0fc83927b79c..f0a0103205d5 100644
--- a/lib/Sema/SemaTemplate.cpp
+++ b/lib/Sema/SemaTemplate.cpp
@@ -20,6 +20,8 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/TypeVisitor.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Basic/LangOptions.h"
@@ -28,6 +30,14 @@
 using namespace clang;
 using namespace sema;
 
+// Exported for use by Parser.
+SourceRange
+clang::getTemplateParamsRange(TemplateParameterList const * const *Ps,
+                              unsigned N) {
+  if (!N) return SourceRange();
+  return SourceRange(Ps[0]->getTemplateLoc(), Ps[N-1]->getRAngleLoc());
+}
+
 /// \brief Determine whether the declaration found is acceptable as the name
 /// of a template and, if so, return that template declaration. Otherwise,
 /// returns NULL.
@@ -78,12 +88,12 @@ static void FilterAcceptableTemplateNames(ASTContext &C, LookupResult &R) {
     else if (Repl != Orig) {
 
       // C++ [temp.local]p3:
-      //   A lookup that finds an injected-class-name (10.2) can result in an 
+      //   A lookup that finds an injected-class-name (10.2) can result in an
       //   ambiguity in certain cases (for example, if it is found in more than
-      //   one base class). If all of the injected-class-names that are found 
-      //   refer to specializations of the same class template, and if the name 
-      //   is followed by a template-argument-list, the reference refers to the 
-      //   class template itself and not a specialization thereof, and is not 
+      //   one base class). If all of the injected-class-names that are found
+      //   refer to specializations of the same class template, and if the name
+      //   is followed by a template-argument-list, the reference refers to the
+      //   class template itself and not a specialization thereof, and is not
       //   ambiguous.
       //
       // FIXME: Will we eventually have to do the same for alias templates?
@@ -116,12 +126,12 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
 
   DeclarationName TName;
   MemberOfUnknownSpecialization = false;
-  
+
   switch (Name.getKind()) {
   case UnqualifiedId::IK_Identifier:
     TName = DeclarationName(Name.Identifier);
     break;
-      
+
   case UnqualifiedId::IK_OperatorFunctionId:
     TName = Context.DeclarationNames.getCXXOperatorName(
                                               Name.OperatorFunctionId.Operator);
@@ -137,7 +147,7 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
 
   QualType ObjectType = ObjectTypePtr.get();
 
-  LookupResult R(*this, TName, Name.getSourceRange().getBegin(), 
+  LookupResult R(*this, TName, Name.getSourceRange().getBegin(),
                  LookupOrdinaryName);
   LookupTemplateName(R, S, SS, ObjectType, EnteringContext,
                      MemberOfUnknownSpecialization);
@@ -190,7 +200,7 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
   return TemplateKind;
 }
 
-bool Sema::DiagnoseUnknownTemplateName(const IdentifierInfo &II, 
+bool Sema::DiagnoseUnknownTemplateName(const IdentifierInfo &II,
                                        SourceLocation IILoc,
                                        Scope *S,
                                        const CXXScopeSpec *SS,
@@ -202,14 +212,14 @@ bool Sema::DiagnoseUnknownTemplateName(const IdentifierInfo &II,
   if (!SS || !SS->isSet() || !isDependentScopeSpecifier(*SS) ||
       computeDeclContext(*SS))
     return false;
-  
+
   // The code is missing a 'template' keyword prior to the dependent template
   // name.
   NestedNameSpecifier *Qualifier = (NestedNameSpecifier*)SS->getScopeRep();
   Diag(IILoc, diag::err_template_kw_missing)
     << Qualifier << II.getName()
     << FixItHint::CreateInsertion(IILoc, "template ");
-  SuggestedTemplate 
+  SuggestedTemplate
     = TemplateTy::make(Context.getDependentTemplateName(Qualifier, &II));
   SuggestedKind = TNK_Dependent_template_name;
   return true;
@@ -230,14 +240,14 @@ void Sema::LookupTemplateName(LookupResult &Found,
     assert(!SS.isSet() && "ObjectType and scope specifier cannot coexist");
     LookupCtx = computeDeclContext(ObjectType);
     isDependent = ObjectType->isDependentType();
-    assert((isDependent || !ObjectType->isIncompleteType()) && 
+    assert((isDependent || !ObjectType->isIncompleteType()) &&
            "Caller should have completed object type");
   } else if (SS.isSet()) {
     // This nested-name-specifier occurs after another nested-name-specifier,
     // so long into the context associated with the prior nested-name-specifier.
     LookupCtx = computeDeclContext(SS, EnteringContext);
     isDependent = isDependentScopeSpecifier(SS);
-    
+
     // The declaration context must be complete.
     if (LookupCtx && RequireCompleteDeclContext(SS, LookupCtx))
       return;
@@ -277,7 +287,7 @@ void Sema::LookupTemplateName(LookupResult &Found,
   if (Found.empty() && !isDependent) {
     // If we did not find any names, attempt to correct any typos.
     DeclarationName Name = Found.getLookupName();
-    if (DeclarationName Corrected = CorrectTypo(Found, S, &SS, LookupCtx, 
+    if (DeclarationName Corrected = CorrectTypo(Found, S, &SS, LookupCtx,
                                                 false, CTC_CXXCasts)) {
       FilterAcceptableTemplateNames(Context, Found);
       if (!Found.empty()) {
@@ -318,7 +328,7 @@ void Sema::LookupTemplateName(LookupResult &Found,
                             LookupOrdinaryName);
     LookupName(FoundOuter, S);
     FilterAcceptableTemplateNames(Context, FoundOuter);
-    
+
     if (FoundOuter.empty()) {
       //   - if the name is not found, the name found in the class of the
       //     object expression is used, otherwise
@@ -333,7 +343,7 @@ void Sema::LookupTemplateName(LookupResult &Found,
       if (!Found.isSingleResult() ||
           Found.getFoundDecl()->getCanonicalDecl()
             != FoundOuter.getFoundDecl()->getCanonicalDecl()) {
-        Diag(Found.getNameLoc(), 
+        Diag(Found.getNameLoc(),
              diag::ext_nested_name_member_ref_lookup_ambiguous)
           << Found.getLookupName()
           << ObjectType;
@@ -362,12 +372,12 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
     = static_cast<NestedNameSpecifier*>(SS.getScopeRep());
 
   DeclContext *DC = getFunctionLevelDeclContext();
-    
+
   if (!isAddressOfOperand &&
       isa<CXXMethodDecl>(DC) &&
       cast<CXXMethodDecl>(DC)->isInstance()) {
     QualType ThisType = cast<CXXMethodDecl>(DC)->getThisType(Context);
-    
+
     // Since the 'this' expression is synthesized, we don't need to
     // perform the double-lookup check.
     NamedDecl *FirstQualifierInScope = 0;
@@ -427,35 +437,52 @@ TemplateDecl *Sema::AdjustDeclIfTemplate(Decl *&D) {
   return 0;
 }
 
+ParsedTemplateArgument ParsedTemplateArgument::getTemplatePackExpansion(
+                                             SourceLocation EllipsisLoc) const {
+  assert(Kind == Template &&
+         "Only template template arguments can be pack expansions here");
+  assert(getAsTemplate().get().containsUnexpandedParameterPack() &&
+         "Template template argument pack expansion without packs");
+  ParsedTemplateArgument Result(*this);
+  Result.EllipsisLoc = EllipsisLoc;
+  return Result;
+}
+
 static TemplateArgumentLoc translateTemplateArgument(Sema &SemaRef,
                                             const ParsedTemplateArgument &Arg) {
-  
+
   switch (Arg.getKind()) {
   case ParsedTemplateArgument::Type: {
     TypeSourceInfo *DI;
     QualType T = SemaRef.GetTypeFromParser(Arg.getAsType(), &DI);
-    if (!DI) 
+    if (!DI)
       DI = SemaRef.Context.getTrivialTypeSourceInfo(T, Arg.getLocation());
     return TemplateArgumentLoc(TemplateArgument(T), DI);
   }
-    
+
   case ParsedTemplateArgument::NonType: {
     Expr *E = static_cast<Expr *>(Arg.getAsExpr());
     return TemplateArgumentLoc(TemplateArgument(E), E);
   }
-    
+
   case ParsedTemplateArgument::Template: {
     TemplateName Template = Arg.getAsTemplate().get();
-    return TemplateArgumentLoc(TemplateArgument(Template),
+    TemplateArgument TArg;
+    if (Arg.getEllipsisLoc().isValid())
+      TArg = TemplateArgument(Template, llvm::Optional<unsigned int>());
+    else
+      TArg = Template;
+    return TemplateArgumentLoc(TArg,
                                Arg.getScopeSpec().getRange(),
-                               Arg.getLocation());
+                               Arg.getLocation(),
+                               Arg.getEllipsisLoc());
   }
   }
-  
+
   llvm_unreachable("Unhandled parsed template argument");
   return TemplateArgumentLoc();
 }
-                                                     
+
 /// \brief Translates template arguments as provided by the parser
 /// into template arguments used by semantic analysis.
 void Sema::translateTemplateArguments(const ASTTemplateArgsPtr &TemplateArgsIn,
@@ -464,7 +491,7 @@ void Sema::translateTemplateArguments(const ASTTemplateArgsPtr &TemplateArgsIn,
    TemplateArgs.addArgument(translateTemplateArgument(*this,
                                                       TemplateArgsIn[I]));
 }
-                                                     
+
 /// ActOnTypeParameter - Called when a C++ template type parameter
 /// (e.g., "typename T") has been parsed. Typename specifies whether
 /// the keyword "typename" was used to declare the type parameter
@@ -512,30 +539,35 @@ Decl *Sema::ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
     IdResolver.AddDecl(Param);
   }
 
+  // C++0x [temp.param]p9:
+  //   A default template-argument may be specified for any kind of
+  //   template-parameter that is not a template parameter pack.
+  if (DefaultArg && Ellipsis) {
+    Diag(EqualLoc, diag::err_template_param_pack_default_arg);
+    DefaultArg = ParsedType();
+  }
+
   // Handle the default argument, if provided.
   if (DefaultArg) {
     TypeSourceInfo *DefaultTInfo;
     GetTypeFromParser(DefaultArg, &DefaultTInfo);
-    
+
     assert(DefaultTInfo && "expected source information for type");
-    
-    // C++0x [temp.param]p9:
-    // A default template-argument may be specified for any kind of
-    // template-parameter that is not a template parameter pack.
-    if (Ellipsis) {
-      Diag(EqualLoc, diag::err_template_param_pack_default_arg);
+
+    // Check for unexpanded parameter packs.
+    if (DiagnoseUnexpandedParameterPack(Loc, DefaultTInfo,
+                                        UPPC_DefaultArgument))
       return Param;
-    }
-    
+
     // Check the template argument itself.
     if (CheckTemplateArgument(Param, DefaultTInfo)) {
       Param->setInvalidDecl();
       return Param;
     }
-    
+
     Param->setDefaultArgument(DefaultTInfo, false);
   }
-  
+
   return Param;
 }
 
@@ -582,7 +614,7 @@ Sema::CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc) {
   else if (T->isFunctionType())
     // FIXME: Keep the type prior to promotion?
     return Context.getPointerType(T);
-  
+
   Diag(Loc, diag::err_template_nontype_parm_bad_type)
     << T;
 
@@ -617,10 +649,12 @@ Decl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
     Invalid = true;
   }
 
+  bool IsParameterPack = D.hasEllipsis();
   NonTypeTemplateParmDecl *Param
     = NonTypeTemplateParmDecl::Create(Context, Context.getTranslationUnitDecl(),
                                       D.getIdentifierLoc(),
-                                      Depth, Position, ParamName, T, TInfo);
+                                      Depth, Position, ParamName, T,
+                                      IsParameterPack, TInfo);
   if (Invalid)
     Param->setInvalidDecl();
 
@@ -629,18 +663,30 @@ Decl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
     S->AddDecl(Param);
     IdResolver.AddDecl(Param);
   }
-  
+
+  // C++0x [temp.param]p9:
+  //   A default template-argument may be specified for any kind of
+  //   template-parameter that is not a template parameter pack.
+  if (Default && IsParameterPack) {
+    Diag(EqualLoc, diag::err_template_param_pack_default_arg);
+    Default = 0;
+  }
+
   // Check the well-formedness of the default template argument, if provided.
-  if (Default) {  
+  if (Default) {
+    // Check for unexpanded parameter packs.
+    if (DiagnoseUnexpandedParameterPack(Default, UPPC_DefaultArgument))
+      return Param;
+
     TemplateArgument Converted;
     if (CheckTemplateArgument(Param, Param->getType(), Default, Converted)) {
       Param->setInvalidDecl();
       return Param;
     }
-  
+
     Param->setDefaultArgument(Default, false);
   }
-  
+
   return Param;
 }
 
@@ -650,29 +696,46 @@ Decl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
 Decl *Sema::ActOnTemplateTemplateParameter(Scope* S,
                                            SourceLocation TmpLoc,
                                            TemplateParamsTy *Params,
+                                           SourceLocation EllipsisLoc,
                                            IdentifierInfo *Name,
                                            SourceLocation NameLoc,
                                            unsigned Depth,
                                            unsigned Position,
                                            SourceLocation EqualLoc,
-                                       const ParsedTemplateArgument &Default) {
+                                           ParsedTemplateArgument Default) {
   assert(S->isTemplateParamScope() &&
          "Template template parameter not in template parameter scope!");
 
   // Construct the parameter object.
+  bool IsParameterPack = EllipsisLoc.isValid();
+  // FIXME: Pack-ness is dropped
   TemplateTemplateParmDecl *Param =
     TemplateTemplateParmDecl::Create(Context, Context.getTranslationUnitDecl(),
-                                     NameLoc.isInvalid()? TmpLoc : NameLoc, 
-                                     Depth, Position, Name,
-                                     (TemplateParameterList*)Params);
+                                     NameLoc.isInvalid()? TmpLoc : NameLoc,
+                                     Depth, Position, IsParameterPack,
+                                     Name, Params);
 
-  // If the template template parameter has a name, then link the identifier 
+  // If the template template parameter has a name, then link the identifier
   // into the scope and lookup mechanisms.
   if (Name) {
     S->AddDecl(Param);
     IdResolver.AddDecl(Param);
   }
 
+  if (Params->size() == 0) {
+    Diag(Param->getLocation(), diag::err_template_template_parm_no_parms)
+    << SourceRange(Params->getLAngleLoc(), Params->getRAngleLoc());
+    Param->setInvalidDecl();
+  }
+
+  // C++0x [temp.param]p9:
+  //   A default template-argument may be specified for any kind of
+  //   template-parameter that is not a template parameter pack.
+  if (IsParameterPack && !Default.isInvalid()) {
+    Diag(EqualLoc, diag::err_template_param_pack_default_arg);
+    Default = ParsedTemplateArgument();
+  }
+
   if (!Default.isInvalid()) {
     // Check only that we have a template template argument. We don't want to
     // try to check well-formedness now, because our template template parameter
@@ -688,10 +751,16 @@ Decl *Sema::ActOnTemplateTemplateParameter(Scope* S,
         << DefaultArg.getSourceRange();
       return Param;
     }
-    
+
+    // Check for unexpanded parameter packs.
+    if (DiagnoseUnexpandedParameterPack(DefaultArg.getLocation(),
+                                        DefaultArg.getArgument().getAsTemplate(),
+                                        UPPC_DefaultArgument))
+      return Param;
+
     Param->setDefaultArgument(DefaultArg, false);
   }
-  
+
   return Param;
 }
 
@@ -708,7 +777,7 @@ Sema::ActOnTemplateParameterList(unsigned Depth,
     Diag(ExportLoc, diag::warn_template_export_unsupported);
 
   return TemplateParameterList::Create(Context, TemplateLoc, LAngleLoc,
-                                       (NamedDecl**)Params, NumParams, 
+                                       (NamedDecl**)Params, NumParams,
                                        RAngleLoc);
 }
 
@@ -765,7 +834,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
 
   if (Previous.isAmbiguous())
     return true;
-  
+
   NamedDecl *PrevDecl = 0;
   if (Previous.begin() != Previous.end())
     PrevDecl = (*Previous.begin())->getUnderlyingDecl();
@@ -776,12 +845,12 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     = dyn_cast_or_null<ClassTemplateDecl>(PrevDecl);
 
   // We may have found the injected-class-name of a class template,
-  // class template partial specialization, or class template specialization. 
+  // class template partial specialization, or class template specialization.
   // In these cases, grab the template that is being defined or specialized.
-  if (!PrevClassTemplate && PrevDecl && isa<CXXRecordDecl>(PrevDecl) && 
+  if (!PrevClassTemplate && PrevDecl && isa<CXXRecordDecl>(PrevDecl) &&
       cast<CXXRecordDecl>(PrevDecl)->isInjectedClassName()) {
     PrevDecl = cast<CXXRecordDecl>(PrevDecl->getDeclContext());
-    PrevClassTemplate 
+    PrevClassTemplate
       = cast<CXXRecordDecl>(PrevDecl)->getDescribedClassTemplate();
     if (!PrevClassTemplate && isa<ClassTemplateSpecializationDecl>(PrevDecl)) {
       PrevClassTemplate
@@ -792,8 +861,8 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
 
   if (TUK == TUK_Friend) {
     // C++ [namespace.memdef]p3:
-    //   [...] When looking for a prior declaration of a class or a function 
-    //   declared as a friend, and when the name of the friend class or 
+    //   [...] When looking for a prior declaration of a class or a function
+    //   declared as a friend, and when the name of the friend class or
     //   function is neither a qualified name nor a template-id, scopes outside
     //   the innermost enclosing namespace scope are not considered.
     if (!SS.isSet()) {
@@ -807,7 +876,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
         SemanticContext = PrevDecl->getDeclContext();
       } else {
         // Declarations in outer scopes don't matter. However, the outermost
-        // context we computed is the semantic context for our new 
+        // context we computed is the semantic context for our new
         // declaration.
         PrevDecl = PrevClassTemplate = 0;
         SemanticContext = OutermostContext;
@@ -823,7 +892,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     }
   } else if (PrevDecl && !isDeclInScope(PrevDecl, SemanticContext, S))
     PrevDecl = PrevClassTemplate = 0;
-  
+
   if (PrevClassTemplate) {
     // Ensure that the template parameter lists are compatible.
     if (!TemplateParameterListsAreEqual(TemplateParams,
@@ -877,18 +946,22 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
   // template declaration.
   if (CheckTemplateParameterList(TemplateParams,
             PrevClassTemplate? PrevClassTemplate->getTemplateParameters() : 0,
-                                 TPC_ClassTemplate))
+                                 (SS.isSet() && SemanticContext &&
+                                  SemanticContext->isRecord() &&
+                                  SemanticContext->isDependentContext())
+                                   ? TPC_ClassTemplateMember
+                                   : TPC_ClassTemplate))
     Invalid = true;
 
   if (SS.isSet()) {
-    // If the name of the template was qualified, we must be defining the 
+    // If the name of the template was qualified, we must be defining the
     // template out-of-line.
     if (!SS.isInvalid() && !Invalid && !PrevClassTemplate &&
         !(TUK == TUK_Friend && CurContext->isDependentContext()))
       Diag(NameLoc, diag::err_member_def_does_not_match)
         << Name << SemanticContext << SS.getRange();
-  } 
-  
+  }
+
   CXXRecordDecl *NewClass =
     CXXRecordDecl::Create(Context, Kind, SemanticContext, NameLoc, Name, KWLoc,
                           PrevClassTemplate?
@@ -908,12 +981,12 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
   assert(T->isDependentType() && "Class template type is not dependent?");
   (void)T;
 
-  // If we are providing an explicit specialization of a member that is a 
+  // If we are providing an explicit specialization of a member that is a
   // class template, make a note of that.
-  if (PrevClassTemplate && 
+  if (PrevClassTemplate &&
       PrevClassTemplate->getInstantiatedFromMemberTemplate())
     PrevClassTemplate->setMemberSpecialization();
-  
+
   // Set the access specifier.
   if (!Invalid && TUK != TUK_Friend)
     SetMemberAccessSpecifier(NewTemplate, PrevClassTemplate, AS);
@@ -938,16 +1011,16 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
 
     NewTemplate->setObjectOfFriendDecl(/* PreviouslyDeclared = */
                                        PrevClassTemplate != NULL);
-    
+
     // Friend templates are visible in fairly strange ways.
     if (!CurContext->isDependentContext()) {
       DeclContext *DC = SemanticContext->getRedeclContext();
       DC->makeDeclVisibleInContext(NewTemplate, /* Recoverable = */ false);
       if (Scope *EnclosingScope = getScopeForDeclContext(S, DC))
         PushOnScopeChains(NewTemplate, EnclosingScope,
-                          /* AddToContext = */ false);      
+                          /* AddToContext = */ false);
     }
-    
+
     FriendDecl *Friend = FriendDecl::Create(Context, CurContext,
                                             NewClass->getLocation(),
                                             NewTemplate,
@@ -967,7 +1040,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
 /// template parameter, which is ill-formed in certain contexts.
 ///
 /// \returns true if the default template argument should be dropped.
-static bool DiagnoseDefaultTemplateArgument(Sema &S, 
+static bool DiagnoseDefaultTemplateArgument(Sema &S,
                                             Sema::TemplateParamListContext TPC,
                                             SourceLocation ParamLoc,
                                             SourceRange DefArgRange) {
@@ -976,14 +1049,18 @@ static bool DiagnoseDefaultTemplateArgument(Sema &S,
     return false;
 
   case Sema::TPC_FunctionTemplate:
-    // C++ [temp.param]p9: 
+  case Sema::TPC_FriendFunctionTemplateDefinition:
+    // C++ [temp.param]p9:
     //   A default template-argument shall not be specified in a
     //   function template declaration or a function template
     //   definition [...]
-    // (This sentence is not in C++0x, per DR226).
+    //   If a friend function template declaration specifies a default 
+    //   template-argument, that declaration shall be a definition and shall be
+    //   the only declaration of the function template in the translation unit.
+    // (C++98/03 doesn't have this wording; see DR226).
     if (!S.getLangOptions().CPlusPlus0x)
-      S.Diag(ParamLoc, 
-             diag::err_template_parameter_default_in_function_template)
+      S.Diag(ParamLoc,
+             diag::ext_template_parameter_default_in_function_template)
         << DefArgRange;
     return false;
 
@@ -1012,6 +1089,30 @@ static bool DiagnoseDefaultTemplateArgument(Sema &S,
   return false;
 }
 
+/// \brief Check for unexpanded parameter packs within the template parameters
+/// of a template template parameter, recursively.
+bool DiagnoseUnexpandedParameterPacks(Sema &S, TemplateTemplateParmDecl *TTP){
+  TemplateParameterList *Params = TTP->getTemplateParameters();
+  for (unsigned I = 0, N = Params->size(); I != N; ++I) {
+    NamedDecl *P = Params->getParam(I);
+    if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(P)) {
+      if (S.DiagnoseUnexpandedParameterPack(NTTP->getLocation(),
+                                            NTTP->getTypeSourceInfo(),
+                                      Sema::UPPC_NonTypeTemplateParameterType))
+        return true;
+
+      continue;
+    }
+
+    if (TemplateTemplateParmDecl *InnerTTP
+                                        = dyn_cast<TemplateTemplateParmDecl>(P))
+      if (DiagnoseUnexpandedParameterPacks(S, InnerTTP))
+        return true;
+  }
+
+  return false;
+}
+
 /// \brief Checks the validity of a template parameter list, possibly
 /// considering the template parameter list from a previous
 /// declaration.
@@ -1056,6 +1157,7 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
   if (OldParams)
     OldParam = OldParams->begin();
 
+  bool RemoveDefaultArguments = false;
   for (TemplateParameterList::iterator NewParam = NewParams->begin(),
                                     NewParamEnd = NewParams->end();
        NewParam != NewParamEnd; ++NewParam) {
@@ -1068,9 +1170,9 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
     bool MissingDefaultArg = false;
 
     // C++0x [temp.param]p11:
-    // If a template parameter of a class template is a template parameter pack,
-    // it must be the last template parameter.
-    if (SawParameterPack) {
+    //   If a template parameter of a primary class template is a template
+    //   parameter pack, it shall be the last template parameter.
+    if (SawParameterPack && TPC == TPC_ClassTemplate) {
       Diag(ParameterPackLoc,
            diag::err_template_param_pack_must_be_last_template_parameter);
       Invalid = true;
@@ -1079,9 +1181,9 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
     if (TemplateTypeParmDecl *NewTypeParm
           = dyn_cast<TemplateTypeParmDecl>(*NewParam)) {
       // Check the presence of a default argument here.
-      if (NewTypeParm->hasDefaultArgument() && 
-          DiagnoseDefaultTemplateArgument(*this, TPC, 
-                                          NewTypeParm->getLocation(), 
+      if (NewTypeParm->hasDefaultArgument() &&
+          DiagnoseDefaultTemplateArgument(*this, TPC,
+                                          NewTypeParm->getLocation(),
                NewTypeParm->getDefaultArgumentInfo()->getTypeLoc()
                                                        .getSourceRange()))
         NewTypeParm->removeDefaultArgument();
@@ -1116,10 +1218,18 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
         MissingDefaultArg = true;
     } else if (NonTypeTemplateParmDecl *NewNonTypeParm
                = dyn_cast<NonTypeTemplateParmDecl>(*NewParam)) {
+      // Check for unexpanded parameter packs.
+      if (DiagnoseUnexpandedParameterPack(NewNonTypeParm->getLocation(),
+                                          NewNonTypeParm->getTypeSourceInfo(),
+                                          UPPC_NonTypeTemplateParameterType)) {
+        Invalid = true;
+        continue;
+      }
+
       // Check the presence of a default argument here.
-      if (NewNonTypeParm->hasDefaultArgument() && 
-          DiagnoseDefaultTemplateArgument(*this, TPC, 
-                                          NewNonTypeParm->getLocation(), 
+      if (NewNonTypeParm->hasDefaultArgument() &&
+          DiagnoseDefaultTemplateArgument(*this, TPC,
+                                          NewNonTypeParm->getLocation(),
                     NewNonTypeParm->getDefaultArgument()->getSourceRange())) {
         NewNonTypeParm->removeDefaultArgument();
       }
@@ -1127,7 +1237,12 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       // Merge default arguments for non-type template parameters
       NonTypeTemplateParmDecl *OldNonTypeParm
         = OldParams? cast<NonTypeTemplateParmDecl>(*OldParam) : 0;
-      if (OldNonTypeParm && OldNonTypeParm->hasDefaultArgument() &&
+      if (NewNonTypeParm->isParameterPack()) {
+        assert(!NewNonTypeParm->hasDefaultArgument() &&
+               "Parameter packs can't have a default argument!");
+        SawParameterPack = true;
+        ParameterPackLoc = NewNonTypeParm->getLocation();
+      } else if (OldNonTypeParm && OldNonTypeParm->hasDefaultArgument() &&
           NewNonTypeParm->hasDefaultArgument()) {
         OldDefaultLoc = OldNonTypeParm->getDefaultArgumentLoc();
         NewDefaultLoc = NewNonTypeParm->getDefaultArgumentLoc();
@@ -1139,7 +1254,7 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
         // new declaration.
         SawDefaultArgument = true;
         // FIXME: We need to create a new kind of "default argument"
-        // expression that points to a previous template template
+        // expression that points to a previous non-type template
         // parameter.
         NewNonTypeParm->setDefaultArgument(
                                          OldNonTypeParm->getDefaultArgument(),
@@ -1154,16 +1269,28 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       // Check the presence of a default argument here.
       TemplateTemplateParmDecl *NewTemplateParm
         = cast<TemplateTemplateParmDecl>(*NewParam);
-      if (NewTemplateParm->hasDefaultArgument() && 
-          DiagnoseDefaultTemplateArgument(*this, TPC, 
-                                          NewTemplateParm->getLocation(), 
+
+      // Check for unexpanded parameter packs, recursively.
+      if (DiagnoseUnexpandedParameterPacks(*this, NewTemplateParm)) {
+        Invalid = true;
+        continue;
+      }
+
+      if (NewTemplateParm->hasDefaultArgument() &&
+          DiagnoseDefaultTemplateArgument(*this, TPC,
+                                          NewTemplateParm->getLocation(),
                      NewTemplateParm->getDefaultArgument().getSourceRange()))
         NewTemplateParm->removeDefaultArgument();
 
       // Merge default arguments for template template parameters
       TemplateTemplateParmDecl *OldTemplateParm
         = OldParams? cast<TemplateTemplateParmDecl>(*OldParam) : 0;
-      if (OldTemplateParm && OldTemplateParm->hasDefaultArgument() &&
+      if (NewTemplateParm->isParameterPack()) {
+        assert(!NewTemplateParm->hasDefaultArgument() &&
+               "Parameter packs can't have a default argument!");
+        SawParameterPack = true;
+        ParameterPackLoc = NewTemplateParm->getLocation();
+      } else if (OldTemplateParm && OldTemplateParm->hasDefaultArgument() &&
           NewTemplateParm->hasDefaultArgument()) {
         OldDefaultLoc = OldTemplateParm->getDefaultArgument().getLocation();
         NewDefaultLoc = NewTemplateParm->getDefaultArgument().getLocation();
@@ -1196,15 +1323,17 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       Diag(NewDefaultLoc, diag::err_template_param_default_arg_redefinition);
       Diag(OldDefaultLoc, diag::note_template_param_prev_default_arg);
       Invalid = true;
-    } else if (MissingDefaultArg) {
+    } else if (MissingDefaultArg && TPC != TPC_FunctionTemplate) {
       // C++ [temp.param]p11:
-      //   If a template-parameter has a default template-argument,
-      //   all subsequent template-parameters shall have a default
-      //   template-argument supplied.
+      //   If a template-parameter of a class template has a default
+      //   template-argument, each subsequent template-parameter shall either
+      //   have a default template-argument supplied or be a template parameter
+      //   pack.
       Diag((*NewParam)->getLocation(),
            diag::err_template_param_default_arg_missing);
       Diag(PreviousDefaultArgLoc, diag::note_template_param_prev_default_arg);
       Invalid = true;
+      RemoveDefaultArguments = true;
     }
 
     // If we have an old template parameter list that we're merging
@@ -1213,9 +1342,89 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       ++OldParam;
   }
 
+  // We were missing some default arguments at the end of the list, so remove
+  // all of the default arguments.
+  if (RemoveDefaultArguments) {
+    for (TemplateParameterList::iterator NewParam = NewParams->begin(),
+                                      NewParamEnd = NewParams->end();
+         NewParam != NewParamEnd; ++NewParam) {
+      if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*NewParam))
+        TTP->removeDefaultArgument();
+      else if (NonTypeTemplateParmDecl *NTTP
+                                = dyn_cast<NonTypeTemplateParmDecl>(*NewParam))
+        NTTP->removeDefaultArgument();
+      else
+        cast<TemplateTemplateParmDecl>(*NewParam)->removeDefaultArgument();
+    }
+  }
+
   return Invalid;
 }
 
+namespace {
+
+/// A class which looks for a use of a certain level of template
+/// parameter.
+struct DependencyChecker : RecursiveASTVisitor<DependencyChecker> {
+  typedef RecursiveASTVisitor<DependencyChecker> super;
+
+  unsigned Depth;
+  bool Match;
+
+  DependencyChecker(TemplateParameterList *Params) : Match(false) {
+    NamedDecl *ND = Params->getParam(0);
+    if (TemplateTypeParmDecl *PD = dyn_cast<TemplateTypeParmDecl>(ND)) {
+      Depth = PD->getDepth();
+    } else if (NonTypeTemplateParmDecl *PD =
+                 dyn_cast<NonTypeTemplateParmDecl>(ND)) {
+      Depth = PD->getDepth();
+    } else {
+      Depth = cast<TemplateTemplateParmDecl>(ND)->getDepth();
+    }
+  }
+
+  bool Matches(unsigned ParmDepth) {
+    if (ParmDepth >= Depth) {
+      Match = true;
+      return true;
+    }
+    return false;
+  }
+
+  bool VisitTemplateTypeParmType(const TemplateTypeParmType *T) {
+    return !Matches(T->getDepth());
+  }
+
+  bool TraverseTemplateName(TemplateName N) {
+    if (TemplateTemplateParmDecl *PD =
+          dyn_cast_or_null<TemplateTemplateParmDecl>(N.getAsTemplateDecl()))
+      if (Matches(PD->getDepth())) return false;
+    return super::TraverseTemplateName(N);
+  }
+
+  bool VisitDeclRefExpr(DeclRefExpr *E) {
+    if (NonTypeTemplateParmDecl *PD =
+          dyn_cast<NonTypeTemplateParmDecl>(E->getDecl())) {
+      if (PD->getDepth() == Depth) {
+        Match = true;
+        return false;
+      }
+    }
+    return super::VisitDeclRefExpr(E);
+  }
+};
+}
+
+/// Determines whether a template-id depends on the given parameter
+/// list.
+static bool
+DependsOnTemplateParameters(const TemplateSpecializationType *TemplateId,
+                            TemplateParameterList *Params) {
+  DependencyChecker Checker(Params);
+  Checker.TraverseType(QualType(TemplateId, 0));
+  return Checker.Match;
+}
+
 /// \brief Match the given template parameter lists to the given scope
 /// specifier, returning the template parameter list that applies to the
 /// name.
@@ -1254,7 +1463,7 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
                                               bool &IsExplicitSpecialization,
                                               bool &Invalid) {
   IsExplicitSpecialization = false;
-  
+
   // Find the template-ids that occur within the nested-name-specifier. These
   // template-ids will match up with the template parameter lists.
   llvm::SmallVector<const TemplateSpecializationType *, 4>
@@ -1275,8 +1484,8 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
     //
     // Following the existing practice of GNU and EDG, we allow a typedef of a
     // template specialization type.
-    if (const TypedefType *TT = dyn_cast<TypedefType>(T))
-      T = TT->LookThroughTypedefs().getTypePtr();
+    while (const TypedefType *TT = dyn_cast<TypedefType>(T))
+      T = TT->getDecl()->getUnderlyingType().getTypePtr();
 
     if (const TemplateSpecializationType *SpecType
                                   = dyn_cast<TemplateSpecializationType>(T)) {
@@ -1309,12 +1518,24 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
 
   // Match the template-ids found in the specifier to the template parameter
   // lists.
-  unsigned Idx = 0;
+  unsigned ParamIdx = 0, TemplateIdx = 0;
   for (unsigned NumTemplateIds = TemplateIdsInSpecifier.size();
-       Idx != NumTemplateIds; ++Idx) {
-    QualType TemplateId = QualType(TemplateIdsInSpecifier[Idx], 0);
+       TemplateIdx != NumTemplateIds; ++TemplateIdx) {
+    const TemplateSpecializationType *TemplateId
+      = TemplateIdsInSpecifier[TemplateIdx];
     bool DependentTemplateId = TemplateId->isDependentType();
-    if (Idx >= NumParamLists) {
+
+    // In friend declarations we can have template-ids which don't
+    // depend on the corresponding template parameter lists.  But
+    // assume that empty parameter lists are supposed to match this
+    // template-id.
+    if (IsFriend && ParamIdx < NumParamLists && ParamLists[ParamIdx]->size()) {
+      if (!DependentTemplateId ||
+          !DependsOnTemplateParameters(TemplateId, ParamLists[ParamIdx]))
+        continue;
+    }
+
+    if (ParamIdx >= NumParamLists) {
       // We have a template-id without a corresponding template parameter
       // list.
 
@@ -1328,7 +1549,7 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
         // FIXME: the location information here isn't great.
         Diag(SS.getRange().getBegin(),
              diag::err_template_spec_needs_template_parameters)
-          << TemplateId
+          << QualType(TemplateId, 0)
           << SS.getRange();
         Invalid = true;
       } else {
@@ -1357,35 +1578,38 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
       }
 
       if (ExpectedTemplateParams)
-        TemplateParameterListsAreEqual(ParamLists[Idx],
+        TemplateParameterListsAreEqual(ParamLists[ParamIdx],
                                        ExpectedTemplateParams,
                                        true, TPL_TemplateMatch);
 
-      CheckTemplateParameterList(ParamLists[Idx], 0, TPC_ClassTemplateMember);
-    } else if (ParamLists[Idx]->size() > 0)
-      Diag(ParamLists[Idx]->getTemplateLoc(),
+      CheckTemplateParameterList(ParamLists[ParamIdx], 0,
+                                 TPC_ClassTemplateMember);
+    } else if (ParamLists[ParamIdx]->size() > 0)
+      Diag(ParamLists[ParamIdx]->getTemplateLoc(),
            diag::err_template_param_list_matches_nontemplate)
         << TemplateId
-        << ParamLists[Idx]->getSourceRange();
+        << ParamLists[ParamIdx]->getSourceRange();
     else
       IsExplicitSpecialization = true;
+
+    ++ParamIdx;
   }
 
   // If there were at least as many template-ids as there were template
   // parameter lists, then there are no template parameter lists remaining for
   // the declaration itself.
-  if (Idx >= NumParamLists)
+  if (ParamIdx >= NumParamLists)
     return 0;
 
   // If there were too many template parameter lists, complain about that now.
-  if (Idx != NumParamLists - 1) {
-    while (Idx < NumParamLists - 1) {
-      bool isExplicitSpecHeader = ParamLists[Idx]->size() == 0;
-      Diag(ParamLists[Idx]->getTemplateLoc(),
+  if (ParamIdx != NumParamLists - 1) {
+    while (ParamIdx < NumParamLists - 1) {
+      bool isExplicitSpecHeader = ParamLists[ParamIdx]->size() == 0;
+      Diag(ParamLists[ParamIdx]->getTemplateLoc(),
            isExplicitSpecHeader? diag::warn_template_spec_extra_headers
                                : diag::err_template_spec_extra_headers)
-        << SourceRange(ParamLists[Idx]->getTemplateLoc(),
-                       ParamLists[Idx]->getRAngleLoc());
+        << SourceRange(ParamLists[ParamIdx]->getTemplateLoc(),
+                       ParamLists[ParamIdx]->getRAngleLoc());
 
       if (isExplicitSpecHeader && !ExplicitSpecializationsInSpecifier.empty()) {
         Diag(ExplicitSpecializationsInSpecifier.back()->getLocation(),
@@ -1394,13 +1618,13 @@ Sema::MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
         ExplicitSpecializationsInSpecifier.pop_back();
       }
 
-      // We have a template parameter list with no corresponding scope, which 
+      // We have a template parameter list with no corresponding scope, which
       // means that the resulting template declaration can't be instantiated
       // properly (we'll end up with dependent nodes when we shouldn't).
       if (!isExplicitSpecHeader)
         Invalid = true;
-      
-      ++Idx;
+
+      ++ParamIdx;
     }
   }
 
@@ -1421,14 +1645,12 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
 
   // Check that the template argument list is well-formed for this
   // template.
-  TemplateArgumentListBuilder Converted(Template->getTemplateParameters(),
-                                        TemplateArgs.size());
+  llvm::SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(Template, TemplateLoc, TemplateArgs,
                                 false, Converted))
     return QualType();
 
-  assert((Converted.structuredSize() ==
-            Template->getTemplateParameters()->size()) &&
+  assert((Converted.size() == Template->getTemplateParameters()->size()) &&
          "Converted template argument list is too short!");
 
   QualType CanonType;
@@ -1445,8 +1667,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     //   template<typename T, typename U = T> struct A;
     TemplateName CanonName = Context.getCanonicalTemplateName(Name);
     CanonType = Context.getTemplateSpecializationType(CanonName,
-                                                   Converted.getFlatArguments(),
-                                                   Converted.flatSize());
+                                                      Converted.data(),
+                                                      Converted.size());
 
     // FIXME: CanonType is not actually the canonical type, and unfortunately
     // it is a TemplateSpecializationType that we will never use again.
@@ -1474,7 +1696,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
         if (!isa<ClassTemplatePartialSpecializationDecl>(Record) &&
             !Record->getDescribedClassTemplate())
           continue;
-          
+
         // Fetch the injected class name type and check whether its
         // injected type is equal to the type we just built.
         QualType ICNT = Context.getTypeDeclType(Record);
@@ -1497,8 +1719,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     // corresponds to these arguments.
     void *InsertPos = 0;
     ClassTemplateSpecializationDecl *Decl
-      = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
-                                          Converted.flatSize(), InsertPos);
+      = ClassTemplate->findSpecialization(Converted.data(), Converted.size(),
+                                          InsertPos);
     if (!Decl) {
       // This is the first time we have referenced this class template
       // specialization. Create the canonical declaration and add it to
@@ -1507,8 +1729,9 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
                             ClassTemplate->getTemplatedDecl()->getTagKind(),
                                                 ClassTemplate->getDeclContext(),
                                                 ClassTemplate->getLocation(),
-                                                ClassTemplate,
-                                                Converted, 0);
+                                                     ClassTemplate,
+                                                     Converted.data(),
+                                                     Converted.size(), 0);
       ClassTemplate->AddSpecialization(Decl, InsertPos);
       Decl->setLexicalDeclContext(CurContext);
     }
@@ -1553,14 +1776,14 @@ Sema::ActOnTemplateIdType(TemplateTy TemplateD, SourceLocation TemplateLoc,
   return CreateParsedType(Result, DI);
 }
 
-TypeResult Sema::ActOnTagTemplateIdType(TypeResult TypeResult,
+TypeResult Sema::ActOnTagTemplateIdType(CXXScopeSpec &SS,
+                                        TypeResult TypeResult,
                                         TagUseKind TUK,
                                         TypeSpecifierType TagSpec,
                                         SourceLocation TagLoc) {
   if (TypeResult.isInvalid())
     return ::TypeResult();
 
-  // FIXME: preserve source info, ideally without copying the DI.
   TypeSourceInfo *DI;
   QualType Type = GetTypeFromParser(TypeResult.get(), &DI);
 
@@ -1585,7 +1808,12 @@ TypeResult Sema::ActOnTagTemplateIdType(TypeResult TypeResult,
     = TypeWithKeyword::getKeywordForTagTypeKind(TagKind);
   QualType ElabType = Context.getElaboratedType(Keyword, /*NNS=*/0, Type);
 
-  return ParsedType::make(ElabType);
+  TypeSourceInfo *ElabDI = Context.CreateTypeSourceInfo(ElabType);
+  ElaboratedTypeLoc TL = cast<ElaboratedTypeLoc>(ElabDI->getTypeLoc());
+  TL.setKeywordLoc(TagLoc);
+  TL.setQualifierRange(SS.getRange());
+  TL.getNamedTypeLoc().initializeFullCopy(DI->getTypeLoc());
+  return CreateParsedType(ElabType, ElabDI);
 }
 
 ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
@@ -1596,6 +1824,11 @@ ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
   // template arguments that we have against the template name, if the template
   // name refers to a single template. That's not a terribly common case,
   // though.
+  // foo<int> could identify a single function unambiguously
+  // This approach does NOT work, since f<int>(1);
+  // gets resolved prior to resorting to overload resolution
+  // i.e., template<class T> void f(double);
+  //       vs template<class T, class U> void f(U);
 
   // These should be filtered out by our callers.
   assert(!R.empty() && "empty lookup results when building templateid");
@@ -1610,15 +1843,12 @@ ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
 
   // We don't want lookup warnings at this point.
   R.suppressDiagnostics();
-  
-  bool Dependent
-    = UnresolvedLookupExpr::ComputeDependence(R.begin(), R.end(),
-                                              &TemplateArgs);
+
   UnresolvedLookupExpr *ULE
-    = UnresolvedLookupExpr::Create(Context, Dependent, R.getNamingClass(),
+    = UnresolvedLookupExpr::Create(Context, R.getNamingClass(),
                                    Qualifier, QualifierRange,
                                    R.getLookupNameInfo(),
-                                   RequiresADL, TemplateArgs, 
+                                   RequiresADL, TemplateArgs,
                                    R.begin(), R.end());
 
   return Owned(ULE);
@@ -1642,7 +1872,7 @@ Sema::BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
 
   if (R.isAmbiguous())
     return ExprError();
-  
+
   if (R.empty()) {
     Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_non_template)
       << NameInfo.getName() << SS.getRange();
@@ -1667,7 +1897,7 @@ Sema::BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
 /// example, given "MetaFun::template apply", the scope specifier \p
 /// SS will be "MetaFun::", \p TemplateKWLoc contains the location
 /// of the "template" keyword, and "apply" is the \p Name.
-TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S, 
+TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
                                                   SourceLocation TemplateKWLoc,
                                                   CXXScopeSpec &SS,
                                                   UnqualifiedId &Name,
@@ -1677,8 +1907,8 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
   if (TemplateKWLoc.isValid() && S && !S->getTemplateParamParent() &&
       !getLangOptions().CPlusPlus0x)
     Diag(TemplateKWLoc, diag::ext_template_outside_of_template)
-      << FixItHint::CreateRemoval(TemplateKWLoc);    
-  
+      << FixItHint::CreateRemoval(TemplateKWLoc);
+
   DeclContext *LookupCtx = 0;
   if (SS.isSet())
     LookupCtx = computeDeclContext(SS, EnteringContext);
@@ -1710,7 +1940,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
         cast<CXXRecordDecl>(LookupCtx)->hasAnyDependentBases()) {
       // This is a dependent template. Handle it below.
     } else if (TNK == TNK_Non_template) {
-      Diag(Name.getSourceRange().getBegin(), 
+      Diag(Name.getSourceRange().getBegin(),
            diag::err_template_kw_refers_to_non_template)
         << GetNameFromUnqualifiedId(Name).getName()
         << Name.getSourceRange()
@@ -1724,13 +1954,13 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
 
   NestedNameSpecifier *Qualifier
     = static_cast<NestedNameSpecifier *>(SS.getScopeRep());
-  
+
   switch (Name.getKind()) {
   case UnqualifiedId::IK_Identifier:
-    Result = TemplateTy::make(Context.getDependentTemplateName(Qualifier, 
+    Result = TemplateTy::make(Context.getDependentTemplateName(Qualifier,
                                                               Name.Identifier));
     return TNK_Dependent_template_name;
-    
+
   case UnqualifiedId::IK_OperatorFunctionId:
     Result = TemplateTy::make(Context.getDependentTemplateName(Qualifier,
                                              Name.OperatorFunctionId.Operator));
@@ -1742,8 +1972,8 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
   default:
     break;
   }
-  
-  Diag(Name.getSourceRange().getBegin(), 
+
+  Diag(Name.getSourceRange().getBegin(),
        diag::err_template_kw_refers_to_non_template)
     << GetNameFromUnqualifiedId(Name).getName()
     << Name.getSourceRange()
@@ -1753,7 +1983,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
 
 bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
                                      const TemplateArgumentLoc &AL,
-                                     TemplateArgumentListBuilder &Converted) {
+                          llvm::SmallVectorImpl<TemplateArgument> &Converted) {
   const TemplateArgument &Arg = AL.getArgument();
 
   // Check template type parameter.
@@ -1790,7 +2020,7 @@ bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
     return true;
 
   // Add the converted template type argument.
-  Converted.Append(
+  Converted.push_back(
                  TemplateArgument(Context.getCanonicalType(Arg.getAsType())));
   return false;
 }
@@ -1801,7 +2031,7 @@ bool Sema::CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
 /// \param SemaRef the semantic analysis object for which we are performing
 /// the substitution.
 ///
-/// \param Template the template that we are synthesizing template arguments 
+/// \param Template the template that we are synthesizing template arguments
 /// for.
 ///
 /// \param TemplateLoc the location of the template name that started the
@@ -1823,23 +2053,23 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              TemplateTypeParmDecl *Param,
-                             TemplateArgumentListBuilder &Converted) {
+                         llvm::SmallVectorImpl<TemplateArgument> &Converted) {
   TypeSourceInfo *ArgType = Param->getDefaultArgumentInfo();
 
   // If the argument type is dependent, instantiate it now based
   // on the previously-computed template arguments.
   if (ArgType->getType()->isDependentType()) {
-    TemplateArgumentList TemplateArgs(SemaRef.Context, Converted,
-                                      /*TakeArgs=*/false);
-    
+    TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                      Converted.data(), Converted.size());
+
     MultiLevelTemplateArgumentList AllTemplateArgs
       = SemaRef.getTemplateInstantiationArgs(Template, &TemplateArgs);
 
     Sema::InstantiatingTemplate Inst(SemaRef, TemplateLoc,
-                                     Template, Converted.getFlatArguments(),
-                                     Converted.flatSize(),
+                                     Template, Converted.data(),
+                                     Converted.size(),
                                      SourceRange(TemplateLoc, RAngleLoc));
-    
+
     ArgType = SemaRef.SubstType(ArgType, AllTemplateArgs,
                                 Param->getDefaultArgumentLoc(),
                                 Param->getDeclName());
@@ -1854,7 +2084,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
 /// \param SemaRef the semantic analysis object for which we are performing
 /// the substitution.
 ///
-/// \param Template the template that we are synthesizing template arguments 
+/// \param Template the template that we are synthesizing template arguments
 /// for.
 ///
 /// \param TemplateLoc the location of the template name that started the
@@ -1876,16 +2106,16 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              NonTypeTemplateParmDecl *Param,
-                             TemplateArgumentListBuilder &Converted) {
-  TemplateArgumentList TemplateArgs(SemaRef.Context, Converted,
-                                    /*TakeArgs=*/false);
-    
+                        llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+  TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                    Converted.data(), Converted.size());
+
   MultiLevelTemplateArgumentList AllTemplateArgs
     = SemaRef.getTemplateInstantiationArgs(Template, &TemplateArgs);
-    
+
   Sema::InstantiatingTemplate Inst(SemaRef, TemplateLoc,
-                                   Template, Converted.getFlatArguments(),
-                                   Converted.flatSize(),
+                                   Template, Converted.data(),
+                                   Converted.size(),
                                    SourceRange(TemplateLoc, RAngleLoc));
 
   return SemaRef.SubstExpr(Param->getDefaultArgument(), AllTemplateArgs);
@@ -1897,7 +2127,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
 /// \param SemaRef the semantic analysis object for which we are performing
 /// the substitution.
 ///
-/// \param Template the template that we are synthesizing template arguments 
+/// \param Template the template that we are synthesizing template arguments
 /// for.
 ///
 /// \param TemplateLoc the location of the template name that started the
@@ -1919,34 +2149,34 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
                              SourceLocation TemplateLoc,
                              SourceLocation RAngleLoc,
                              TemplateTemplateParmDecl *Param,
-                             TemplateArgumentListBuilder &Converted) {
-  TemplateArgumentList TemplateArgs(SemaRef.Context, Converted,
-                                    /*TakeArgs=*/false);
-  
+                       llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+  TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                    Converted.data(), Converted.size());
+
   MultiLevelTemplateArgumentList AllTemplateArgs
     = SemaRef.getTemplateInstantiationArgs(Template, &TemplateArgs);
-  
+
   Sema::InstantiatingTemplate Inst(SemaRef, TemplateLoc,
-                                   Template, Converted.getFlatArguments(),
-                                   Converted.flatSize(),
+                                   Template, Converted.data(),
+                                   Converted.size(),
                                    SourceRange(TemplateLoc, RAngleLoc));
-  
+
   return SemaRef.SubstTemplateName(
                       Param->getDefaultArgument().getArgument().getAsTemplate(),
-                              Param->getDefaultArgument().getTemplateNameLoc(), 
+                              Param->getDefaultArgument().getTemplateNameLoc(),
                                    AllTemplateArgs);
 }
 
 /// \brief If the given template parameter has a default template
 /// argument, substitute into that default template argument and
 /// return the corresponding template argument.
-TemplateArgumentLoc 
+TemplateArgumentLoc
 Sema::SubstDefaultTemplateArgumentIfAvailable(TemplateDecl *Template,
                                               SourceLocation TemplateLoc,
                                               SourceLocation RAngleLoc,
                                               Decl *Param,
-                                     TemplateArgumentListBuilder &Converted) {
-  if (TemplateTypeParmDecl *TypeParm = dyn_cast<TemplateTypeParmDecl>(Param)) {
+                      llvm::SmallVectorImpl<TemplateArgument> &Converted) {
+   if (TemplateTypeParmDecl *TypeParm = dyn_cast<TemplateTypeParmDecl>(Param)) {
     if (!TypeParm->hasDefaultArgument())
       return TemplateArgumentLoc();
 
@@ -1984,45 +2214,75 @@ Sema::SubstDefaultTemplateArgumentIfAvailable(TemplateDecl *Template,
     return TemplateArgumentLoc();
 
   TemplateName TName = SubstDefaultTemplateArgument(*this, Template,
-                                                    TemplateLoc, 
+                                                    TemplateLoc,
                                                     RAngleLoc,
                                                     TempTempParm,
                                                     Converted);
   if (TName.isNull())
     return TemplateArgumentLoc();
 
-  return TemplateArgumentLoc(TemplateArgument(TName), 
+  return TemplateArgumentLoc(TemplateArgument(TName),
                 TempTempParm->getDefaultArgument().getTemplateQualifierRange(),
                 TempTempParm->getDefaultArgument().getTemplateNameLoc());
 }
 
 /// \brief Check that the given template argument corresponds to the given
 /// template parameter.
+///
+/// \param Param The template parameter against which the argument will be
+/// checked.
+///
+/// \param Arg The template argument.
+///
+/// \param Template The template in which the template argument resides.
+///
+/// \param TemplateLoc The location of the template name for the template
+/// whose argument list we're matching.
+///
+/// \param RAngleLoc The location of the right angle bracket ('>') that closes
+/// the template argument list.
+///
+/// \param ArgumentPackIndex The index into the argument pack where this
+/// argument will be placed. Only valid if the parameter is a parameter pack.
+///
+/// \param Converted The checked, converted argument will be added to the
+/// end of this small vector.
+///
+/// \param CTAK Describes how we arrived at this particular template argument:
+/// explicitly written, deduced, etc.
+///
+/// \returns true on error, false otherwise.
 bool Sema::CheckTemplateArgument(NamedDecl *Param,
                                  const TemplateArgumentLoc &Arg,
-                                 TemplateDecl *Template,
+                                 NamedDecl *Template,
                                  SourceLocation TemplateLoc,
                                  SourceLocation RAngleLoc,
-                                 TemplateArgumentListBuilder &Converted,
+                                 unsigned ArgumentPackIndex,
+                            llvm::SmallVectorImpl<TemplateArgument> &Converted,
                                  CheckTemplateArgumentKind CTAK) {
   // Check template type parameters.
   if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
     return CheckTemplateTypeArgument(TTP, Arg, Converted);
-  
+
   // Check non-type template parameters.
-  if (NonTypeTemplateParmDecl *NTTP =dyn_cast<NonTypeTemplateParmDecl>(Param)) {    
+  if (NonTypeTemplateParmDecl *NTTP =dyn_cast<NonTypeTemplateParmDecl>(Param)) {
     // Do substitution on the type of the non-type template parameter
-    // with the template arguments we've seen thus far.
+    // with the template arguments we've seen thus far.  But if the
+    // template has a dependent context then we cannot substitute yet.
     QualType NTTPType = NTTP->getType();
-    if (NTTPType->isDependentType()) {
+    if (NTTP->isParameterPack() && NTTP->isExpandedParameterPack())
+      NTTPType = NTTP->getExpansionType(ArgumentPackIndex);
+
+    if (NTTPType->isDependentType() &&
+        !isa<TemplateTemplateParmDecl>(Template) &&
+        !Template->getDeclContext()->isDependentContext()) {
       // Do substitution on the type of the non-type template parameter.
       InstantiatingTemplate Inst(*this, TemplateLoc, Template,
-                                 NTTP, Converted.getFlatArguments(),
-                                 Converted.flatSize(),
+                                 NTTP, Converted.data(), Converted.size(),
                                  SourceRange(TemplateLoc, RAngleLoc));
-      
-      TemplateArgumentList TemplateArgs(Context, Converted,
-                                        /*TakeArgs=*/false);
+
+      TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                        Converted.data(), Converted.size());
       NTTPType = SubstType(NTTPType,
                            MultiLevelTemplateArgumentList(TemplateArgs),
                            NTTP->getLocation(),
@@ -2035,34 +2295,36 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
       if (NTTPType.isNull())
         return true;
     }
-    
+
     switch (Arg.getArgument().getKind()) {
     case TemplateArgument::Null:
       assert(false && "Should never see a NULL template argument here");
       return true;
-      
+
     case TemplateArgument::Expression: {
       Expr *E = Arg.getArgument().getAsExpr();
       TemplateArgument Result;
       if (CheckTemplateArgument(NTTP, NTTPType, E, Result, CTAK))
         return true;
-      
-      Converted.Append(Result);
+
+      Converted.push_back(Result);
       break;
     }
-      
+
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
       // We've already checked this template argument, so just copy
       // it to the list of converted arguments.
-      Converted.Append(Arg.getArgument());
+      Converted.push_back(Arg.getArgument());
       break;
-      
+
     case TemplateArgument::Template:
+    case TemplateArgument::TemplateExpansion:
       // We were given a template template argument. It may not be ill-formed;
       // see below.
       if (DependentTemplateName *DTN
-            = Arg.getArgument().getAsTemplate().getAsDependentTemplateName()) {
+            = Arg.getArgument().getAsTemplateOrTemplatePattern()
+                                              .getAsDependentTemplateName()) {
         // We have a template argument such as \c T::template X, which we
         // parsed as a template template argument. However, since we now
         // know that we need a non-type template argument, convert this
@@ -2075,28 +2337,39 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
                                                     DTN->getQualifier(),
                                                Arg.getTemplateQualifierRange(),
                                                     NameInfo);
-        
+
+        // If we parsed the template argument as a pack expansion, create a
+        // pack expansion expression.
+        if (Arg.getArgument().getKind() == TemplateArgument::TemplateExpansion){
+          ExprResult Expansion = ActOnPackExpansion(E,
+                                                  Arg.getTemplateEllipsisLoc());
+          if (Expansion.isInvalid())
+            return true;
+
+          E = Expansion.get();
+        }
+
         TemplateArgument Result;
         if (CheckTemplateArgument(NTTP, NTTPType, E, Result))
           return true;
-        
-        Converted.Append(Result);
+
+        Converted.push_back(Result);
         break;
       }
-      
+
       // We have a template argument that actually does refer to a class
       // template, template alias, or template template parameter, and
       // therefore cannot be a non-type template argument.
       Diag(Arg.getLocation(), diag::err_template_arg_must_be_expr)
         << Arg.getSourceRange();
-      
+
       Diag(Param->getLocation(), diag::note_template_param_here);
       return true;
-      
+
     case TemplateArgument::Type: {
       // We have a non-type template parameter but the template
       // argument is a type.
-      
+
       // C++ [temp.arg]p2:
       //   In a template-argument, an ambiguity between a type-id and
       //   an expression is resolved to a type-id, regardless of the
@@ -2113,19 +2386,19 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
       Diag(Param->getLocation(), diag::note_template_param_here);
       return true;
     }
-      
+
     case TemplateArgument::Pack:
       llvm_unreachable("Caller must expand template argument packs");
       break;
     }
-    
+
     return false;
-  } 
-  
-  
+  }
+
+
   // Check template template parameters.
   TemplateTemplateParmDecl *TempParm = cast<TemplateTemplateParmDecl>(Param);
-    
+
   // Substitute into the template parameter list of the template
   // template parameter, since previously-supplied template arguments
   // may appear within the template template parameter.
@@ -2133,40 +2406,38 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
     // Set up a template instantiation context.
     LocalInstantiationScope Scope(*this);
     InstantiatingTemplate Inst(*this, TemplateLoc, Template,
-                               TempParm, Converted.getFlatArguments(),
-                               Converted.flatSize(),
+                               TempParm, Converted.data(), Converted.size(),
                                SourceRange(TemplateLoc, RAngleLoc));
-    
-    TemplateArgumentList TemplateArgs(Context, Converted,
-                                      /*TakeArgs=*/false);
+
+    TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                      Converted.data(), Converted.size());
     TempParm = cast_or_null<TemplateTemplateParmDecl>(
-                      SubstDecl(TempParm, CurContext, 
+                      SubstDecl(TempParm, CurContext,
                                 MultiLevelTemplateArgumentList(TemplateArgs)));
     if (!TempParm)
       return true;
-    
-    // FIXME: TempParam is leaked.
   }
-    
+
   switch (Arg.getArgument().getKind()) {
   case TemplateArgument::Null:
     assert(false && "Should never see a NULL template argument here");
     return true;
-    
+
   case TemplateArgument::Template:
+  case TemplateArgument::TemplateExpansion:
     if (CheckTemplateArgument(TempParm, Arg))
       return true;
-      
-    Converted.Append(Arg.getArgument());
+
+    Converted.push_back(Arg.getArgument());
     break;
-    
+
   case TemplateArgument::Expression:
   case TemplateArgument::Type:
     // We have a template template parameter but the template
     // argument does not refer to a template.
     Diag(Arg.getLocation(), diag::err_template_arg_must_be_template);
     return true;
-      
+
   case TemplateArgument::Declaration:
     llvm_unreachable(
                        "Declaration argument with template template parameter");
@@ -2175,12 +2446,12 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
     llvm_unreachable(
                           "Integral argument with template template parameter");
     break;
-    
+
   case TemplateArgument::Pack:
     llvm_unreachable("Caller must expand template argument packs");
     break;
   }
-  
+
   return false;
 }
 
@@ -2190,7 +2461,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
                                      SourceLocation TemplateLoc,
                                 const TemplateArgumentListInfo &TemplateArgs,
                                      bool PartialTemplateArgs,
-                                     TemplateArgumentListBuilder &Converted) {
+                          llvm::SmallVectorImpl<TemplateArgument> &Converted) {
   TemplateParameterList *Params = Template->getTemplateParameters();
   unsigned NumParams = Params->size();
   unsigned NumArgs = TemplateArgs.size();
@@ -2226,44 +2497,67 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
   //   a template-id shall match the type and form specified for the
   //   corresponding parameter declared by the template in its
   //   template-parameter-list.
+  llvm::SmallVector<TemplateArgument, 2> ArgumentPack;
+  TemplateParameterList::iterator Param = Params->begin(),
+                               ParamEnd = Params->end();
   unsigned ArgIdx = 0;
-  for (TemplateParameterList::iterator Param = Params->begin(),
-                                       ParamEnd = Params->end();
-       Param != ParamEnd; ++Param, ++ArgIdx) {
+  LocalInstantiationScope InstScope(*this, true);
+  while (Param != ParamEnd) {
     if (ArgIdx > NumArgs && PartialTemplateArgs)
       break;
 
-    // If we have a template parameter pack, check every remaining template
-    // argument against that template parameter pack.
-    if ((*Param)->isTemplateParameterPack()) {
-      Converted.BeginPack();
-      for (; ArgIdx < NumArgs; ++ArgIdx) {
-        if (CheckTemplateArgument(*Param, TemplateArgs[ArgIdx], Template,
-                                  TemplateLoc, RAngleLoc, Converted)) {
-          Invalid = true;
-          break;
+    if (ArgIdx < NumArgs) {
+      // If we have an expanded parameter pack, make sure we don't have too
+      // many arguments.
+      if (NonTypeTemplateParmDecl *NTTP
+                                = dyn_cast<NonTypeTemplateParmDecl>(*Param)) {
+        if (NTTP->isExpandedParameterPack() &&
+            ArgumentPack.size() >= NTTP->getNumExpansionTypes()) {
+          Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
+            << true
+            << (isa<ClassTemplateDecl>(Template)? 0 :
+                isa<FunctionTemplateDecl>(Template)? 1 :
+                isa<TemplateTemplateParmDecl>(Template)? 2 : 3)
+            << Template;
+          Diag(Template->getLocation(), diag::note_template_decl_here)
+            << Params->getSourceRange();
+          return true;
         }
       }
-      Converted.EndPack();
-      continue;
-    }
-    
-    if (ArgIdx < NumArgs) {
+
       // Check the template argument we were given.
-      if (CheckTemplateArgument(*Param, TemplateArgs[ArgIdx], Template, 
-                                TemplateLoc, RAngleLoc, Converted))
+      if (CheckTemplateArgument(*Param, TemplateArgs[ArgIdx], Template,
+                                TemplateLoc, RAngleLoc,
+                                ArgumentPack.size(), Converted))
         return true;
-      
+
+      if ((*Param)->isTemplateParameterPack()) {
+        // The template parameter was a template parameter pack, so take the
+        // deduced argument and place it on the argument pack. Note that we
+        // stay on the same template parameter so that we can deduce more
+        // arguments.
+        ArgumentPack.push_back(Converted.back());
+        Converted.pop_back();
+      } else {
+        // Move to the next template parameter.
+        ++Param;
+      }
+      ++ArgIdx;
       continue;
     }
-    
+
+    // If we have a template parameter pack with no more corresponding
+    // arguments, just break out now and we'll fill in the argument pack below.
+    if ((*Param)->isTemplateParameterPack())
+      break;
+
     // We have a default template argument that we will use.
     TemplateArgumentLoc Arg;
-    
+
     // Retrieve the default template argument from the template
     // parameter. For each kind of template parameter, we substitute the
     // template arguments provided thus far and any "outer" template arguments
-    // (when the template parameter was part of a nested template) into 
+    // (when the template parameter was part of a nested template) into
     // the default argument.
     if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(*Param)) {
       if (!TTP->hasDefaultArgument()) {
@@ -2271,7 +2565,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
         break;
       }
 
-      TypeSourceInfo *ArgType = SubstDefaultTemplateArgument(*this, 
+      TypeSourceInfo *ArgType = SubstDefaultTemplateArgument(*this,
                                                              Template,
                                                              TemplateLoc,
                                                              RAngleLoc,
@@ -2279,7 +2573,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
                                                              Converted);
       if (!ArgType)
         return true;
-                                                             
+
       Arg = TemplateArgumentLoc(TemplateArgument(ArgType->getType()),
                                 ArgType);
     } else if (NonTypeTemplateParmDecl *NTTP
@@ -2290,9 +2584,9 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
       }
 
       ExprResult E = SubstDefaultTemplateArgument(*this, Template,
-                                                              TemplateLoc, 
-                                                              RAngleLoc, 
-                                                              NTTP, 
+                                                              TemplateLoc,
+                                                              RAngleLoc,
+                                                              NTTP,
                                                               Converted);
       if (E.isInvalid())
         return true;
@@ -2309,34 +2603,282 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
       }
 
       TemplateName Name = SubstDefaultTemplateArgument(*this, Template,
-                                                       TemplateLoc, 
-                                                       RAngleLoc, 
+                                                       TemplateLoc,
+                                                       RAngleLoc,
                                                        TempParm,
                                                        Converted);
       if (Name.isNull())
         return true;
-      
-      Arg = TemplateArgumentLoc(TemplateArgument(Name), 
+
+      Arg = TemplateArgumentLoc(TemplateArgument(Name),
                   TempParm->getDefaultArgument().getTemplateQualifierRange(),
                   TempParm->getDefaultArgument().getTemplateNameLoc());
     }
-    
+
     // Introduce an instantiation record that describes where we are using
     // the default template argument.
     InstantiatingTemplate Instantiating(*this, RAngleLoc, Template, *Param,
-                                        Converted.getFlatArguments(),
-                                        Converted.flatSize(),
-                                        SourceRange(TemplateLoc, RAngleLoc));    
-    
+                                        Converted.data(), Converted.size(),
+                                        SourceRange(TemplateLoc, RAngleLoc));
+
     // Check the default template argument.
     if (CheckTemplateArgument(*Param, Arg, Template, TemplateLoc,
-                              RAngleLoc, Converted))
+                              RAngleLoc, 0, Converted))
       return true;
+
+    // Move to the next template parameter and argument.
+    ++Param;
+    ++ArgIdx;
+  }
+
+  // Form argument packs for each of the parameter packs remaining.
+  while (Param != ParamEnd) {
+    // If we're checking a partial list of template arguments, don't fill
+    // in arguments for non-template parameter packs.
+
+    if ((*Param)->isTemplateParameterPack()) {
+      if (PartialTemplateArgs && ArgumentPack.empty()) {
+        Converted.push_back(TemplateArgument());
+      } else if (ArgumentPack.empty())
+        Converted.push_back(TemplateArgument(0, 0));
+      else {
+        Converted.push_back(TemplateArgument::CreatePackCopy(Context,
+                                                          ArgumentPack.data(),
+                                                         ArgumentPack.size()));
+        ArgumentPack.clear();
+      }
+    }
+
+    ++Param;
   }
 
   return Invalid;
 }
 
+namespace {
+  class UnnamedLocalNoLinkageFinder
+    : public TypeVisitor<UnnamedLocalNoLinkageFinder, bool>
+  {
+    Sema &S;
+    SourceRange SR;
+
+    typedef TypeVisitor<UnnamedLocalNoLinkageFinder, bool> inherited;
+
+  public:
+    UnnamedLocalNoLinkageFinder(Sema &S, SourceRange SR) : S(S), SR(SR) { }
+
+    bool Visit(QualType T) {
+      return inherited::Visit(T.getTypePtr());
+    }
+
+#define TYPE(Class, Parent) \
+    bool Visit##Class##Type(const Class##Type *);
+#define ABSTRACT_TYPE(Class, Parent) \
+    bool Visit##Class##Type(const Class##Type *) { return false; }
+#define NON_CANONICAL_TYPE(Class, Parent) \
+    bool Visit##Class##Type(const Class##Type *) { return false; }
+#include "clang/AST/TypeNodes.def"
+
+    bool VisitTagDecl(const TagDecl *Tag);
+    bool VisitNestedNameSpecifier(NestedNameSpecifier *NNS);
+  };
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitBuiltinType(const BuiltinType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitComplexType(const ComplexType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitPointerType(const PointerType* T) {
+  return Visit(T->getPointeeType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitBlockPointerType(
+                                                    const BlockPointerType* T) {
+  return Visit(T->getPointeeType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitLValueReferenceType(
+                                                const LValueReferenceType* T) {
+  return Visit(T->getPointeeType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitRValueReferenceType(
+                                                const RValueReferenceType* T) {
+  return Visit(T->getPointeeType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitMemberPointerType(
+                                                  const MemberPointerType* T) {
+  return Visit(T->getPointeeType()) || Visit(QualType(T->getClass(), 0));
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitConstantArrayType(
+                                                  const ConstantArrayType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitIncompleteArrayType(
+                                                 const IncompleteArrayType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitVariableArrayType(
+                                                   const VariableArrayType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitDependentSizedArrayType(
+                                            const DependentSizedArrayType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitDependentSizedExtVectorType(
+                                         const DependentSizedExtVectorType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitVectorType(const VectorType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitExtVectorType(const ExtVectorType* T) {
+  return Visit(T->getElementType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitFunctionProtoType(
+                                                  const FunctionProtoType* T) {
+  for (FunctionProtoType::arg_type_iterator A = T->arg_type_begin(),
+                                         AEnd = T->arg_type_end();
+       A != AEnd; ++A) {
+    if (Visit(*A))
+      return true;
+  }
+
+  return Visit(T->getResultType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitFunctionNoProtoType(
+                                               const FunctionNoProtoType* T) {
+  return Visit(T->getResultType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitUnresolvedUsingType(
+                                                  const UnresolvedUsingType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitTypeOfExprType(const TypeOfExprType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitTypeOfType(const TypeOfType* T) {
+  return Visit(T->getUnderlyingType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitDecltypeType(const DecltypeType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitAutoType(const AutoType *T) {
+  return Visit(T->getDeducedType());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitRecordType(const RecordType* T) {
+  return VisitTagDecl(T->getDecl());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitEnumType(const EnumType* T) {
+  return VisitTagDecl(T->getDecl());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitTemplateTypeParmType(
+                                                 const TemplateTypeParmType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitSubstTemplateTypeParmPackType(
+                                        const SubstTemplateTypeParmPackType *) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitTemplateSpecializationType(
+                                            const TemplateSpecializationType*) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitInjectedClassNameType(
+                                              const InjectedClassNameType* T) {
+  return VisitTagDecl(T->getDecl());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitDependentNameType(
+                                                   const DependentNameType* T) {
+  return VisitNestedNameSpecifier(T->getQualifier());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitDependentTemplateSpecializationType(
+                                 const DependentTemplateSpecializationType* T) {
+  return VisitNestedNameSpecifier(T->getQualifier());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitPackExpansionType(
+                                                   const PackExpansionType* T) {
+  return Visit(T->getPattern());
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitObjCObjectType(const ObjCObjectType *) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitObjCInterfaceType(
+                                                   const ObjCInterfaceType *) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitObjCObjectPointerType(
+                                                const ObjCObjectPointerType *) {
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitTagDecl(const TagDecl *Tag) {
+  if (Tag->getDeclContext()->isFunctionOrMethod()) {
+    S.Diag(SR.getBegin(), diag::ext_template_arg_local_type)
+      << S.Context.getTypeDeclType(Tag) << SR;
+    return true;
+  }
+
+  if (!Tag->getDeclName() && !Tag->getTypedefForAnonDecl()) {
+    S.Diag(SR.getBegin(), diag::ext_template_arg_unnamed_type) << SR;
+    S.Diag(Tag->getLocation(), diag::note_template_unnamed_type_here);
+    return true;
+  }
+
+  return false;
+}
+
+bool UnnamedLocalNoLinkageFinder::VisitNestedNameSpecifier(
+                                                    NestedNameSpecifier *NNS) {
+  if (NNS->getPrefix() && VisitNestedNameSpecifier(NNS->getPrefix()))
+    return true;
+
+  switch (NNS->getKind()) {
+  case NestedNameSpecifier::Identifier:
+  case NestedNameSpecifier::Namespace:
+  case NestedNameSpecifier::Global:
+    return false;
+
+  case NestedNameSpecifier::TypeSpec:
+  case NestedNameSpecifier::TypeSpecWithTemplate:
+    return Visit(QualType(NNS->getAsType(), 0));
+  }
+  return false;
+}
+
+
 /// \brief Check a template argument against its corresponding
 /// template type parameter.
 ///
@@ -2346,36 +2888,24 @@ bool Sema::CheckTemplateArgument(TemplateTypeParmDecl *Param,
                                  TypeSourceInfo *ArgInfo) {
   assert(ArgInfo && "invalid TypeSourceInfo");
   QualType Arg = ArgInfo->getType();
+  SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
+
+  if (Arg->isVariablyModifiedType()) {
+    return Diag(SR.getBegin(), diag::err_variably_modified_template_arg) << Arg;
+  } else if (Context.hasSameUnqualifiedType(Arg, Context.OverloadTy)) {
+    return Diag(SR.getBegin(), diag::err_template_arg_overload_type) << SR;
+  }
 
   // C++03 [temp.arg.type]p2:
   //   A local type, a type with no linkage, an unnamed type or a type
   //   compounded from any of these types shall not be used as a
   //   template-argument for a template type-parameter.
+  //
   // C++0x allows these, and even in C++03 we allow them as an extension with
   // a warning.
-  SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
-  if (!LangOpts.CPlusPlus0x) {
-    const TagType *Tag = 0;
-    if (const EnumType *EnumT = Arg->getAs<EnumType>())
-      Tag = EnumT;
-    else if (const RecordType *RecordT = Arg->getAs<RecordType>())
-      Tag = RecordT;
-    if (Tag && Tag->getDecl()->getDeclContext()->isFunctionOrMethod()) {
-      SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
-      Diag(SR.getBegin(), diag::ext_template_arg_local_type)
-        << QualType(Tag, 0) << SR;
-    } else if (Tag && !Tag->getDecl()->getDeclName() &&
-               !Tag->getDecl()->getTypedefForAnonDecl()) {
-      Diag(SR.getBegin(), diag::ext_template_arg_unnamed_type) << SR;
-      Diag(Tag->getDecl()->getLocation(),
-           diag::note_template_unnamed_type_here);
-    }
-  }
-
-  if (Arg->isVariablyModifiedType()) {
-    return Diag(SR.getBegin(), diag::err_variably_modified_template_arg) << Arg;
-  } else if (Context.hasSameUnqualifiedType(Arg, Context.OverloadTy)) {
-    return Diag(SR.getBegin(), diag::err_template_arg_overload_type) << SR;
+  if (!LangOpts.CPlusPlus0x && Arg->hasUnnamedOrLocalType()) {
+    UnnamedLocalNoLinkageFinder Finder(*this, SR);
+    (void)Finder.Visit(Context.getCanonicalType(Arg));
   }
 
   return false;
@@ -2383,7 +2913,7 @@ bool Sema::CheckTemplateArgument(TemplateTypeParmDecl *Param,
 
 /// \brief Checks whether the given template argument is the address
 /// of an object or function according to C++ [temp.arg.nontype]p1.
-static bool 
+static bool
 CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
                                                NonTypeTemplateParmDecl *Param,
                                                QualType ParamType,
@@ -2410,13 +2940,15 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
   //        corresponding template-parameter is a reference; or
   DeclRefExpr *DRE = 0;
 
-  // Ignore (and complain about) any excess parentheses.
+  // In C++98/03 mode, give an extension warning on any extra parentheses.
+  // See http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#773
+  bool ExtraParens = false;
   while (ParenExpr *Parens = dyn_cast<ParenExpr>(Arg)) {
-    if (!Invalid) {
+    if (!Invalid && !ExtraParens && !S.getLangOptions().CPlusPlus0x) {
       S.Diag(Arg->getSourceRange().getBegin(),
-             diag::err_template_arg_extra_parens)
+             diag::ext_template_arg_extra_parens)
         << Arg->getSourceRange();
-      Invalid = true;
+      ExtraParens = true;
     }
 
     Arg = Parens->getSubExpr();
@@ -2443,7 +2975,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
   // Stop checking the precise nature of the argument if it is value dependent,
   // it should be checked when instantiated.
   if (Arg->isValueDependent()) {
-    Converted = TemplateArgument(ArgIn->Retain());
+    Converted = TemplateArgument(ArgIn);
     return false;
   }
 
@@ -2522,7 +3054,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
 
     // A value of reference type is not an object.
     if (Var->getType()->isReferenceType()) {
-      S.Diag(Arg->getSourceRange().getBegin(), 
+      S.Diag(Arg->getSourceRange().getBegin(),
              diag::err_template_arg_reference_var)
         << Var->getType() << Arg->getSourceRange();
       S.Diag(Param->getLocation(), diag::note_template_param_here);
@@ -2586,9 +3118,9 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
     return true;
   }
 
-  if (ParamType->isPointerType() && 
+  if (ParamType->isPointerType() &&
       !ParamType->getAs<PointerType>()->getPointeeType()->isFunctionType() &&
-      S.IsQualificationConversion(ArgType, ParamType)) {
+      S.IsQualificationConversion(ArgType, ParamType, false)) {
     // For pointer-to-object types, qualification conversions are
     // permitted.
   } else {
@@ -2613,7 +3145,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
             << Arg->getSourceRange();
           S.Diag(Param->getLocation(), diag::note_template_param_here);
           return true;
-        }     
+        }
       }
     }
 
@@ -2642,7 +3174,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
 
 /// \brief Checks whether the given template argument is a pointer to
 /// member constant according to C++ [temp.arg.nontype]p1.
-bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg, 
+bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg,
                                                 TemplateArgument &Converted) {
   bool Invalid = false;
 
@@ -2658,13 +3190,15 @@ bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg,
   //     -- a pointer to member expressed as described in 5.3.1.
   DeclRefExpr *DRE = 0;
 
-  // Ignore (and complain about) any excess parentheses.
+  // In C++98/03 mode, give an extension warning on any extra parentheses.
+  // See http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#773
+  bool ExtraParens = false;
   while (ParenExpr *Parens = dyn_cast<ParenExpr>(Arg)) {
-    if (!Invalid) {
+    if (!Invalid && !ExtraParens && !getLangOptions().CPlusPlus0x) {
       Diag(Arg->getSourceRange().getBegin(),
-           diag::err_template_arg_extra_parens)
+           diag::ext_template_arg_extra_parens)
         << Arg->getSourceRange();
-      Invalid = true;
+      ExtraParens = true;
     }
 
     Arg = Parens->getSubExpr();
@@ -2677,26 +3211,26 @@ bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg,
       if (DRE && !DRE->getQualifier())
         DRE = 0;
     }
-  } 
+  }
   // A constant of pointer-to-member type.
   else if ((DRE = dyn_cast<DeclRefExpr>(Arg))) {
     if (ValueDecl *VD = dyn_cast<ValueDecl>(DRE->getDecl())) {
       if (VD->getType()->isMemberPointerType()) {
         if (isa<NonTypeTemplateParmDecl>(VD) ||
-            (isa<VarDecl>(VD) && 
+            (isa<VarDecl>(VD) &&
              Context.getCanonicalType(VD->getType()).isConstQualified())) {
           if (Arg->isTypeDependent() || Arg->isValueDependent())
-            Converted = TemplateArgument(Arg->Retain());
+            Converted = TemplateArgument(Arg);
           else
             Converted = TemplateArgument(VD->getCanonicalDecl());
           return Invalid;
         }
       }
     }
-    
+
     DRE = 0;
   }
-  
+
   if (!DRE)
     return Diag(Arg->getSourceRange().getBegin(),
                 diag::err_template_arg_not_pointer_to_member_form)
@@ -2710,7 +3244,7 @@ bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg,
     // Okay: this is the address of a non-static member, and therefore
     // a member pointer constant.
     if (Arg->isTypeDependent() || Arg->isValueDependent())
-      Converted = TemplateArgument(Arg->Retain());
+      Converted = TemplateArgument(Arg);
     else
       Converted = TemplateArgument(DRE->getDecl()->getCanonicalDecl());
     return Invalid;
@@ -2803,7 +3337,10 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       Diag(StartLoc, diag::err_deduced_non_type_template_arg_type_mismatch)
         << ArgType << ParamType;
       Diag(Param->getLocation(), diag::note_template_param_here);
-      return true;      
+      return true;
+    } else if (ParamType->isBooleanType()) {
+      // This is an integral-to-boolean conversion.
+      ImpCastExprToType(Arg, ParamType, CK_IntegralToBoolean);
     } else if (IsIntegralPromotion(Arg, ArgType, ParamType) ||
                !ParamType->isEnumeralType()) {
       // This is an integral promotion or conversion.
@@ -2823,12 +3360,12 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
 
     if (!Arg->isValueDependent()) {
       llvm::APSInt OldValue = Value;
-      
-      // Coerce the template argument's value to the value it will have 
+
+      // Coerce the template argument's value to the value it will have
       // based on the template parameter's type.
       unsigned AllowedBits = Context.getTypeSize(IntegerType);
       if (Value.getBitWidth() != AllowedBits)
-        Value.extOrTrunc(AllowedBits);
+        Value = Value.extOrTrunc(AllowedBits);
       Value.setIsSigned(IntegerType->isSignedIntegerType());
 
       // Complain if an unsigned parameter received a negative value.
@@ -2879,7 +3416,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
   // from a template argument of type std::nullptr_t to a non-type
   // template parameter of type pointer to object, pointer to
   // function, or pointer-to-member, respectively.
-  if (ArgType->isNullPtrType() && 
+  if (ArgType->isNullPtrType() &&
       (ParamType->isPointerType() || ParamType->isMemberPointerType())) {
     Converted = TemplateArgument((NamedDecl *)0);
     return false;
@@ -2910,7 +3447,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
          ->isFunctionType())) {
 
     if (Arg->getType() == Context.OverloadTy) {
-      if (FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(Arg, ParamType, 
+      if (FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(Arg, ParamType,
                                                                 true,
                                                                 FoundResult)) {
         if (DiagnoseUseOfDecl(Fn, Arg->getSourceRange().getBegin()))
@@ -2921,13 +3458,14 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       } else
         return true;
     }
-        
+
     if (!ParamType->isMemberPointerType())
       return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param,
-                                                            ParamType, 
+                                                            ParamType,
                                                             Arg, Converted);
 
-    if (IsQualificationConversion(ArgType, ParamType.getNonReferenceType())) {
+    if (IsQualificationConversion(ArgType, ParamType.getNonReferenceType(),
+                                  false)) {
       ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg));
     } else if (!Context.hasSameUnqualifiedType(ArgType,
                                            ParamType.getNonReferenceType())) {
@@ -2950,7 +3488,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     assert(ParamType->getPointeeType()->isIncompleteOrObjectType() &&
            "Only object pointers allowed here");
 
-    return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param, 
+    return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param,
                                                           ParamType,
                                                           Arg, Converted);
   }
@@ -2966,8 +3504,8 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
            "Only object references allowed here");
 
     if (Arg->getType() == Context.OverloadTy) {
-      if (FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(Arg, 
-                                                 ParamRefType->getPointeeType(), 
+      if (FunctionDecl *Fn = ResolveAddressOfOverloadedFunction(Arg,
+                                                 ParamRefType->getPointeeType(),
                                                                 true,
                                                                 FoundResult)) {
         if (DiagnoseUseOfDecl(Fn, Arg->getSourceRange().getBegin()))
@@ -2978,8 +3516,8 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       } else
         return true;
     }
-    
-    return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param, 
+
+    return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param,
                                                           ParamType,
                                                           Arg, Converted);
   }
@@ -2990,7 +3528,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
 
   if (Context.hasSameUnqualifiedType(ParamType, ArgType)) {
     // Types match exactly: nothing more to do here.
-  } else if (IsQualificationConversion(ArgType, ParamType)) {
+  } else if (IsQualificationConversion(ArgType, ParamType, false)) {
     ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg));
   } else {
     // We can't perform this conversion.
@@ -3041,7 +3579,7 @@ bool Sema::CheckTemplateArgument(TemplateTemplateParmDecl *Param,
 
   return !TemplateParameterListsAreEqual(Template->getTemplateParameters(),
                                          Param->getTemplateParameters(),
-                                         true, 
+                                         true,
                                          TPL_TemplateTemplateArgumentMatch,
                                          Arg.getLocation());
 }
@@ -3050,7 +3588,7 @@ bool Sema::CheckTemplateArgument(TemplateTemplateParmDecl *Param,
 /// declaration and the type of its corresponding non-type template
 /// parameter, produce an expression that properly refers to that
 /// declaration.
-ExprResult 
+ExprResult
 Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
                                               QualType ParamType,
                                               SourceLocation Loc) {
@@ -3058,7 +3596,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
          "Only declaration template arguments permitted here");
   ValueDecl *VD = cast<ValueDecl>(Arg.getAsDecl());
 
-  if (VD->getDeclContext()->isRecord() && 
+  if (VD->getDeclContext()->isRecord() &&
       (isa<CXXMethodDecl>(VD) || isa<FieldDecl>(VD))) {
     // If the value is a class member, we might have a pointer-to-member.
     // Determine whether the non-type template template parameter is of
@@ -3073,37 +3611,46 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
                                       ClassType.getTypePtr());
       CXXScopeSpec SS;
       SS.setScopeRep(Qualifier);
-      ExprResult RefExpr = BuildDeclRefExpr(VD, 
-                                           VD->getType().getNonReferenceType(), 
-                                                  Loc,
-                                                  &SS);
+
+      // The actual value-ness of this is unimportant, but for
+      // internal consistency's sake, references to instance methods
+      // are r-values.
+      ExprValueKind VK = VK_LValue;
+      if (isa<CXXMethodDecl>(VD) && cast<CXXMethodDecl>(VD)->isInstance())
+        VK = VK_RValue;
+
+      ExprResult RefExpr = BuildDeclRefExpr(VD,
+                                            VD->getType().getNonReferenceType(),
+                                            VK,
+                                            Loc,
+                                            &SS);
       if (RefExpr.isInvalid())
         return ExprError();
-      
+
       RefExpr = CreateBuiltinUnaryOp(Loc, UO_AddrOf, RefExpr.get());
-      
+
       // We might need to perform a trailing qualification conversion, since
       // the element type on the parameter could be more qualified than the
       // element type in the expression we constructed.
       if (IsQualificationConversion(((Expr*) RefExpr.get())->getType(),
-                                    ParamType.getUnqualifiedType())) {
+                                    ParamType.getUnqualifiedType(), false)) {
         Expr *RefE = RefExpr.takeAs<Expr>();
         ImpCastExprToType(RefE, ParamType.getUnqualifiedType(), CK_NoOp);
         RefExpr = Owned(RefE);
       }
-      
+
       assert(!RefExpr.isInvalid() &&
              Context.hasSameType(((Expr*) RefExpr.get())->getType(),
                                  ParamType.getUnqualifiedType()));
       return move(RefExpr);
     }
   }
-  
+
   QualType T = VD->getType().getNonReferenceType();
   if (ParamType->isPointerType()) {
     // When the non-type template parameter is a pointer, take the
     // address of the declaration.
-    ExprResult RefExpr = BuildDeclRefExpr(VD, T, Loc);
+    ExprResult RefExpr = BuildDeclRefExpr(VD, T, VK_LValue, Loc);
     if (RefExpr.isInvalid())
       return ExprError();
 
@@ -3118,18 +3665,23 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
 
       return move(RefExpr);
     }
-    
+
     // Take the address of everything else
     return CreateBuiltinUnaryOp(Loc, UO_AddrOf, RefExpr.get());
   }
 
+  ExprValueKind VK = VK_RValue;
+
   // If the non-type template parameter has reference type, qualify the
   // resulting declaration reference with the extra qualifiers on the
   // type that the reference refers to.
-  if (const ReferenceType *TargetRef = ParamType->getAs<ReferenceType>())
-    T = Context.getQualifiedType(T, TargetRef->getPointeeType().getQualifiers());
-    
-  return BuildDeclRefExpr(VD, T, Loc);
+  if (const ReferenceType *TargetRef = ParamType->getAs<ReferenceType>()) {
+    VK = VK_LValue;
+    T = Context.getQualifiedType(T,
+                              TargetRef->getPointeeType().getQualifiers());
+  }
+
+  return BuildDeclRefExpr(VD, T, VK, Loc);
 }
 
 /// \brief Construct a new expression that refers to the given
@@ -3139,11 +3691,11 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
 /// This routine takes care of the mapping from an integral template
 /// argument (which may have any integral type) to the appropriate
 /// literal value.
-ExprResult 
+ExprResult
 Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
                                                   SourceLocation Loc) {
   assert(Arg.getKind() == TemplateArgument::Integral &&
-         "Operation is only value for integral template arguments");
+         "Operation is only valid for integral template arguments");
   QualType T = Arg.getIntegralType();
   if (T->isCharType() || T->isWideCharType())
     return Owned(new (Context) CharacterLiteral(
@@ -3157,9 +3709,149 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
                                             T,
                                             Loc));
 
-  return Owned(IntegerLiteral::Create(Context, *Arg.getAsIntegral(), T, Loc));
+  QualType BT;
+  if (const EnumType *ET = T->getAs<EnumType>())
+    BT = ET->getDecl()->getPromotionType();
+  else
+    BT = T;
+
+  Expr *E = IntegerLiteral::Create(Context, *Arg.getAsIntegral(), BT, Loc);
+  if (T->isEnumeralType()) {
+    // FIXME: This is a hack. We need a better way to handle substituted
+    // non-type template parameters.
+    E = CStyleCastExpr::Create(Context, T, VK_RValue, CK_IntegralCast,
+                                      E, 0, 
+                                      Context.getTrivialTypeSourceInfo(T, Loc),
+                                      Loc, Loc);
+  }
+  
+  return Owned(E);
 }
 
+/// \brief Match two template parameters within template parameter lists.
+static bool MatchTemplateParameterKind(Sema &S, NamedDecl *New, NamedDecl *Old,
+                                       bool Complain,
+                                     Sema::TemplateParameterListEqualKind Kind,
+                                       SourceLocation TemplateArgLoc) {
+  // Check the actual kind (type, non-type, template).
+  if (Old->getKind() != New->getKind()) {
+    if (Complain) {
+      unsigned NextDiag = diag::err_template_param_different_kind;
+      if (TemplateArgLoc.isValid()) {
+        S.Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
+        NextDiag = diag::note_template_param_different_kind;
+      }
+      S.Diag(New->getLocation(), NextDiag)
+        << (Kind != Sema::TPL_TemplateMatch);
+      S.Diag(Old->getLocation(), diag::note_template_prev_declaration)
+        << (Kind != Sema::TPL_TemplateMatch);
+    }
+
+    return false;
+  }
+
+  // Check that both are parameter packs are neither are parameter packs.
+  // However, if we are matching a template template argument to a
+  // template template parameter, the template template parameter can have
+  // a parameter pack where the template template argument does not.
+  if (Old->isTemplateParameterPack() != New->isTemplateParameterPack() &&
+      !(Kind == Sema::TPL_TemplateTemplateArgumentMatch &&
+        Old->isTemplateParameterPack())) {
+    if (Complain) {
+      unsigned NextDiag = diag::err_template_parameter_pack_non_pack;
+      if (TemplateArgLoc.isValid()) {
+        S.Diag(TemplateArgLoc,
+             diag::err_template_arg_template_params_mismatch);
+        NextDiag = diag::note_template_parameter_pack_non_pack;
+      }
+
+      unsigned ParamKind = isa<TemplateTypeParmDecl>(New)? 0
+                      : isa<NonTypeTemplateParmDecl>(New)? 1
+                      : 2;
+      S.Diag(New->getLocation(), NextDiag)
+        << ParamKind << New->isParameterPack();
+      S.Diag(Old->getLocation(), diag::note_template_parameter_pack_here)
+        << ParamKind << Old->isParameterPack();
+    }
+
+    return false;
+  }
+
+  // For non-type template parameters, check the type of the parameter.
+  if (NonTypeTemplateParmDecl *OldNTTP
+                                    = dyn_cast<NonTypeTemplateParmDecl>(Old)) {
+    NonTypeTemplateParmDecl *NewNTTP = cast<NonTypeTemplateParmDecl>(New);
+
+    // If we are matching a template template argument to a template
+    // template parameter and one of the non-type template parameter types
+    // is dependent, then we must wait until template instantiation time
+    // to actually compare the arguments.
+    if (Kind == Sema::TPL_TemplateTemplateArgumentMatch &&
+        (OldNTTP->getType()->isDependentType() ||
+         NewNTTP->getType()->isDependentType()))
+      return true;
+
+    if (!S.Context.hasSameType(OldNTTP->getType(), NewNTTP->getType())) {
+      if (Complain) {
+        unsigned NextDiag = diag::err_template_nontype_parm_different_type;
+        if (TemplateArgLoc.isValid()) {
+          S.Diag(TemplateArgLoc,
+                 diag::err_template_arg_template_params_mismatch);
+          NextDiag = diag::note_template_nontype_parm_different_type;
+        }
+        S.Diag(NewNTTP->getLocation(), NextDiag)
+          << NewNTTP->getType()
+          << (Kind != Sema::TPL_TemplateMatch);
+        S.Diag(OldNTTP->getLocation(),
+               diag::note_template_nontype_parm_prev_declaration)
+          << OldNTTP->getType();
+      }
+
+      return false;
+    }
+
+    return true;
+  }
+
+  // For template template parameters, check the template parameter types.
+  // The template parameter lists of template template
+  // parameters must agree.
+  if (TemplateTemplateParmDecl *OldTTP
+                                    = dyn_cast<TemplateTemplateParmDecl>(Old)) {
+    TemplateTemplateParmDecl *NewTTP = cast<TemplateTemplateParmDecl>(New);
+    return S.TemplateParameterListsAreEqual(NewTTP->getTemplateParameters(),
+                                            OldTTP->getTemplateParameters(),
+                                            Complain,
+                                        (Kind == Sema::TPL_TemplateMatch
+                                           ? Sema::TPL_TemplateTemplateParmMatch
+                                           : Kind),
+                                            TemplateArgLoc);
+  }
+
+  return true;
+}
+
+/// \brief Diagnose a known arity mismatch when comparing template argument
+/// lists.
+static
+void DiagnoseTemplateParameterListArityMismatch(Sema &S,
+                                                TemplateParameterList *New,
+                                                TemplateParameterList *Old,
+                                      Sema::TemplateParameterListEqualKind Kind,
+                                                SourceLocation TemplateArgLoc) {
+  unsigned NextDiag = diag::err_template_param_list_different_arity;
+  if (TemplateArgLoc.isValid()) {
+    S.Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
+    NextDiag = diag::note_template_param_list_different_arity;
+  }
+  S.Diag(New->getTemplateLoc(), NextDiag)
+    << (New->size() > Old->size())
+    << (Kind != Sema::TPL_TemplateMatch)
+    << SourceRange(New->getTemplateLoc(), New->getRAngleLoc());
+  S.Diag(Old->getTemplateLoc(), diag::note_template_prev_declaration)
+    << (Kind != Sema::TPL_TemplateMatch)
+    << SourceRange(Old->getTemplateLoc(), Old->getRAngleLoc());
+}
 
 /// \brief Determine whether the given template parameter lists are
 /// equivalent.
@@ -3190,120 +3882,66 @@ Sema::TemplateParameterListsAreEqual(TemplateParameterList *New,
                                      bool Complain,
                                      TemplateParameterListEqualKind Kind,
                                      SourceLocation TemplateArgLoc) {
-  if (Old->size() != New->size()) {
-    if (Complain) {
-      unsigned NextDiag = diag::err_template_param_list_different_arity;
-      if (TemplateArgLoc.isValid()) {
-        Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
-        NextDiag = diag::note_template_param_list_different_arity;
-      }
-      Diag(New->getTemplateLoc(), NextDiag)
-          << (New->size() > Old->size())
-          << (Kind != TPL_TemplateMatch)
-          << SourceRange(New->getTemplateLoc(), New->getRAngleLoc());
-      Diag(Old->getTemplateLoc(), diag::note_template_prev_declaration)
-        << (Kind != TPL_TemplateMatch)
-        << SourceRange(Old->getTemplateLoc(), Old->getRAngleLoc());
-    }
+  if (Old->size() != New->size() && Kind != TPL_TemplateTemplateArgumentMatch) {
+    if (Complain)
+      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                 TemplateArgLoc);
 
     return false;
   }
 
+  // C++0x [temp.arg.template]p3:
+  //   A template-argument matches a template template-parameter (call it P)
+  //   when each of the template parameters in the template-parameter-list of
+  //   the template-argument's corresponding class template or template alias
+  //   (call it A) matches the corresponding template parameter in the
+  //   template-parameter-list of P. [...]
+  TemplateParameterList::iterator NewParm = New->begin();
+  TemplateParameterList::iterator NewParmEnd = New->end();
   for (TemplateParameterList::iterator OldParm = Old->begin(),
-         OldParmEnd = Old->end(), NewParm = New->begin();
-       OldParm != OldParmEnd; ++OldParm, ++NewParm) {
-    if ((*OldParm)->getKind() != (*NewParm)->getKind()) {
-      if (Complain) {
-        unsigned NextDiag = diag::err_template_param_different_kind;
-        if (TemplateArgLoc.isValid()) {
-          Diag(TemplateArgLoc, diag::err_template_arg_template_params_mismatch);
-          NextDiag = diag::note_template_param_different_kind;
-        }
-        Diag((*NewParm)->getLocation(), NextDiag)
-          << (Kind != TPL_TemplateMatch);
-        Diag((*OldParm)->getLocation(), diag::note_template_prev_declaration)
-          << (Kind != TPL_TemplateMatch);
-      }
-      return false;
-    }
+                                    OldParmEnd = Old->end();
+       OldParm != OldParmEnd; ++OldParm) {
+    if (Kind != TPL_TemplateTemplateArgumentMatch ||
+        !(*OldParm)->isTemplateParameterPack()) {
+      if (NewParm == NewParmEnd) {
+        if (Complain)
+          DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                     TemplateArgLoc);
 
-    if (TemplateTypeParmDecl *OldTTP
-                                  = dyn_cast<TemplateTypeParmDecl>(*OldParm)) {
-      // Template type parameters are equivalent if either both are template
-      // type parameter packs or neither are (since we know we're at the same 
-      // index).
-      TemplateTypeParmDecl *NewTTP = cast<TemplateTypeParmDecl>(*NewParm);
-      if (OldTTP->isParameterPack() != NewTTP->isParameterPack()) {
-        // FIXME: Implement the rules in C++0x [temp.arg.template]p5 that
-        // allow one to match a template parameter pack in the template
-        // parameter list of a template template parameter to one or more
-        // template parameters in the template parameter list of the 
-        // corresponding template template argument.        
-        if (Complain) {
-          unsigned NextDiag = diag::err_template_parameter_pack_non_pack;
-          if (TemplateArgLoc.isValid()) {
-            Diag(TemplateArgLoc,
-                 diag::err_template_arg_template_params_mismatch);
-            NextDiag = diag::note_template_parameter_pack_non_pack;
-          }
-          Diag(NewTTP->getLocation(), NextDiag)
-            << 0 << NewTTP->isParameterPack();
-          Diag(OldTTP->getLocation(), diag::note_template_parameter_pack_here)
-            << 0 << OldTTP->isParameterPack();
-        }
         return false;
       }
-    } else if (NonTypeTemplateParmDecl *OldNTTP
-                 = dyn_cast<NonTypeTemplateParmDecl>(*OldParm)) {
-      // The types of non-type template parameters must agree.
-      NonTypeTemplateParmDecl *NewNTTP
-        = cast<NonTypeTemplateParmDecl>(*NewParm);
-      
-      // If we are matching a template template argument to a template
-      // template parameter and one of the non-type template parameter types
-      // is dependent, then we must wait until template instantiation time
-      // to actually compare the arguments.
-      if (Kind == TPL_TemplateTemplateArgumentMatch &&
-          (OldNTTP->getType()->isDependentType() ||
-           NewNTTP->getType()->isDependentType()))
-        continue;
-      
-      if (Context.getCanonicalType(OldNTTP->getType()) !=
-            Context.getCanonicalType(NewNTTP->getType())) {
-        if (Complain) {
-          unsigned NextDiag = diag::err_template_nontype_parm_different_type;
-          if (TemplateArgLoc.isValid()) {
-            Diag(TemplateArgLoc,
-                 diag::err_template_arg_template_params_mismatch);
-            NextDiag = diag::note_template_nontype_parm_different_type;
-          }
-          Diag(NewNTTP->getLocation(), NextDiag)
-            << NewNTTP->getType()
-            << (Kind != TPL_TemplateMatch);
-          Diag(OldNTTP->getLocation(),
-               diag::note_template_nontype_parm_prev_declaration)
-            << OldNTTP->getType();
-        }
+
+      if (!MatchTemplateParameterKind(*this, *NewParm, *OldParm, Complain,
+                                      Kind, TemplateArgLoc))
         return false;
-      }
-    } else {
-      // The template parameter lists of template template
-      // parameters must agree.
-      assert(isa<TemplateTemplateParmDecl>(*OldParm) &&
-             "Only template template parameters handled here");
-      TemplateTemplateParmDecl *OldTTP
-        = cast<TemplateTemplateParmDecl>(*OldParm);
-      TemplateTemplateParmDecl *NewTTP
-        = cast<TemplateTemplateParmDecl>(*NewParm);
-      if (!TemplateParameterListsAreEqual(NewTTP->getTemplateParameters(),
-                                          OldTTP->getTemplateParameters(),
-                                          Complain,
-              (Kind == TPL_TemplateMatch? TPL_TemplateTemplateParmMatch : Kind),
-                                          TemplateArgLoc))
+
+      ++NewParm;
+      continue;
+    }
+
+    // C++0x [temp.arg.template]p3:
+    //   [...] When P's template- parameter-list contains a template parameter
+    //   pack (14.5.3), the template parameter pack will match zero or more
+    //   template parameters or template parameter packs in the
+    //   template-parameter-list of A with the same type and form as the
+    //   template parameter pack in P (ignoring whether those template
+    //   parameters are template parameter packs).
+    for (; NewParm != NewParmEnd; ++NewParm) {
+      if (!MatchTemplateParameterKind(*this, *NewParm, *OldParm, Complain,
+                                      Kind, TemplateArgLoc))
         return false;
     }
   }
 
+  // Make sure we exhausted all of the arguments.
+  if (NewParm != NewParmEnd) {
+    if (Complain)
+      DiagnoseTemplateParameterListArityMismatch(*this, New, Old, Kind,
+                                                 TemplateArgLoc);
+
+    return false;
+  }
+
   return true;
 }
 
@@ -3343,18 +3981,18 @@ Sema::CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams) {
 static TemplateSpecializationKind getTemplateSpecializationKind(NamedDecl *D) {
   if (!D)
     return TSK_Undeclared;
-  
+
   if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(D))
     return Record->getTemplateSpecializationKind();
   if (FunctionDecl *Function = dyn_cast<FunctionDecl>(D))
     return Function->getTemplateSpecializationKind();
   if (VarDecl *Var = dyn_cast<VarDecl>(D))
     return Var->getTemplateSpecializationKind();
-  
+
   return TSK_Undeclared;
 }
 
-/// \brief Check whether a specialization is well-formed in the current 
+/// \brief Check whether a specialization is well-formed in the current
 /// context.
 ///
 /// This routine determines whether a template specialization can be declared
@@ -3365,7 +4003,7 @@ static TemplateSpecializationKind getTemplateSpecializationKind(NamedDecl *D) {
 ///
 /// \param Specialized the entity being specialized or instantiated, which
 /// may be a kind of template (class template, function template, etc.) or
-/// a member of a class template (member function, static data member, 
+/// a member of a class template (member function, static data member,
 /// member class).
 ///
 /// \param PrevDecl the previous declaration of this entity, if any.
@@ -3386,14 +4024,11 @@ static bool CheckTemplateSpecializationScope(Sema &S,
   // Keep these "kind" numbers in sync with the %select statements in the
   // various diagnostics emitted by this routine.
   int EntityKind = 0;
-  bool isTemplateSpecialization = false;
-  if (isa<ClassTemplateDecl>(Specialized)) {
+  if (isa<ClassTemplateDecl>(Specialized))
     EntityKind = IsPartialSpecialization? 1 : 0;
-    isTemplateSpecialization = true;
-  } else if (isa<FunctionTemplateDecl>(Specialized)) {
+  else if (isa<FunctionTemplateDecl>(Specialized))
     EntityKind = 2;
-    isTemplateSpecialization = true;
-  } else if (isa<CXXMethodDecl>(Specialized))
+  else if (isa<CXXMethodDecl>(Specialized))
     EntityKind = 3;
   else if (isa<VarDecl>(Specialized))
     EntityKind = 4;
@@ -3429,38 +4064,53 @@ static bool CheckTemplateSpecializationScope(Sema &S,
       << Specialized;
     return true;
   }
-  
+
   // C++ [temp.class.spec]p6:
   //   A class template partial specialization may be declared or redeclared
-  //   in any namespace scope in which its definition may be defined (14.5.1 
-  //   and 14.5.2).  
+  //   in any namespace scope in which its definition may be defined (14.5.1
+  //   and 14.5.2).
   bool ComplainedAboutScope = false;
-  DeclContext *SpecializedContext 
+  DeclContext *SpecializedContext
     = Specialized->getDeclContext()->getEnclosingNamespaceContext();
   DeclContext *DC = S.CurContext->getEnclosingNamespaceContext();
-  if ((!PrevDecl || 
+  if ((!PrevDecl ||
        getTemplateSpecializationKind(PrevDecl) == TSK_Undeclared ||
        getTemplateSpecializationKind(PrevDecl) == TSK_ImplicitInstantiation)){
-    // There is no prior declaration of this entity, so this
-    // specialization must be in the same context as the template
-    // itself, or in the enclosing namespace set.
-    if (!DC->InEnclosingNamespaceSetOf(SpecializedContext)) {
+    // C++ [temp.exp.spec]p2:
+    //   An explicit specialization shall be declared in the namespace of which
+    //   the template is a member, or, for member templates, in the namespace
+    //   of which the enclosing class or enclosing class template is a member.
+    //   An explicit specialization of a member function, member class or
+    //   static data member of a class template shall be declared in the
+    //   namespace of which the class template is a member.
+    //
+    // C++0x [temp.expl.spec]p2:
+    //   An explicit specialization shall be declared in a namespace enclosing
+    //   the specialized template.
+    if (!DC->InEnclosingNamespaceSetOf(SpecializedContext) &&
+        !(S.getLangOptions().CPlusPlus0x && DC->Encloses(SpecializedContext))) {
+      bool IsCPlusPlus0xExtension
+        = !S.getLangOptions().CPlusPlus0x && DC->Encloses(SpecializedContext);
       if (isa<TranslationUnitDecl>(SpecializedContext))
-        S.Diag(Loc, diag::err_template_spec_decl_out_of_scope_global)
-        << EntityKind << Specialized;
+        S.Diag(Loc, IsCPlusPlus0xExtension
+                      ? diag::ext_template_spec_decl_out_of_scope_global
+                      : diag::err_template_spec_decl_out_of_scope_global)
+          << EntityKind << Specialized;
       else if (isa<NamespaceDecl>(SpecializedContext))
-        S.Diag(Loc, diag::err_template_spec_decl_out_of_scope)
-        << EntityKind << Specialized
-        << cast<NamedDecl>(SpecializedContext);
-      
+        S.Diag(Loc, IsCPlusPlus0xExtension
+                      ? diag::ext_template_spec_decl_out_of_scope
+                      : diag::err_template_spec_decl_out_of_scope)
+          << EntityKind << Specialized
+          << cast<NamedDecl>(SpecializedContext);
+
       S.Diag(Specialized->getLocation(), diag::note_specialized_entity);
       ComplainedAboutScope = true;
     }
   }
-  
-  // Make sure that this redeclaration (or definition) occurs in an enclosing 
+
+  // Make sure that this redeclaration (or definition) occurs in an enclosing
   // namespace.
-  // Note that HandleDeclarator() performs this check for explicit 
+  // Note that HandleDeclarator() performs this check for explicit
   // specializations of function templates, static data members, and member
   // functions, so we skip the check here for those kinds of entities.
   // FIXME: HandleDeclarator's diagnostics aren't quite as good, though.
@@ -3475,77 +4125,44 @@ static bool CheckTemplateSpecializationScope(Sema &S,
       S.Diag(Loc, diag::err_template_spec_redecl_out_of_scope)
         << EntityKind << Specialized
         << cast<NamedDecl>(SpecializedContext);
-  
+
     S.Diag(Specialized->getLocation(), diag::note_specialized_entity);
   }
-  
+
   // FIXME: check for specialization-after-instantiation errors and such.
-  
+
   return false;
 }
-                                             
-/// \brief Check the non-type template arguments of a class template
-/// partial specialization according to C++ [temp.class.spec]p9.
-///
-/// \param TemplateParams the template parameters of the primary class
-/// template.
-///
-/// \param TemplateArg the template arguments of the class template
-/// partial specialization.
-///
-/// \param MirrorsPrimaryTemplate will be set true if the class
-/// template partial specialization arguments are identical to the
-/// implicit template arguments of the primary template. This is not
-/// necessarily an error (C++0x), and it is left to the caller to diagnose
-/// this condition when it is an error.
-///
-/// \returns true if there was an error, false otherwise.
-bool Sema::CheckClassTemplatePartialSpecializationArgs(
-                                        TemplateParameterList *TemplateParams,
-                             const TemplateArgumentListBuilder &TemplateArgs,
-                                        bool &MirrorsPrimaryTemplate) {
-  // FIXME: the interface to this function will have to change to
-  // accommodate variadic templates.
-  MirrorsPrimaryTemplate = true;
-
-  const TemplateArgument *ArgList = TemplateArgs.getFlatArguments();
 
-  for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
-    // Determine whether the template argument list of the partial
-    // specialization is identical to the implicit argument list of
-    // the primary template. The caller may need to diagnostic this as
-    // an error per C++ [temp.class.spec]p9b3.
-    if (MirrorsPrimaryTemplate) {
-      if (TemplateTypeParmDecl *TTP
-            = dyn_cast<TemplateTypeParmDecl>(TemplateParams->getParam(I))) {
-        if (Context.getCanonicalType(Context.getTypeDeclType(TTP)) !=
-              Context.getCanonicalType(ArgList[I].getAsType()))
-          MirrorsPrimaryTemplate = false;
-      } else if (TemplateTemplateParmDecl *TTP
-                   = dyn_cast<TemplateTemplateParmDecl>(
-                                                 TemplateParams->getParam(I))) {
-        TemplateName Name = ArgList[I].getAsTemplate();
-        TemplateTemplateParmDecl *ArgDecl
-          = dyn_cast_or_null<TemplateTemplateParmDecl>(Name.getAsTemplateDecl());
-        if (!ArgDecl ||
-            ArgDecl->getIndex() != TTP->getIndex() ||
-            ArgDecl->getDepth() != TTP->getDepth())
-          MirrorsPrimaryTemplate = false;
-      }
-    }
+/// \brief Subroutine of Sema::CheckClassTemplatePartialSpecializationArgs
+/// that checks non-type template partial specialization arguments.
+static bool CheckNonTypeClassTemplatePartialSpecializationArgs(Sema &S,
+                                                NonTypeTemplateParmDecl *Param,
+                                                  const TemplateArgument *Args,
+                                                        unsigned NumArgs) {
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    if (Args[I].getKind() == TemplateArgument::Pack) {
+      if (CheckNonTypeClassTemplatePartialSpecializationArgs(S, Param,
+                                                           Args[I].pack_begin(),
+                                                           Args[I].pack_size()))
+        return true;
 
-    NonTypeTemplateParmDecl *Param
-      = dyn_cast<NonTypeTemplateParmDecl>(TemplateParams->getParam(I));
-    if (!Param) {
       continue;
     }
 
-    Expr *ArgExpr = ArgList[I].getAsExpr();
+    Expr *ArgExpr = Args[I].getAsExpr();
     if (!ArgExpr) {
-      MirrorsPrimaryTemplate = false;
       continue;
     }
 
+    // We can have a pack expansion of any of the bullets below.
+    if (PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(ArgExpr))
+      ArgExpr = Expansion->getPattern();
+
+    // Strip off any implicit casts we added as part of type checking.
+    while (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(ArgExpr))
+      ArgExpr = ICE->getSubExpr();
+
     // C++ [temp.class.spec]p8:
     //   A non-type argument is non-specialized if it is the name of a
     //   non-type parameter. All other non-type arguments are
@@ -3555,15 +4172,8 @@ bool Sema::CheckClassTemplatePartialSpecializationArgs(
     // specialized non-type arguments, so skip any non-specialized
     // arguments.
     if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(ArgExpr))
-      if (NonTypeTemplateParmDecl *NTTP
-            = dyn_cast<NonTypeTemplateParmDecl>(DRE->getDecl())) {
-        if (MirrorsPrimaryTemplate &&
-            (Param->getIndex() != NTTP->getIndex() ||
-             Param->getDepth() != NTTP->getDepth()))
-          MirrorsPrimaryTemplate = false;
-
+      if (isa<NonTypeTemplateParmDecl>(DRE->getDecl()))
         continue;
-      }
 
     // C++ [temp.class.spec]p9:
     //   Within the argument list of a class template partial
@@ -3573,7 +4183,7 @@ bool Sema::CheckClassTemplatePartialSpecializationArgs(
     //        specialization except when the argument expression is a
     //        simple identifier.
     if (ArgExpr->isTypeDependent() || ArgExpr->isValueDependent()) {
-      Diag(ArgExpr->getLocStart(),
+      S.Diag(ArgExpr->getLocStart(),
            diag::err_dependent_non_type_arg_in_partial_spec)
         << ArgExpr->getSourceRange();
       return true;
@@ -3583,15 +4193,42 @@ bool Sema::CheckClassTemplatePartialSpecializationArgs(
     //        specialized non-type argument shall not be dependent on a
     //        parameter of the specialization.
     if (Param->getType()->isDependentType()) {
-      Diag(ArgExpr->getLocStart(),
+      S.Diag(ArgExpr->getLocStart(),
            diag::err_dependent_typed_non_type_arg_in_partial_spec)
         << Param->getType()
         << ArgExpr->getSourceRange();
-      Diag(Param->getLocation(), diag::note_template_param_here);
+      S.Diag(Param->getLocation(), diag::note_template_param_here);
       return true;
     }
+  }
+
+  return false;
+}
 
-    MirrorsPrimaryTemplate = false;
+/// \brief Check the non-type template arguments of a class template
+/// partial specialization according to C++ [temp.class.spec]p9.
+///
+/// \param TemplateParams the template parameters of the primary class
+/// template.
+///
+/// \param TemplateArg the template arguments of the class template
+/// partial specialization.
+///
+/// \returns true if there was an error, false otherwise.
+static bool CheckClassTemplatePartialSpecializationArgs(Sema &S,
+                                        TemplateParameterList *TemplateParams,
+                       llvm::SmallVectorImpl<TemplateArgument> &TemplateArgs) {
+  const TemplateArgument *ArgList = TemplateArgs.data();
+
+  for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
+    NonTypeTemplateParmDecl *Param
+      = dyn_cast<NonTypeTemplateParmDecl>(TemplateParams->getParam(I));
+    if (!Param)
+      continue;
+
+    if (CheckNonTypeClassTemplatePartialSpecializationArgs(S, Param,
+                                                           &ArgList[I], 1))
+      return true;
   }
 
   return false;
@@ -3635,7 +4272,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 
   if (!ClassTemplate) {
     Diag(TemplateNameLoc, diag::err_not_class_template_specialization)
-      << (Name.getAsTemplateDecl() && 
+      << (Name.getAsTemplateDecl() &&
           isa<TemplateTemplateParmDecl>(Name.getAsTemplateDecl()));
     return true;
   }
@@ -3657,7 +4294,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                               Invalid);
   if (Invalid)
     return true;
-    
+
   unsigned NumMatchedTemplateParamLists = TemplateParameterLists.size();
   if (TemplateParams)
     --NumMatchedTemplateParamLists;
@@ -3665,6 +4302,12 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   if (TemplateParams && TemplateParams->size() > 0) {
     isPartialSpecialization = true;
 
+    if (TUK == TUK_Friend) {
+      Diag(KWLoc, diag::err_partial_specialization_friend)
+        << SourceRange(LAngleLoc, RAngleLoc);
+      return true;
+    }
+
     // C++ [temp.class.spec]p10:
     //   The template parameter list of a specialization shall not
     //   contain default template argument values.
@@ -3731,48 +4374,33 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   TemplateArgs.setRAngleLoc(RAngleLoc);
   translateTemplateArguments(TemplateArgsIn, TemplateArgs);
 
+  // Check for unexpanded parameter packs in any of the template arguments.
+  for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
+    if (DiagnoseUnexpandedParameterPack(TemplateArgs[I],
+                                        UPPC_PartialSpecialization))
+      return true;
+
   // Check that the template argument list is well-formed for this
   // template.
-  TemplateArgumentListBuilder Converted(ClassTemplate->getTemplateParameters(),
-                                        TemplateArgs.size());
+  llvm::SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc,
                                 TemplateArgs, false, Converted))
     return true;
 
-  assert((Converted.structuredSize() ==
-            ClassTemplate->getTemplateParameters()->size()) &&
+  assert((Converted.size() == ClassTemplate->getTemplateParameters()->size()) &&
          "Converted template argument list is too short!");
 
   // Find the class template (partial) specialization declaration that
   // corresponds to these arguments.
   if (isPartialSpecialization) {
-    bool MirrorsPrimaryTemplate;
-    if (CheckClassTemplatePartialSpecializationArgs(
+    if (CheckClassTemplatePartialSpecializationArgs(*this,
                                          ClassTemplate->getTemplateParameters(),
-                                         Converted, MirrorsPrimaryTemplate))
+                                         Converted))
       return true;
 
-    if (MirrorsPrimaryTemplate) {
-      // C++ [temp.class.spec]p9b3:
-      //
-      //   -- The argument list of the specialization shall not be identical
-      //      to the implicit argument list of the primary template.
-      Diag(TemplateNameLoc, diag::err_partial_spec_args_match_primary_template)
-        << (TUK == TUK_Definition)
-        << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
-      return CheckClassTemplate(S, TagSpec, TUK, KWLoc, SS,
-                                ClassTemplate->getIdentifier(),
-                                TemplateNameLoc,
-                                Attr,
-                                TemplateParams,
-                                AS_none);
-    }
-
-    // FIXME: Diagnose friend partial specializations
-
-    if (!Name.isDependent() && 
+    if (!Name.isDependent() &&
         !TemplateSpecializationType::anyDependentTemplateArguments(
-                                             TemplateArgs.getArgumentArray(), 
+                                             TemplateArgs.getArgumentArray(),
                                                          TemplateArgs.size())) {
       Diag(TemplateNameLoc, diag::err_partial_spec_fully_specialized)
         << ClassTemplate->getDeclName();
@@ -3786,27 +4414,27 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   if (isPartialSpecialization)
     // FIXME: Template parameter list matters, too
     PrevDecl
-      = ClassTemplate->findPartialSpecialization(Converted.getFlatArguments(),
-                                                 Converted.flatSize(),
+      = ClassTemplate->findPartialSpecialization(Converted.data(),
+                                                 Converted.size(),
                                                  InsertPos);
   else
     PrevDecl
-      = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
-                                          Converted.flatSize(), InsertPos);
+      = ClassTemplate->findSpecialization(Converted.data(),
+                                          Converted.size(), InsertPos);
 
   ClassTemplateSpecializationDecl *Specialization = 0;
 
   // Check whether we can declare a class template specialization in
   // the current scope.
   if (TUK != TUK_Friend &&
-      CheckTemplateSpecializationScope(*this, ClassTemplate, PrevDecl, 
-                                       TemplateNameLoc, 
+      CheckTemplateSpecializationScope(*this, ClassTemplate, PrevDecl,
+                                       TemplateNameLoc,
                                        isPartialSpecialization))
     return true;
-  
+
   // The canonical type
   QualType CanonType;
-  if (PrevDecl && 
+  if (PrevDecl &&
       (PrevDecl->getSpecializationKind() == TSK_Undeclared ||
                TUK == TUK_Friend)) {
     // Since the only prior class template specialization with these
@@ -3823,8 +4451,25 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     // arguments of the class template partial specialization.
     TemplateName CanonTemplate = Context.getCanonicalTemplateName(Name);
     CanonType = Context.getTemplateSpecializationType(CanonTemplate,
-                                                  Converted.getFlatArguments(),
-                                                  Converted.flatSize());
+                                                      Converted.data(),
+                                                      Converted.size());
+
+    if (Context.hasSameType(CanonType,
+                        ClassTemplate->getInjectedClassNameSpecialization())) {
+      // C++ [temp.class.spec]p9b3:
+      //
+      //   -- The argument list of the specialization shall not be identical
+      //      to the implicit argument list of the primary template.
+      Diag(TemplateNameLoc, diag::err_partial_spec_args_match_primary_template)
+      << (TUK == TUK_Definition)
+      << FixItHint::CreateRemoval(SourceRange(LAngleLoc, RAngleLoc));
+      return CheckClassTemplate(S, TagSpec, TUK, KWLoc, SS,
+                                ClassTemplate->getIdentifier(),
+                                TemplateNameLoc,
+                                Attr,
+                                TemplateParams,
+                                AS_none);
+    }
 
     // Create a new class template partial specialization declaration node.
     ClassTemplatePartialSpecializationDecl *PrevPartial
@@ -3837,7 +4482,8 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                                        TemplateNameLoc,
                                                        TemplateParams,
                                                        ClassTemplate,
-                                                       Converted,
+                                                       Converted.data(),
+                                                       Converted.size(),
                                                        TemplateArgs,
                                                        CanonType,
                                                        PrevPartial,
@@ -3853,18 +4499,18 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
       ClassTemplate->AddPartialSpecialization(Partial, InsertPos);
     Specialization = Partial;
 
-    // If we are providing an explicit specialization of a member class 
+    // If we are providing an explicit specialization of a member class
     // template specialization, make a note of that.
     if (PrevPartial && PrevPartial->getInstantiatedFromMember())
       PrevPartial->setMemberSpecialization();
-    
+
     // Check that all of the template parameters of the class template
     // partial specialization are deducible from the template
     // arguments. If not, this class template partial specialization
     // will never be used.
     llvm::SmallVector<bool, 8> DeducibleParams;
     DeducibleParams.resize(TemplateParams->size());
-    MarkUsedTemplateParameters(Partial->getTemplateArgs(), true, 
+    MarkUsedTemplateParameters(Partial->getTemplateArgs(), true,
                                TemplateParams->getDepth(),
                                DeducibleParams);
     unsigned NumNonDeducible = 0;
@@ -3898,7 +4544,8 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                              ClassTemplate->getDeclContext(),
                                                 TemplateNameLoc,
                                                 ClassTemplate,
-                                                Converted,
+                                                Converted.data(),
+                                                Converted.size(),
                                                 PrevDecl);
     SetNestedNameSpecifier(Specialization, SS);
     if (NumMatchedTemplateParamLists > 0 && SS.isSet()) {
@@ -3915,9 +4562,9 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 
   // C++ [temp.expl.spec]p6:
   //   If a template, a member template or the member of a class template is
-  //   explicitly specialized then that specialization shall be declared 
+  //   explicitly specialized then that specialization shall be declared
   //   before the first use of that specialization that would cause an implicit
-  //   instantiation to take place, in every translation unit in which such a 
+  //   instantiation to take place, in every translation unit in which such a
   //   use occurs; no diagnostic is required.
   if (PrevDecl && PrevDecl->getPointOfInstantiation().isValid()) {
     bool Okay = false;
@@ -3934,14 +4581,14 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
       Diag(TemplateNameLoc, diag::err_specialization_after_instantiation)
         << Context.getTypeDeclType(Specialization) << Range;
 
-      Diag(PrevDecl->getPointOfInstantiation(), 
+      Diag(PrevDecl->getPointOfInstantiation(),
            diag::note_instantiation_required_here)
-        << (PrevDecl->getTemplateSpecializationKind() 
+        << (PrevDecl->getTemplateSpecializationKind()
                                                 != TSK_ImplicitInstantiation);
       return true;
     }
   }
-  
+
   // If this is not a friend, note that this is an explicit specialization.
   if (TUK != TUK_Friend)
     Specialization->setSpecializationKind(TSK_ExplicitSpecialization);
@@ -3958,6 +4605,9 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     }
   }
 
+  if (Attr)
+    ProcessDeclAttributeList(S, Specialization, Attr);
+
   // Build the fully-sugared type for this class template
   // specialization as the user wrote in the specialization
   // itself. This means that we'll pretty-print the type retrieved
@@ -4015,9 +4665,7 @@ Decl *Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
                                MultiTemplateParamsArg TemplateParameterLists,
                                             Declarator &D) {
   assert(getCurFunctionDecl() == 0 && "Function parsing confused");
-  assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
-         "Not a function declarator!");
-  DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
+  DeclaratorChunk::FunctionTypeInfo &FTI = D.getFunctionTypeInfo();
 
   if (FTI.hasPrototype) {
     // FIXME: Diagnose arguments without names in C.
@@ -4047,12 +4695,12 @@ static void StripImplicitInstantiation(NamedDecl *D) {
   }
 }
 
-/// \brief Diagnose cases where we have an explicit template specialization 
+/// \brief Diagnose cases where we have an explicit template specialization
 /// before/after an explicit template instantiation, producing diagnostics
-/// for those cases where they are required and determining whether the 
+/// for those cases where they are required and determining whether the
 /// new specialization/instantiation will have any effect.
 ///
-/// \param NewLoc the location of the new explicit specialization or 
+/// \param NewLoc the location of the new explicit specialization or
 /// instantiation.
 ///
 /// \param NewTSK the kind of the new explicit specialization or instantiation.
@@ -4061,10 +4709,10 @@ static void StripImplicitInstantiation(NamedDecl *D) {
 ///
 /// \param PrevTSK the kind of the old explicit specialization or instantiatin.
 ///
-/// \param PrevPointOfInstantiation if valid, indicates where the previus 
+/// \param PrevPointOfInstantiation if valid, indicates where the previus
 /// declaration was instantiated (either implicitly or explicitly).
 ///
-/// \param HasNoEffect will be set to true to indicate that the new 
+/// \param HasNoEffect will be set to true to indicate that the new
 /// specialization or instantiation has no effect and should be ignored.
 ///
 /// \returns true if there was an error that should prevent the introduction of
@@ -4077,18 +4725,18 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
                                         SourceLocation PrevPointOfInstantiation,
                                              bool &HasNoEffect) {
   HasNoEffect = false;
-  
+
   switch (NewTSK) {
   case TSK_Undeclared:
   case TSK_ImplicitInstantiation:
     assert(false && "Don't check implicit instantiations here");
     return false;
-    
+
   case TSK_ExplicitSpecialization:
     switch (PrevTSK) {
     case TSK_Undeclared:
     case TSK_ExplicitSpecialization:
-      // Okay, we're just specializing something that is either already 
+      // Okay, we're just specializing something that is either already
       // explicitly specialized or has merely been mentioned without any
       // instantiation.
       return false;
@@ -4101,17 +4749,17 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
         return false;
       }
       // Fall through
-        
+
     case TSK_ExplicitInstantiationDeclaration:
     case TSK_ExplicitInstantiationDefinition:
-      assert((PrevTSK == TSK_ImplicitInstantiation || 
-              PrevPointOfInstantiation.isValid()) && 
+      assert((PrevTSK == TSK_ImplicitInstantiation ||
+              PrevPointOfInstantiation.isValid()) &&
              "Explicit instantiation without point of instantiation?");
-        
+
       // C++ [temp.expl.spec]p6:
-      //   If a template, a member template or the member of a class template 
+      //   If a template, a member template or the member of a class template
       //   is explicitly specialized then that specialization shall be declared
-      //   before the first use of that specialization that would cause an 
+      //   before the first use of that specialization that would cause an
       //   implicit instantiation to take place, in every translation unit in
       //   which such a use occurs; no diagnostic is required.
       for (NamedDecl *Prev = PrevDecl; Prev; Prev = getPreviousDecl(Prev)) {
@@ -4124,41 +4772,41 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
         << PrevDecl;
       Diag(PrevPointOfInstantiation, diag::note_instantiation_required_here)
         << (PrevTSK != TSK_ImplicitInstantiation);
-      
+
       return true;
     }
     break;
-      
+
   case TSK_ExplicitInstantiationDeclaration:
     switch (PrevTSK) {
     case TSK_ExplicitInstantiationDeclaration:
       // This explicit instantiation declaration is redundant (that's okay).
       HasNoEffect = true;
       return false;
-        
+
     case TSK_Undeclared:
     case TSK_ImplicitInstantiation:
       // We're explicitly instantiating something that may have already been
       // implicitly instantiated; that's fine.
       return false;
-        
+
     case TSK_ExplicitSpecialization:
       // C++0x [temp.explicit]p4:
       //   For a given set of template parameters, if an explicit instantiation
-      //   of a template appears after a declaration of an explicit 
+      //   of a template appears after a declaration of an explicit
       //   specialization for that template, the explicit instantiation has no
       //   effect.
       HasNoEffect = true;
       return false;
-        
+
     case TSK_ExplicitInstantiationDefinition:
       // C++0x [temp.explicit]p10:
-      //   If an entity is the subject of both an explicit instantiation 
-      //   declaration and an explicit instantiation definition in the same 
+      //   If an entity is the subject of both an explicit instantiation
+      //   declaration and an explicit instantiation definition in the same
       //   translation unit, the definition shall follow the declaration.
-      Diag(NewLoc, 
+      Diag(NewLoc,
            diag::err_explicit_instantiation_declaration_after_definition);
-      Diag(PrevPointOfInstantiation, 
+      Diag(PrevPointOfInstantiation,
            diag::note_explicit_instantiation_definition_here);
       assert(PrevPointOfInstantiation.isValid() &&
              "Explicit instantiation without point of instantiation?");
@@ -4166,7 +4814,7 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
       return false;
     }
     break;
-      
+
   case TSK_ExplicitInstantiationDefinition:
     switch (PrevTSK) {
     case TSK_Undeclared:
@@ -4174,7 +4822,7 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
       // We're explicitly instantiating something that may have already been
       // implicitly instantiated; that's fine.
       return false;
-        
+
     case TSK_ExplicitSpecialization:
       // C++ DR 259, C++0x [temp.explicit]p4:
       //   For a given set of template parameters, if an explicit
@@ -4182,7 +4830,7 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
       //   an explicit specialization for that template, the explicit
       //   instantiation has no effect.
       //
-      // In C++98/03 mode, we only give an extension warning here, because it 
+      // In C++98/03 mode, we only give an extension warning here, because it
       // is not harmful to try to explicitly instantiate something that
       // has been explicitly specialized.
       if (!getLangOptions().CPlusPlus0x) {
@@ -4193,12 +4841,12 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
       }
       HasNoEffect = true;
       return false;
-        
+
     case TSK_ExplicitInstantiationDeclaration:
       // We're explicity instantiating a definition for something for which we
-      // were previously asked to suppress instantiations. That's fine. 
+      // were previously asked to suppress instantiations. That's fine.
       return false;
-        
+
     case TSK_ExplicitInstantiationDefinition:
       // C++0x [temp.spec]p5:
       //   For a given template and a given set of template-arguments,
@@ -4206,16 +4854,16 @@ Sema::CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
       //       in a program,
       Diag(NewLoc, diag::err_explicit_instantiation_duplicate)
         << PrevDecl;
-      Diag(PrevPointOfInstantiation, 
+      Diag(PrevPointOfInstantiation,
            diag::note_previous_explicit_instantiation);
       HasNoEffect = true;
-      return false;        
+      return false;
     }
     break;
   }
-  
+
   assert(false && "Missing specialization/instantiation case?");
-         
+
   return false;
 }
 
@@ -4281,21 +4929,21 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
   // The set of function template specializations that could match this
   // explicit function template specialization.
   UnresolvedSet<8> Candidates;
-  
+
   DeclContext *FDLookupContext = FD->getDeclContext()->getRedeclContext();
   for (LookupResult::iterator I = Previous.begin(), E = Previous.end();
          I != E; ++I) {
     NamedDecl *Ovl = (*I)->getUnderlyingDecl();
     if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(Ovl)) {
-      // Only consider templates found within the same semantic lookup scope as 
+      // Only consider templates found within the same semantic lookup scope as
       // FD.
       if (!FDLookupContext->InEnclosingNamespaceSetOf(
                                 Ovl->getDeclContext()->getRedeclContext()))
         continue;
-      
+
       // C++ [temp.expl.spec]p11:
-      //   A trailing template-argument can be left unspecified in the 
-      //   template-id naming an explicit function template specialization 
+      //   A trailing template-argument can be left unspecified in the
+      //   template-id naming an explicit function template specialization
       //   provided it can be deduced from the function argument type.
       // Perform template argument deduction to determine whether we may be
       // specializing this template.
@@ -4312,17 +4960,17 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
         (void)TDK;
         continue;
       }
-      
+
       // Record this candidate.
       Candidates.addDecl(Specialization, I.getAccess());
     }
   }
-  
+
   // Find the most specialized function template.
   UnresolvedSetIterator Result
     = getMostSpecialized(Candidates.begin(), Candidates.end(),
-                         TPOC_Other, FD->getLocation(),
-                  PDiag(diag::err_function_template_spec_no_match) 
+                         TPOC_Other, 0, FD->getLocation(),
+                  PDiag(diag::err_function_template_spec_no_match)
                     << FD->getDeclName(),
                   PDiag(diag::err_function_template_spec_ambiguous)
                     << FD->getDeclName() << (ExplicitTemplateArgs != 0),
@@ -4333,27 +4981,27 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
   // Ignore access information;  it doesn't figure into redeclaration checking.
   FunctionDecl *Specialization = cast<FunctionDecl>(*Result);
   Specialization->setLocation(FD->getLocation());
-  
+
   // FIXME: Check if the prior specialization has a point of instantiation.
   // If so, we have run afoul of .
 
   // If this is a friend declaration, then we're not really declaring
   // an explicit specialization.
   bool isFriend = (FD->getFriendObjectKind() != Decl::FOK_None);
-  
+
   // Check the scope of this explicit specialization.
   if (!isFriend &&
-      CheckTemplateSpecializationScope(*this, 
+      CheckTemplateSpecializationScope(*this,
                                        Specialization->getPrimaryTemplate(),
-                                       Specialization, FD->getLocation(), 
+                                       Specialization, FD->getLocation(),
                                        false))
     return true;
 
   // C++ [temp.expl.spec]p6:
   //   If a template, a member template or the member of a class template is
-  //   explicitly specialized then that specialization shall be declared 
+  //   explicitly specialized then that specialization shall be declared
   //   before the first use of that specialization that would cause an implicit
-  //   instantiation to take place, in every translation unit in which such a 
+  //   instantiation to take place, in every translation unit in which such a
   //   use occurs; no diagnostic is required.
   FunctionTemplateSpecializationInfo *SpecInfo
     = Specialization->getTemplateSpecializationInfo();
@@ -4368,14 +5016,14 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
                                          SpecInfo->getPointOfInstantiation(),
                                              HasNoEffect))
     return true;
-  
+
   // Mark the prior declaration as an explicit specialization, so that later
   // clients know that this is an explicit specialization.
   if (!isFriend) {
     SpecInfo->setTemplateSpecializationKind(TSK_ExplicitSpecialization);
     MarkUnusedFileScopedDecl(Specialization);
   }
-  
+
   // Turn the given function declaration into a function template
   // specialization, with the template arguments from the previous
   // specialization.
@@ -4399,7 +5047,7 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
 /// \brief Perform semantic analysis for the given non-template member
 /// specialization.
 ///
-/// This routine performs all of the semantic analysis required for an 
+/// This routine performs all of the semantic analysis required for an
 /// explicit member function specialization. On successful completion,
 /// the function declaration \p FD will become a member function
 /// specialization.
@@ -4410,7 +5058,7 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
 /// \param Previous the set of declarations, one of which may be specialized
 /// by this function specialization;  the set will be modified to contain the
 /// redeclared member.
-bool 
+bool
 Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
   assert(!isa<TemplateDecl>(Member) && "Only for non-template members");
 
@@ -4452,7 +5100,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
       MSInfo = PrevRecord->getMemberSpecializationInfo();
     }
   }
-  
+
   if (!Instantiation) {
     // There is no previous declaration that matches. Since member
     // specializations are always out-of-line, the caller will complain about
@@ -4478,7 +5126,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
     Previous.addDecl(Instantiation);
     return false;
   }
-  
+
   // Make sure that this is a specialization of a member.
   if (!InstantiatedFrom) {
     Diag(Member->getLocation(), diag::err_spec_member_not_instantiated)
@@ -4486,12 +5134,12 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
     Diag(Instantiation->getLocation(), diag::note_specialized_decl);
     return true;
   }
-  
+
   // C++ [temp.expl.spec]p6:
   //   If a template, a member template or the member of a class template is
-  //   explicitly specialized then that spe- cialization shall be declared 
+  //   explicitly specialized then that spe- cialization shall be declared
   //   before the first use of that specialization that would cause an implicit
-  //   instantiation to take place, in every translation unit in which such a 
+  //   instantiation to take place, in every translation unit in which such a
   //   use occurs; no diagnostic is required.
   assert(MSInfo && "Member specialization info missing?");
 
@@ -4503,11 +5151,11 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
                                            MSInfo->getPointOfInstantiation(),
                                              HasNoEffect))
     return true;
-  
+
   // Check the scope of this explicit specialization.
-  if (CheckTemplateSpecializationScope(*this, 
+  if (CheckTemplateSpecializationScope(*this,
                                        InstantiatedFrom,
-                                       Instantiation, Member->getLocation(), 
+                                       Instantiation, Member->getLocation(),
                                        false))
     return true;
 
@@ -4523,7 +5171,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
                                                   TSK_ExplicitSpecialization);
       InstantiationFunction->setLocation(Member->getLocation());
     }
-    
+
     cast<FunctionDecl>(Member)->setInstantiationOfMemberFunction(
                                         cast<CXXMethodDecl>(InstantiatedFrom),
                                                   TSK_ExplicitSpecialization);
@@ -4536,7 +5184,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
                                                   TSK_ExplicitSpecialization);
       InstantiationVar->setLocation(Member->getLocation());
     }
-    
+
     Context.setInstantiatedFromStaticDataMember(cast<VarDecl>(Member),
                                                 cast<VarDecl>(InstantiatedFrom),
                                                 TSK_ExplicitSpecialization);
@@ -4550,12 +5198,12 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
                                                    TSK_ExplicitSpecialization);
       InstantiationClass->setLocation(Member->getLocation());
     }
-    
+
     cast<CXXRecordDecl>(Member)->setInstantiationOfMemberClass(
                                         cast<CXXRecordDecl>(InstantiatedFrom),
                                                    TSK_ExplicitSpecialization);
   }
-             
+
   // Save the caller the trouble of having to figure out which declaration
   // this specialization matches.
   Previous.clear();
@@ -4571,28 +5219,28 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
                                             bool WasQualifiedName) {
   DeclContext *OrigContext= D->getDeclContext()->getEnclosingNamespaceContext();
   DeclContext *CurContext = S.CurContext->getRedeclContext();
-  
+
   if (CurContext->isRecord()) {
     S.Diag(InstLoc, diag::err_explicit_instantiation_in_class)
       << D;
     return true;
   }
-  
+
   // C++0x [temp.explicit]p2:
-  //   An explicit instantiation shall appear in an enclosing namespace of its 
+  //   An explicit instantiation shall appear in an enclosing namespace of its
   //   template.
   //
   // This is DR275, which we do not retroactively apply to C++98/03.
-  if (S.getLangOptions().CPlusPlus0x && 
+  if (S.getLangOptions().CPlusPlus0x &&
       !CurContext->Encloses(OrigContext)) {
     if (NamespaceDecl *NS = dyn_cast<NamespaceDecl>(OrigContext))
-      S.Diag(InstLoc, 
-             S.getLangOptions().CPlusPlus0x? 
+      S.Diag(InstLoc,
+             S.getLangOptions().CPlusPlus0x?
                  diag::err_explicit_instantiation_out_of_scope
                : diag::warn_explicit_instantiation_out_of_scope_0x)
         << D << NS;
     else
-      S.Diag(InstLoc, 
+      S.Diag(InstLoc,
              S.getLangOptions().CPlusPlus0x?
                  diag::err_explicit_instantiation_must_be_global
                : diag::warn_explicit_instantiation_out_of_scope_0x)
@@ -4602,8 +5250,8 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
   }
 
   // C++0x [temp.explicit]p2:
-  //   If the name declared in the explicit instantiation is an unqualified 
-  //   name, the explicit instantiation shall appear in the namespace where 
+  //   If the name declared in the explicit instantiation is an unqualified
+  //   name, the explicit instantiation shall appear in the namespace where
   //   its template is declared or, if that namespace is inline (7.3.1), any
   //   namespace from its enclosing namespace set.
   if (WasQualifiedName)
@@ -4612,7 +5260,7 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
   if (CurContext->InEnclosingNamespaceSetOf(OrigContext))
     return false;
 
-  S.Diag(InstLoc, 
+  S.Diag(InstLoc,
          S.getLangOptions().CPlusPlus0x?
              diag::err_explicit_instantiation_unqualified_wrong_namespace
            : diag::warn_explicit_instantiation_unqualified_wrong_namespace_0x)
@@ -4625,9 +5273,9 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
 static bool ScopeSpecifierHasTemplateId(const CXXScopeSpec &SS) {
   if (!SS.isSet())
     return false;
-  
+
   // C++0x [temp.explicit]p2:
-  //   If the explicit instantiation is for a member function, a member class 
+  //   If the explicit instantiation is for a member function, a member class
   //   or a static data member of a class template specialization, the name of
   //   the class template specialization in the qualified-id for the member
   //   name shall be a simple-template-id.
@@ -4635,7 +5283,7 @@ static bool ScopeSpecifierHasTemplateId(const CXXScopeSpec &SS) {
   // C++98 has the same restriction, just worded differently.
   for (NestedNameSpecifier *NNS = (NestedNameSpecifier *)SS.getScopeRep();
        NNS; NNS = NNS->getPrefix())
-    if (Type *T = NNS->getAsType())
+    if (const Type *T = NNS->getAsType())
       if (isa<TemplateSpecializationType>(T))
         return true;
 
@@ -4680,34 +5328,32 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   // C++0x [temp.explicit]p2:
   //   There are two forms of explicit instantiation: an explicit instantiation
-  //   definition and an explicit instantiation declaration. An explicit 
-  //   instantiation declaration begins with the extern keyword. [...]  
+  //   definition and an explicit instantiation declaration. An explicit
+  //   instantiation declaration begins with the extern keyword. [...]
   TemplateSpecializationKind TSK
     = ExternLoc.isInvalid()? TSK_ExplicitInstantiationDefinition
                            : TSK_ExplicitInstantiationDeclaration;
-  
+
   // Translate the parser's template argument list in our AST format.
   TemplateArgumentListInfo TemplateArgs(LAngleLoc, RAngleLoc);
   translateTemplateArguments(TemplateArgsIn, TemplateArgs);
 
   // Check that the template argument list is well-formed for this
   // template.
-  TemplateArgumentListBuilder Converted(ClassTemplate->getTemplateParameters(),
-                                        TemplateArgs.size());
+  llvm::SmallVector<TemplateArgument, 4> Converted;
   if (CheckTemplateArgumentList(ClassTemplate, TemplateNameLoc,
                                 TemplateArgs, false, Converted))
     return true;
 
-  assert((Converted.structuredSize() ==
-            ClassTemplate->getTemplateParameters()->size()) &&
+  assert((Converted.size() == ClassTemplate->getTemplateParameters()->size()) &&
          "Converted template argument list is too short!");
 
   // Find the class template specialization declaration that
   // corresponds to these arguments.
   void *InsertPos = 0;
   ClassTemplateSpecializationDecl *PrevDecl
-    = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
-                                        Converted.flatSize(), InsertPos);
+    = ClassTemplate->findSpecialization(Converted.data(),
+                                        Converted.size(), InsertPos);
 
   TemplateSpecializationKind PrevDecl_TSK
     = PrevDecl ? PrevDecl->getTemplateSpecializationKind() : TSK_Undeclared;
@@ -4720,10 +5366,9 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   if (CheckExplicitInstantiationScope(*this, ClassTemplate, TemplateNameLoc,
                                       SS.isSet()))
     return true;
-  
+
   ClassTemplateSpecializationDecl *Specialization = 0;
 
-  bool ReusedDecl = false;
   bool HasNoEffect = false;
   if (PrevDecl) {
     if (CheckSpecializationInstantiationRedecl(TemplateNameLoc, TSK,
@@ -4745,7 +5390,6 @@ Sema::ActOnExplicitInstantiation(Scope *S,
       Specialization = PrevDecl;
       Specialization->setLocation(TemplateNameLoc);
       PrevDecl = 0;
-      ReusedDecl = true;
     }
   }
 
@@ -4757,7 +5401,9 @@ Sema::ActOnExplicitInstantiation(Scope *S,
                                              ClassTemplate->getDeclContext(),
                                                 TemplateNameLoc,
                                                 ClassTemplate,
-                                                Converted, PrevDecl);
+                                                Converted.data(),
+                                                Converted.size(),
+                                                PrevDecl);
     SetNestedNameSpecifier(Specialization, SS);
 
     if (!HasNoEffect && !PrevDecl) {
@@ -4851,7 +5497,8 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   Decl *TagD = ActOnTag(S, TagSpec, Sema::TUK_Reference,
                         KWLoc, SS, Name, NameLoc, Attr, AS_none,
                         MultiTemplateParamsArg(*this, 0, 0),
-                        Owned, IsDependent);
+                        Owned, IsDependent, false, false,
+                        TypeResult());
   assert(!IsDependent && "explicit instantiation of dependent name not yet handled");
 
   if (!TagD)
@@ -4866,7 +5513,7 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   if (Tag->isInvalidDecl())
     return true;
-    
+
   CXXRecordDecl *Record = cast<CXXRecordDecl>(Tag);
   CXXRecordDecl *Pattern = Record->getInstantiatedFromMemberClass();
   if (!Pattern) {
@@ -4877,32 +5524,32 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   }
 
   // C++0x [temp.explicit]p2:
-  //   If the explicit instantiation is for a class or member class, the 
-  //   elaborated-type-specifier in the declaration shall include a 
+  //   If the explicit instantiation is for a class or member class, the
+  //   elaborated-type-specifier in the declaration shall include a
   //   simple-template-id.
   //
   // C++98 has the same restriction, just worded differently.
   if (!ScopeSpecifierHasTemplateId(SS))
     Diag(TemplateLoc, diag::ext_explicit_instantiation_without_qualified_id)
       << Record << SS.getRange();
-           
+
   // C++0x [temp.explicit]p2:
   //   There are two forms of explicit instantiation: an explicit instantiation
-  //   definition and an explicit instantiation declaration. An explicit 
+  //   definition and an explicit instantiation declaration. An explicit
   //   instantiation declaration begins with the extern keyword. [...]
   TemplateSpecializationKind TSK
     = ExternLoc.isInvalid()? TSK_ExplicitInstantiationDefinition
                            : TSK_ExplicitInstantiationDeclaration;
-  
+
   // C++0x [temp.explicit]p2:
   //   [...] An explicit instantiation shall appear in an enclosing
   //   namespace of its template. [...]
   //
   // This is C++ DR 275.
   CheckExplicitInstantiationScope(*this, Record, NameLoc, true);
-  
+
   // Verify that it is okay to explicitly instantiate here.
-  CXXRecordDecl *PrevDecl 
+  CXXRecordDecl *PrevDecl
     = cast_or_null<CXXRecordDecl>(Record->getPreviousDeclaration());
   if (!PrevDecl && Record->getDefinition())
     PrevDecl = Record;
@@ -4910,23 +5557,23 @@ Sema::ActOnExplicitInstantiation(Scope *S,
     MemberSpecializationInfo *MSInfo = PrevDecl->getMemberSpecializationInfo();
     bool HasNoEffect = false;
     assert(MSInfo && "No member specialization information?");
-    if (CheckSpecializationInstantiationRedecl(TemplateLoc, TSK, 
+    if (CheckSpecializationInstantiationRedecl(TemplateLoc, TSK,
                                                PrevDecl,
                                         MSInfo->getTemplateSpecializationKind(),
-                                             MSInfo->getPointOfInstantiation(), 
+                                             MSInfo->getPointOfInstantiation(),
                                                HasNoEffect))
       return true;
     if (HasNoEffect)
       return TagD;
   }
-  
+
   CXXRecordDecl *RecordDef
     = cast_or_null<CXXRecordDecl>(Record->getDefinition());
   if (!RecordDef) {
     // C++ [temp.explicit]p3:
-    //   A definition of a member class of a class template shall be in scope 
+    //   A definition of a member class of a class template shall be in scope
     //   at the point of an explicit instantiation of the member class.
-    CXXRecordDecl *Def 
+    CXXRecordDecl *Def
       = cast_or_null<CXXRecordDecl>(Pattern->getDefinition());
     if (!Def) {
       Diag(TemplateLoc, diag::err_explicit_instantiation_undefined_member)
@@ -4944,8 +5591,8 @@ Sema::ActOnExplicitInstantiation(Scope *S,
       if (!RecordDef)
         return true;
     }
-  } 
-  
+  }
+
   // Instantiate all of the members of the class.
   InstantiateClassMembers(NameLoc, RecordDef,
                           getTemplateInstantiationArgs(Record), TSK);
@@ -4974,7 +5621,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
            diag::err_explicit_instantiation_requires_name)
         << D.getDeclSpec().getSourceRange()
         << D.getSourceRange();
-    
+
     return true;
   }
 
@@ -4989,7 +5636,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   QualType R = T->getType();
   if (R.isNull())
     return true;
-  
+
   if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef) {
     // Cannot explicitly instantiate a typedef.
     Diag(D.getIdentifierLoc(), diag::err_explicit_instantiation_of_typedef)
@@ -5003,31 +5650,31 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   // Presumably, this also applies to member functions of class templates as
   // well.
   if (D.getDeclSpec().isInlineSpecified() && getLangOptions().CPlusPlus0x)
-    Diag(D.getDeclSpec().getInlineSpecLoc(), 
+    Diag(D.getDeclSpec().getInlineSpecLoc(),
          diag::err_explicit_instantiation_inline)
       <<FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
-  
+
   // FIXME: check for constexpr specifier.
-  
+
   // C++0x [temp.explicit]p2:
   //   There are two forms of explicit instantiation: an explicit instantiation
-  //   definition and an explicit instantiation declaration. An explicit 
-  //   instantiation declaration begins with the extern keyword. [...]  
+  //   definition and an explicit instantiation declaration. An explicit
+  //   instantiation declaration begins with the extern keyword. [...]
   TemplateSpecializationKind TSK
     = ExternLoc.isInvalid()? TSK_ExplicitInstantiationDefinition
                            : TSK_ExplicitInstantiationDeclaration;
-    
+
   LookupResult Previous(*this, NameInfo, LookupOrdinaryName);
   LookupParsedName(Previous, S, &D.getCXXScopeSpec());
 
   if (!R->isFunctionType()) {
     // C++ [temp.explicit]p1:
-    //   A [...] static data member of a class template can be explicitly 
-    //   instantiated from the member definition associated with its class 
+    //   A [...] static data member of a class template can be explicitly
+    //   instantiated from the member definition associated with its class
     //   template.
     if (Previous.isAmbiguous())
       return true;
-    
+
     VarDecl *Prev = Previous.getAsSingle<VarDecl>();
     if (!Prev || !Prev->isStaticDataMember()) {
       // We expect to see a data data member here.
@@ -5038,54 +5685,54 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
         Diag((*P)->getLocation(), diag::note_explicit_instantiation_here);
       return true;
     }
-    
+
     if (!Prev->getInstantiatedFromStaticDataMember()) {
       // FIXME: Check for explicit specialization?
-      Diag(D.getIdentifierLoc(), 
+      Diag(D.getIdentifierLoc(),
            diag::err_explicit_instantiation_data_member_not_instantiated)
         << Prev;
       Diag(Prev->getLocation(), diag::note_explicit_instantiation_here);
       // FIXME: Can we provide a note showing where this was declared?
       return true;
     }
-    
+
     // C++0x [temp.explicit]p2:
-    //   If the explicit instantiation is for a member function, a member class 
+    //   If the explicit instantiation is for a member function, a member class
     //   or a static data member of a class template specialization, the name of
     //   the class template specialization in the qualified-id for the member
     //   name shall be a simple-template-id.
     //
     // C++98 has the same restriction, just worded differently.
     if (!ScopeSpecifierHasTemplateId(D.getCXXScopeSpec()))
-      Diag(D.getIdentifierLoc(), 
+      Diag(D.getIdentifierLoc(),
            diag::ext_explicit_instantiation_without_qualified_id)
         << Prev << D.getCXXScopeSpec().getRange();
-    
+
     // Check the scope of this explicit instantiation.
     CheckExplicitInstantiationScope(*this, Prev, D.getIdentifierLoc(), true);
-    
+
     // Verify that it is okay to explicitly instantiate here.
     MemberSpecializationInfo *MSInfo = Prev->getMemberSpecializationInfo();
     assert(MSInfo && "Missing static data member specialization info?");
     bool HasNoEffect = false;
     if (CheckSpecializationInstantiationRedecl(D.getIdentifierLoc(), TSK, Prev,
                                         MSInfo->getTemplateSpecializationKind(),
-                                              MSInfo->getPointOfInstantiation(), 
+                                              MSInfo->getPointOfInstantiation(),
                                                HasNoEffect))
       return true;
     if (HasNoEffect)
       return (Decl*) 0;
-    
+
     // Instantiate static data member.
     Prev->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
     if (TSK == TSK_ExplicitInstantiationDefinition)
       InstantiateStaticDataMemberDefinition(D.getIdentifierLoc(), Prev);
-    
+
     // FIXME: Create an ExplicitInstantiation node?
     return (Decl*) 0;
   }
-  
-  // If the declarator is a template-id, translate the parser's template 
+
+  // If the declarator is a template-id, translate the parser's template
   // argument list into our AST format.
   bool HasExplicitTemplateArgs = false;
   TemplateArgumentListInfo TemplateArgs;
@@ -5100,11 +5747,11 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     HasExplicitTemplateArgs = true;
     TemplateArgsPtr.release();
   }
-    
+
   // C++ [temp.explicit]p1:
-  //   A [...] function [...] can be explicitly instantiated from its template. 
-  //   A member function [...] of a class template can be explicitly 
-  //  instantiated from the member definition associated with its class 
+  //   A [...] function [...] can be explicitly instantiated from its template.
+  //   A member function [...] of a class template can be explicitly
+  //  instantiated from the member definition associated with its class
   //  template.
   UnresolvedSet<8> Matches;
   for (LookupResult::iterator P = Previous.begin(), PEnd = Previous.end();
@@ -5121,7 +5768,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
         }
       }
     }
-    
+
     FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(Prev);
     if (!FunTmpl)
       continue;
@@ -5129,21 +5776,21 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     TemplateDeductionInfo Info(Context, D.getIdentifierLoc());
     FunctionDecl *Specialization = 0;
     if (TemplateDeductionResult TDK
-          = DeduceTemplateArguments(FunTmpl, 
+          = DeduceTemplateArguments(FunTmpl,
                                (HasExplicitTemplateArgs ? &TemplateArgs : 0),
                                     R, Specialization, Info)) {
       // FIXME: Keep track of almost-matches?
       (void)TDK;
       continue;
     }
-    
+
     Matches.addDecl(Specialization, P.getAccess());
   }
-  
+
   // Find the most specialized function template specialization.
   UnresolvedSetIterator Result
-    = getMostSpecialized(Matches.begin(), Matches.end(), TPOC_Other, 
-                         D.getIdentifierLoc(), 
+    = getMostSpecialized(Matches.begin(), Matches.end(), TPOC_Other, 0,
+                         D.getIdentifierLoc(),
                      PDiag(diag::err_explicit_instantiation_not_known) << Name,
                      PDiag(diag::err_explicit_instantiation_ambiguous) << Name,
                          PDiag(diag::note_explicit_instantiation_candidate));
@@ -5153,17 +5800,17 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
 
   // Ignore access control bits, we don't need them for redeclaration checking.
   FunctionDecl *Specialization = cast<FunctionDecl>(*Result);
-  
+
   if (Specialization->getTemplateSpecializationKind() == TSK_Undeclared) {
-    Diag(D.getIdentifierLoc(), 
+    Diag(D.getIdentifierLoc(),
          diag::err_explicit_instantiation_member_function_not_instantiated)
       << Specialization
       << (Specialization->getTemplateSpecializationKind() ==
           TSK_ExplicitSpecialization);
     Diag(Specialization->getLocation(), diag::note_explicit_instantiation_here);
     return true;
-  } 
-  
+  }
+
   FunctionDecl *PrevDecl = Specialization->getPreviousDeclaration();
   if (!PrevDecl && Specialization->isThisDeclarationADefinition())
     PrevDecl = Specialization;
@@ -5171,12 +5818,12 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   if (PrevDecl) {
     bool HasNoEffect = false;
     if (CheckSpecializationInstantiationRedecl(D.getIdentifierLoc(), TSK,
-                                               PrevDecl, 
-                                     PrevDecl->getTemplateSpecializationKind(), 
+                                               PrevDecl,
+                                     PrevDecl->getTemplateSpecializationKind(),
                                           PrevDecl->getPointOfInstantiation(),
                                                HasNoEffect))
       return true;
-    
+
     // FIXME: We may still want to build some representation of this
     // explicit specialization.
     if (HasNoEffect)
@@ -5184,12 +5831,12 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   }
 
   Specialization->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
-  
+
   if (TSK == TSK_ExplicitInstantiationDefinition)
     InstantiateFunctionDefinition(D.getIdentifierLoc(), Specialization);
- 
+
   // C++0x [temp.explicit]p2:
-  //   If the explicit instantiation is for a member function, a member class 
+  //   If the explicit instantiation is for a member function, a member class
   //   or a static data member of a class template specialization, the name of
   //   the class template specialization in the qualified-id for the member
   //   name shall be a simple-template-id.
@@ -5197,18 +5844,18 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   // C++98 has the same restriction, just worded differently.
   FunctionTemplateDecl *FunTmpl = Specialization->getPrimaryTemplate();
   if (D.getName().getKind() != UnqualifiedId::IK_TemplateId && !FunTmpl &&
-      D.getCXXScopeSpec().isSet() && 
+      D.getCXXScopeSpec().isSet() &&
       !ScopeSpecifierHasTemplateId(D.getCXXScopeSpec()))
-    Diag(D.getIdentifierLoc(), 
+    Diag(D.getIdentifierLoc(),
          diag::ext_explicit_instantiation_without_qualified_id)
     << Specialization << D.getCXXScopeSpec().getRange();
-  
+
   CheckExplicitInstantiationScope(*this,
-                   FunTmpl? (NamedDecl *)FunTmpl 
+                   FunTmpl? (NamedDecl *)FunTmpl
                           : Specialization->getInstantiatedFromMemberFunction(),
-                                  D.getIdentifierLoc(), 
+                                  D.getIdentifierLoc(),
                                   D.getCXXScopeSpec().isSet());
-  
+
   // FIXME: Create some kind of ExplicitInstantiationDecl here.
   return (Decl*) 0;
 }
@@ -5238,8 +5885,8 @@ Sema::ActOnDependentTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
 }
 
 TypeResult
-Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
-                        const CXXScopeSpec &SS, const IdentifierInfo &II, 
+Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
+                        const CXXScopeSpec &SS, const IdentifierInfo &II,
                         SourceLocation IdLoc) {
   NestedNameSpecifier *NNS
     = static_cast<NestedNameSpecifier *>(SS.getScopeRep());
@@ -5249,8 +5896,8 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   if (TypenameLoc.isValid() && S && !S->getTemplateParamParent() &&
       !getLangOptions().CPlusPlus0x)
     Diag(TypenameLoc, diag::ext_typename_outside_of_template)
-      << FixItHint::CreateRemoval(TypenameLoc);    
-  
+      << FixItHint::CreateRemoval(TypenameLoc);
+
   QualType T = CheckTypenameType(ETK_Typename, NNS, II,
                                  TypenameLoc, SS.getRange(), IdLoc);
   if (T.isNull())
@@ -5268,23 +5915,23 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     TL.setQualifierRange(SS.getRange());
     cast<TypeSpecTypeLoc>(TL.getNamedTypeLoc()).setNameLoc(IdLoc);
   }
-  
+
   return CreateParsedType(T, TSI);
 }
 
 TypeResult
-Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
-                        const CXXScopeSpec &SS, SourceLocation TemplateLoc, 
+Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
+                        const CXXScopeSpec &SS, SourceLocation TemplateLoc,
                         ParsedType Ty) {
   if (TypenameLoc.isValid() && S && !S->getTemplateParamParent() &&
       !getLangOptions().CPlusPlus0x)
     Diag(TypenameLoc, diag::ext_typename_outside_of_template)
-      << FixItHint::CreateRemoval(TypenameLoc);    
-  
+      << FixItHint::CreateRemoval(TypenameLoc);
+
   TypeSourceInfo *InnerTSI = 0;
   QualType T = GetTypeFromParser(Ty, &InnerTSI);
 
-  assert(isa<TemplateSpecializationType>(T) && 
+  assert(isa<TemplateSpecializationType>(T) &&
          "Expected a template specialization type");
 
   if (computeDeclContext(SS, false)) {
@@ -5297,7 +5944,8 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     if (InnerTSI)
       Builder.pushFullCopy(InnerTSI->getTypeLoc());
     else
-      Builder.push<TemplateSpecializationTypeLoc>(T).initialize(TemplateLoc);
+      Builder.push<TemplateSpecializationTypeLoc>(T).initialize(Context,
+                                                                TemplateLoc);
 
     /* Note: NNS already embedded in template specialization type T. */
     T = Context.getElaboratedType(ETK_Typename, /*NNS=*/0, T);
@@ -5311,7 +5959,7 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
 
   // TODO: it's really silly that we make a template specialization
   // type earlier only to drop it again here.
-  TemplateSpecializationType *TST = cast<TemplateSpecializationType>(T);
+  const TemplateSpecializationType *TST = cast<TemplateSpecializationType>(T);
   DependentTemplateName *DTN =
     TST->getTemplateName().getAsDependentTemplateName();
   assert(DTN && "dependent template has non-dependent name?");
@@ -5333,7 +5981,8 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     for (unsigned I = 0, E = TST->getNumArgs(); I != E; ++I)
       TL.setArgLocInfo(I, TSTL.getArgLocInfo(I));
   } else {
-    TL.initializeLocal(SourceLocation());
+    // FIXME: Poor source-location information here.
+    TL.initializeLocal(Context, TemplateLoc);
   }
   TL.setKeywordLoc(TypenameLoc);
   TL.setQualifierRange(SS.getRange());
@@ -5377,13 +6026,30 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
   case LookupResult::NotFound:
     DiagID = diag::err_typename_nested_not_found;
     break;
-      
+
+  case LookupResult::FoundUnresolvedValue: {
+    // We found a using declaration that is a value. Most likely, the using
+    // declaration itself is meant to have the 'typename' keyword.
+    SourceRange FullRange(KeywordLoc.isValid() ? KeywordLoc : NNSRange.getBegin(),
+                          IILoc);
+    Diag(IILoc, diag::err_typename_refers_to_using_value_decl)
+      << Name << Ctx << FullRange;
+    if (UnresolvedUsingValueDecl *Using
+          = dyn_cast<UnresolvedUsingValueDecl>(Result.getRepresentativeDecl())){
+      SourceLocation Loc = Using->getTargetNestedNameRange().getBegin();
+      Diag(Loc, diag::note_using_value_decl_missing_typename)
+        << FixItHint::CreateInsertion(Loc, "typename ");
+    }
+  }
+  // Fall through to create a dependent typename type, from which we can recover
+  // better.
+
   case LookupResult::NotFoundInCurrentInstantiation:
     // Okay, it's a member of an unknown instantiation.
     return Context.getDependentNameType(Keyword, NNS, &II);
 
   case LookupResult::Found:
-    if (TypeDecl *Type = dyn_cast<TypeDecl>(Result.getFoundDecl())) {      
+    if (TypeDecl *Type = dyn_cast<TypeDecl>(Result.getFoundDecl())) {
       // We found a type. Build an ElaboratedType, since the
       // typename-specifier was just sugar.
       return Context.getElaboratedType(ETK_Typename, NNS,
@@ -5394,7 +6060,7 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
     Referenced = Result.getFoundDecl();
     break;
 
-  case LookupResult::FoundUnresolvedValue:
+
     llvm_unreachable("unresolved using decl in non-dependent context");
     return QualType();
 
@@ -5427,7 +6093,7 @@ namespace {
 
   public:
     typedef TreeTransform<CurrentInstantiationRebuilder> inherited;
-      
+
     CurrentInstantiationRebuilder(Sema &SemaRef,
                                   SourceLocation Loc,
                                   DeclarationName Entity)
@@ -5507,7 +6173,7 @@ bool Sema::RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS) {
   NestedNameSpecifier *NNS = static_cast<NestedNameSpecifier*>(SS.getScopeRep());
   CurrentInstantiationRebuilder Rebuilder(*this, SS.getRange().getBegin(),
                                           DeclarationName());
-  NestedNameSpecifier *Rebuilt = 
+  NestedNameSpecifier *Rebuilt =
     Rebuilder.TransformNestedNameSpecifier(NNS, SS.getRange());
   if (!Rebuilt) return true;
 
@@ -5520,99 +6186,38 @@ bool Sema::RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS) {
 std::string
 Sema::getTemplateArgumentBindingsText(const TemplateParameterList *Params,
                                       const TemplateArgumentList &Args) {
-  // FIXME: For variadic templates, we'll need to get the structured list.
-  return getTemplateArgumentBindingsText(Params, Args.getFlatArgumentList(),
-                                         Args.flat_size());
+  return getTemplateArgumentBindingsText(Params, Args.data(), Args.size());
 }
 
 std::string
 Sema::getTemplateArgumentBindingsText(const TemplateParameterList *Params,
                                       const TemplateArgument *Args,
                                       unsigned NumArgs) {
-  std::string Result;
+  llvm::SmallString<128> Str;
+  llvm::raw_svector_ostream Out(Str);
 
   if (!Params || Params->size() == 0 || NumArgs == 0)
-    return Result;
-  
+    return std::string();
+
   for (unsigned I = 0, N = Params->size(); I != N; ++I) {
     if (I >= NumArgs)
       break;
-    
+
     if (I == 0)
-      Result += "[with ";
+      Out << "[with ";
     else
-      Result += ", ";
-    
+      Out << ", ";
+
     if (const IdentifierInfo *Id = Params->getParam(I)->getIdentifier()) {
-      Result += Id->getName();
+      Out << Id->getName();
     } else {
-      Result += '$';
-      Result += llvm::utostr(I);
-    }
-    
-    Result += " = ";
-    
-    switch (Args[I].getKind()) {
-      case TemplateArgument::Null:
-        Result += "<no value>";
-        break;
-        
-      case TemplateArgument::Type: {
-        std::string TypeStr;
-        Args[I].getAsType().getAsStringInternal(TypeStr, 
-                                                Context.PrintingPolicy);
-        Result += TypeStr;
-        break;
-      }
-        
-      case TemplateArgument::Declaration: {
-        bool Unnamed = true;
-        if (NamedDecl *ND = dyn_cast_or_null<NamedDecl>(Args[I].getAsDecl())) {
-          if (ND->getDeclName()) {
-            Unnamed = false;
-            Result += ND->getNameAsString();
-          }
-        }
-        
-        if (Unnamed) {
-          Result += "<anonymous>";
-        }
-        break;
-      }
-        
-      case TemplateArgument::Template: {
-        std::string Str;
-        llvm::raw_string_ostream OS(Str);
-        Args[I].getAsTemplate().print(OS, Context.PrintingPolicy);
-        Result += OS.str();
-        break;
-      }
-        
-      case TemplateArgument::Integral: {
-        Result += Args[I].getAsIntegral()->toString(10);
-        break;
-      }
-        
-      case TemplateArgument::Expression: {
-        // FIXME: This is non-optimal, since we're regurgitating the
-        // expression we were given.
-        std::string Str; 
-        {
-          llvm::raw_string_ostream OS(Str);
-          Args[I].getAsExpr()->printPretty(OS, Context, 0,
-                                           Context.PrintingPolicy);
-        }
-        Result += Str;
-        break;
-      }
-        
-      case TemplateArgument::Pack:
-        // FIXME: Format template argument packs
-        Result += "<template argument pack>";
-        break;        
+      Out << '$' << I;
     }
+
+    Out << " = ";
+    Args[I].print(Context.PrintingPolicy, Out);
   }
-  
-  Result += ']';
-  return Result;
+
+  Out << ']';
+  return Out.str();
 }
diff --git a/lib/Sema/SemaTemplateDeduction.cpp b/lib/Sema/SemaTemplateDeduction.cpp
index 5c77ed61060b..bd0a618283b3 100644
--- a/lib/Sema/SemaTemplateDeduction.cpp
+++ b/lib/Sema/SemaTemplateDeduction.cpp
@@ -12,6 +12,7 @@
 
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/SemaDiagnostic.h" // FIXME: temporary!
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTContext.h"
@@ -20,6 +21,8 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
+#include "llvm/ADT/BitVector.h"
+#include "TreeTransform.h"
 #include <algorithm>
 
 namespace clang {
@@ -47,7 +50,10 @@ namespace clang {
     /// \brief Allow non-dependent types to differ, e.g., when performing
     /// template argument deduction from a function call where conversions
     /// may apply.
-    TDF_SkipNonDependent = 0x08
+    TDF_SkipNonDependent = 0x08,
+    /// \brief Whether we are performing template argument deduction for
+    /// parameters and arguments in a top-level template argument
+    TDF_TopLevelParameterTypeList = 0x10
   };
 }
 
@@ -57,9 +63,9 @@ using namespace clang;
 /// necessary to compare their values regardless of underlying type.
 static bool hasSameExtendedValue(llvm::APSInt X, llvm::APSInt Y) {
   if (Y.getBitWidth() > X.getBitWidth())
-    X.extend(Y.getBitWidth());
+    X = X.extend(Y.getBitWidth());
   else if (Y.getBitWidth() < X.getBitWidth())
-    Y.extend(X.getBitWidth());
+    Y = Y.extend(X.getBitWidth());
 
   // If there is a signedness mismatch, correct it.
   if (X.isSigned() != Y.isSigned()) {
@@ -78,9 +84,54 @@ static Sema::TemplateDeductionResult
 DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateArgument &Param,
-                        const TemplateArgument &Arg,
+                        TemplateArgument Arg,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced);
+                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced);
+
+/// \brief Whether template argument deduction for two reference parameters
+/// resulted in the argument type, parameter type, or neither type being more
+/// qualified than the other.
+enum DeductionQualifierComparison {
+  NeitherMoreQualified = 0,
+  ParamMoreQualified,
+  ArgMoreQualified
+};
+
+/// \brief Stores the result of comparing two reference parameters while
+/// performing template argument deduction for partial ordering of function
+/// templates.
+struct RefParamPartialOrderingComparison {
+  /// \brief Whether the parameter type is an rvalue reference type.
+  bool ParamIsRvalueRef;
+  /// \brief Whether the argument type is an rvalue reference type.
+  bool ArgIsRvalueRef;
+
+  /// \brief Whether the parameter or argument (or neither) is more qualified.
+  DeductionQualifierComparison Qualifiers;
+};
+
+
+
+static Sema::TemplateDeductionResult
+DeduceTemplateArguments(Sema &S,
+                        TemplateParameterList *TemplateParams,
+                        QualType Param,
+                        QualType Arg,
+                        TemplateDeductionInfo &Info,
+                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        unsigned TDF,
+                        bool PartialOrdering = false,
+                      llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *
+                                                      RefParamComparisons = 0);
+
+static Sema::TemplateDeductionResult
+DeduceTemplateArguments(Sema &S,
+                        TemplateParameterList *TemplateParams,
+                        const TemplateArgument *Params, unsigned NumParams,
+                        const TemplateArgument *Args, unsigned NumArgs,
+                        TemplateDeductionInfo &Info,
+                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        bool NumberOfArgumentsMustMatch = true);
 
 /// \brief If the given expression is of a form that permits the deduction
 /// of a non-type template parameter, return the declaration of that
@@ -95,6 +146,141 @@ static NonTypeTemplateParmDecl *getDeducedParameterFromExpr(Expr *E) {
   return 0;
 }
 
+/// \brief Determine whether two declaration pointers refer to the same
+/// declaration.
+static bool isSameDeclaration(Decl *X, Decl *Y) {
+  if (!X || !Y)
+    return !X && !Y;
+
+  if (NamedDecl *NX = dyn_cast<NamedDecl>(X))
+    X = NX->getUnderlyingDecl();
+  if (NamedDecl *NY = dyn_cast<NamedDecl>(Y))
+    Y = NY->getUnderlyingDecl();
+
+  return X->getCanonicalDecl() == Y->getCanonicalDecl();
+}
+
+/// \brief Verify that the given, deduced template arguments are compatible.
+///
+/// \returns The deduced template argument, or a NULL template argument if
+/// the deduced template arguments were incompatible.
+static DeducedTemplateArgument
+checkDeducedTemplateArguments(ASTContext &Context,
+                              const DeducedTemplateArgument &X,
+                              const DeducedTemplateArgument &Y) {
+  // We have no deduction for one or both of the arguments; they're compatible.
+  if (X.isNull())
+    return Y;
+  if (Y.isNull())
+    return X;
+
+  switch (X.getKind()) {
+  case TemplateArgument::Null:
+    llvm_unreachable("Non-deduced template arguments handled above");
+
+  case TemplateArgument::Type:
+    // If two template type arguments have the same type, they're compatible.
+    if (Y.getKind() == TemplateArgument::Type &&
+        Context.hasSameType(X.getAsType(), Y.getAsType()))
+      return X;
+
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::Integral:
+    // If we deduced a constant in one case and either a dependent expression or
+    // declaration in another case, keep the integral constant.
+    // If both are integral constants with the same value, keep that value.
+    if (Y.getKind() == TemplateArgument::Expression ||
+        Y.getKind() == TemplateArgument::Declaration ||
+        (Y.getKind() == TemplateArgument::Integral &&
+         hasSameExtendedValue(*X.getAsIntegral(), *Y.getAsIntegral())))
+      return DeducedTemplateArgument(X,
+                                     X.wasDeducedFromArrayBound() &&
+                                     Y.wasDeducedFromArrayBound());
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::Template:
+    if (Y.getKind() == TemplateArgument::Template &&
+        Context.hasSameTemplateName(X.getAsTemplate(), Y.getAsTemplate()))
+      return X;
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::TemplateExpansion:
+    if (Y.getKind() == TemplateArgument::TemplateExpansion &&
+        Context.hasSameTemplateName(X.getAsTemplateOrTemplatePattern(),
+                                    Y.getAsTemplateOrTemplatePattern()))
+      return X;
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::Expression:
+    // If we deduced a dependent expression in one case and either an integral
+    // constant or a declaration in another case, keep the integral constant
+    // or declaration.
+    if (Y.getKind() == TemplateArgument::Integral ||
+        Y.getKind() == TemplateArgument::Declaration)
+      return DeducedTemplateArgument(Y, X.wasDeducedFromArrayBound() &&
+                                     Y.wasDeducedFromArrayBound());
+
+    if (Y.getKind() == TemplateArgument::Expression) {
+      // Compare the expressions for equality
+      llvm::FoldingSetNodeID ID1, ID2;
+      X.getAsExpr()->Profile(ID1, Context, true);
+      Y.getAsExpr()->Profile(ID2, Context, true);
+      if (ID1 == ID2)
+        return X;
+    }
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::Declaration:
+    // If we deduced a declaration and a dependent expression, keep the
+    // declaration.
+    if (Y.getKind() == TemplateArgument::Expression)
+      return X;
+
+    // If we deduced a declaration and an integral constant, keep the
+    // integral constant.
+    if (Y.getKind() == TemplateArgument::Integral)
+      return Y;
+
+    // If we deduced two declarations, make sure they they refer to the
+    // same declaration.
+    if (Y.getKind() == TemplateArgument::Declaration &&
+        isSameDeclaration(X.getAsDecl(), Y.getAsDecl()))
+      return X;
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::Pack:
+    if (Y.getKind() != TemplateArgument::Pack ||
+        X.pack_size() != Y.pack_size())
+      return DeducedTemplateArgument();
+
+    for (TemplateArgument::pack_iterator XA = X.pack_begin(),
+                                      XAEnd = X.pack_end(),
+                                         YA = Y.pack_begin();
+         XA != XAEnd; ++XA, ++YA) {
+      if (checkDeducedTemplateArguments(Context,
+                    DeducedTemplateArgument(*XA, X.wasDeducedFromArrayBound()),
+                    DeducedTemplateArgument(*YA, Y.wasDeducedFromArrayBound()))
+            .isNull())
+        return DeducedTemplateArgument();
+    }
+
+    return X;
+  }
+
+  return DeducedTemplateArgument();
+}
+
 /// \brief Deduce the value of the given non-type template parameter
 /// from the given constant.
 static Sema::TemplateDeductionResult
@@ -107,31 +293,18 @@ DeduceNonTypeTemplateArgument(Sema &S,
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
 
-  if (Deduced[NTTP->getIndex()].isNull()) {
-    Deduced[NTTP->getIndex()] = DeducedTemplateArgument(Value, ValueType,
-                                                        DeducedFromArrayBound);
-    return Sema::TDK_Success;
-  }
-
-  if (Deduced[NTTP->getIndex()].getKind() != TemplateArgument::Integral) {
+  DeducedTemplateArgument NewDeduced(Value, ValueType, DeducedFromArrayBound);
+  DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
+                                                     Deduced[NTTP->getIndex()],
+                                                                 NewDeduced);
+  if (Result.isNull()) {
     Info.Param = NTTP;
     Info.FirstArg = Deduced[NTTP->getIndex()];
-    Info.SecondArg = TemplateArgument(Value, ValueType);
-    return Sema::TDK_Inconsistent;    
-  }
-
-  // Extent the smaller of the two values.
-  llvm::APSInt PrevValue = *Deduced[NTTP->getIndex()].getAsIntegral();
-  if (!hasSameExtendedValue(PrevValue, Value)) {
-    Info.Param = NTTP;
-    Info.FirstArg = Deduced[NTTP->getIndex()];
-    Info.SecondArg = TemplateArgument(Value, ValueType);
+    Info.SecondArg = NewDeduced;
     return Sema::TDK_Inconsistent;
   }
 
-  if (!DeducedFromArrayBound)
-    Deduced[NTTP->getIndex()].setDeducedFromArrayBound(false);
-
+  Deduced[NTTP->getIndex()] = Result;
   return Sema::TDK_Success;
 }
 
@@ -150,30 +323,19 @@ DeduceNonTypeTemplateArgument(Sema &S,
   assert((Value->isTypeDependent() || Value->isValueDependent()) &&
          "Expression template argument must be type- or value-dependent.");
 
-  if (Deduced[NTTP->getIndex()].isNull()) {
-    Deduced[NTTP->getIndex()] = TemplateArgument(Value->Retain());
-    return Sema::TDK_Success;
-  }
-
-  if (Deduced[NTTP->getIndex()].getKind() == TemplateArgument::Integral) {
-    // Okay, we deduced a constant in one case and a dependent expression
-    // in another case. FIXME: Later, we will check that instantiating the
-    // dependent expression gives us the constant value.
-    return Sema::TDK_Success;
-  }
+  DeducedTemplateArgument NewDeduced(Value);
+  DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
+                                                     Deduced[NTTP->getIndex()],
+                                                                 NewDeduced);
 
-  if (Deduced[NTTP->getIndex()].getKind() == TemplateArgument::Expression) {
-    // Compare the expressions for equality
-    llvm::FoldingSetNodeID ID1, ID2;
-    Deduced[NTTP->getIndex()].getAsExpr()->Profile(ID1, S.Context, true);
-    Value->Profile(ID2, S.Context, true);
-    if (ID1 == ID2)
-      return Sema::TDK_Success;
-   
-    // FIXME: Fill in argument mismatch information
-    return Sema::TDK_NonDeducedMismatch;
+  if (Result.isNull()) {
+    Info.Param = NTTP;
+    Info.FirstArg = Deduced[NTTP->getIndex()];
+    Info.SecondArg = NewDeduced;
+    return Sema::TDK_Inconsistent;
   }
 
+  Deduced[NTTP->getIndex()] = Result;
   return Sema::TDK_Success;
 }
 
@@ -189,28 +351,19 @@ DeduceNonTypeTemplateArgument(Sema &S,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
-  
-  if (Deduced[NTTP->getIndex()].isNull()) {
-    Deduced[NTTP->getIndex()] = TemplateArgument(D->getCanonicalDecl());
-    return Sema::TDK_Success;
-  }
-  
-  if (Deduced[NTTP->getIndex()].getKind() == TemplateArgument::Expression) {
-    // Okay, we deduced a declaration in one case and a dependent expression
-    // in another case.
-    return Sema::TDK_Success;
-  }
-  
-  if (Deduced[NTTP->getIndex()].getKind() == TemplateArgument::Declaration) {
-    // Compare the declarations for equality
-    if (Deduced[NTTP->getIndex()].getAsDecl()->getCanonicalDecl() ==
-          D->getCanonicalDecl())
-      return Sema::TDK_Success;
-    
-    // FIXME: Fill in argument mismatch information
-    return Sema::TDK_NonDeducedMismatch;
+
+  DeducedTemplateArgument NewDeduced(D? D->getCanonicalDecl() : 0);
+  DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
+                                                     Deduced[NTTP->getIndex()],
+                                                                 NewDeduced);
+  if (Result.isNull()) {
+    Info.Param = NTTP;
+    Info.FirstArg = Deduced[NTTP->getIndex()];
+    Info.SecondArg = NewDeduced;
+    return Sema::TDK_Inconsistent;
   }
-  
+
+  Deduced[NTTP->getIndex()] = Result;
   return Sema::TDK_Success;
 }
 
@@ -227,33 +380,28 @@ DeduceTemplateArguments(Sema &S,
     // so there is nothing that we can deduce.
     return Sema::TDK_Success;
   }
-  
+
   if (TemplateTemplateParmDecl *TempParam
         = dyn_cast<TemplateTemplateParmDecl>(ParamDecl)) {
-    // Bind the template template parameter to the given template name.
-    TemplateArgument &ExistingArg = Deduced[TempParam->getIndex()];
-    if (ExistingArg.isNull()) {
-      // This is the first deduction for this template template parameter.
-      ExistingArg = TemplateArgument(S.Context.getCanonicalTemplateName(Arg));
-      return Sema::TDK_Success;
+    DeducedTemplateArgument NewDeduced(S.Context.getCanonicalTemplateName(Arg));
+    DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
+                                                 Deduced[TempParam->getIndex()],
+                                                                   NewDeduced);
+    if (Result.isNull()) {
+      Info.Param = TempParam;
+      Info.FirstArg = Deduced[TempParam->getIndex()];
+      Info.SecondArg = NewDeduced;
+      return Sema::TDK_Inconsistent;
     }
-    
-    // Verify that the previous binding matches this deduction.
-    assert(ExistingArg.getKind() == TemplateArgument::Template);
-    if (S.Context.hasSameTemplateName(ExistingArg.getAsTemplate(), Arg))
-      return Sema::TDK_Success;
-    
-    // Inconsistent deduction.
-    Info.Param = TempParam;
-    Info.FirstArg = ExistingArg;
-    Info.SecondArg = TemplateArgument(Arg);
-    return Sema::TDK_Inconsistent;
+
+    Deduced[TempParam->getIndex()] = Result;
+    return Sema::TDK_Success;
   }
-  
+
   // Verify that the two template names are equivalent.
   if (S.Context.hasSameTemplateName(Param, Arg))
     return Sema::TDK_Success;
-  
+
   // Mismatch of non-dependent template parameter to argument.
   Info.FirstArg = TemplateArgument(Param);
   Info.SecondArg = TemplateArgument(Arg);
@@ -300,17 +448,13 @@ DeduceTemplateArguments(Sema &S,
 
 
     // Perform template argument deduction on each template
-    // argument.
-    unsigned NumArgs = std::min(SpecArg->getNumArgs(), Param->getNumArgs());
-    for (unsigned I = 0; I != NumArgs; ++I)
-      if (Sema::TemplateDeductionResult Result
-            = DeduceTemplateArguments(S, TemplateParams,
-                                      Param->getArg(I),
-                                      SpecArg->getArg(I),
-                                      Info, Deduced))
-        return Result;
-
-    return Sema::TDK_Success;
+    // argument. Ignore any missing/extra arguments, since they could be
+    // filled in by default arguments.
+    return DeduceTemplateArguments(S, TemplateParams,
+                                   Param->getArgs(), Param->getNumArgs(),
+                                   SpecArg->getArgs(), SpecArg->getNumArgs(),
+                                   Info, Deduced,
+                                   /*NumberOfArgumentsMustMatch=*/false);
   }
 
   // If the argument type is a class template specialization, we
@@ -334,20 +478,12 @@ DeduceTemplateArguments(Sema &S,
                                   Info, Deduced))
     return Result;
 
-  unsigned NumArgs = Param->getNumArgs();
-  const TemplateArgumentList &ArgArgs = SpecArg->getTemplateArgs();
-  if (NumArgs != ArgArgs.size())
-    return Sema::TDK_NonDeducedMismatch;
-
-  for (unsigned I = 0; I != NumArgs; ++I)
-    if (Sema::TemplateDeductionResult Result
-          = DeduceTemplateArguments(S, TemplateParams,
-                                    Param->getArg(I),
-                                    ArgArgs.get(I),
-                                    Info, Deduced))
-      return Result;
-
-  return Sema::TDK_Success;
+  // Perform template argument deduction for the template arguments.
+  return DeduceTemplateArguments(S, TemplateParams,
+                                 Param->getArgs(), Param->getNumArgs(),
+                                 SpecArg->getTemplateArgs().data(),
+                                 SpecArg->getTemplateArgs().size(),
+                                 Info, Deduced);
 }
 
 /// \brief Determines whether the given type is an opaque type that
@@ -359,6 +495,7 @@ static bool IsPossiblyOpaquelyQualifiedType(QualType T) {
   case Type::DependentName:
   case Type::Decltype:
   case Type::UnresolvedUsing:
+  case Type::TemplateTypeParm:
     return true;
 
   case Type::ConstantArray:
@@ -373,6 +510,296 @@ static bool IsPossiblyOpaquelyQualifiedType(QualType T) {
   }
 }
 
+/// \brief Retrieve the depth and index of a template parameter.
+static std::pair<unsigned, unsigned>
+getDepthAndIndex(NamedDecl *ND) {
+  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(ND))
+    return std::make_pair(TTP->getDepth(), TTP->getIndex());
+
+  if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(ND))
+    return std::make_pair(NTTP->getDepth(), NTTP->getIndex());
+
+  TemplateTemplateParmDecl *TTP = cast<TemplateTemplateParmDecl>(ND);
+  return std::make_pair(TTP->getDepth(), TTP->getIndex());
+}
+
+/// \brief Retrieve the depth and index of an unexpanded parameter pack.
+static std::pair<unsigned, unsigned>
+getDepthAndIndex(UnexpandedParameterPack UPP) {
+  if (const TemplateTypeParmType *TTP
+                          = UPP.first.dyn_cast<const TemplateTypeParmType *>())
+    return std::make_pair(TTP->getDepth(), TTP->getIndex());
+
+  return getDepthAndIndex(UPP.first.get<NamedDecl *>());
+}
+
+/// \brief Helper function to build a TemplateParameter when we don't
+/// know its type statically.
+static TemplateParameter makeTemplateParameter(Decl *D) {
+  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(D))
+    return TemplateParameter(TTP);
+  else if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(D))
+    return TemplateParameter(NTTP);
+
+  return TemplateParameter(cast<TemplateTemplateParmDecl>(D));
+}
+
+/// \brief Prepare to perform template argument deduction for all of the
+/// arguments in a set of argument packs.
+static void PrepareArgumentPackDeduction(Sema &S,
+                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                             const llvm::SmallVectorImpl<unsigned> &PackIndices,
+                     llvm::SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
+         llvm::SmallVectorImpl<
+           llvm::SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks) {
+  // Save the deduced template arguments for each parameter pack expanded
+  // by this pack expansion, then clear out the deduction.
+  for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
+    // Save the previously-deduced argument pack, then clear it out so that we
+    // can deduce a new argument pack.
+    SavedPacks[I] = Deduced[PackIndices[I]];
+    Deduced[PackIndices[I]] = TemplateArgument();
+
+    // If the template arugment pack was explicitly specified, add that to
+    // the set of deduced arguments.
+    const TemplateArgument *ExplicitArgs;
+    unsigned NumExplicitArgs;
+    if (NamedDecl *PartiallySubstitutedPack
+        = S.CurrentInstantiationScope->getPartiallySubstitutedPack(
+                                                           &ExplicitArgs,
+                                                           &NumExplicitArgs)) {
+      if (getDepthAndIndex(PartiallySubstitutedPack).second == PackIndices[I])
+        NewlyDeducedPacks[I].append(ExplicitArgs,
+                                    ExplicitArgs + NumExplicitArgs);
+    }
+  }
+}
+
+/// \brief Finish template argument deduction for a set of argument packs,
+/// producing the argument packs and checking for consistency with prior
+/// deductions.
+static Sema::TemplateDeductionResult
+FinishArgumentPackDeduction(Sema &S,
+                            TemplateParameterList *TemplateParams,
+                            bool HasAnyArguments,
+                        llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                            const llvm::SmallVectorImpl<unsigned> &PackIndices,
+                    llvm::SmallVectorImpl<DeducedTemplateArgument> &SavedPacks,
+        llvm::SmallVectorImpl<
+          llvm::SmallVector<DeducedTemplateArgument, 4> > &NewlyDeducedPacks,
+                            TemplateDeductionInfo &Info) {
+  // Build argument packs for each of the parameter packs expanded by this
+  // pack expansion.
+  for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
+    if (HasAnyArguments && NewlyDeducedPacks[I].empty()) {
+      // We were not able to deduce anything for this parameter pack,
+      // so just restore the saved argument pack.
+      Deduced[PackIndices[I]] = SavedPacks[I];
+      continue;
+    }
+
+    DeducedTemplateArgument NewPack;
+
+    if (NewlyDeducedPacks[I].empty()) {
+      // If we deduced an empty argument pack, create it now.
+      NewPack = DeducedTemplateArgument(TemplateArgument(0, 0));
+    } else {
+      TemplateArgument *ArgumentPack
+        = new (S.Context) TemplateArgument [NewlyDeducedPacks[I].size()];
+      std::copy(NewlyDeducedPacks[I].begin(), NewlyDeducedPacks[I].end(),
+                ArgumentPack);
+      NewPack
+        = DeducedTemplateArgument(TemplateArgument(ArgumentPack,
+                                                   NewlyDeducedPacks[I].size()),
+                            NewlyDeducedPacks[I][0].wasDeducedFromArrayBound());
+    }
+
+    DeducedTemplateArgument Result
+      = checkDeducedTemplateArguments(S.Context, SavedPacks[I], NewPack);
+    if (Result.isNull()) {
+      Info.Param
+        = makeTemplateParameter(TemplateParams->getParam(PackIndices[I]));
+      Info.FirstArg = SavedPacks[I];
+      Info.SecondArg = NewPack;
+      return Sema::TDK_Inconsistent;
+    }
+
+    Deduced[PackIndices[I]] = Result;
+  }
+
+  return Sema::TDK_Success;
+}
+
+/// \brief Deduce the template arguments by comparing the list of parameter
+/// types to the list of argument types, as in the parameter-type-lists of
+/// function types (C++ [temp.deduct.type]p10).
+///
+/// \param S The semantic analysis object within which we are deducing
+///
+/// \param TemplateParams The template parameters that we are deducing
+///
+/// \param Params The list of parameter types
+///
+/// \param NumParams The number of types in \c Params
+///
+/// \param Args The list of argument types
+///
+/// \param NumArgs The number of types in \c Args
+///
+/// \param Info information about the template argument deduction itself
+///
+/// \param Deduced the deduced template arguments
+///
+/// \param TDF bitwise OR of the TemplateDeductionFlags bits that describe
+/// how template argument deduction is performed.
+///
+/// \param PartialOrdering If true, we are performing template argument
+/// deduction for during partial ordering for a call
+/// (C++0x [temp.deduct.partial]).
+///
+/// \param RefParamComparisons If we're performing template argument deduction
+/// in the context of partial ordering, the set of qualifier comparisons.
+///
+/// \returns the result of template argument deduction so far. Note that a
+/// "success" result means that template argument deduction has not yet failed,
+/// but it may still fail, later, for other reasons.
+static Sema::TemplateDeductionResult
+DeduceTemplateArguments(Sema &S,
+                        TemplateParameterList *TemplateParams,
+                        const QualType *Params, unsigned NumParams,
+                        const QualType *Args, unsigned NumArgs,
+                        TemplateDeductionInfo &Info,
+                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        unsigned TDF,
+                        bool PartialOrdering = false,
+                        llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *
+                                                     RefParamComparisons = 0) {
+  // Fast-path check to see if we have too many/too few arguments.
+  if (NumParams != NumArgs &&
+      !(NumParams && isa<PackExpansionType>(Params[NumParams - 1])) &&
+      !(NumArgs && isa<PackExpansionType>(Args[NumArgs - 1])))
+    return Sema::TDK_NonDeducedMismatch;
+
+  // C++0x [temp.deduct.type]p10:
+  //   Similarly, if P has a form that contains (T), then each parameter type
+  //   Pi of the respective parameter-type- list of P is compared with the
+  //   corresponding parameter type Ai of the corresponding parameter-type-list
+  //   of A. [...]
+  unsigned ArgIdx = 0, ParamIdx = 0;
+  for (; ParamIdx != NumParams; ++ParamIdx) {
+    // Check argument types.
+    const PackExpansionType *Expansion
+                                = dyn_cast<PackExpansionType>(Params[ParamIdx]);
+    if (!Expansion) {
+      // Simple case: compare the parameter and argument types at this point.
+
+      // Make sure we have an argument.
+      if (ArgIdx >= NumArgs)
+        return Sema::TDK_NonDeducedMismatch;
+
+      if (isa<PackExpansionType>(Args[ArgIdx])) {
+        // C++0x [temp.deduct.type]p22:
+        //   If the original function parameter associated with A is a function
+        //   parameter pack and the function parameter associated with P is not
+        //   a function parameter pack, then template argument deduction fails.
+        return Sema::TDK_NonDeducedMismatch;
+      }
+
+      if (Sema::TemplateDeductionResult Result
+            = DeduceTemplateArguments(S, TemplateParams,
+                                      Params[ParamIdx],
+                                      Args[ArgIdx],
+                                      Info, Deduced, TDF,
+                                      PartialOrdering,
+                                      RefParamComparisons))
+        return Result;
+
+      ++ArgIdx;
+      continue;
+    }
+
+    // C++0x [temp.deduct.type]p5:
+    //   The non-deduced contexts are:
+    //     - A function parameter pack that does not occur at the end of the
+    //       parameter-declaration-clause.
+    if (ParamIdx + 1 < NumParams)
+      return Sema::TDK_Success;
+
+    // C++0x [temp.deduct.type]p10:
+    //   If the parameter-declaration corresponding to Pi is a function
+    //   parameter pack, then the type of its declarator- id is compared with
+    //   each remaining parameter type in the parameter-type-list of A. Each
+    //   comparison deduces template arguments for subsequent positions in the
+    //   template parameter packs expanded by the function parameter pack.
+
+    // Compute the set of template parameter indices that correspond to
+    // parameter packs expanded by the pack expansion.
+    llvm::SmallVector<unsigned, 2> PackIndices;
+    QualType Pattern = Expansion->getPattern();
+    {
+      llvm::BitVector SawIndices(TemplateParams->size());
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      S.collectUnexpandedParameterPacks(Pattern, Unexpanded);
+      for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
+        unsigned Depth, Index;
+        llvm::tie(Depth, Index) = getDepthAndIndex(Unexpanded[I]);
+        if (Depth == 0 && !SawIndices[Index]) {
+          SawIndices[Index] = true;
+          PackIndices.push_back(Index);
+        }
+      }
+    }
+    assert(!PackIndices.empty() && "Pack expansion without unexpanded packs?");
+
+    // Keep track of the deduced template arguments for each parameter pack
+    // expanded by this pack expansion (the outer index) and for each
+    // template argument (the inner SmallVectors).
+    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+      NewlyDeducedPacks(PackIndices.size());
+    llvm::SmallVector<DeducedTemplateArgument, 2>
+      SavedPacks(PackIndices.size());
+    PrepareArgumentPackDeduction(S, Deduced, PackIndices, SavedPacks,
+                                 NewlyDeducedPacks);
+
+    bool HasAnyArguments = false;
+    for (; ArgIdx < NumArgs; ++ArgIdx) {
+      HasAnyArguments = true;
+
+      // Deduce template arguments from the pattern.
+      if (Sema::TemplateDeductionResult Result
+            = DeduceTemplateArguments(S, TemplateParams, Pattern, Args[ArgIdx],
+                                      Info, Deduced, TDF, PartialOrdering,
+                                      RefParamComparisons))
+        return Result;
+
+      // Capture the deduced template arguments for each parameter pack expanded
+      // by this pack expansion, add them to the list of arguments we've deduced
+      // for that pack, then clear out the deduced argument.
+      for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
+        DeducedTemplateArgument &DeducedArg = Deduced[PackIndices[I]];
+        if (!DeducedArg.isNull()) {
+          NewlyDeducedPacks[I].push_back(DeducedArg);
+          DeducedArg = DeducedTemplateArgument();
+        }
+      }
+    }
+
+    // Build argument packs for each of the parameter packs expanded by this
+    // pack expansion.
+    if (Sema::TemplateDeductionResult Result
+          = FinishArgumentPackDeduction(S, TemplateParams, HasAnyArguments,
+                                        Deduced, PackIndices, SavedPacks,
+                                        NewlyDeducedPacks, Info))
+      return Result;
+  }
+
+  // Make sure we don't have any extra arguments.
+  if (ArgIdx < NumArgs)
+    return Sema::TDK_NonDeducedMismatch;
+
+  return Sema::TDK_Success;
+}
+
 /// \brief Deduce the template arguments by comparing the parameter type and
 /// the argument type (C++ [temp.deduct.type]).
 ///
@@ -391,6 +818,12 @@ static bool IsPossiblyOpaquelyQualifiedType(QualType T) {
 /// \param TDF bitwise OR of the TemplateDeductionFlags bits that describe
 /// how template argument deduction is performed.
 ///
+/// \param PartialOrdering Whether we're performing template argument deduction
+/// in the context of partial ordering (C++0x [temp.deduct.partial]).
+///
+/// \param RefParamComparisons If we're performing template argument deduction
+/// in the context of partial ordering, the set of qualifier comparisons.
+///
 /// \returns the result of template argument deduction so far. Note that a
 /// "success" result means that template argument deduction has not yet failed,
 /// but it may still fail, later, for other reasons.
@@ -400,31 +833,105 @@ DeduceTemplateArguments(Sema &S,
                         QualType ParamIn, QualType ArgIn,
                         TemplateDeductionInfo &Info,
                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                        unsigned TDF) {
+                        unsigned TDF,
+                        bool PartialOrdering,
+    llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
   // We only want to look at the canonical types, since typedefs and
   // sugar are not part of template argument deduction.
   QualType Param = S.Context.getCanonicalType(ParamIn);
   QualType Arg = S.Context.getCanonicalType(ArgIn);
 
-  // C++0x [temp.deduct.call]p4 bullet 1:
-  //   - If the original P is a reference type, the deduced A (i.e., the type
-  //     referred to by the reference) can be more cv-qualified than the
-  //     transformed A.
-  if (TDF & TDF_ParamWithReferenceType) {
-    Qualifiers Quals;
-    QualType UnqualParam = S.Context.getUnqualifiedArrayType(Param, Quals);
-    Quals.setCVRQualifiers(Quals.getCVRQualifiers() &
-                           Arg.getCVRQualifiersThroughArrayTypes());
-    Param = S.Context.getQualifiedType(UnqualParam, Quals);
+  // If the argument type is a pack expansion, look at its pattern.
+  // This isn't explicitly called out
+  if (const PackExpansionType *ArgExpansion
+                                            = dyn_cast<PackExpansionType>(Arg))
+    Arg = ArgExpansion->getPattern();
+
+  if (PartialOrdering) {
+    // C++0x [temp.deduct.partial]p5:
+    //   Before the partial ordering is done, certain transformations are
+    //   performed on the types used for partial ordering:
+    //     - If P is a reference type, P is replaced by the type referred to.
+    const ReferenceType *ParamRef = Param->getAs<ReferenceType>();
+    if (ParamRef)
+      Param = ParamRef->getPointeeType();
+
+    //     - If A is a reference type, A is replaced by the type referred to.
+    const ReferenceType *ArgRef = Arg->getAs<ReferenceType>();
+    if (ArgRef)
+      Arg = ArgRef->getPointeeType();
+
+    if (RefParamComparisons && ParamRef && ArgRef) {
+      // C++0x [temp.deduct.partial]p6:
+      //   If both P and A were reference types (before being replaced with the
+      //   type referred to above), determine which of the two types (if any) is
+      //   more cv-qualified than the other; otherwise the types are considered
+      //   to be equally cv-qualified for partial ordering purposes. The result
+      //   of this determination will be used below.
+      //
+      // We save this information for later, using it only when deduction
+      // succeeds in both directions.
+      RefParamPartialOrderingComparison Comparison;
+      Comparison.ParamIsRvalueRef = ParamRef->getAs<RValueReferenceType>();
+      Comparison.ArgIsRvalueRef = ArgRef->getAs<RValueReferenceType>();
+      Comparison.Qualifiers = NeitherMoreQualified;
+      if (Param.isMoreQualifiedThan(Arg))
+        Comparison.Qualifiers = ParamMoreQualified;
+      else if (Arg.isMoreQualifiedThan(Param))
+        Comparison.Qualifiers = ArgMoreQualified;
+      RefParamComparisons->push_back(Comparison);
+    }
+
+    // C++0x [temp.deduct.partial]p7:
+    //   Remove any top-level cv-qualifiers:
+    //     - If P is a cv-qualified type, P is replaced by the cv-unqualified
+    //       version of P.
+    Param = Param.getUnqualifiedType();
+    //     - If A is a cv-qualified type, A is replaced by the cv-unqualified
+    //       version of A.
+    Arg = Arg.getUnqualifiedType();
+  } else {
+    // C++0x [temp.deduct.call]p4 bullet 1:
+    //   - If the original P is a reference type, the deduced A (i.e., the type
+    //     referred to by the reference) can be more cv-qualified than the
+    //     transformed A.
+    if (TDF & TDF_ParamWithReferenceType) {
+      Qualifiers Quals;
+      QualType UnqualParam = S.Context.getUnqualifiedArrayType(Param, Quals);
+      Quals.setCVRQualifiers(Quals.getCVRQualifiers() &
+                             Arg.getCVRQualifiers());
+      Param = S.Context.getQualifiedType(UnqualParam, Quals);
+    }
+
+    if ((TDF & TDF_TopLevelParameterTypeList) && !Param->isFunctionType()) {
+      // C++0x [temp.deduct.type]p10:
+      //   If P and A are function types that originated from deduction when
+      //   taking the address of a function template (14.8.2.2) or when deducing
+      //   template arguments from a function declaration (14.8.2.6) and Pi and
+      //   Ai are parameters of the top-level parameter-type-list of P and A,
+      //   respectively, Pi is adjusted if it is an rvalue reference to a
+      //   cv-unqualified template parameter and Ai is an lvalue reference, in
+      //   which case the type of Pi is changed to be the template parameter
+      //   type (i.e., T&& is changed to simply T). [ Note: As a result, when
+      //   Pi is T&& and Ai is X&, the adjusted Pi will be T, causing T to be
+      //   deduced as X&. - end note ]
+      TDF &= ~TDF_TopLevelParameterTypeList;
+
+      if (const RValueReferenceType *ParamRef
+                                        = Param->getAs<RValueReferenceType>()) {
+        if (isa<TemplateTypeParmType>(ParamRef->getPointeeType()) &&
+            !ParamRef->getPointeeType().getQualifiers())
+          if (Arg->isLValueReferenceType())
+            Param = ParamRef->getPointeeType();
+      }
+    }
   }
 
   // If the parameter type is not dependent, there is nothing to deduce.
   if (!Param->isDependentType()) {
-    if (!(TDF & TDF_SkipNonDependent) && Param != Arg) {
-      
+    if (!(TDF & TDF_SkipNonDependent) && Param != Arg)
       return Sema::TDK_NonDeducedMismatch;
-    }
-    
+
     return Sema::TDK_Success;
   }
 
@@ -464,27 +971,24 @@ DeduceTemplateArguments(Sema &S,
     assert(TemplateTypeParm->getDepth() == 0 && "Can't deduce with depth > 0");
     assert(Arg != S.Context.OverloadTy && "Unresolved overloaded function");
     QualType DeducedType = Arg;
-    DeducedType.removeCVRQualifiers(Param.getCVRQualifiers());
+
+    // local manipulation is okay because it's canonical
+    DeducedType.removeLocalCVRQualifiers(Param.getCVRQualifiers());
     if (RecanonicalizeArg)
       DeducedType = S.Context.getCanonicalType(DeducedType);
 
-    if (Deduced[Index].isNull())
-      Deduced[Index] = TemplateArgument(DeducedType);
-    else {
-      // C++ [temp.deduct.type]p2:
-      //   [...] If type deduction cannot be done for any P/A pair, or if for
-      //   any pair the deduction leads to more than one possible set of
-      //   deduced values, or if different pairs yield different deduced
-      //   values, or if any template argument remains neither deduced nor
-      //   explicitly specified, template argument deduction fails.
-      if (Deduced[Index].getAsType() != DeducedType) {
-        Info.Param
-          = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
-        Info.FirstArg = Deduced[Index];
-        Info.SecondArg = TemplateArgument(Arg);
-        return Sema::TDK_Inconsistent;
-      }
+    DeducedTemplateArgument NewDeduced(DeducedType);
+    DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
+                                                                 Deduced[Index],
+                                                                   NewDeduced);
+    if (Result.isNull()) {
+      Info.Param = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
+      Info.FirstArg = Deduced[Index];
+      Info.SecondArg = NewDeduced;
+      return Sema::TDK_Inconsistent;
     }
+
+    Deduced[Index] = Result;
     return Sema::TDK_Success;
   }
 
@@ -492,6 +996,13 @@ DeduceTemplateArguments(Sema &S,
   Info.FirstArg = TemplateArgument(ParamIn);
   Info.SecondArg = TemplateArgument(ArgIn);
 
+  // If the parameter is an already-substituted template parameter
+  // pack, do nothing: we don't know which of its arguments to look
+  // at, so we have to wait until all of the parameter packs in this
+  // expansion have arguments.
+  if (isa<SubstTemplateTypeParmPackType>(Param))
+    return Sema::TDK_Success;
+
   // Check the cv-qualifiers on the parameter and argument types.
   if (!(TDF & TDF_IgnoreQualifiers)) {
     if (TDF & TDF_ParamWithReferenceType) {
@@ -615,16 +1126,17 @@ DeduceTemplateArguments(Sema &S,
       if (const ConstantArrayType *ConstantArrayArg
             = dyn_cast<ConstantArrayType>(ArrayArg)) {
         llvm::APSInt Size(ConstantArrayArg->getSize());
-        return DeduceNonTypeTemplateArgument(S, NTTP, Size, 
+        return DeduceNonTypeTemplateArgument(S, NTTP, Size,
                                              S.Context.getSizeType(),
                                              /*ArrayBound=*/true,
                                              Info, Deduced);
       }
       if (const DependentSizedArrayType *DependentArrayArg
             = dyn_cast<DependentSizedArrayType>(ArrayArg))
-        return DeduceNonTypeTemplateArgument(S, NTTP,
-                                             DependentArrayArg->getSizeExpr(),
-                                             Info, Deduced);
+        if (DependentArrayArg->getSizeExpr())
+          return DeduceNonTypeTemplateArgument(S, NTTP,
+                                               DependentArrayArg->getSizeExpr(),
+                                               Info, Deduced);
 
       // Incomplete type does not match a dependently-sized array type
       return Sema::TDK_NonDeducedMismatch;
@@ -634,6 +1146,7 @@ DeduceTemplateArguments(Sema &S,
     //     T(*)()
     //     T(*)(T)
     case Type::FunctionProto: {
+      unsigned SubTDF = TDF & TDF_TopLevelParameterTypeList;
       const FunctionProtoType *FunctionProtoArg =
         dyn_cast<FunctionProtoType>(Arg);
       if (!FunctionProtoArg)
@@ -642,14 +1155,11 @@ DeduceTemplateArguments(Sema &S,
       const FunctionProtoType *FunctionProtoParam =
         cast<FunctionProtoType>(Param);
 
-      if (FunctionProtoParam->getTypeQuals() !=
-          FunctionProtoArg->getTypeQuals())
-        return Sema::TDK_NonDeducedMismatch;
-
-      if (FunctionProtoParam->getNumArgs() != FunctionProtoArg->getNumArgs())
-        return Sema::TDK_NonDeducedMismatch;
-
-      if (FunctionProtoParam->isVariadic() != FunctionProtoArg->isVariadic())
+      if (FunctionProtoParam->getTypeQuals()
+            != FunctionProtoArg->getTypeQuals() ||
+          FunctionProtoParam->getRefQualifier()
+            != FunctionProtoArg->getRefQualifier() ||
+          FunctionProtoParam->isVariadic() != FunctionProtoArg->isVariadic())
         return Sema::TDK_NonDeducedMismatch;
 
       // Check return types.
@@ -660,17 +1170,12 @@ DeduceTemplateArguments(Sema &S,
                                       Info, Deduced, 0))
         return Result;
 
-      for (unsigned I = 0, N = FunctionProtoParam->getNumArgs(); I != N; ++I) {
-        // Check argument types.
-        if (Sema::TemplateDeductionResult Result
-              = DeduceTemplateArguments(S, TemplateParams,
-                                        FunctionProtoParam->getArgType(I),
-                                        FunctionProtoArg->getArgType(I),
-                                        Info, Deduced, 0))
-          return Result;
-      }
-
-      return Sema::TDK_Success;
+      return DeduceTemplateArguments(S, TemplateParams,
+                                     FunctionProtoParam->arg_type_begin(),
+                                     FunctionProtoParam->getNumArgs(),
+                                     FunctionProtoArg->arg_type_begin(),
+                                     FunctionProtoArg->getNumArgs(),
+                                     Info, Deduced, SubTDF);
     }
 
     case Type::InjectedClassName: {
@@ -721,6 +1226,8 @@ DeduceTemplateArguments(Sema &S,
           llvm::SmallVector<const RecordType *, 8> ToVisit;
           ToVisit.push_back(RecordT);
           bool Successful = false;
+          llvm::SmallVectorImpl<DeducedTemplateArgument> DeducedOrig(0);
+          DeducedOrig = Deduced;
           while (!ToVisit.empty()) {
             // Retrieve the next class in the inheritance hierarchy.
             const RecordType *NextT = ToVisit.back();
@@ -738,9 +1245,14 @@ DeduceTemplateArguments(Sema &S,
                                           QualType(NextT, 0), Info, Deduced);
 
               // If template argument deduction for this base was successful,
-              // note that we had some success.
-              if (BaseResult == Sema::TDK_Success)
+              // note that we had some success. Otherwise, ignore any deductions
+              // from this base class.
+              if (BaseResult == Sema::TDK_Success) {
                 Successful = true;
+                DeducedOrig = Deduced;
+              }
+              else
+                Deduced = DeducedOrig;
             }
 
             // Visit base classes
@@ -828,9 +1340,15 @@ static Sema::TemplateDeductionResult
 DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateArgument &Param,
-                        const TemplateArgument &Arg,
+                        TemplateArgument Arg,
                         TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+  // If the template argument is a pack expansion, perform template argument
+  // deduction against the pattern of that expansion. This only occurs during
+  // partial ordering.
+  if (Arg.isPackExpansion())
+    Arg = Arg.getPackExpansionPattern();
+
   switch (Param.getKind()) {
   case TemplateArgument::Null:
     assert(false && "Null template argument in parameter list");
@@ -843,22 +1361,26 @@ DeduceTemplateArguments(Sema &S,
     Info.FirstArg = Param;
     Info.SecondArg = Arg;
     return Sema::TDK_NonDeducedMismatch;
-      
+
   case TemplateArgument::Template:
     if (Arg.getKind() == TemplateArgument::Template)
-      return DeduceTemplateArguments(S, TemplateParams, 
+      return DeduceTemplateArguments(S, TemplateParams,
                                      Param.getAsTemplate(),
                                      Arg.getAsTemplate(), Info, Deduced);
     Info.FirstArg = Param;
     Info.SecondArg = Arg;
     return Sema::TDK_NonDeducedMismatch;
-      
+
+  case TemplateArgument::TemplateExpansion:
+    llvm_unreachable("caller should handle pack expansions");
+    break;
+
   case TemplateArgument::Declaration:
     if (Arg.getKind() == TemplateArgument::Declaration &&
         Param.getAsDecl()->getCanonicalDecl() ==
           Arg.getAsDecl()->getCanonicalDecl())
       return Sema::TDK_Success;
-      
+
     Info.FirstArg = Param;
     Info.SecondArg = Arg;
     return Sema::TDK_NonDeducedMismatch;
@@ -898,7 +1420,7 @@ DeduceTemplateArguments(Sema &S,
       if (Arg.getKind() == TemplateArgument::Declaration)
         return DeduceNonTypeTemplateArgument(S, NTTP, Arg.getAsDecl(),
                                              Info, Deduced);
-      
+
       Info.FirstArg = Param;
       Info.SecondArg = Arg;
       return Sema::TDK_NonDeducedMismatch;
@@ -908,31 +1430,209 @@ DeduceTemplateArguments(Sema &S,
     return Sema::TDK_Success;
   }
   case TemplateArgument::Pack:
-    assert(0 && "FIXME: Implement!");
-    break;
+    llvm_unreachable("Argument packs should be expanded by the caller!");
   }
 
   return Sema::TDK_Success;
 }
 
+/// \brief Determine whether there is a template argument to be used for
+/// deduction.
+///
+/// This routine "expands" argument packs in-place, overriding its input
+/// parameters so that \c Args[ArgIdx] will be the available template argument.
+///
+/// \returns true if there is another template argument (which will be at
+/// \c Args[ArgIdx]), false otherwise.
+static bool hasTemplateArgumentForDeduction(const TemplateArgument *&Args,
+                                            unsigned &ArgIdx,
+                                            unsigned &NumArgs) {
+  if (ArgIdx == NumArgs)
+    return false;
+
+  const TemplateArgument &Arg = Args[ArgIdx];
+  if (Arg.getKind() != TemplateArgument::Pack)
+    return true;
+
+  assert(ArgIdx == NumArgs - 1 && "Pack not at the end of argument list?");
+  Args = Arg.pack_begin();
+  NumArgs = Arg.pack_size();
+  ArgIdx = 0;
+  return ArgIdx < NumArgs;
+}
+
+/// \brief Determine whether the given set of template arguments has a pack
+/// expansion that is not the last template argument.
+static bool hasPackExpansionBeforeEnd(const TemplateArgument *Args,
+                                      unsigned NumArgs) {
+  unsigned ArgIdx = 0;
+  while (ArgIdx < NumArgs) {
+    const TemplateArgument &Arg = Args[ArgIdx];
+
+    // Unwrap argument packs.
+    if (Args[ArgIdx].getKind() == TemplateArgument::Pack) {
+      Args = Arg.pack_begin();
+      NumArgs = Arg.pack_size();
+      ArgIdx = 0;
+      continue;
+    }
+
+    ++ArgIdx;
+    if (ArgIdx == NumArgs)
+      return false;
+
+    if (Arg.isPackExpansion())
+      return true;
+  }
+
+  return false;
+}
+
 static Sema::TemplateDeductionResult
 DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
-                        const TemplateArgumentList &ParamList,
-                        const TemplateArgumentList &ArgList,
+                        const TemplateArgument *Params, unsigned NumParams,
+                        const TemplateArgument *Args, unsigned NumArgs,
                         TemplateDeductionInfo &Info,
-                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
-  assert(ParamList.size() == ArgList.size());
-  for (unsigned I = 0, N = ParamList.size(); I != N; ++I) {
+                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
+                        bool NumberOfArgumentsMustMatch) {
+  // C++0x [temp.deduct.type]p9:
+  //   If the template argument list of P contains a pack expansion that is not
+  //   the last template argument, the entire template argument list is a
+  //   non-deduced context.
+  if (hasPackExpansionBeforeEnd(Params, NumParams))
+    return Sema::TDK_Success;
+
+  // C++0x [temp.deduct.type]p9:
+  //   If P has a form that contains <T> or <i>, then each argument Pi of the
+  //   respective template argument list P is compared with the corresponding
+  //   argument Ai of the corresponding template argument list of A.
+  unsigned ArgIdx = 0, ParamIdx = 0;
+  for (; hasTemplateArgumentForDeduction(Params, ParamIdx, NumParams);
+       ++ParamIdx) {
+    if (!Params[ParamIdx].isPackExpansion()) {
+      // The simple case: deduce template arguments by matching Pi and Ai.
+
+      // Check whether we have enough arguments.
+      if (!hasTemplateArgumentForDeduction(Args, ArgIdx, NumArgs))
+        return NumberOfArgumentsMustMatch? Sema::TDK_NonDeducedMismatch
+                                         : Sema::TDK_Success;
+
+      if (Args[ArgIdx].isPackExpansion()) {
+        // FIXME: We follow the logic of C++0x [temp.deduct.type]p22 here,
+        // but applied to pack expansions that are template arguments.
+        return Sema::TDK_NonDeducedMismatch;
+      }
+
+      // Perform deduction for this Pi/Ai pair.
+      if (Sema::TemplateDeductionResult Result
+            = DeduceTemplateArguments(S, TemplateParams,
+                                      Params[ParamIdx], Args[ArgIdx],
+                                      Info, Deduced))
+        return Result;
+
+      // Move to the next argument.
+      ++ArgIdx;
+      continue;
+    }
+
+    // The parameter is a pack expansion.
+
+    // C++0x [temp.deduct.type]p9:
+    //   If Pi is a pack expansion, then the pattern of Pi is compared with
+    //   each remaining argument in the template argument list of A. Each
+    //   comparison deduces template arguments for subsequent positions in the
+    //   template parameter packs expanded by Pi.
+    TemplateArgument Pattern = Params[ParamIdx].getPackExpansionPattern();
+
+    // Compute the set of template parameter indices that correspond to
+    // parameter packs expanded by the pack expansion.
+    llvm::SmallVector<unsigned, 2> PackIndices;
+    {
+      llvm::BitVector SawIndices(TemplateParams->size());
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      S.collectUnexpandedParameterPacks(Pattern, Unexpanded);
+      for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
+        unsigned Depth, Index;
+        llvm::tie(Depth, Index) = getDepthAndIndex(Unexpanded[I]);
+        if (Depth == 0 && !SawIndices[Index]) {
+          SawIndices[Index] = true;
+          PackIndices.push_back(Index);
+        }
+      }
+    }
+    assert(!PackIndices.empty() && "Pack expansion without unexpanded packs?");
+
+    // FIXME: If there are no remaining arguments, we can bail out early
+    // and set any deduced parameter packs to an empty argument pack.
+    // The latter part of this is a (minor) correctness issue.
+
+    // Save the deduced template arguments for each parameter pack expanded
+    // by this pack expansion, then clear out the deduction.
+    llvm::SmallVector<DeducedTemplateArgument, 2>
+      SavedPacks(PackIndices.size());
+    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+      NewlyDeducedPacks(PackIndices.size());
+    PrepareArgumentPackDeduction(S, Deduced, PackIndices, SavedPacks,
+                                 NewlyDeducedPacks);
+
+    // Keep track of the deduced template arguments for each parameter pack
+    // expanded by this pack expansion (the outer index) and for each
+    // template argument (the inner SmallVectors).
+    bool HasAnyArguments = false;
+    while (hasTemplateArgumentForDeduction(Args, ArgIdx, NumArgs)) {
+      HasAnyArguments = true;
+
+      // Deduce template arguments from the pattern.
+      if (Sema::TemplateDeductionResult Result
+            = DeduceTemplateArguments(S, TemplateParams, Pattern, Args[ArgIdx],
+                                      Info, Deduced))
+        return Result;
+
+      // Capture the deduced template arguments for each parameter pack expanded
+      // by this pack expansion, add them to the list of arguments we've deduced
+      // for that pack, then clear out the deduced argument.
+      for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
+        DeducedTemplateArgument &DeducedArg = Deduced[PackIndices[I]];
+        if (!DeducedArg.isNull()) {
+          NewlyDeducedPacks[I].push_back(DeducedArg);
+          DeducedArg = DeducedTemplateArgument();
+        }
+      }
+
+      ++ArgIdx;
+    }
+
+    // Build argument packs for each of the parameter packs expanded by this
+    // pack expansion.
     if (Sema::TemplateDeductionResult Result
-          = DeduceTemplateArguments(S, TemplateParams,
-                                    ParamList[I], ArgList[I],
-                                    Info, Deduced))
+          = FinishArgumentPackDeduction(S, TemplateParams, HasAnyArguments,
+                                        Deduced, PackIndices, SavedPacks,
+                                        NewlyDeducedPacks, Info))
       return Result;
   }
+
+  // If there is an argument remaining, then we had too many arguments.
+  if (NumberOfArgumentsMustMatch &&
+      hasTemplateArgumentForDeduction(Args, ArgIdx, NumArgs))
+    return Sema::TDK_NonDeducedMismatch;
+
   return Sema::TDK_Success;
 }
 
+static Sema::TemplateDeductionResult
+DeduceTemplateArguments(Sema &S,
+                        TemplateParameterList *TemplateParams,
+                        const TemplateArgumentList &ParamList,
+                        const TemplateArgumentList &ArgList,
+                        TemplateDeductionInfo &Info,
+                    llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
+  return DeduceTemplateArguments(S, TemplateParams,
+                                 ParamList.data(), ParamList.size(),
+                                 ArgList.data(), ArgList.size(),
+                                 Info, Deduced);
+}
+
 /// \brief Determine whether two template arguments are the same.
 static bool isSameTemplateArg(ASTContext &Context,
                               const TemplateArgument &X,
@@ -954,18 +1654,19 @@ static bool isSameTemplateArg(ASTContext &Context,
              Y.getAsDecl()->getCanonicalDecl();
 
     case TemplateArgument::Template:
-      return Context.getCanonicalTemplateName(X.getAsTemplate())
-               .getAsVoidPointer() ==
-             Context.getCanonicalTemplateName(Y.getAsTemplate())
-               .getAsVoidPointer();
-      
+    case TemplateArgument::TemplateExpansion:
+      return Context.getCanonicalTemplateName(
+                    X.getAsTemplateOrTemplatePattern()).getAsVoidPointer() ==
+             Context.getCanonicalTemplateName(
+                    Y.getAsTemplateOrTemplatePattern()).getAsVoidPointer();
+
     case TemplateArgument::Integral:
       return *X.getAsIntegral() == *Y.getAsIntegral();
 
     case TemplateArgument::Expression: {
       llvm::FoldingSetNodeID XID, YID;
       X.getAsExpr()->Profile(XID, Context, true);
-      Y.getAsExpr()->Profile(YID, Context, true);      
+      Y.getAsExpr()->Profile(YID, Context, true);
       return XID == YID;
     }
 
@@ -986,51 +1687,192 @@ static bool isSameTemplateArg(ASTContext &Context,
   return false;
 }
 
-/// \brief Helper function to build a TemplateParameter when we don't
-/// know its type statically.
-static TemplateParameter makeTemplateParameter(Decl *D) {
-  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(D))
-    return TemplateParameter(TTP);
-  else if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(D))
-    return TemplateParameter(NTTP);
+/// \brief Allocate a TemplateArgumentLoc where all locations have
+/// been initialized to the given location.
+///
+/// \param S The semantic analysis object.
+///
+/// \param The template argument we are producing template argument
+/// location information for.
+///
+/// \param NTTPType For a declaration template argument, the type of
+/// the non-type template parameter that corresponds to this template
+/// argument.
+///
+/// \param Loc The source location to use for the resulting template
+/// argument.
+static TemplateArgumentLoc
+getTrivialTemplateArgumentLoc(Sema &S,
+                              const TemplateArgument &Arg,
+                              QualType NTTPType,
+                              SourceLocation Loc) {
+  switch (Arg.getKind()) {
+  case TemplateArgument::Null:
+    llvm_unreachable("Can't get a NULL template argument here");
+    break;
 
-  return TemplateParameter(cast<TemplateTemplateParmDecl>(D));
+  case TemplateArgument::Type:
+    return TemplateArgumentLoc(Arg,
+                     S.Context.getTrivialTypeSourceInfo(Arg.getAsType(), Loc));
+
+  case TemplateArgument::Declaration: {
+    Expr *E
+      = S.BuildExpressionFromDeclTemplateArgument(Arg, NTTPType, Loc)
+    .takeAs<Expr>();
+    return TemplateArgumentLoc(TemplateArgument(E), E);
+  }
+
+  case TemplateArgument::Integral: {
+    Expr *E
+      = S.BuildExpressionFromIntegralTemplateArgument(Arg, Loc).takeAs<Expr>();
+    return TemplateArgumentLoc(TemplateArgument(E), E);
+  }
+
+  case TemplateArgument::Template:
+    return TemplateArgumentLoc(Arg, SourceRange(), Loc);
+
+  case TemplateArgument::TemplateExpansion:
+    return TemplateArgumentLoc(Arg, SourceRange(), Loc, Loc);
+
+  case TemplateArgument::Expression:
+    return TemplateArgumentLoc(Arg, Arg.getAsExpr());
+
+  case TemplateArgument::Pack:
+    return TemplateArgumentLoc(Arg, TemplateArgumentLocInfo());
+  }
+
+  return TemplateArgumentLoc();
+}
+
+
+/// \brief Convert the given deduced template argument and add it to the set of
+/// fully-converted template arguments.
+static bool ConvertDeducedTemplateArgument(Sema &S, NamedDecl *Param,
+                                           DeducedTemplateArgument Arg,
+                                           NamedDecl *Template,
+                                           QualType NTTPType,
+                                           unsigned ArgumentPackIndex,
+                                           TemplateDeductionInfo &Info,
+                                           bool InFunctionTemplate,
+                             llvm::SmallVectorImpl<TemplateArgument> &Output) {
+  if (Arg.getKind() == TemplateArgument::Pack) {
+    // This is a template argument pack, so check each of its arguments against
+    // the template parameter.
+    llvm::SmallVector<TemplateArgument, 2> PackedArgsBuilder;
+    for (TemplateArgument::pack_iterator PA = Arg.pack_begin(),
+                                      PAEnd = Arg.pack_end();
+         PA != PAEnd; ++PA) {
+      // When converting the deduced template argument, append it to the
+      // general output list. We need to do this so that the template argument
+      // checking logic has all of the prior template arguments available.
+      DeducedTemplateArgument InnerArg(*PA);
+      InnerArg.setDeducedFromArrayBound(Arg.wasDeducedFromArrayBound());
+      if (ConvertDeducedTemplateArgument(S, Param, InnerArg, Template,
+                                         NTTPType, PackedArgsBuilder.size(),
+                                         Info, InFunctionTemplate, Output))
+        return true;
+
+      // Move the converted template argument into our argument pack.
+      PackedArgsBuilder.push_back(Output.back());
+      Output.pop_back();
+    }
+
+    // Create the resulting argument pack.
+    Output.push_back(TemplateArgument::CreatePackCopy(S.Context,
+                                                      PackedArgsBuilder.data(),
+                                                     PackedArgsBuilder.size()));
+    return false;
+  }
+
+  // Convert the deduced template argument into a template
+  // argument that we can check, almost as if the user had written
+  // the template argument explicitly.
+  TemplateArgumentLoc ArgLoc = getTrivialTemplateArgumentLoc(S, Arg, NTTPType,
+                                                             Info.getLocation());
+
+  // Check the template argument, converting it as necessary.
+  return S.CheckTemplateArgument(Param, ArgLoc,
+                                 Template,
+                                 Template->getLocation(),
+                                 Template->getSourceRange().getEnd(),
+                                 ArgumentPackIndex,
+                                 Output,
+                                 InFunctionTemplate
+                                  ? (Arg.wasDeducedFromArrayBound()
+                                       ? Sema::CTAK_DeducedFromArrayBound
+                                       : Sema::CTAK_Deduced)
+                                 : Sema::CTAK_Specified);
 }
 
 /// Complete template argument deduction for a class template partial
 /// specialization.
 static Sema::TemplateDeductionResult
-FinishTemplateArgumentDeduction(Sema &S, 
+FinishTemplateArgumentDeduction(Sema &S,
                                 ClassTemplatePartialSpecializationDecl *Partial,
                                 const TemplateArgumentList &TemplateArgs,
                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                                 TemplateDeductionInfo &Info) {
   // Trap errors.
   Sema::SFINAETrap Trap(S);
-  
+
   Sema::ContextRAII SavedContext(S, Partial);
 
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  TemplateArgumentListBuilder Builder(Partial->getTemplateParameters(),
-                                      Deduced.size());
-  for (unsigned I = 0, N = Deduced.size(); I != N; ++I) {
+  llvm::SmallVector<TemplateArgument, 4> Builder;
+  TemplateParameterList *PartialParams = Partial->getTemplateParameters();
+  for (unsigned I = 0, N = PartialParams->size(); I != N; ++I) {
+    NamedDecl *Param = PartialParams->getParam(I);
     if (Deduced[I].isNull()) {
-      Decl *Param
-      = const_cast<NamedDecl *>(
-                                Partial->getTemplateParameters()->getParam(I));
       Info.Param = makeTemplateParameter(Param);
       return Sema::TDK_Incomplete;
     }
-    
-    Builder.Append(Deduced[I]);
+
+    // We have deduced this argument, so it still needs to be
+    // checked and converted.
+
+    // First, for a non-type template parameter type that is
+    // initialized by a declaration, we need the type of the
+    // corresponding non-type template parameter.
+    QualType NTTPType;
+    if (NonTypeTemplateParmDecl *NTTP
+                                  = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
+      NTTPType = NTTP->getType();
+      if (NTTPType->isDependentType()) {
+        TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                          Builder.data(), Builder.size());
+        NTTPType = S.SubstType(NTTPType,
+                               MultiLevelTemplateArgumentList(TemplateArgs),
+                               NTTP->getLocation(),
+                               NTTP->getDeclName());
+        if (NTTPType.isNull()) {
+          Info.Param = makeTemplateParameter(Param);
+          // FIXME: These template arguments are temporary. Free them!
+          Info.reset(TemplateArgumentList::CreateCopy(S.Context,
+                                                      Builder.data(),
+                                                      Builder.size()));
+          return Sema::TDK_SubstitutionFailure;
+        }
+      }
+    }
+
+    if (ConvertDeducedTemplateArgument(S, Param, Deduced[I],
+                                       Partial, NTTPType, 0, Info, false,
+                                       Builder)) {
+      Info.Param = makeTemplateParameter(Param);
+      // FIXME: These template arguments are temporary. Free them!
+      Info.reset(TemplateArgumentList::CreateCopy(S.Context, Builder.data(),
+                                                  Builder.size()));
+      return Sema::TDK_SubstitutionFailure;
+    }
   }
-  
+
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *DeducedArgumentList
-    = new (S.Context) TemplateArgumentList(S.Context, Builder, 
-                                           /*TakeArgs=*/true);
+    = TemplateArgumentList::CreateCopy(S.Context, Builder.data(),
+                                       Builder.size());
+
   Info.reset(DeducedArgumentList);
 
   // Substitute the deduced template arguments into the template
@@ -1038,60 +1880,40 @@ FinishTemplateArgumentDeduction(Sema &S,
   // verify that the instantiated template arguments are both valid
   // and are equivalent to the template arguments originally provided
   // to the class template.
-  // FIXME: Do we have to correct the types of deduced non-type template 
-  // arguments (in particular, integral non-type template arguments?).
   LocalInstantiationScope InstScope(S);
   ClassTemplateDecl *ClassTemplate = Partial->getSpecializedTemplate();
   const TemplateArgumentLoc *PartialTemplateArgs
     = Partial->getTemplateArgsAsWritten();
-  unsigned N = Partial->getNumTemplateArgsAsWritten();
 
   // Note that we don't provide the langle and rangle locations.
   TemplateArgumentListInfo InstArgs;
 
-  for (unsigned I = 0; I != N; ++I) {
-    Decl *Param = const_cast<NamedDecl *>(
-                    ClassTemplate->getTemplateParameters()->getParam(I));
-    TemplateArgumentLoc InstArg;
-    if (S.Subst(PartialTemplateArgs[I], InstArg,
-                MultiLevelTemplateArgumentList(*DeducedArgumentList))) {
-      Info.Param = makeTemplateParameter(Param);
-      Info.FirstArg = PartialTemplateArgs[I].getArgument();
-      return Sema::TDK_SubstitutionFailure;
-    }
-    InstArgs.addArgument(InstArg);
-  }
+  if (S.Subst(PartialTemplateArgs,
+              Partial->getNumTemplateArgsAsWritten(),
+              InstArgs, MultiLevelTemplateArgumentList(*DeducedArgumentList))) {
+    unsigned ArgIdx = InstArgs.size(), ParamIdx = ArgIdx;
+    if (ParamIdx >= Partial->getTemplateParameters()->size())
+      ParamIdx = Partial->getTemplateParameters()->size() - 1;
 
-  TemplateArgumentListBuilder ConvertedInstArgs(
-                                  ClassTemplate->getTemplateParameters(), N);
+    Decl *Param
+      = const_cast<NamedDecl *>(
+                          Partial->getTemplateParameters()->getParam(ParamIdx));
+    Info.Param = makeTemplateParameter(Param);
+    Info.FirstArg = PartialTemplateArgs[ArgIdx].getArgument();
+    return Sema::TDK_SubstitutionFailure;
+  }
 
+  llvm::SmallVector<TemplateArgument, 4> ConvertedInstArgs;
   if (S.CheckTemplateArgumentList(ClassTemplate, Partial->getLocation(),
-                                InstArgs, false, ConvertedInstArgs))
+                                  InstArgs, false, ConvertedInstArgs))
     return Sema::TDK_SubstitutionFailure;
-  
-  for (unsigned I = 0, E = ConvertedInstArgs.flatSize(); I != E; ++I) {
-    TemplateArgument InstArg = ConvertedInstArgs.getFlatArguments()[I];
-
-    Decl *Param = const_cast<NamedDecl *>(
-                    ClassTemplate->getTemplateParameters()->getParam(I));
-
-    if (InstArg.getKind() == TemplateArgument::Expression) {
-      // When the argument is an expression, check the expression result
-      // against the actual template parameter to get down to the canonical
-      // template argument.
-      Expr *InstExpr = InstArg.getAsExpr();
-      if (NonTypeTemplateParmDecl *NTTP
-            = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
-        if (S.CheckTemplateArgument(NTTP, NTTP->getType(), InstExpr, InstArg)) {
-          Info.Param = makeTemplateParameter(Param);
-          Info.FirstArg = Partial->getTemplateArgs()[I];
-          return Sema::TDK_SubstitutionFailure;
-        }
-      }
-    }
 
+  TemplateParameterList *TemplateParams
+    = ClassTemplate->getTemplateParameters();
+  for (unsigned I = 0, E = TemplateParams->size(); I != E; ++I) {
+    TemplateArgument InstArg = ConvertedInstArgs.data()[I];
     if (!isSameTemplateArg(S.Context, TemplateArgs[I], InstArg)) {
-      Info.Param = makeTemplateParameter(Param);
+      Info.Param = makeTemplateParameter(TemplateParams->getParam(I));
       Info.FirstArg = TemplateArgs[I];
       Info.SecondArg = InstArg;
       return Sema::TDK_NonDeducedMismatch;
@@ -1127,14 +1949,14 @@ Sema::DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial,
     return Result;
 
   InstantiatingTemplate Inst(*this, Partial->getLocation(), Partial,
-                             Deduced.data(), Deduced.size());
+                             Deduced.data(), Deduced.size(), Info);
   if (Inst)
     return TDK_InstantiationDepth;
 
   if (Trap.hasErrorOccurred())
     return Sema::TDK_SubstitutionFailure;
- 
-  return ::FinishTemplateArgumentDeduction(*this, Partial, TemplateArgs, 
+
+  return ::FinishTemplateArgumentDeduction(*this, Partial, TemplateArgs,
                                            Deduced, Info);
 }
 
@@ -1206,26 +2028,24 @@ Sema::SubstituteExplicitTemplateArguments(
   //   declaration order of their corresponding template-parameters. The
   //   template argument list shall not specify more template-arguments than
   //   there are corresponding template-parameters.
-  TemplateArgumentListBuilder Builder(TemplateParams,
-                                      ExplicitTemplateArgs.size());
+  llvm::SmallVector<TemplateArgument, 4> Builder;
 
   // Enter a new template instantiation context where we check the
   // explicitly-specified template arguments against this function template,
   // and then substitute them into the function parameter types.
   InstantiatingTemplate Inst(*this, FunctionTemplate->getLocation(),
                              FunctionTemplate, Deduced.data(), Deduced.size(),
-           ActiveTemplateInstantiation::ExplicitTemplateArgumentSubstitution);
+           ActiveTemplateInstantiation::ExplicitTemplateArgumentSubstitution,
+                             Info);
   if (Inst)
     return TDK_InstantiationDepth;
 
-  ContextRAII SavedContext(*this, FunctionTemplate->getTemplatedDecl());
-
   if (CheckTemplateArgumentList(FunctionTemplate,
                                 SourceLocation(),
                                 ExplicitTemplateArgs,
                                 true,
                                 Builder) || Trap.hasErrorOccurred()) {
-    unsigned Index = Builder.structuredSize();
+    unsigned Index = Builder.size();
     if (Index >= TemplateParams->size())
       Index = TemplateParams->size() - 1;
     Info.Param = makeTemplateParameter(TemplateParams->getParam(Index));
@@ -1235,25 +2055,38 @@ Sema::SubstituteExplicitTemplateArguments(
   // Form the template argument list from the explicitly-specified
   // template arguments.
   TemplateArgumentList *ExplicitArgumentList
-    = new (Context) TemplateArgumentList(Context, Builder, /*TakeArgs=*/true);
+    = TemplateArgumentList::CreateCopy(Context, Builder.data(), Builder.size());
   Info.reset(ExplicitArgumentList);
 
-  // Instantiate the types of each of the function parameters given the
-  // explicitly-specified template arguments.
-  for (FunctionDecl::param_iterator P = Function->param_begin(),
-                                PEnd = Function->param_end();
-       P != PEnd;
-       ++P) {
-    QualType ParamType
-      = SubstType((*P)->getType(),
-                  MultiLevelTemplateArgumentList(*ExplicitArgumentList),
-                  (*P)->getLocation(), (*P)->getDeclName());
-    if (ParamType.isNull() || Trap.hasErrorOccurred())
-      return TDK_SubstitutionFailure;
+  // Template argument deduction and the final substitution should be
+  // done in the context of the templated declaration.  Explicit
+  // argument substitution, on the other hand, needs to happen in the
+  // calling context.
+  ContextRAII SavedContext(*this, FunctionTemplate->getTemplatedDecl());
 
-    ParamTypes.push_back(ParamType);
+  // If we deduced template arguments for a template parameter pack,
+  // note that the template argument pack is partially substituted and record
+  // the explicit template arguments. They'll be used as part of deduction
+  // for this template parameter pack.
+  for (unsigned I = 0, N = Builder.size(); I != N; ++I) {
+    const TemplateArgument &Arg = Builder[I];
+    if (Arg.getKind() == TemplateArgument::Pack) {
+      CurrentInstantiationScope->SetPartiallySubstitutedPack(
+                                                 TemplateParams->getParam(I),
+                                                             Arg.pack_begin(),
+                                                             Arg.pack_size());
+      break;
+    }
   }
 
+  // Instantiate the types of each of the function parameters given the
+  // explicitly-specified template arguments.
+  if (SubstParmTypes(Function->getLocation(),
+                     Function->param_begin(), Function->getNumParams(),
+                     MultiLevelTemplateArgumentList(*ExplicitArgumentList),
+                     ParamTypes))
+    return TDK_SubstitutionFailure;
+
   // If the caller wants a full function type back, instantiate the return
   // type and form that function type.
   if (FunctionType) {
@@ -1274,6 +2107,7 @@ Sema::SubstituteExplicitTemplateArguments(
                                       ParamTypes.data(), ParamTypes.size(),
                                       Proto->isVariadic(),
                                       Proto->getTypeQuals(),
+                                      Proto->getRefQualifier(),
                                       Function->getLocation(),
                                       Function->getDeclName(),
                                       Proto->getExtInfo());
@@ -1287,67 +2121,20 @@ Sema::SubstituteExplicitTemplateArguments(
   //   template arguments can be deduced, they may all be omitted; in this
   //   case, the empty template argument list <> itself may also be omitted.
   //
-  // Take all of the explicitly-specified arguments and put them into the
-  // set of deduced template arguments.
+  // Take all of the explicitly-specified arguments and put them into
+  // the set of deduced template arguments. Explicitly-specified
+  // parameter packs, however, will be set to NULL since the deduction
+  // mechanisms handle explicitly-specified argument packs directly.
   Deduced.reserve(TemplateParams->size());
-  for (unsigned I = 0, N = ExplicitArgumentList->size(); I != N; ++I)
-    Deduced.push_back(ExplicitArgumentList->get(I));
-
-  return TDK_Success;
-}
-
-/// \brief Allocate a TemplateArgumentLoc where all locations have
-/// been initialized to the given location.
-///
-/// \param S The semantic analysis object.
-///
-/// \param The template argument we are producing template argument
-/// location information for.
-///
-/// \param NTTPType For a declaration template argument, the type of
-/// the non-type template parameter that corresponds to this template
-/// argument.
-///
-/// \param Loc The source location to use for the resulting template
-/// argument.
-static TemplateArgumentLoc 
-getTrivialTemplateArgumentLoc(Sema &S,
-                              const TemplateArgument &Arg, 
-                              QualType NTTPType,
-                              SourceLocation Loc) {
-  switch (Arg.getKind()) {
-  case TemplateArgument::Null:
-    llvm_unreachable("Can't get a NULL template argument here");
-    break;
-
-  case TemplateArgument::Type:
-    return TemplateArgumentLoc(Arg, 
-                    S.Context.getTrivialTypeSourceInfo(Arg.getAsType(), Loc));
-
-  case TemplateArgument::Declaration: {
-    Expr *E
-      = S.BuildExpressionFromDeclTemplateArgument(Arg, NTTPType, Loc)
-                                                              .takeAs<Expr>();
-    return TemplateArgumentLoc(TemplateArgument(E), E);
-  }
-
-  case TemplateArgument::Integral: {
-    Expr *E
-      = S.BuildExpressionFromIntegralTemplateArgument(Arg, Loc).takeAs<Expr>();
-    return TemplateArgumentLoc(TemplateArgument(E), E);
-  }
-
-  case TemplateArgument::Template:
-    return TemplateArgumentLoc(Arg, SourceRange(), Loc);
-
-  case TemplateArgument::Expression:
-    return TemplateArgumentLoc(Arg, Arg.getAsExpr());
-
-  case TemplateArgument::Pack:
-    llvm_unreachable("Template parameter packs are not yet supported");
+  for (unsigned I = 0, N = ExplicitArgumentList->size(); I != N; ++I) {
+    const TemplateArgument &Arg = ExplicitArgumentList->get(I);
+    if (Arg.getKind() == TemplateArgument::Pack)
+      Deduced.push_back(DeducedTemplateArgument());
+    else
+      Deduced.push_back(Arg);
   }
 
-  return TemplateArgumentLoc();
+  return TDK_Success;
 }
 
 /// \brief Finish template argument deduction for a function template,
@@ -1370,7 +2157,8 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   // actual function declaration.
   InstantiatingTemplate Inst(*this, FunctionTemplate->getLocation(),
                              FunctionTemplate, Deduced.data(), Deduced.size(),
-              ActiveTemplateInstantiation::DeducedTemplateArgumentSubstitution);
+              ActiveTemplateInstantiation::DeducedTemplateArgumentSubstitution,
+                             Info);
   if (Inst)
     return TDK_InstantiationDepth;
 
@@ -1379,18 +2167,16 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   // C++ [temp.deduct.type]p2:
   //   [...] or if any template argument remains neither deduced nor
   //   explicitly specified, template argument deduction fails.
-  TemplateArgumentListBuilder Builder(TemplateParams, Deduced.size());
-  for (unsigned I = 0, N = Deduced.size(); I != N; ++I) {
-    NamedDecl *Param = FunctionTemplate->getTemplateParameters()->getParam(I);
+  llvm::SmallVector<TemplateArgument, 4> Builder;
+  for (unsigned I = 0, N = TemplateParams->size(); I != N; ++I) {
+    NamedDecl *Param = TemplateParams->getParam(I);
+
     if (!Deduced[I].isNull()) {
-      if (I < NumExplicitlySpecified || 
-          Deduced[I].getKind() == TemplateArgument::Type) {
+      if (I < NumExplicitlySpecified) {
         // We have already fully type-checked and converted this
-        // argument (because it was explicitly-specified) or no
-        // additional checking is necessary (because it's a template
-        // type parameter). Just record the presence of this
-        // parameter.
-        Builder.Append(Deduced[I]);
+        // argument, because it was explicitly-specified. Just record the
+        // presence of this argument.
+        Builder.push_back(Deduced[I]);
         continue;
       }
 
@@ -1401,55 +2187,61 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
       // initialized by a declaration, we need the type of the
       // corresponding non-type template parameter.
       QualType NTTPType;
-      if (NonTypeTemplateParmDecl *NTTP 
-                                = dyn_cast<NonTypeTemplateParmDecl>(Param)) { 
-        if (Deduced[I].getKind() == TemplateArgument::Declaration) {
-          NTTPType = NTTP->getType();
-          if (NTTPType->isDependentType()) {
-            TemplateArgumentList TemplateArgs(Context, Builder, 
-                                              /*TakeArgs=*/false);
-            NTTPType = SubstType(NTTPType,
-                                 MultiLevelTemplateArgumentList(TemplateArgs),
-                                 NTTP->getLocation(),
-                                 NTTP->getDeclName());
-            if (NTTPType.isNull()) {
-              Info.Param = makeTemplateParameter(Param);
-              Info.reset(new (Context) TemplateArgumentList(Context, Builder, 
-                                                            /*TakeArgs=*/true));
-              return TDK_SubstitutionFailure;
-            }
+      if (NonTypeTemplateParmDecl *NTTP
+                                = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
+        NTTPType = NTTP->getType();
+        if (NTTPType->isDependentType()) {
+          TemplateArgumentList TemplateArgs(TemplateArgumentList::OnStack,
+                                            Builder.data(), Builder.size());
+          NTTPType = SubstType(NTTPType,
+                               MultiLevelTemplateArgumentList(TemplateArgs),
+                               NTTP->getLocation(),
+                               NTTP->getDeclName());
+          if (NTTPType.isNull()) {
+            Info.Param = makeTemplateParameter(Param);
+            // FIXME: These template arguments are temporary. Free them!
+            Info.reset(TemplateArgumentList::CreateCopy(Context,
+                                                        Builder.data(),
+                                                        Builder.size()));
+            return TDK_SubstitutionFailure;
           }
         }
       }
 
-      // Convert the deduced template argument into a template
-      // argument that we can check, almost as if the user had written
-      // the template argument explicitly.
-      TemplateArgumentLoc Arg = getTrivialTemplateArgumentLoc(*this,
-                                                              Deduced[I],
-                                                              NTTPType,
-                                                           SourceLocation());
-
-      // Check the template argument, converting it as necessary.
-      if (CheckTemplateArgument(Param, Arg,
-                                FunctionTemplate,
-                                FunctionTemplate->getLocation(),
-                                FunctionTemplate->getSourceRange().getEnd(),
-                                Builder,
-                                Deduced[I].wasDeducedFromArrayBound()
-                                  ? CTAK_DeducedFromArrayBound 
-                                  : CTAK_Deduced)) {
-        Info.Param = makeTemplateParameter(
-                         const_cast<NamedDecl *>(TemplateParams->getParam(I)));
-        Info.reset(new (Context) TemplateArgumentList(Context, Builder, 
-                                                      /*TakeArgs=*/true));
+      if (ConvertDeducedTemplateArgument(*this, Param, Deduced[I],
+                                         FunctionTemplate, NTTPType, 0, Info,
+                                         true, Builder)) {
+        Info.Param = makeTemplateParameter(Param);
+        // FIXME: These template arguments are temporary. Free them!
+        Info.reset(TemplateArgumentList::CreateCopy(Context, Builder.data(),
+                                                    Builder.size()));
         return TDK_SubstitutionFailure;
       }
 
       continue;
     }
 
-    // Substitute into the default template argument, if available. 
+    // C++0x [temp.arg.explicit]p3:
+    //    A trailing template parameter pack (14.5.3) not otherwise deduced will
+    //    be deduced to an empty sequence of template arguments.
+    // FIXME: Where did the word "trailing" come from?
+    if (Param->isTemplateParameterPack()) {
+      // We may have had explicitly-specified template arguments for this
+      // template parameter pack. If so, our empty deduction extends the
+      // explicitly-specified set (C++0x [temp.arg.explicit]p9).
+      const TemplateArgument *ExplicitArgs;
+      unsigned NumExplicitArgs;
+      if (CurrentInstantiationScope->getPartiallySubstitutedPack(&ExplicitArgs,
+                                                             &NumExplicitArgs)
+          == Param)
+        Builder.push_back(TemplateArgument(ExplicitArgs, NumExplicitArgs));
+      else
+        Builder.push_back(TemplateArgument(0, 0));
+
+      continue;
+    }
+
+    // Substitute into the default template argument, if available.
     TemplateArgumentLoc DefArg
       = SubstDefaultTemplateArgumentIfAvailable(FunctionTemplate,
                                               FunctionTemplate->getLocation(),
@@ -1463,18 +2255,19 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
                          const_cast<NamedDecl *>(TemplateParams->getParam(I)));
       return TDK_Incomplete;
     }
-    
+
     // Check whether we can actually use the default argument.
     if (CheckTemplateArgument(Param, DefArg,
                               FunctionTemplate,
                               FunctionTemplate->getLocation(),
                               FunctionTemplate->getSourceRange().getEnd(),
-                              Builder,
+                              0, Builder,
                               CTAK_Deduced)) {
       Info.Param = makeTemplateParameter(
                          const_cast<NamedDecl *>(TemplateParams->getParam(I)));
-      Info.reset(new (Context) TemplateArgumentList(Context, Builder, 
-                                                    /*TakeArgs=*/true));
+      // FIXME: These template arguments are temporary. Free them!
+      Info.reset(TemplateArgumentList::CreateCopy(Context, Builder.data(),
+                                                  Builder.size()));
       return TDK_SubstitutionFailure;
     }
 
@@ -1483,7 +2276,7 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
 
   // Form the template argument list from the deduced template arguments.
   TemplateArgumentList *DeducedArgumentList
-    = new (Context) TemplateArgumentList(Context, Builder, /*TakeArgs=*/true);
+    = TemplateArgumentList::CreateCopy(Context, Builder.data(), Builder.size());
   Info.reset(DeducedArgumentList);
 
   // Substitute the deduced template arguments into the function template
@@ -1497,9 +2290,9 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   if (!Specialization)
     return TDK_SubstitutionFailure;
 
-  assert(Specialization->getPrimaryTemplate()->getCanonicalDecl() == 
+  assert(Specialization->getPrimaryTemplate()->getCanonicalDecl() ==
          FunctionTemplate->getCanonicalDecl());
-  
+
   // If the template argument list is owned by the function template
   // specialization, release it.
   if (Specialization->getTemplateSpecializationArgs() == DeducedArgumentList &&
@@ -1514,6 +2307,18 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
     return TDK_SubstitutionFailure;
   }
 
+  // If we suppressed any diagnostics while performing template argument
+  // deduction, and if we haven't already instantiated this declaration,
+  // keep track of these diagnostics. They'll be emitted if this specialization
+  // is actually used.
+  if (Info.diag_begin() != Info.diag_end()) {
+    llvm::DenseMap<Decl *, llvm::SmallVector<PartialDiagnosticAt, 1> >::iterator
+      Pos = SuppressedDiagnostics.find(Specialization->getCanonicalDecl());
+    if (Pos == SuppressedDiagnostics.end())
+        SuppressedDiagnostics[Specialization->getCanonicalDecl()]
+          .append(Info.diag_begin(), Info.diag_end());
+  }
+
   return TDK_Success;
 }
 
@@ -1544,7 +2349,7 @@ static QualType
 ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
                             Expr *Arg, QualType ParamType,
                             bool ParamWasReference) {
-  
+
   OverloadExpr::FindResult R = OverloadExpr::find(Arg);
 
   OverloadExpr *Ovl = R.Expression;
@@ -1592,10 +2397,10 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     if (ArgType.isNull()) continue;
 
     // Function-to-pointer conversion.
-    if (!ParamWasReference && ParamType->isPointerType() && 
+    if (!ParamWasReference && ParamType->isPointerType() &&
         ArgType->isFunctionType())
       ArgType = S.Context.getPointerType(ArgType);
-    
+
     //   - If the argument is an overload set (not containing function
     //     templates), trial argument deduction is attempted using each
     //     of the members of the set. If deduction succeeds for only one
@@ -1608,7 +2413,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     //   Type deduction is done independently for each P/A pair, and
     //   the deduced template argument values are then combined.
     // So we do not reject deductions which were made elsewhere.
-    llvm::SmallVector<DeducedTemplateArgument, 8> 
+    llvm::SmallVector<DeducedTemplateArgument, 8>
       Deduced(TemplateParams->size());
     TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
     Sema::TemplateDeductionResult Result
@@ -1623,6 +2428,113 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
   return Match;
 }
 
+/// \brief Perform the adjustments to the parameter and argument types
+/// described in C++ [temp.deduct.call].
+///
+/// \returns true if the caller should not attempt to perform any template
+/// argument deduction based on this P/A pair.
+static bool AdjustFunctionParmAndArgTypesForDeduction(Sema &S,
+                                          TemplateParameterList *TemplateParams,
+                                                      QualType &ParamType,
+                                                      QualType &ArgType,
+                                                      Expr *Arg,
+                                                      unsigned &TDF) {
+  // C++0x [temp.deduct.call]p3:
+  //   If P is a cv-qualified type, the top level cv-qualifiers of P's type
+  //   are ignored for type deduction.
+  if (ParamType.getCVRQualifiers())
+    ParamType = ParamType.getLocalUnqualifiedType();
+  const ReferenceType *ParamRefType = ParamType->getAs<ReferenceType>();
+  if (ParamRefType) {
+    QualType PointeeType = ParamRefType->getPointeeType();
+
+    //   [C++0x] If P is an rvalue reference to a cv-unqualified
+    //   template parameter and the argument is an lvalue, the type
+    //   "lvalue reference to A" is used in place of A for type
+    //   deduction.
+    if (isa<RValueReferenceType>(ParamType)) {
+      if (!PointeeType.getQualifiers() &&
+          isa<TemplateTypeParmType>(PointeeType) &&
+          Arg->Classify(S.Context).isLValue())
+        ArgType = S.Context.getLValueReferenceType(ArgType);
+    }
+
+    //   [...] If P is a reference type, the type referred to by P is used
+    //   for type deduction.
+    ParamType = PointeeType;
+  }
+
+  // Overload sets usually make this parameter an undeduced
+  // context, but there are sometimes special circumstances.
+  if (ArgType == S.Context.OverloadTy) {
+    ArgType = ResolveOverloadForDeduction(S, TemplateParams,
+                                          Arg, ParamType,
+                                          ParamRefType != 0);
+    if (ArgType.isNull())
+      return true;
+  }
+
+  if (ParamRefType) {
+    // C++0x [temp.deduct.call]p3:
+    //   [...] If P is of the form T&&, where T is a template parameter, and
+    //   the argument is an lvalue, the type A& is used in place of A for
+    //   type deduction.
+    if (ParamRefType->isRValueReferenceType() &&
+        ParamRefType->getAs<TemplateTypeParmType>() &&
+        Arg->isLValue())
+      ArgType = S.Context.getLValueReferenceType(ArgType);
+  } else {
+    // C++ [temp.deduct.call]p2:
+    //   If P is not a reference type:
+    //   - If A is an array type, the pointer type produced by the
+    //     array-to-pointer standard conversion (4.2) is used in place of
+    //     A for type deduction; otherwise,
+    if (ArgType->isArrayType())
+      ArgType = S.Context.getArrayDecayedType(ArgType);
+    //   - If A is a function type, the pointer type produced by the
+    //     function-to-pointer standard conversion (4.3) is used in place
+    //     of A for type deduction; otherwise,
+    else if (ArgType->isFunctionType())
+      ArgType = S.Context.getPointerType(ArgType);
+    else {
+      // - If A is a cv-qualified type, the top level cv-qualifiers of A's
+      //   type are ignored for type deduction.
+      if (ArgType.getCVRQualifiers())
+        ArgType = ArgType.getUnqualifiedType();
+    }
+  }
+
+  // C++0x [temp.deduct.call]p4:
+  //   In general, the deduction process attempts to find template argument
+  //   values that will make the deduced A identical to A (after the type A
+  //   is transformed as described above). [...]
+  TDF = TDF_SkipNonDependent;
+
+  //     - If the original P is a reference type, the deduced A (i.e., the
+  //       type referred to by the reference) can be more cv-qualified than
+  //       the transformed A.
+  if (ParamRefType)
+    TDF |= TDF_ParamWithReferenceType;
+  //     - The transformed A can be another pointer or pointer to member
+  //       type that can be converted to the deduced A via a qualification
+  //       conversion (4.4).
+  if (ArgType->isPointerType() || ArgType->isMemberPointerType() ||
+      ArgType->isObjCObjectPointerType())
+    TDF |= TDF_IgnoreQualifiers;
+  //     - If P is a class and P has the form simple-template-id, then the
+  //       transformed A can be a derived class of the deduced A. Likewise,
+  //       if P is a pointer to a class of the form simple-template-id, the
+  //       transformed A can be a pointer to a derived class pointed to by
+  //       the deduced A.
+  if (isSimpleTemplateIdType(ParamType) ||
+      (isa<PointerType>(ParamType) &&
+       isSimpleTemplateIdType(
+                              ParamType->getAs<PointerType>()->getPointeeType())))
+    TDF |= TDF_DerivedClass;
+
+  return false;
+}
+
 /// \brief Perform template argument deduction from a function call
 /// (C++ [temp.deduct.call]).
 ///
@@ -1639,7 +2551,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
 /// \param Name the name of the function being called. This is only significant
 /// when the function template is a conversion function template, in which
 /// case this routine will also perform template argument deduction based on
-/// the function to which 
+/// the function to which
 ///
 /// \param Specialization if template argument deduction was successful,
 /// this will be set to the function template specialization produced by
@@ -1667,10 +2579,12 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   else if (NumArgs > Function->getNumParams()) {
     const FunctionProtoType *Proto
       = Function->getType()->getAs<FunctionProtoType>();
-    if (!Proto->isVariadic())
+    if (Proto->isTemplateVariadic())
+      /* Do nothing */;
+    else if (Proto->isVariadic())
+      CheckArgs = Function->getNumParams();
+    else
       return TDK_TooManyArguments;
-
-    CheckArgs = Function->getNumParams();
   }
 
   // The types of the parameters from which we will perform template argument
@@ -1695,105 +2609,126 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     NumExplicitlySpecified = Deduced.size();
   } else {
     // Just fill in the parameter types from the function declaration.
-    for (unsigned I = 0; I != CheckArgs; ++I)
+    for (unsigned I = 0, N = Function->getNumParams(); I != N; ++I)
       ParamTypes.push_back(Function->getParamDecl(I)->getType());
   }
 
   // Deduce template arguments from the function parameters.
   Deduced.resize(TemplateParams->size());
-  for (unsigned I = 0; I != CheckArgs; ++I) {
-    QualType ParamType = ParamTypes[I];
-    QualType ArgType = Args[I]->getType();
+  unsigned ArgIdx = 0;
+  for (unsigned ParamIdx = 0, NumParams = ParamTypes.size();
+       ParamIdx != NumParams; ++ParamIdx) {
+    QualType ParamType = ParamTypes[ParamIdx];
+
+    const PackExpansionType *ParamExpansion
+      = dyn_cast<PackExpansionType>(ParamType);
+    if (!ParamExpansion) {
+      // Simple case: matching a function parameter to a function argument.
+      if (ArgIdx >= CheckArgs)
+        break;
 
-    // C++0x [temp.deduct.call]p3:
-    //   If P is a cv-qualified type, the top level cv-qualifiers of P’s type
-    //   are ignored for type deduction.
-    if (ParamType.getCVRQualifiers())
-      ParamType = ParamType.getLocalUnqualifiedType();
-    const ReferenceType *ParamRefType = ParamType->getAs<ReferenceType>();
-    if (ParamRefType) {
-      //   [...] If P is a reference type, the type referred to by P is used
-      //   for type deduction.
-      ParamType = ParamRefType->getPointeeType();
-    }
-    
-    // Overload sets usually make this parameter an undeduced
-    // context, but there are sometimes special circumstances.
-    if (ArgType == Context.OverloadTy) {
-      ArgType = ResolveOverloadForDeduction(*this, TemplateParams,
-                                            Args[I], ParamType,
-                                            ParamRefType != 0);
-      if (ArgType.isNull())
+      Expr *Arg = Args[ArgIdx++];
+      QualType ArgType = Arg->getType();
+      unsigned TDF = 0;
+      if (AdjustFunctionParmAndArgTypesForDeduction(*this, TemplateParams,
+                                                    ParamType, ArgType, Arg,
+                                                    TDF))
         continue;
+
+      if (TemplateDeductionResult Result
+          = ::DeduceTemplateArguments(*this, TemplateParams,
+                                      ParamType, ArgType, Info, Deduced,
+                                      TDF))
+        return Result;
+
+      // FIXME: we need to check that the deduced A is the same as A,
+      // modulo the various allowed differences.
+      continue;
     }
 
-    if (ParamRefType) {
-      // C++0x [temp.deduct.call]p3:
-      //   [...] If P is of the form T&&, where T is a template parameter, and
-      //   the argument is an lvalue, the type A& is used in place of A for
-      //   type deduction.
-      if (ParamRefType->isRValueReferenceType() &&
-          ParamRefType->getAs<TemplateTypeParmType>() &&
-          Args[I]->isLvalue(Context) == Expr::LV_Valid)
-        ArgType = Context.getLValueReferenceType(ArgType);
-    } else {
-      // C++ [temp.deduct.call]p2:
-      //   If P is not a reference type:
-      //   - If A is an array type, the pointer type produced by the
-      //     array-to-pointer standard conversion (4.2) is used in place of
-      //     A for type deduction; otherwise,
-      if (ArgType->isArrayType())
-        ArgType = Context.getArrayDecayedType(ArgType);
-      //   - If A is a function type, the pointer type produced by the
-      //     function-to-pointer standard conversion (4.3) is used in place
-      //     of A for type deduction; otherwise,
-      else if (ArgType->isFunctionType())
-        ArgType = Context.getPointerType(ArgType);
-      else {
-        // - If A is a cv-qualified type, the top level cv-qualifiers of A’s
-        //   type are ignored for type deduction.
-        QualType CanonArgType = Context.getCanonicalType(ArgType);
-        if (ArgType.getCVRQualifiers())
-          ArgType = ArgType.getUnqualifiedType();
+    // C++0x [temp.deduct.call]p1:
+    //   For a function parameter pack that occurs at the end of the
+    //   parameter-declaration-list, the type A of each remaining argument of
+    //   the call is compared with the type P of the declarator-id of the
+    //   function parameter pack. Each comparison deduces template arguments
+    //   for subsequent positions in the template parameter packs expanded by
+    //   the function parameter pack. For a function parameter pack that does
+    //   not occur at the end of the parameter-declaration-list, the type of
+    //   the parameter pack is a non-deduced context.
+    if (ParamIdx + 1 < NumParams)
+      break;
+
+    QualType ParamPattern = ParamExpansion->getPattern();
+    llvm::SmallVector<unsigned, 2> PackIndices;
+    {
+      llvm::BitVector SawIndices(TemplateParams->size());
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      collectUnexpandedParameterPacks(ParamPattern, Unexpanded);
+      for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
+        unsigned Depth, Index;
+        llvm::tie(Depth, Index) = getDepthAndIndex(Unexpanded[I]);
+        if (Depth == 0 && !SawIndices[Index]) {
+          SawIndices[Index] = true;
+          PackIndices.push_back(Index);
+        }
       }
     }
+    assert(!PackIndices.empty() && "Pack expansion without unexpanded packs?");
+
+    // Keep track of the deduced template arguments for each parameter pack
+    // expanded by this pack expansion (the outer index) and for each
+    // template argument (the inner SmallVectors).
+    llvm::SmallVector<llvm::SmallVector<DeducedTemplateArgument, 4>, 2>
+      NewlyDeducedPacks(PackIndices.size());
+    llvm::SmallVector<DeducedTemplateArgument, 2>
+      SavedPacks(PackIndices.size());
+    PrepareArgumentPackDeduction(*this, Deduced, PackIndices, SavedPacks,
+                                 NewlyDeducedPacks);
+    bool HasAnyArguments = false;
+    for (; ArgIdx < NumArgs; ++ArgIdx) {
+      HasAnyArguments = true;
+
+      ParamType = ParamPattern;
+      Expr *Arg = Args[ArgIdx];
+      QualType ArgType = Arg->getType();
+      unsigned TDF = 0;
+      if (AdjustFunctionParmAndArgTypesForDeduction(*this, TemplateParams,
+                                                    ParamType, ArgType, Arg,
+                                                    TDF)) {
+        // We can't actually perform any deduction for this argument, so stop
+        // deduction at this point.
+        ++ArgIdx;
+        break;
+      }
 
-    // C++0x [temp.deduct.call]p4:
-    //   In general, the deduction process attempts to find template argument
-    //   values that will make the deduced A identical to A (after the type A
-    //   is transformed as described above). [...]
-    unsigned TDF = TDF_SkipNonDependent;
-
-    //     - If the original P is a reference type, the deduced A (i.e., the
-    //       type referred to by the reference) can be more cv-qualified than
-    //       the transformed A.
-    if (ParamRefType)
-      TDF |= TDF_ParamWithReferenceType;
-    //     - The transformed A can be another pointer or pointer to member
-    //       type that can be converted to the deduced A via a qualification
-    //       conversion (4.4).
-    if (ArgType->isPointerType() || ArgType->isMemberPointerType() ||
-        ArgType->isObjCObjectPointerType())
-      TDF |= TDF_IgnoreQualifiers;
-    //     - If P is a class and P has the form simple-template-id, then the
-    //       transformed A can be a derived class of the deduced A. Likewise,
-    //       if P is a pointer to a class of the form simple-template-id, the
-    //       transformed A can be a pointer to a derived class pointed to by
-    //       the deduced A.
-    if (isSimpleTemplateIdType(ParamType) ||
-        (isa<PointerType>(ParamType) &&
-         isSimpleTemplateIdType(
-                              ParamType->getAs<PointerType>()->getPointeeType())))
-      TDF |= TDF_DerivedClass;
+      if (TemplateDeductionResult Result
+          = ::DeduceTemplateArguments(*this, TemplateParams,
+                                      ParamType, ArgType, Info, Deduced,
+                                      TDF))
+        return Result;
 
-    if (TemplateDeductionResult Result
-        = ::DeduceTemplateArguments(*this, TemplateParams,
-                                    ParamType, ArgType, Info, Deduced,
-                                    TDF))
+      // Capture the deduced template arguments for each parameter pack expanded
+      // by this pack expansion, add them to the list of arguments we've deduced
+      // for that pack, then clear out the deduced argument.
+      for (unsigned I = 0, N = PackIndices.size(); I != N; ++I) {
+        DeducedTemplateArgument &DeducedArg = Deduced[PackIndices[I]];
+        if (!DeducedArg.isNull()) {
+          NewlyDeducedPacks[I].push_back(DeducedArg);
+          DeducedArg = DeducedTemplateArgument();
+        }
+      }
+    }
+
+    // Build argument packs for each of the parameter packs expanded by this
+    // pack expansion.
+    if (Sema::TemplateDeductionResult Result
+          = FinishArgumentPackDeduction(*this, TemplateParams, HasAnyArguments,
+                                        Deduced, PackIndices, SavedPacks,
+                                        NewlyDeducedPacks, Info))
       return Result;
 
-    // FIXME: we need to check that the deduced A is the same as A,
-    // modulo the various allowed differences.
+    // After we've matching against a parameter pack, we're done.
+    break;
   }
 
   return FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
@@ -1808,7 +2743,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
 /// \param FunctionTemplate the function template for which we are performing
 /// template argument deduction.
 ///
-/// \param ExplicitTemplateArguments the explicitly-specified template 
+/// \param ExplicitTemplateArguments the explicitly-specified template
 /// arguments.
 ///
 /// \param ArgFunctionType the function type that will be used as the
@@ -1862,13 +2797,23 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     if (TemplateDeductionResult Result
           = ::DeduceTemplateArguments(*this, TemplateParams,
                                       FunctionType, ArgFunctionType, Info,
-                                      Deduced, 0))
+                                      Deduced, TDF_TopLevelParameterTypeList))
       return Result;
   }
-  
-  return FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
-                                         NumExplicitlySpecified,
-                                         Specialization, Info);
+
+  if (TemplateDeductionResult Result
+        = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced,
+                                          NumExplicitlySpecified,
+                                          Specialization, Info))
+    return Result;
+
+  // If the requested function type does not match the actual type of the
+  // specialization, template argument deduction fails.
+  if (!ArgFunctionType.isNull() &&
+      !Context.hasSameType(ArgFunctionType, Specialization->getType()))
+    return TDK_NonDeducedMismatch;
+
+  return TDK_Success;
 }
 
 /// \brief Deduce template arguments for a templated conversion
@@ -1915,12 +2860,12 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     else if (P->isFunctionType())
       P = Context.getPointerType(P);
     //   - If P is a cv-qualified type, the top level cv-qualifiers of
-    //     P’s type are ignored for type deduction.
+    //     P's type are ignored for type deduction.
     else
       P = P.getUnqualifiedType();
 
     // C++0x [temp.deduct.conv]p3:
-    //   If A is a cv-qualified type, the top level cv-qualifiers of A’s
+    //   If A is a cv-qualified type, the top level cv-qualifiers of A's
     //   type are ignored for type deduction.
     A = A.getUnqualifiedType();
   }
@@ -1950,7 +2895,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   if (ToType->isReferenceType())
     TDF |= TDF_ParamWithReferenceType;
   //     - The deduced A can be another pointer or pointer to member
-  //       type that can be converted to A via a qualification
+  //       type that can be converted to A via a qualification
   //       conversion.
   //
   // (C++0x [temp.deduct.conv]p6 clarifies that this only happens when
@@ -1971,7 +2916,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   LocalInstantiationScope InstScope(*this);
   FunctionDecl *Spec = 0;
   TemplateDeductionResult Result
-    = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced, 0, Spec, 
+    = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced, 0, Spec,
                                       Info);
   Specialization = cast_or_null<CXXConversionDecl>(Spec);
   return Result;
@@ -1983,7 +2928,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
 /// \param FunctionTemplate the function template for which we are performing
 /// template argument deduction.
 ///
-/// \param ExplicitTemplateArguments the explicitly-specified template 
+/// \param ExplicitTemplateArguments the explicitly-specified template
 /// arguments.
 ///
 /// \param Specialization if template argument deduction was successful,
@@ -2003,88 +2948,93 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                                  QualType(), Specialization, Info);
 }
 
-/// \brief Stores the result of comparing the qualifiers of two types.
-enum DeductionQualifierComparison { 
-  NeitherMoreQualified = 0, 
-  ParamMoreQualified, 
-  ArgMoreQualified 
-};
+namespace {
+  /// Substitute the 'auto' type specifier within a type for a given replacement
+  /// type.
+  class SubstituteAutoTransform :
+    public TreeTransform<SubstituteAutoTransform> {
+    QualType Replacement;
+  public:
+    SubstituteAutoTransform(Sema &SemaRef, QualType Replacement) :
+      TreeTransform<SubstituteAutoTransform>(SemaRef), Replacement(Replacement) {
+    }
+    QualType TransformAutoType(TypeLocBuilder &TLB, AutoTypeLoc TL) {
+      // If we're building the type pattern to deduce against, don't wrap the
+      // substituted type in an AutoType. Certain template deduction rules
+      // apply only when a template type parameter appears directly (and not if
+      // the parameter is found through desugaring). For instance:
+      //   auto &&lref = lvalue;
+      // must transform into "rvalue reference to T" not "rvalue reference to
+      // auto type deduced as T" in order for [temp.deduct.call]p3 to apply.
+      if (isa<TemplateTypeParmType>(Replacement)) {
+        QualType Result = Replacement;
+        TemplateTypeParmTypeLoc NewTL = TLB.push<TemplateTypeParmTypeLoc>(Result);
+        NewTL.setNameLoc(TL.getNameLoc());
+        return Result;
+      } else {
+        QualType Result = RebuildAutoType(Replacement);
+        AutoTypeLoc NewTL = TLB.push<AutoTypeLoc>(Result);
+        NewTL.setNameLoc(TL.getNameLoc());
+        return Result;
+      }
+    }
+  };
+}
 
-/// \brief Deduce the template arguments during partial ordering by comparing 
-/// the parameter type and the argument type (C++0x [temp.deduct.partial]).
-///
-/// \param S the semantic analysis object within which we are deducing
+/// \brief Deduce the type for an auto type-specifier (C++0x [dcl.spec.auto]p6)
 ///
-/// \param TemplateParams the template parameters that we are deducing
-///
-/// \param ParamIn the parameter type
-///
-/// \param ArgIn the argument type
+/// \param Type the type pattern using the auto type-specifier.
 ///
-/// \param Info information about the template argument deduction itself
+/// \param Init the initializer for the variable whose type is to be deduced.
 ///
-/// \param Deduced the deduced template arguments
+/// \param Result if type deduction was successful, this will be set to the
+/// deduced type. This may still contain undeduced autos if the type is
+/// dependent.
 ///
-/// \returns the result of template argument deduction so far. Note that a
-/// "success" result means that template argument deduction has not yet failed,
-/// but it may still fail, later, for other reasons.
-static Sema::TemplateDeductionResult
-DeduceTemplateArgumentsDuringPartialOrdering(Sema &S,
-                                        TemplateParameterList *TemplateParams,
-                                             QualType ParamIn, QualType ArgIn,
-                                             TemplateDeductionInfo &Info,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-   llvm::SmallVectorImpl<DeductionQualifierComparison> *QualifierComparisons) {
-  CanQualType Param = S.Context.getCanonicalType(ParamIn);
-  CanQualType Arg = S.Context.getCanonicalType(ArgIn);
-
-  // C++0x [temp.deduct.partial]p5:
-  //   Before the partial ordering is done, certain transformations are 
-  //   performed on the types used for partial ordering: 
-  //     - If P is a reference type, P is replaced by the type referred to. 
-  CanQual<ReferenceType> ParamRef = Param->getAs<ReferenceType>();
-  if (!ParamRef.isNull())
-    Param = ParamRef->getPointeeType();
-  
-  //     - If A is a reference type, A is replaced by the type referred to.
-  CanQual<ReferenceType> ArgRef = Arg->getAs<ReferenceType>();
-  if (!ArgRef.isNull())
-    Arg = ArgRef->getPointeeType();
-  
-  if (QualifierComparisons && !ParamRef.isNull() && !ArgRef.isNull()) {
-    // C++0x [temp.deduct.partial]p6:
-    //   If both P and A were reference types (before being replaced with the 
-    //   type referred to above), determine which of the two types (if any) is 
-    //   more cv-qualified than the other; otherwise the types are considered to 
-    //   be equally cv-qualified for partial ordering purposes. The result of this
-    //   determination will be used below.
-    //
-    // We save this information for later, using it only when deduction 
-    // succeeds in both directions.
-    DeductionQualifierComparison QualifierResult = NeitherMoreQualified;
-    if (Param.isMoreQualifiedThan(Arg))
-      QualifierResult = ParamMoreQualified;
-    else if (Arg.isMoreQualifiedThan(Param))
-      QualifierResult = ArgMoreQualified;
-    QualifierComparisons->push_back(QualifierResult);
-  }
-  
-  // C++0x [temp.deduct.partial]p7:
-  //   Remove any top-level cv-qualifiers:
-  //     - If P is a cv-qualified type, P is replaced by the cv-unqualified 
-  //       version of P.
-  Param = Param.getUnqualifiedType();
-  //     - If A is a cv-qualified type, A is replaced by the cv-unqualified 
-  //       version of A.
-  Arg = Arg.getUnqualifiedType();
-  
-  // C++0x [temp.deduct.partial]p8:
-  //   Using the resulting types P and A the deduction is then done as 
-  //   described in 14.9.2.5. If deduction succeeds for a given type, the type
-  //   from the argument template is considered to be at least as specialized
-  //   as the type from the parameter template.
-  return DeduceTemplateArguments(S, TemplateParams, Param, Arg, Info,
-                                 Deduced, TDF_None);
+/// \returns true if deduction succeeded, false if it failed.
+bool
+Sema::DeduceAutoType(QualType Type, Expr *Init, QualType &Result) {
+  if (Init->isTypeDependent()) {
+    Result = Type;
+    return true;
+  }
+
+  SourceLocation Loc = Init->getExprLoc();
+
+  LocalInstantiationScope InstScope(*this);
+
+  // Build template<class TemplParam> void Func(FuncParam);
+  NamedDecl *TemplParam
+    = TemplateTypeParmDecl::Create(Context, 0, Loc, 0, 0, 0, false, false);
+  TemplateParameterList *TemplateParams
+    = TemplateParameterList::Create(Context, Loc, Loc, &TemplParam, 1, Loc);
+
+  QualType TemplArg = Context.getTemplateTypeParmType(0, 0, false);
+  QualType FuncParam =
+    SubstituteAutoTransform(*this, TemplArg).TransformType(Type);
+
+  // Deduce type of TemplParam in Func(Init)
+  llvm::SmallVector<DeducedTemplateArgument, 1> Deduced;
+  Deduced.resize(1);
+  QualType InitType = Init->getType();
+  unsigned TDF = 0;
+  if (AdjustFunctionParmAndArgTypesForDeduction(*this, TemplateParams,
+                                                FuncParam, InitType, Init,
+                                                TDF))
+    return false;
+
+  TemplateDeductionInfo Info(Context, Loc);
+  if (::DeduceTemplateArguments(*this, TemplateParams,
+                                FuncParam, InitType, Info, Deduced,
+                                TDF))
+    return false;
+
+  QualType DeducedType = Deduced[0].getAsType();
+  if (DeducedType.isNull())
+    return false;
+
+  Result = SubstituteAutoTransform(*this, DeducedType).TransformType(Type);
+  return true;
 }
 
 static void
@@ -2092,7 +3042,31 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                            bool OnlyDeduced,
                            unsigned Level,
                            llvm::SmallVectorImpl<bool> &Deduced);
-  
+
+/// \brief If this is a non-static member function,
+static void MaybeAddImplicitObjectParameterType(ASTContext &Context,
+                                                CXXMethodDecl *Method,
+                                 llvm::SmallVectorImpl<QualType> &ArgTypes) {
+  if (Method->isStatic())
+    return;
+
+  // C++ [over.match.funcs]p4:
+  //
+  //   For non-static member functions, the type of the implicit
+  //   object parameter is
+  //     - "lvalue reference to cv X" for functions declared without a
+  //       ref-qualifier or with the & ref-qualifier
+  //     - "rvalue reference to cv X" for functions declared with the
+  //       && ref-qualifier
+  //
+  // FIXME: We don't have ref-qualifiers yet, so we don't do that part.
+  QualType ArgTy = Context.getTypeDeclType(Method->getParent());
+  ArgTy = Context.getQualifiedType(ArgTy,
+                        Qualifiers::fromCVRMask(Method->getTypeQualifiers()));
+  ArgTy = Context.getLValueReferenceType(ArgTy);
+  ArgTypes.push_back(ArgTy);
+}
+
 /// \brief Determine whether the function template \p FT1 is at least as
 /// specialized as \p FT2.
 static bool isAtLeastAsSpecializedAs(Sema &S,
@@ -2100,12 +3074,13 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
                                      FunctionTemplateDecl *FT1,
                                      FunctionTemplateDecl *FT2,
                                      TemplatePartialOrderingContext TPOC,
-    llvm::SmallVectorImpl<DeductionQualifierComparison> *QualifierComparisons) {
+                                     unsigned NumCallArguments,
+    llvm::SmallVectorImpl<RefParamPartialOrderingComparison> *RefParamComparisons) {
   FunctionDecl *FD1 = FT1->getTemplatedDecl();
-  FunctionDecl *FD2 = FT2->getTemplatedDecl();  
+  FunctionDecl *FD2 = FT2->getTemplatedDecl();
   const FunctionProtoType *Proto1 = FD1->getType()->getAs<FunctionProtoType>();
   const FunctionProtoType *Proto2 = FD2->getType()->getAs<FunctionProtoType>();
-  
+
   assert(Proto1 && Proto2 && "Function templates must have prototypes");
   TemplateParameterList *TemplateParams = FT2->getTemplateParameters();
   llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
@@ -2115,55 +3090,89 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
   //   The types used to determine the ordering depend on the context in which
   //   the partial ordering is done:
   TemplateDeductionInfo Info(S.Context, Loc);
+  CXXMethodDecl *Method1 = 0;
+  CXXMethodDecl *Method2 = 0;
+  bool IsNonStatic2 = false;
+  bool IsNonStatic1 = false;
+  unsigned Skip2 = 0;
   switch (TPOC) {
   case TPOC_Call: {
     //   - In the context of a function call, the function parameter types are
     //     used.
-    unsigned NumParams = std::min(Proto1->getNumArgs(), Proto2->getNumArgs());
-    for (unsigned I = 0; I != NumParams; ++I)
-      if (DeduceTemplateArgumentsDuringPartialOrdering(S,
-                                                       TemplateParams,
-                                                       Proto2->getArgType(I),
-                                                       Proto1->getArgType(I),
-                                                       Info,
-                                                       Deduced,
-                                                       QualifierComparisons))
+    Method1 = dyn_cast<CXXMethodDecl>(FD1);
+    Method2 = dyn_cast<CXXMethodDecl>(FD2);
+    IsNonStatic1 = Method1 && !Method1->isStatic();
+    IsNonStatic2 = Method2 && !Method2->isStatic();
+
+    // C++0x [temp.func.order]p3:
+    //   [...] If only one of the function templates is a non-static
+    //   member, that function template is considered to have a new
+    //   first parameter inserted in its function parameter list. The
+    //   new parameter is of type "reference to cv A," where cv are
+    //   the cv-qualifiers of the function template (if any) and A is
+    //   the class of which the function template is a member.
+    //
+    // C++98/03 doesn't have this provision, so instead we drop the
+    // first argument of the free function or static member, which
+    // seems to match existing practice.
+    llvm::SmallVector<QualType, 4> Args1;
+    unsigned Skip1 = !S.getLangOptions().CPlusPlus0x &&
+      IsNonStatic2 && !IsNonStatic1;
+    if (S.getLangOptions().CPlusPlus0x && IsNonStatic1 && !IsNonStatic2)
+      MaybeAddImplicitObjectParameterType(S.Context, Method1, Args1);
+    Args1.insert(Args1.end(),
+                 Proto1->arg_type_begin() + Skip1, Proto1->arg_type_end());
+
+    llvm::SmallVector<QualType, 4> Args2;
+    Skip2 = !S.getLangOptions().CPlusPlus0x &&
+      IsNonStatic1 && !IsNonStatic2;
+    if (S.getLangOptions().CPlusPlus0x && IsNonStatic2 && !IsNonStatic1)
+      MaybeAddImplicitObjectParameterType(S.Context, Method2, Args2);
+    Args2.insert(Args2.end(),
+                 Proto2->arg_type_begin() + Skip2, Proto2->arg_type_end());
+
+    // C++ [temp.func.order]p5:
+    //   The presence of unused ellipsis and default arguments has no effect on
+    //   the partial ordering of function templates.
+    if (Args1.size() > NumCallArguments)
+      Args1.resize(NumCallArguments);
+    if (Args2.size() > NumCallArguments)
+      Args2.resize(NumCallArguments);
+    if (DeduceTemplateArguments(S, TemplateParams, Args2.data(), Args2.size(),
+                                Args1.data(), Args1.size(), Info, Deduced,
+                                TDF_None, /*PartialOrdering=*/true,
+                                RefParamComparisons))
         return false;
-    
+
     break;
   }
-    
+
   case TPOC_Conversion:
     //   - In the context of a call to a conversion operator, the return types
     //     of the conversion function templates are used.
-    if (DeduceTemplateArgumentsDuringPartialOrdering(S,
-                                                     TemplateParams,
-                                                     Proto2->getResultType(),
-                                                     Proto1->getResultType(),
-                                                     Info,
-                                                     Deduced,
-                                                     QualifierComparisons))
+    if (DeduceTemplateArguments(S, TemplateParams, Proto2->getResultType(),
+                                Proto1->getResultType(), Info, Deduced,
+                                TDF_None, /*PartialOrdering=*/true,
+                                RefParamComparisons))
       return false;
     break;
-    
+
   case TPOC_Other:
-    //   - In other contexts (14.6.6.2) the function template’s function type 
+    //   - In other contexts (14.6.6.2) the function template's function type
     //     is used.
-    if (DeduceTemplateArgumentsDuringPartialOrdering(S,
-                                                     TemplateParams,
-                                                     FD2->getType(),
-                                                     FD1->getType(),
-                                                     Info,
-                                                     Deduced,
-                                                     QualifierComparisons))
+    // FIXME: Don't we actually want to perform the adjustments on the parameter
+    // types?
+    if (DeduceTemplateArguments(S, TemplateParams, FD2->getType(),
+                                FD1->getType(), Info, Deduced, TDF_None,
+                                /*PartialOrdering=*/true, RefParamComparisons))
       return false;
     break;
   }
-  
+
   // C++0x [temp.deduct.partial]p11:
-  //   In most cases, all template parameters must have values in order for 
-  //   deduction to succeed, but for partial ordering purposes a template 
-  //   parameter may remain without a value provided it is not used in the 
+  //   In most cases, all template parameters must have values in order for
+  //   deduction to succeed, but for partial ordering purposes a template
+  //   parameter may remain without a value provided it is not used in the
   //   types being used for partial ordering. [ Note: a template parameter used
   //   in a non-deduced context is considered used. -end note]
   unsigned ArgIdx = 0, NumArgs = Deduced.size();
@@ -2172,7 +3181,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
       break;
 
   if (ArgIdx == NumArgs) {
-    // All template arguments were deduced. FT1 is at least as specialized 
+    // All template arguments were deduced. FT1 is at least as specialized
     // as FT2.
     return true;
   }
@@ -2182,37 +3191,62 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
   UsedParameters.resize(TemplateParams->size());
   switch (TPOC) {
   case TPOC_Call: {
-    unsigned NumParams = std::min(Proto1->getNumArgs(), Proto2->getNumArgs());
-    for (unsigned I = 0; I != NumParams; ++I)
-      ::MarkUsedTemplateParameters(S, Proto2->getArgType(I), false, 
+    unsigned NumParams = std::min(NumCallArguments,
+                                  std::min(Proto1->getNumArgs(),
+                                           Proto2->getNumArgs()));
+    if (S.getLangOptions().CPlusPlus0x && IsNonStatic2 && !IsNonStatic1)
+      ::MarkUsedTemplateParameters(S, Method2->getThisType(S.Context), false,
+                                   TemplateParams->getDepth(), UsedParameters);
+    for (unsigned I = Skip2; I < NumParams; ++I)
+      ::MarkUsedTemplateParameters(S, Proto2->getArgType(I), false,
                                    TemplateParams->getDepth(),
                                    UsedParameters);
     break;
   }
-    
+
   case TPOC_Conversion:
-    ::MarkUsedTemplateParameters(S, Proto2->getResultType(), false, 
+    ::MarkUsedTemplateParameters(S, Proto2->getResultType(), false,
                                  TemplateParams->getDepth(),
                                  UsedParameters);
     break;
-    
+
   case TPOC_Other:
-    ::MarkUsedTemplateParameters(S, FD2->getType(), false, 
+    ::MarkUsedTemplateParameters(S, FD2->getType(), false,
                                  TemplateParams->getDepth(),
                                  UsedParameters);
     break;
   }
-  
+
   for (; ArgIdx != NumArgs; ++ArgIdx)
     // If this argument had no value deduced but was used in one of the types
     // used for partial ordering, then deduction fails.
     if (Deduced[ArgIdx].isNull() && UsedParameters[ArgIdx])
       return false;
-  
+
+  return true;
+}
+
+/// \brief Determine whether this a function template whose parameter-type-list
+/// ends with a function parameter pack.
+static bool isVariadicFunctionTemplate(FunctionTemplateDecl *FunTmpl) {
+  FunctionDecl *Function = FunTmpl->getTemplatedDecl();
+  unsigned NumParams = Function->getNumParams();
+  if (NumParams == 0)
+    return false;
+
+  ParmVarDecl *Last = Function->getParamDecl(NumParams - 1);
+  if (!Last->isParameterPack())
+    return false;
+
+  // Make sure that no previous parameter is a parameter pack.
+  while (--NumParams > 0) {
+    if (Function->getParamDecl(NumParams - 1)->isParameterPack())
+      return false;
+  }
+
   return true;
 }
-                                    
-                                     
+
 /// \brief Returns the more specialized function template according
 /// to the rules of function template partial ordering (C++ [temp.func.order]).
 ///
@@ -2223,77 +3257,113 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
 /// \param TPOC the context in which we are performing partial ordering of
 /// function templates.
 ///
+/// \param NumCallArguments The number of arguments in a call, used only
+/// when \c TPOC is \c TPOC_Call.
+///
 /// \returns the more specialized function template. If neither
 /// template is more specialized, returns NULL.
 FunctionTemplateDecl *
 Sema::getMoreSpecializedTemplate(FunctionTemplateDecl *FT1,
                                  FunctionTemplateDecl *FT2,
                                  SourceLocation Loc,
-                                 TemplatePartialOrderingContext TPOC) {
-  llvm::SmallVector<DeductionQualifierComparison, 4> QualifierComparisons;
-  bool Better1 = isAtLeastAsSpecializedAs(*this, Loc, FT1, FT2, TPOC, 0);
-  bool Better2 = isAtLeastAsSpecializedAs(*this, Loc, FT2, FT1, TPOC, 
-                                          &QualifierComparisons);
-  
+                                 TemplatePartialOrderingContext TPOC,
+                                 unsigned NumCallArguments) {
+  llvm::SmallVector<RefParamPartialOrderingComparison, 4> RefParamComparisons;
+  bool Better1 = isAtLeastAsSpecializedAs(*this, Loc, FT1, FT2, TPOC,
+                                          NumCallArguments, 0);
+  bool Better2 = isAtLeastAsSpecializedAs(*this, Loc, FT2, FT1, TPOC,
+                                          NumCallArguments,
+                                          &RefParamComparisons);
+
   if (Better1 != Better2) // We have a clear winner
     return Better1? FT1 : FT2;
-  
+
   if (!Better1 && !Better2) // Neither is better than the other
     return 0;
 
-
   // C++0x [temp.deduct.partial]p10:
-  //   If for each type being considered a given template is at least as 
+  //   If for each type being considered a given template is at least as
   //   specialized for all types and more specialized for some set of types and
-  //   the other template is not more specialized for any types or is not at 
+  //   the other template is not more specialized for any types or is not at
   //   least as specialized for any types, then the given template is more
   //   specialized than the other template. Otherwise, neither template is more
   //   specialized than the other.
   Better1 = false;
   Better2 = false;
-  for (unsigned I = 0, N = QualifierComparisons.size(); I != N; ++I) {
+  for (unsigned I = 0, N = RefParamComparisons.size(); I != N; ++I) {
     // C++0x [temp.deduct.partial]p9:
     //   If, for a given type, deduction succeeds in both directions (i.e., the
-    //   types are identical after the transformations above) and if the type
-    //   from the argument template is more cv-qualified than the type from the
-    //   parameter template (as described above) that type is considered to be
-    //   more specialized than the other. If neither type is more cv-qualified 
-    //   than the other then neither type is more specialized than the other.
-    switch (QualifierComparisons[I]) {
-      case NeitherMoreQualified:
-        break;
-        
-      case ParamMoreQualified:
-        Better1 = true;
-        if (Better2)
-          return 0;
-        break;
-        
-      case ArgMoreQualified:
-        Better2 = true;
-        if (Better1)
-          return 0;
-        break;
+    //   types are identical after the transformations above) and both P and A
+    //   were reference types (before being replaced with the type referred to
+    //   above):
+
+    //     -- if the type from the argument template was an lvalue reference
+    //        and the type from the parameter template was not, the argument
+    //        type is considered to be more specialized than the other;
+    //        otherwise,
+    if (!RefParamComparisons[I].ArgIsRvalueRef &&
+        RefParamComparisons[I].ParamIsRvalueRef) {
+      Better2 = true;
+      if (Better1)
+        return 0;
+      continue;
+    } else if (!RefParamComparisons[I].ParamIsRvalueRef &&
+               RefParamComparisons[I].ArgIsRvalueRef) {
+      Better1 = true;
+      if (Better2)
+        return 0;
+      continue;
     }
+
+    //     -- if the type from the argument template is more cv-qualified than
+    //        the type from the parameter template (as described above), the
+    //        argument type is considered to be more specialized than the
+    //        other; otherwise,
+    switch (RefParamComparisons[I].Qualifiers) {
+    case NeitherMoreQualified:
+      break;
+
+    case ParamMoreQualified:
+      Better1 = true;
+      if (Better2)
+        return 0;
+      continue;
+
+    case ArgMoreQualified:
+      Better2 = true;
+      if (Better1)
+        return 0;
+      continue;
+    }
+
+    //     -- neither type is more specialized than the other.
   }
-   
+
   assert(!(Better1 && Better2) && "Should have broken out in the loop above");
   if (Better1)
     return FT1;
   else if (Better2)
     return FT2;
-  else
-    return 0;
+
+  // FIXME: This mimics what GCC implements, but doesn't match up with the
+  // proposed resolution for core issue 692. This area needs to be sorted out,
+  // but for now we attempt to maintain compatibility.
+  bool Variadic1 = isVariadicFunctionTemplate(FT1);
+  bool Variadic2 = isVariadicFunctionTemplate(FT2);
+  if (Variadic1 != Variadic2)
+    return Variadic1? FT2 : FT1;
+
+  return 0;
 }
 
 /// \brief Determine if the two templates are equivalent.
 static bool isSameTemplate(TemplateDecl *T1, TemplateDecl *T2) {
   if (T1 == T2)
     return true;
-  
+
   if (!T1 || !T2)
     return false;
-  
+
   return T1->getCanonicalDecl() == T2->getCanonicalDecl();
 }
 
@@ -2309,7 +3379,10 @@ static bool isSameTemplate(TemplateDecl *T1, TemplateDecl *T2) {
 /// \param TPOC the partial ordering context to use to compare the function
 /// template specializations.
 ///
-/// \param Loc the location where the ambiguity or no-specializations 
+/// \param NumCallArguments The number of arguments in a call, used only
+/// when \c TPOC is \c TPOC_Call.
+///
+/// \param Loc the location where the ambiguity or no-specializations
 /// diagnostic should occur.
 ///
 /// \param NoneDiag partial diagnostic used to diagnose cases where there are
@@ -2323,35 +3396,38 @@ static bool isSameTemplate(TemplateDecl *T1, TemplateDecl *T2) {
 /// in this diagnostic should be unbound, which will correspond to the string
 /// describing the template arguments for the function template specialization.
 ///
-/// \param Index if non-NULL and the result of this function is non-nULL, 
+/// \param Index if non-NULL and the result of this function is non-nULL,
 /// receives the index corresponding to the resulting function template
 /// specialization.
 ///
-/// \returns the most specialized function template specialization, if 
+/// \returns the most specialized function template specialization, if
 /// found. Otherwise, returns SpecEnd.
 ///
-/// \todo FIXME: Consider passing in the "also-ran" candidates that failed 
+/// \todo FIXME: Consider passing in the "also-ran" candidates that failed
 /// template argument deduction.
 UnresolvedSetIterator
 Sema::getMostSpecialized(UnresolvedSetIterator SpecBegin,
-                         UnresolvedSetIterator SpecEnd,
+                        UnresolvedSetIterator SpecEnd,
                          TemplatePartialOrderingContext TPOC,
+                         unsigned NumCallArguments,
                          SourceLocation Loc,
                          const PartialDiagnostic &NoneDiag,
                          const PartialDiagnostic &AmbigDiag,
-                         const PartialDiagnostic &CandidateDiag) {
+                         const PartialDiagnostic &CandidateDiag,
+                         bool Complain) {
   if (SpecBegin == SpecEnd) {
-    Diag(Loc, NoneDiag);
+    if (Complain)
+      Diag(Loc, NoneDiag);
     return SpecEnd;
   }
-  
-  if (SpecBegin + 1 == SpecEnd)    
+
+  if (SpecBegin + 1 == SpecEnd)
     return SpecBegin;
-  
+
   // Find the function template that is better than all of the templates it
   // has been compared to.
   UnresolvedSetIterator Best = SpecBegin;
-  FunctionTemplateDecl *BestTemplate 
+  FunctionTemplateDecl *BestTemplate
     = cast<FunctionDecl>(*Best)->getPrimaryTemplate();
   assert(BestTemplate && "Not a function template specialization?");
   for (UnresolvedSetIterator I = SpecBegin + 1; I != SpecEnd; ++I) {
@@ -2359,13 +3435,13 @@ Sema::getMostSpecialized(UnresolvedSetIterator SpecBegin,
       = cast<FunctionDecl>(*I)->getPrimaryTemplate();
     assert(Challenger && "Not a function template specialization?");
     if (isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger,
-                                                  Loc, TPOC),
+                                                  Loc, TPOC, NumCallArguments),
                        Challenger)) {
       Best = I;
       BestTemplate = Challenger;
     }
   }
-  
+
   // Make sure that the "best" function template is more specialized than all
   // of the others.
   bool Ambiguous = false;
@@ -2373,29 +3449,31 @@ Sema::getMostSpecialized(UnresolvedSetIterator SpecBegin,
     FunctionTemplateDecl *Challenger
       = cast<FunctionDecl>(*I)->getPrimaryTemplate();
     if (I != Best &&
-        !isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger, 
-                                                   Loc, TPOC),
+        !isSameTemplate(getMoreSpecializedTemplate(BestTemplate, Challenger,
+                                                   Loc, TPOC, NumCallArguments),
                         BestTemplate)) {
       Ambiguous = true;
       break;
     }
   }
-  
+
   if (!Ambiguous) {
     // We found an answer. Return it.
     return Best;
   }
-  
+
   // Diagnose the ambiguity.
-  Diag(Loc, AmbigDiag);
-  
+  if (Complain)
+    Diag(Loc, AmbigDiag);
+
+  if (Complain)
   // FIXME: Can we order the candidates in some sane way?
-  for (UnresolvedSetIterator I = SpecBegin; I != SpecEnd; ++I)
-    Diag((*I)->getLocation(), CandidateDiag)
-      << getTemplateArgumentBindingsText(
-        cast<FunctionDecl>(*I)->getPrimaryTemplate()->getTemplateParameters(),
+    for (UnresolvedSetIterator I = SpecBegin; I != SpecEnd; ++I)
+      Diag((*I)->getLocation(), CandidateDiag)
+        << getTemplateArgumentBindingsText(
+          cast<FunctionDecl>(*I)->getPrimaryTemplate()->getTemplateParameters(),
                     *cast<FunctionDecl>(*I)->getTemplateSpecializationArgs());
-  
+
   return SpecEnd;
 }
 
@@ -2416,17 +3494,17 @@ Sema::getMoreSpecializedPartialSpecialization(
                                               SourceLocation Loc) {
   // C++ [temp.class.order]p1:
   //   For two class template partial specializations, the first is at least as
-  //   specialized as the second if, given the following rewrite to two 
-  //   function templates, the first function template is at least as 
-  //   specialized as the second according to the ordering rules for function 
+  //   specialized as the second if, given the following rewrite to two
+  //   function templates, the first function template is at least as
+  //   specialized as the second according to the ordering rules for function
   //   templates (14.6.6.2):
   //     - the first function template has the same template parameters as the
-  //       first partial specialization and has a single function parameter 
-  //       whose type is a class template specialization with the template 
+  //       first partial specialization and has a single function parameter
+  //       whose type is a class template specialization with the template
   //       arguments of the first partial specialization, and
   //     - the second function template has the same template parameters as the
-  //       second partial specialization and has a single function parameter 
-  //       whose type is a class template specialization with the template 
+  //       second partial specialization and has a single function parameter
+  //       whose type is a class template specialization with the template
   //       arguments of the second partial specialization.
   //
   // Rather than synthesize function templates, we merely perform the
@@ -2443,39 +3521,39 @@ Sema::getMoreSpecializedPartialSpecialization(
 
   QualType PT1 = PS1->getInjectedSpecializationType();
   QualType PT2 = PS2->getInjectedSpecializationType();
-  
+
   // Determine whether PS1 is at least as specialized as PS2
   Deduced.resize(PS2->getTemplateParameters()->size());
-  bool Better1 = !DeduceTemplateArgumentsDuringPartialOrdering(*this,
-                                                  PS2->getTemplateParameters(),
-                                                               PT2,
-                                                               PT1,
-                                                               Info,
-                                                               Deduced,
-                                                               0);
-  if (Better1)
-    Better1 = !::FinishTemplateArgumentDeduction(*this, PS2, 
-                                                 PS1->getTemplateArgs(), 
+  bool Better1 = !::DeduceTemplateArguments(*this, PS2->getTemplateParameters(),
+                                            PT2, PT1, Info, Deduced, TDF_None,
+                                            /*PartialOrdering=*/true,
+                                            /*RefParamComparisons=*/0);
+  if (Better1) {
+    InstantiatingTemplate Inst(*this, PS2->getLocation(), PS2,
+                               Deduced.data(), Deduced.size(), Info);
+    Better1 = !::FinishTemplateArgumentDeduction(*this, PS2,
+                                                 PS1->getTemplateArgs(),
                                                  Deduced, Info);
-  
+  }
+
   // Determine whether PS2 is at least as specialized as PS1
   Deduced.clear();
   Deduced.resize(PS1->getTemplateParameters()->size());
-  bool Better2 = !DeduceTemplateArgumentsDuringPartialOrdering(*this,
-                                                  PS1->getTemplateParameters(),
-                                                               PT1,
-                                                               PT2,
-                                                               Info,
-                                                               Deduced,
-                                                               0);
-  if (Better2)
-    Better2 = !::FinishTemplateArgumentDeduction(*this, PS1, 
-                                                 PS2->getTemplateArgs(), 
+  bool Better2 = !::DeduceTemplateArguments(*this, PS1->getTemplateParameters(),
+                                            PT1, PT2, Info, Deduced, TDF_None,
+                                            /*PartialOrdering=*/true,
+                                            /*RefParamComparisons=*/0);
+  if (Better2) {
+    InstantiatingTemplate Inst(*this, PS1->getLocation(), PS1,
+                               Deduced.data(), Deduced.size(), Info);
+    Better2 = !::FinishTemplateArgumentDeduction(*this, PS1,
+                                                 PS2->getTemplateArgs(),
                                                  Deduced, Info);
-  
+  }
+
   if (Better1 == Better2)
     return 0;
-  
+
   return Better1? PS1 : PS2;
 }
 
@@ -2494,7 +3572,15 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            bool OnlyDeduced,
                            unsigned Depth,
                            llvm::SmallVectorImpl<bool> &Used) {
-  // FIXME: if !OnlyDeduced, we have to walk the whole subexpression to 
+  // We can deduce from a pack expansion.
+  if (const PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(E))
+    E = Expansion->getPattern();
+
+  // Skip through any implicit casts we added while type-checking.
+  while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E))
+    E = ICE->getSubExpr();
+
+  // FIXME: if !OnlyDeduced, we have to walk the whole subexpression to
   // find other occurrences of template parameters.
   const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E);
   if (!DRE)
@@ -2519,13 +3605,13 @@ MarkUsedTemplateParameters(Sema &SemaRef,
                            llvm::SmallVectorImpl<bool> &Used) {
   if (!NNS)
     return;
-  
+
   MarkUsedTemplateParameters(SemaRef, NNS->getPrefix(), OnlyDeduced, Depth,
                              Used);
-  MarkUsedTemplateParameters(SemaRef, QualType(NNS->getAsType(), 0), 
+  MarkUsedTemplateParameters(SemaRef, QualType(NNS->getAsType(), 0),
                              OnlyDeduced, Depth, Used);
 }
-  
+
 /// \brief Mark the template parameters that are used by the given
 /// template name.
 static void
@@ -2542,12 +3628,12 @@ MarkUsedTemplateParameters(Sema &SemaRef,
     }
     return;
   }
-  
+
   if (QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName())
-    MarkUsedTemplateParameters(SemaRef, QTN->getQualifier(), OnlyDeduced, 
+    MarkUsedTemplateParameters(SemaRef, QTN->getQualifier(), OnlyDeduced,
                                Depth, Used);
   if (DependentTemplateName *DTN = Name.getAsDependentTemplateName())
-    MarkUsedTemplateParameters(SemaRef, DTN->getQualifier(), OnlyDeduced, 
+    MarkUsedTemplateParameters(SemaRef, DTN->getQualifier(), OnlyDeduced,
                                Depth, Used);
 }
 
@@ -2560,7 +3646,7 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                            llvm::SmallVectorImpl<bool> &Used) {
   if (T.isNull())
     return;
-  
+
   // Non-dependent types have nothing deducible
   if (!T->isDependentType())
     return;
@@ -2626,7 +3712,7 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
       = cast<DependentSizedExtVectorType>(T);
     MarkUsedTemplateParameters(SemaRef, VecType->getElementType(), OnlyDeduced,
                                Depth, Used);
-    MarkUsedTemplateParameters(SemaRef, VecType->getSizeExpr(), OnlyDeduced, 
+    MarkUsedTemplateParameters(SemaRef, VecType->getSizeExpr(), OnlyDeduced,
                                Depth, Used);
     break;
   }
@@ -2648,6 +3734,17 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
     break;
   }
 
+  case Type::SubstTemplateTypeParmPack: {
+    const SubstTemplateTypeParmPackType *Subst
+      = cast<SubstTemplateTypeParmPackType>(T);
+    MarkUsedTemplateParameters(SemaRef,
+                               QualType(Subst->getReplacedParameter(), 0),
+                               OnlyDeduced, Depth, Used);
+    MarkUsedTemplateParameters(SemaRef, Subst->getArgumentPack(),
+                               OnlyDeduced, Depth, Used);
+    break;
+  }
+
   case Type::InjectedClassName:
     T = cast<InjectedClassNameType>(T)->getInjectedSpecializationType();
     // fall through
@@ -2657,6 +3754,15 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
       = cast<TemplateSpecializationType>(T);
     MarkUsedTemplateParameters(SemaRef, Spec->getTemplateName(), OnlyDeduced,
                                Depth, Used);
+
+    // C++0x [temp.deduct.type]p9:
+    //   If the template argument list of P contains a pack expansion that is not
+    //   the last template argument, the entire template argument list is a
+    //   non-deduced context.
+    if (OnlyDeduced &&
+        hasPackExpansionBeforeEnd(Spec->getArgs(), Spec->getNumArgs()))
+      break;
+
     for (unsigned I = 0, N = Spec->getNumArgs(); I != N; ++I)
       MarkUsedTemplateParameters(SemaRef, Spec->getArg(I), OnlyDeduced, Depth,
                                  Used);
@@ -2665,7 +3771,7 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
 
   case Type::Complex:
     if (!OnlyDeduced)
-      MarkUsedTemplateParameters(SemaRef, 
+      MarkUsedTemplateParameters(SemaRef,
                                  cast<ComplexType>(T)->getElementType(),
                                  OnlyDeduced, Depth, Used);
     break;
@@ -2683,6 +3789,15 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
     if (!OnlyDeduced)
       MarkUsedTemplateParameters(SemaRef, Spec->getQualifier(),
                                  OnlyDeduced, Depth, Used);
+
+    // C++0x [temp.deduct.type]p9:
+    //   If the template argument list of P contains a pack expansion that is not
+    //   the last template argument, the entire template argument list is a
+    //   non-deduced context.
+    if (OnlyDeduced &&
+        hasPackExpansionBeforeEnd(Spec->getArgs(), Spec->getNumArgs()))
+      break;
+
     for (unsigned I = 0, N = Spec->getNumArgs(); I != N; ++I)
       MarkUsedTemplateParameters(SemaRef, Spec->getArg(I), OnlyDeduced, Depth,
                                  Used);
@@ -2710,6 +3825,17 @@ MarkUsedTemplateParameters(Sema &SemaRef, QualType T,
                                  OnlyDeduced, Depth, Used);
     break;
 
+  case Type::PackExpansion:
+    MarkUsedTemplateParameters(SemaRef,
+                               cast<PackExpansionType>(T)->getPattern(),
+                               OnlyDeduced, Depth, Used);
+    break;
+
+  case Type::Auto:
+    MarkUsedTemplateParameters(SemaRef,
+                               cast<AutoType>(T)->getDeducedType(),
+                               OnlyDeduced, Depth, Used);
+
   // None of these types have any template parameters in them.
   case Type::Builtin:
   case Type::VariableArray:
@@ -2749,15 +3875,17 @@ MarkUsedTemplateParameters(Sema &SemaRef,
     break;
 
   case TemplateArgument::Template:
-    MarkUsedTemplateParameters(SemaRef, TemplateArg.getAsTemplate(), 
+  case TemplateArgument::TemplateExpansion:
+    MarkUsedTemplateParameters(SemaRef,
+                               TemplateArg.getAsTemplateOrTemplatePattern(),
                                OnlyDeduced, Depth, Used);
     break;
 
   case TemplateArgument::Expression:
-    MarkUsedTemplateParameters(SemaRef, TemplateArg.getAsExpr(), OnlyDeduced, 
+    MarkUsedTemplateParameters(SemaRef, TemplateArg.getAsExpr(), OnlyDeduced,
                                Depth, Used);
     break;
-      
+
   case TemplateArgument::Pack:
     for (TemplateArgument::pack_iterator P = TemplateArg.pack_begin(),
                                       PEnd = TemplateArg.pack_end();
@@ -2780,21 +3908,29 @@ void
 Sema::MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs,
                                  bool OnlyDeduced, unsigned Depth,
                                  llvm::SmallVectorImpl<bool> &Used) {
+  // C++0x [temp.deduct.type]p9:
+  //   If the template argument list of P contains a pack expansion that is not
+  //   the last template argument, the entire template argument list is a
+  //   non-deduced context.
+  if (OnlyDeduced &&
+      hasPackExpansionBeforeEnd(TemplateArgs.data(), TemplateArgs.size()))
+    return;
+
   for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
-    ::MarkUsedTemplateParameters(*this, TemplateArgs[I], OnlyDeduced, 
+    ::MarkUsedTemplateParameters(*this, TemplateArgs[I], OnlyDeduced,
                                  Depth, Used);
 }
 
 /// \brief Marks all of the template parameters that will be deduced by a
 /// call to the given function template.
-void 
+void
 Sema::MarkDeducedTemplateParameters(FunctionTemplateDecl *FunctionTemplate,
                                     llvm::SmallVectorImpl<bool> &Deduced) {
-  TemplateParameterList *TemplateParams 
+  TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
   Deduced.clear();
   Deduced.resize(TemplateParams->size());
-  
+
   FunctionDecl *Function = FunctionTemplate->getTemplatedDecl();
   for (unsigned I = 0, N = Function->getNumParams(); I != N; ++I)
     ::MarkUsedTemplateParameters(*this, Function->getParamDecl(I)->getType(),
diff --git a/lib/Sema/SemaTemplateInstantiate.cpp b/lib/Sema/SemaTemplateInstantiate.cpp
index 4d4c18130b01..44f5913d55c7 100644
--- a/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/lib/Sema/SemaTemplateInstantiate.cpp
@@ -147,8 +147,10 @@ Sema::InstantiatingTemplate::
 InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
                       Decl *Entity,
                       SourceRange InstantiationRange)
-  :  SemaRef(SemaRef) {
-
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                        SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = CheckInstantiationDepth(PointOfInstantiation,
                                     InstantiationRange);
   if (!Invalid) {
@@ -159,6 +161,7 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
     Inst.TemplateArgs = 0;
     Inst.NumTemplateArgs = 0;
     Inst.InstantiationRange = InstantiationRange;
+    SemaRef.InNonInstantiationSFINAEContext = false;
     SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   }
 }
@@ -169,8 +172,10 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
                                          const TemplateArgument *TemplateArgs,
                                          unsigned NumTemplateArgs,
                                          SourceRange InstantiationRange)
-  : SemaRef(SemaRef) {
-
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = CheckInstantiationDepth(PointOfInstantiation,
                                     InstantiationRange);
   if (!Invalid) {
@@ -182,6 +187,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
     Inst.TemplateArgs = TemplateArgs;
     Inst.NumTemplateArgs = NumTemplateArgs;
     Inst.InstantiationRange = InstantiationRange;
+    SemaRef.InNonInstantiationSFINAEContext = false;
     SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   }
 }
@@ -192,9 +198,12 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
                                         const TemplateArgument *TemplateArgs,
                                                    unsigned NumTemplateArgs,
                          ActiveTemplateInstantiation::InstantiationKind Kind,
+                                   sema::TemplateDeductionInfo &DeductionInfo,
                                               SourceRange InstantiationRange)
-: SemaRef(SemaRef) {
-
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = CheckInstantiationDepth(PointOfInstantiation,
                                     InstantiationRange);
   if (!Invalid) {
@@ -204,7 +213,9 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
     Inst.Entity = reinterpret_cast<uintptr_t>(FunctionTemplate);
     Inst.TemplateArgs = TemplateArgs;
     Inst.NumTemplateArgs = NumTemplateArgs;
+    Inst.DeductionInfo = &DeductionInfo;
     Inst.InstantiationRange = InstantiationRange;
+    SemaRef.InNonInstantiationSFINAEContext = false;
     SemaRef.ActiveTemplateInstantiations.push_back(Inst);
     
     if (!Inst.isInstantiationRecord())
@@ -217,9 +228,12 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
                           ClassTemplatePartialSpecializationDecl *PartialSpec,
                                          const TemplateArgument *TemplateArgs,
                                          unsigned NumTemplateArgs,
+                                    sema::TemplateDeductionInfo &DeductionInfo,
                                          SourceRange InstantiationRange)
-  : SemaRef(SemaRef) {
-
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = false;
     
   ActiveTemplateInstantiation Inst;
@@ -228,7 +242,9 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
   Inst.Entity = reinterpret_cast<uintptr_t>(PartialSpec);
   Inst.TemplateArgs = TemplateArgs;
   Inst.NumTemplateArgs = NumTemplateArgs;
+  Inst.DeductionInfo = &DeductionInfo;
   Inst.InstantiationRange = InstantiationRange;
+  SemaRef.InNonInstantiationSFINAEContext = false;
   SemaRef.ActiveTemplateInstantiations.push_back(Inst);
       
   assert(!Inst.isInstantiationRecord());
@@ -241,8 +257,10 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
                                           const TemplateArgument *TemplateArgs,
                                           unsigned NumTemplateArgs,
                                           SourceRange InstantiationRange)
-  : SemaRef(SemaRef) {
-
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = CheckInstantiationDepth(PointOfInstantiation, InstantiationRange);
 
   if (!Invalid) {
@@ -254,17 +272,22 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(Sema &SemaRef,
     Inst.TemplateArgs = TemplateArgs;
     Inst.NumTemplateArgs = NumTemplateArgs;
     Inst.InstantiationRange = InstantiationRange;
+    SemaRef.InNonInstantiationSFINAEContext = false;
     SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   }
 }
 
 Sema::InstantiatingTemplate::
 InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                      TemplateDecl *Template,
+                      NamedDecl *Template,
                       NonTypeTemplateParmDecl *Param,
                       const TemplateArgument *TemplateArgs,
                       unsigned NumTemplateArgs,
-                      SourceRange InstantiationRange) : SemaRef(SemaRef) {
+                      SourceRange InstantiationRange) 
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = false;
   
   ActiveTemplateInstantiation Inst;
@@ -275,6 +298,7 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
   Inst.TemplateArgs = TemplateArgs;
   Inst.NumTemplateArgs = NumTemplateArgs;
   Inst.InstantiationRange = InstantiationRange;
+  SemaRef.InNonInstantiationSFINAEContext = false;
   SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   
   assert(!Inst.isInstantiationRecord());
@@ -283,11 +307,15 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
 
 Sema::InstantiatingTemplate::
 InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                      TemplateDecl *Template,
+                      NamedDecl *Template,
                       TemplateTemplateParmDecl *Param,
                       const TemplateArgument *TemplateArgs,
                       unsigned NumTemplateArgs,
-                      SourceRange InstantiationRange) : SemaRef(SemaRef) {
+                      SourceRange InstantiationRange) 
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = false;
   ActiveTemplateInstantiation Inst;
   Inst.Kind = ActiveTemplateInstantiation::PriorTemplateArgumentSubstitution;
@@ -297,6 +325,7 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
   Inst.TemplateArgs = TemplateArgs;
   Inst.NumTemplateArgs = NumTemplateArgs;
   Inst.InstantiationRange = InstantiationRange;
+  SemaRef.InNonInstantiationSFINAEContext = false;
   SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   
   assert(!Inst.isInstantiationRecord());
@@ -309,7 +338,11 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
                       NamedDecl *Param,
                       const TemplateArgument *TemplateArgs,
                       unsigned NumTemplateArgs,
-                      SourceRange InstantiationRange) : SemaRef(SemaRef) {
+                      SourceRange InstantiationRange) 
+  : SemaRef(SemaRef),
+    SavedInNonInstantiationSFINAEContext(
+                                     SemaRef.InNonInstantiationSFINAEContext)
+{
   Invalid = false;
   
   ActiveTemplateInstantiation Inst;
@@ -320,6 +353,7 @@ InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
   Inst.TemplateArgs = TemplateArgs;
   Inst.NumTemplateArgs = NumTemplateArgs;
   Inst.InstantiationRange = InstantiationRange;
+  SemaRef.InNonInstantiationSFINAEContext = false;
   SemaRef.ActiveTemplateInstantiations.push_back(Inst);
   
   assert(!Inst.isInstantiationRecord());
@@ -332,7 +366,8 @@ void Sema::InstantiatingTemplate::Clear() {
       assert(SemaRef.NonInstantiationEntries > 0);
       --SemaRef.NonInstantiationEntries;
     }
-    
+    SemaRef.InNonInstantiationSFINAEContext
+      = SavedInNonInstantiationSFINAEContext;
     SemaRef.ActiveTemplateInstantiations.pop_back();
     Invalid = true;
   }
@@ -379,7 +414,7 @@ void Sema::PrintInstantiationStack() {
     if (InstantiationIdx >= SkipStart && InstantiationIdx < SkipEnd) {
       if (InstantiationIdx == SkipStart) {
         // Note that we're skipping instantiations.
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+        Diags.Report(Active->PointOfInstantiation,
                      diag::note_instantiation_contexts_suppressed)
           << unsigned(ActiveTemplateInstantiations.size() - Limit);
       }
@@ -393,8 +428,7 @@ void Sema::PrintInstantiationStack() {
         unsigned DiagID = diag::note_template_member_class_here;
         if (isa<ClassTemplateSpecializationDecl>(Record))
           DiagID = diag::note_template_class_instantiation_here;
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
-                     DiagID)
+        Diags.Report(Active->PointOfInstantiation, DiagID)
           << Context.getTypeDeclType(Record)
           << Active->InstantiationRange;
       } else if (FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
@@ -403,12 +437,11 @@ void Sema::PrintInstantiationStack() {
           DiagID = diag::note_function_template_spec_here;
         else
           DiagID = diag::note_template_member_function_here;
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
-                     DiagID)
+        Diags.Report(Active->PointOfInstantiation, DiagID)
           << Function
           << Active->InstantiationRange;
       } else {
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+        Diags.Report(Active->PointOfInstantiation,
                      diag::note_template_static_data_member_def_here)
           << cast<VarDecl>(D)
           << Active->InstantiationRange;
@@ -423,7 +456,7 @@ void Sema::PrintInstantiationStack() {
                                                          Active->TemplateArgs,
                                                       Active->NumTemplateArgs,
                                                       Context.PrintingPolicy);
-      Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+      Diags.Report(Active->PointOfInstantiation,
                    diag::note_default_arg_instantiation_here)
         << (Template->getNameAsString() + TemplateArgsStr)
         << Active->InstantiationRange;
@@ -433,7 +466,7 @@ void Sema::PrintInstantiationStack() {
     case ActiveTemplateInstantiation::ExplicitTemplateArgumentSubstitution: {
       FunctionTemplateDecl *FnTmpl
         = cast<FunctionTemplateDecl>((Decl *)Active->Entity);
-      Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+      Diags.Report(Active->PointOfInstantiation,
                    diag::note_explicit_template_arg_substitution_here)
         << FnTmpl 
         << getTemplateArgumentBindingsText(FnTmpl->getTemplateParameters(), 
@@ -447,7 +480,7 @@ void Sema::PrintInstantiationStack() {
       if (ClassTemplatePartialSpecializationDecl *PartialSpec
             = dyn_cast<ClassTemplatePartialSpecializationDecl>(
                                                     (Decl *)Active->Entity)) {
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+        Diags.Report(Active->PointOfInstantiation,
                      diag::note_partial_spec_deduct_instantiation_here)
           << Context.getTypeDeclType(PartialSpec)
           << getTemplateArgumentBindingsText(
@@ -458,7 +491,7 @@ void Sema::PrintInstantiationStack() {
       } else {
         FunctionTemplateDecl *FnTmpl
           = cast<FunctionTemplateDecl>((Decl *)Active->Entity);
-        Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+        Diags.Report(Active->PointOfInstantiation,
                      diag::note_function_template_deduction_instantiation_here)
           << FnTmpl
           << getTemplateArgumentBindingsText(FnTmpl->getTemplateParameters(), 
@@ -477,7 +510,7 @@ void Sema::PrintInstantiationStack() {
                                                          Active->TemplateArgs,
                                                       Active->NumTemplateArgs,
                                                       Context.PrintingPolicy);
-      Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+      Diags.Report(Active->PointOfInstantiation,
                    diag::note_default_function_arg_instantiation_here)
         << (FD->getNameAsString() + TemplateArgsStr)
         << Active->InstantiationRange;
@@ -489,13 +522,19 @@ void Sema::PrintInstantiationStack() {
       std::string Name;
       if (!Parm->getName().empty())
         Name = std::string(" '") + Parm->getName().str() + "'";
-                                        
-      Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+                    
+      TemplateParameterList *TemplateParams = 0;
+      if (TemplateDecl *Template = dyn_cast<TemplateDecl>(Active->Template))
+        TemplateParams = Template->getTemplateParameters();
+      else
+        TemplateParams =
+          cast<ClassTemplatePartialSpecializationDecl>(Active->Template)
+                                                      ->getTemplateParameters();
+      Diags.Report(Active->PointOfInstantiation,
                    diag::note_prior_template_arg_substitution)
         << isa<TemplateTemplateParmDecl>(Parm)
         << Name
-        << getTemplateArgumentBindingsText(
-                                    Active->Template->getTemplateParameters(), 
+        << getTemplateArgumentBindingsText(TemplateParams, 
                                            Active->TemplateArgs, 
                                            Active->NumTemplateArgs)
         << Active->InstantiationRange;
@@ -503,10 +542,17 @@ void Sema::PrintInstantiationStack() {
     }
 
     case ActiveTemplateInstantiation::DefaultTemplateArgumentChecking: {
-      Diags.Report(FullSourceLoc(Active->PointOfInstantiation, SourceMgr),
+      TemplateParameterList *TemplateParams = 0;
+      if (TemplateDecl *Template = dyn_cast<TemplateDecl>(Active->Template))
+        TemplateParams = Template->getTemplateParameters();
+      else
+        TemplateParams =
+          cast<ClassTemplatePartialSpecializationDecl>(Active->Template)
+                                                      ->getTemplateParameters();
+
+      Diags.Report(Active->PointOfInstantiation,
                    diag::note_template_default_arg_checking)
-        << getTemplateArgumentBindingsText(
-                                     Active->Template->getTemplateParameters(), 
+        << getTemplateArgumentBindingsText(TemplateParams, 
                                            Active->TemplateArgs, 
                                            Active->NumTemplateArgs)
         << Active->InstantiationRange;
@@ -516,8 +562,11 @@ void Sema::PrintInstantiationStack() {
   }
 }
 
-bool Sema::isSFINAEContext() const {
+llvm::Optional<TemplateDeductionInfo *> Sema::isSFINAEContext() const {
   using llvm::SmallVector;
+  if (InNonInstantiationSFINAEContext)
+    return llvm::Optional<TemplateDeductionInfo *>(0);
+
   for (SmallVector<ActiveTemplateInstantiation, 16>::const_reverse_iterator
          Active = ActiveTemplateInstantiations.rbegin(),
          ActiveEnd = ActiveTemplateInstantiations.rend();
@@ -525,10 +574,10 @@ bool Sema::isSFINAEContext() const {
        ++Active) 
   {
     switch(Active->Kind) {
-    case ActiveTemplateInstantiation::TemplateInstantiation:
     case ActiveTemplateInstantiation::DefaultFunctionArgumentInstantiation:
+    case ActiveTemplateInstantiation::TemplateInstantiation:
       // This is a template instantiation, so there is no SFINAE.
-      return false;
+      return llvm::Optional<TemplateDeductionInfo *>();
 
     case ActiveTemplateInstantiation::DefaultTemplateArgumentInstantiation:
     case ActiveTemplateInstantiation::PriorTemplateArgumentSubstitution:
@@ -542,11 +591,25 @@ bool Sema::isSFINAEContext() const {
     case ActiveTemplateInstantiation::DeducedTemplateArgumentSubstitution:
       // We're either substitution explicitly-specified template arguments
       // or deduced template arguments, so SFINAE applies.
-      return true;
+      assert(Active->DeductionInfo && "Missing deduction info pointer");
+      return Active->DeductionInfo;
     }
   }
 
-  return false;
+  return llvm::Optional<TemplateDeductionInfo *>();
+}
+
+/// \brief Retrieve the depth and index of a parameter pack.
+static std::pair<unsigned, unsigned> 
+getDepthAndIndex(NamedDecl *ND) {
+  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(ND))
+    return std::make_pair(TTP->getDepth(), TTP->getIndex());
+  
+  if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(ND))
+    return std::make_pair(NTTP->getDepth(), NTTP->getIndex());
+  
+  TemplateTemplateParmDecl *TTP = cast<TemplateTemplateParmDecl>(ND);
+  return std::make_pair(TTP->getDepth(), TTP->getIndex());
 }
 
 //===----------------------------------------------------------------------===/
@@ -587,7 +650,58 @@ namespace {
       this->Loc = Loc;
       this->Entity = Entity;
     }
+
+    bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
+                                 SourceRange PatternRange,
+                                 const UnexpandedParameterPack *Unexpanded,
+                                 unsigned NumUnexpanded,
+                                 bool &ShouldExpand,
+                                 bool &RetainExpansion,
+                                 llvm::Optional<unsigned> &NumExpansions) {
+      return getSema().CheckParameterPacksForExpansion(EllipsisLoc, 
+                                                       PatternRange, Unexpanded,
+                                                       NumUnexpanded, 
+                                                       TemplateArgs, 
+                                                       ShouldExpand,
+                                                       RetainExpansion,
+                                                       NumExpansions);
+    }
+
+    void ExpandingFunctionParameterPack(ParmVarDecl *Pack) { 
+      SemaRef.CurrentInstantiationScope->MakeInstantiatedLocalArgPack(Pack);
+    }
+    
+    TemplateArgument ForgetPartiallySubstitutedPack() {
+      TemplateArgument Result;
+      if (NamedDecl *PartialPack
+            = SemaRef.CurrentInstantiationScope->getPartiallySubstitutedPack()){
+        MultiLevelTemplateArgumentList &TemplateArgs
+          = const_cast<MultiLevelTemplateArgumentList &>(this->TemplateArgs);
+        unsigned Depth, Index;
+        llvm::tie(Depth, Index) = getDepthAndIndex(PartialPack);
+        if (TemplateArgs.hasTemplateArgument(Depth, Index)) {
+          Result = TemplateArgs(Depth, Index);
+          TemplateArgs.setArgument(Depth, Index, TemplateArgument());
+        }
+      }
+      
+      return Result;
+    }
+    
+    void RememberPartiallySubstitutedPack(TemplateArgument Arg) {
+      if (Arg.isNull())
+        return;
       
+      if (NamedDecl *PartialPack
+            = SemaRef.CurrentInstantiationScope->getPartiallySubstitutedPack()){
+        MultiLevelTemplateArgumentList &TemplateArgs
+        = const_cast<MultiLevelTemplateArgumentList &>(this->TemplateArgs);
+        unsigned Depth, Index;
+        llvm::tie(Depth, Index) = getDepthAndIndex(PartialPack);
+        TemplateArgs.setArgument(Depth, Index, Arg);
+      }
+    }
+
     /// \brief Transform the given declaration by instantiating a reference to
     /// this declaration.
     Decl *TransformDecl(SourceLocation Loc, Decl *D);
@@ -602,10 +716,10 @@ namespace {
       
     /// \brief Rebuild the exception declaration and register the declaration
     /// as an instantiated local.
-    VarDecl *RebuildExceptionDecl(VarDecl *ExceptionDecl, QualType T,
+    VarDecl *RebuildExceptionDecl(VarDecl *ExceptionDecl, 
                                   TypeSourceInfo *Declarator,
                                   IdentifierInfo *Name,
-                                  SourceLocation Loc, SourceRange TypeRange);
+                                  SourceLocation Loc);
 
     /// \brief Rebuild the Objective-C exception declaration and register the 
     /// declaration as an instantiated local.
@@ -614,25 +728,37 @@ namespace {
       
     /// \brief Check for tag mismatches when instantiating an
     /// elaborated type.
-    QualType RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
+    QualType RebuildElaboratedType(SourceLocation KeywordLoc,
+                                   ElaboratedTypeKeyword Keyword,
                                    NestedNameSpecifier *NNS, QualType T);
 
+    TemplateName TransformTemplateName(TemplateName Name,
+                                       QualType ObjectType = QualType(),
+                                       NamedDecl *FirstQualifierInScope = 0);
+
     ExprResult TransformPredefinedExpr(PredefinedExpr *E);
     ExprResult TransformDeclRefExpr(DeclRefExpr *E);
     ExprResult TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E);
     ExprResult TransformTemplateParmRefExpr(DeclRefExpr *E,
-                                                NonTypeTemplateParmDecl *D);
-
+                                            NonTypeTemplateParmDecl *D);
+    ExprResult TransformSubstNonTypeTemplateParmPackExpr(
+                                           SubstNonTypeTemplateParmPackExpr *E);
+    
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
-                                        FunctionProtoTypeLoc TL,
-                                        QualType ObjectType);
-    ParmVarDecl *TransformFunctionTypeParam(ParmVarDecl *OldParm);
+                                        FunctionProtoTypeLoc TL);
+    ParmVarDecl *TransformFunctionTypeParam(ParmVarDecl *OldParm,
+                                      llvm::Optional<unsigned> NumExpansions);
 
     /// \brief Transforms a template type parameter type by performing
     /// substitution of the corresponding template type argument.
     QualType TransformTemplateTypeParmType(TypeLocBuilder &TLB,
-                                           TemplateTypeParmTypeLoc TL,
-                                           QualType ObjectType);
+                                           TemplateTypeParmTypeLoc TL);
+
+    /// \brief Transforms an already-substituted template type parameter pack
+    /// into either itself (if we aren't substituting into its pack expansion)
+    /// or the appropriate substituted argument.
+    QualType TransformSubstTemplateTypeParmPackType(TypeLocBuilder &TLB,
+                                           SubstTemplateTypeParmPackTypeLoc TL);
 
     ExprResult TransformCallExpr(CallExpr *CE) {
       getSema().CallsUndergoingInstantiation.push_back(CE);
@@ -669,8 +795,18 @@ Decl *TemplateInstantiator::TransformDecl(SourceLocation Loc, Decl *D) {
                                             TTP->getPosition()))
         return D;
 
-      TemplateName Template
-        = TemplateArgs(TTP->getDepth(), TTP->getPosition()).getAsTemplate();
+      TemplateArgument Arg = TemplateArgs(TTP->getDepth(), TTP->getPosition());
+      
+      if (TTP->isParameterPack()) {
+        assert(Arg.getKind() == TemplateArgument::Pack && 
+               "Missing argument pack");
+        
+        assert(getSema().ArgumentPackSubstitutionIndex >= 0);        
+        assert(getSema().ArgumentPackSubstitutionIndex < (int)Arg.pack_size());
+        Arg = Arg.pack_begin()[getSema().ArgumentPackSubstitutionIndex];
+      }
+
+      TemplateName Template = Arg.getAsTemplate();
       assert(!Template.isNull() && Template.getAsTemplateDecl() &&
              "Wrong kind of template template argument");
       return Template.getAsTemplateDecl();
@@ -700,8 +836,23 @@ TemplateInstantiator::TransformFirstQualifierInScope(NamedDecl *D,
   if (TemplateTypeParmDecl *TTPD = dyn_cast_or_null<TemplateTypeParmDecl>(D)) {
     const TemplateTypeParmType *TTP 
       = cast<TemplateTypeParmType>(getSema().Context.getTypeDeclType(TTPD));
+    
     if (TTP->getDepth() < TemplateArgs.getNumLevels()) {
-      QualType T = TemplateArgs(TTP->getDepth(), TTP->getIndex()).getAsType();
+      // FIXME: This needs testing w/ member access expressions.
+      TemplateArgument Arg = TemplateArgs(TTP->getDepth(), TTP->getIndex());
+      
+      if (TTP->isParameterPack()) {
+        assert(Arg.getKind() == TemplateArgument::Pack && 
+               "Missing argument pack");
+        
+        if (getSema().ArgumentPackSubstitutionIndex == -1)
+          return 0;
+        
+        assert(getSema().ArgumentPackSubstitutionIndex < (int)Arg.pack_size());
+        Arg = Arg.pack_begin()[getSema().ArgumentPackSubstitutionIndex];
+      }
+
+      QualType T = Arg.getAsType();
       if (T.isNull())
         return cast_or_null<NamedDecl>(TransformDecl(Loc, D));
       
@@ -719,13 +870,11 @@ TemplateInstantiator::TransformFirstQualifierInScope(NamedDecl *D,
 
 VarDecl *
 TemplateInstantiator::RebuildExceptionDecl(VarDecl *ExceptionDecl,
-                                           QualType T,
                                            TypeSourceInfo *Declarator,
                                            IdentifierInfo *Name,
-                                           SourceLocation Loc,
-                                           SourceRange TypeRange) {
-  VarDecl *Var = inherited::RebuildExceptionDecl(ExceptionDecl, T, Declarator,
-                                                 Name, Loc, TypeRange);
+                                           SourceLocation Loc) {
+  VarDecl *Var = inherited::RebuildExceptionDecl(ExceptionDecl, Declarator,
+                                                 Name, Loc);
   if (Var)
     getSema().CurrentInstantiationScope->InstantiatedLocal(ExceptionDecl, Var);
   return Var;
@@ -741,14 +890,14 @@ VarDecl *TemplateInstantiator::RebuildObjCExceptionDecl(VarDecl *ExceptionDecl,
 }
 
 QualType
-TemplateInstantiator::RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
+TemplateInstantiator::RebuildElaboratedType(SourceLocation KeywordLoc,
+                                            ElaboratedTypeKeyword Keyword,
                                             NestedNameSpecifier *NNS,
                                             QualType T) {
   if (const TagType *TT = T->getAs<TagType>()) {
     TagDecl* TD = TT->getDecl();
 
-    // FIXME: this location is very wrong;  we really need typelocs.
-    SourceLocation TagLocation = TD->getTagKeywordLoc();
+    SourceLocation TagLocation = KeywordLoc;
 
     // FIXME: type might be anonymous.
     IdentifierInfo *Id = TD->getIdentifier();
@@ -767,14 +916,69 @@ TemplateInstantiator::RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
     }
   }
 
-  return TreeTransform<TemplateInstantiator>::RebuildElaboratedType(Keyword,
+  return TreeTransform<TemplateInstantiator>::RebuildElaboratedType(KeywordLoc,
+                                                                    Keyword,
                                                                     NNS, T);
 }
 
+TemplateName TemplateInstantiator::TransformTemplateName(TemplateName Name,
+                                                         QualType ObjectType,
+                                             NamedDecl *FirstQualifierInScope) {
+  if (TemplateTemplateParmDecl *TTP
+       = dyn_cast_or_null<TemplateTemplateParmDecl>(Name.getAsTemplateDecl())) {
+    if (TTP->getDepth() < TemplateArgs.getNumLevels()) {
+      // If the corresponding template argument is NULL or non-existent, it's
+      // because we are performing instantiation from explicitly-specified
+      // template arguments in a function template, but there were some
+      // arguments left unspecified.
+      if (!TemplateArgs.hasTemplateArgument(TTP->getDepth(),
+                                            TTP->getPosition()))
+        return Name;
+      
+      TemplateArgument Arg = TemplateArgs(TTP->getDepth(), TTP->getPosition());
+      
+      if (TTP->isParameterPack()) {
+        assert(Arg.getKind() == TemplateArgument::Pack && 
+               "Missing argument pack");
+        
+        if (getSema().ArgumentPackSubstitutionIndex == -1) {
+          // We have the template argument pack to substitute, but we're not
+          // actually expanding the enclosing pack expansion yet. So, just
+          // keep the entire argument pack.
+          return getSema().Context.getSubstTemplateTemplateParmPack(TTP, Arg);
+        }
+        
+        assert(getSema().ArgumentPackSubstitutionIndex < (int)Arg.pack_size());
+        Arg = Arg.pack_begin()[getSema().ArgumentPackSubstitutionIndex];
+      }
+      
+      TemplateName Template = Arg.getAsTemplate();
+      assert(!Template.isNull() && Template.getAsTemplateDecl() &&
+             "Wrong kind of template template argument");
+      return Template;
+    }
+  }
+  
+  if (SubstTemplateTemplateParmPackStorage *SubstPack
+                                  = Name.getAsSubstTemplateTemplateParmPack()) {
+    if (getSema().ArgumentPackSubstitutionIndex == -1)
+      return Name;
+
+    const TemplateArgument &ArgPack = SubstPack->getArgumentPack();
+    assert(getSema().ArgumentPackSubstitutionIndex < (int)ArgPack.pack_size() &&
+           "Pack substitution index out-of-range");
+    return ArgPack.pack_begin()[getSema().ArgumentPackSubstitutionIndex]
+                                                               .getAsTemplate();
+  }
+  
+  return inherited::TransformTemplateName(Name, ObjectType, 
+                                          FirstQualifierInScope);
+}
+
 ExprResult 
 TemplateInstantiator::TransformPredefinedExpr(PredefinedExpr *E) {
   if (!E->isTypeDependent())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   FunctionDecl *currentDecl = getSema().getCurFunctionDecl();
   assert(currentDecl && "Must have current function declaration when "
@@ -802,15 +1006,37 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
   // arguments left unspecified.
   if (!TemplateArgs.hasTemplateArgument(NTTP->getDepth(),
                                         NTTP->getPosition()))
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
-  const TemplateArgument &Arg = TemplateArgs(NTTP->getDepth(),
-                                             NTTP->getPosition());
+  TemplateArgument Arg = TemplateArgs(NTTP->getDepth(), NTTP->getPosition());
+  if (NTTP->isParameterPack()) {
+    assert(Arg.getKind() == TemplateArgument::Pack && 
+           "Missing argument pack");
+    
+    if (getSema().ArgumentPackSubstitutionIndex == -1) {
+      // We have an argument pack, but we can't select a particular argument
+      // out of it yet. Therefore, we'll build an expression to hold on to that
+      // argument pack.
+      QualType TargetType = SemaRef.SubstType(NTTP->getType(), TemplateArgs,
+                                              E->getLocation(), 
+                                              NTTP->getDeclName());
+      if (TargetType.isNull())
+        return ExprError();
+      
+      return new (SemaRef.Context) SubstNonTypeTemplateParmPackExpr(TargetType,
+                                                                    NTTP, 
+                                                              E->getLocation(),
+                                                                    Arg);
+    }
+    
+    assert(getSema().ArgumentPackSubstitutionIndex < (int)Arg.pack_size());
+    Arg = Arg.pack_begin()[getSema().ArgumentPackSubstitutionIndex];
+  }
 
   // The template argument itself might be an expression, in which
   // case we just return that expression.
   if (Arg.getKind() == TemplateArgument::Expression)
-    return SemaRef.Owned(Arg.getAsExpr()->Retain());
+    return SemaRef.Owned(Arg.getAsExpr());
 
   if (Arg.getKind() == TemplateArgument::Declaration) {
     ValueDecl *VD = cast<ValueDecl>(Arg.getAsDecl());
@@ -825,9 +1051,19 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
 
     // Derive the type we want the substituted decl to have.  This had
     // better be non-dependent, or these checks will have serious problems.
-    QualType TargetType = SemaRef.SubstType(NTTP->getType(), TemplateArgs,
-                                            E->getLocation(), 
-                                            DeclarationName());
+    QualType TargetType;
+    if (NTTP->isExpandedParameterPack())
+      TargetType = NTTP->getExpansionType(
+                                      getSema().ArgumentPackSubstitutionIndex);
+    else if (NTTP->isParameterPack() && 
+             isa<PackExpansionType>(NTTP->getType())) {
+      TargetType = SemaRef.SubstType(
+                        cast<PackExpansionType>(NTTP->getType())->getPattern(),
+                                     TemplateArgs, E->getLocation(), 
+                                     NTTP->getDeclName());
+    } else
+      TargetType = SemaRef.SubstType(NTTP->getType(), TemplateArgs, 
+                                     E->getLocation(), NTTP->getDeclName());
     assert(!TargetType.isNull() && "type substitution failed for param type");
     assert(!TargetType->isDependentType() && "param type still dependent");
     return SemaRef.BuildExpressionFromDeclTemplateArgument(Arg,
@@ -839,6 +1075,50 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
                                                 E->getSourceRange().getBegin());
 }
                                                    
+ExprResult 
+TemplateInstantiator::TransformSubstNonTypeTemplateParmPackExpr(
+                                          SubstNonTypeTemplateParmPackExpr *E) {
+  if (getSema().ArgumentPackSubstitutionIndex == -1) {
+    // We aren't expanding the parameter pack, so just return ourselves.
+    return getSema().Owned(E);
+  }
+  
+  const TemplateArgument &ArgPack = E->getArgumentPack();
+  unsigned Index = (unsigned)getSema().ArgumentPackSubstitutionIndex;
+  assert(Index < ArgPack.pack_size() && "Substitution index out-of-range");
+  
+  const TemplateArgument &Arg = ArgPack.pack_begin()[Index];
+  if (Arg.getKind() == TemplateArgument::Expression)
+    return SemaRef.Owned(Arg.getAsExpr());
+  
+  if (Arg.getKind() == TemplateArgument::Declaration) {
+    ValueDecl *VD = cast<ValueDecl>(Arg.getAsDecl());
+    
+    // Find the instantiation of the template argument.  This is
+    // required for nested templates.
+    VD = cast_or_null<ValueDecl>(
+                   getSema().FindInstantiatedDecl(E->getParameterPackLocation(),
+                                                  VD, TemplateArgs));
+    if (!VD)
+      return ExprError();
+
+    QualType T;
+    NonTypeTemplateParmDecl *NTTP = E->getParameterPack();
+    if (NTTP->isExpandedParameterPack())
+      T = NTTP->getExpansionType(getSema().ArgumentPackSubstitutionIndex);
+    else if (const PackExpansionType *Expansion 
+                                = dyn_cast<PackExpansionType>(NTTP->getType()))
+      T = SemaRef.SubstType(Expansion->getPattern(), TemplateArgs, 
+                            E->getParameterPackLocation(), NTTP->getDeclName());
+    else
+      T = E->getType();
+    return SemaRef.BuildExpressionFromDeclTemplateArgument(Arg, T,
+                                                 E->getParameterPackLocation());
+  }
+    
+  return SemaRef.BuildExpressionFromIntegralTemplateArgument(Arg, 
+                                                 E->getParameterPackLocation());
+}
 
 ExprResult
 TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
@@ -865,23 +1145,23 @@ ExprResult TemplateInstantiator::TransformCXXDefaultArgExpr(
 }
 
 QualType TemplateInstantiator::TransformFunctionProtoType(TypeLocBuilder &TLB,
-                                                        FunctionProtoTypeLoc TL,
-                                                          QualType ObjectType) {
+                                                      FunctionProtoTypeLoc TL) {
   // We need a local instantiation scope for this function prototype.
   LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
-  return inherited::TransformFunctionProtoType(TLB, TL, ObjectType);
+  return inherited::TransformFunctionProtoType(TLB, TL);
 }
 
 ParmVarDecl *
-TemplateInstantiator::TransformFunctionTypeParam(ParmVarDecl *OldParm) {
-  return SemaRef.SubstParmVarDecl(OldParm, TemplateArgs);
+TemplateInstantiator::TransformFunctionTypeParam(ParmVarDecl *OldParm,
+                                       llvm::Optional<unsigned> NumExpansions) {
+  return SemaRef.SubstParmVarDecl(OldParm, TemplateArgs,
+                                  NumExpansions);
 }
 
 QualType
 TemplateInstantiator::TransformTemplateTypeParmType(TypeLocBuilder &TLB,
-                                                TemplateTypeParmTypeLoc TL, 
-                                                    QualType ObjectType) {
-  TemplateTypeParmType *T = TL.getTypePtr();
+                                                TemplateTypeParmTypeLoc TL) {
+  const TemplateTypeParmType *T = TL.getTypePtr();
   if (T->getDepth() < TemplateArgs.getNumLevels()) {
     // Replace the template type parameter with its corresponding
     // template argument.
@@ -897,12 +1177,32 @@ TemplateInstantiator::TransformTemplateTypeParmType(TypeLocBuilder &TLB,
       return TL.getType();
     }
 
-    assert(TemplateArgs(T->getDepth(), T->getIndex()).getKind()
-             == TemplateArgument::Type &&
+    TemplateArgument Arg = TemplateArgs(T->getDepth(), T->getIndex());
+    
+    if (T->isParameterPack()) {
+      assert(Arg.getKind() == TemplateArgument::Pack && 
+             "Missing argument pack");
+      
+      if (getSema().ArgumentPackSubstitutionIndex == -1) {
+        // We have the template argument pack, but we're not expanding the
+        // enclosing pack expansion yet. Just save the template argument
+        // pack for later substitution.
+        QualType Result
+          = getSema().Context.getSubstTemplateTypeParmPackType(T, Arg);
+        SubstTemplateTypeParmPackTypeLoc NewTL
+          = TLB.push<SubstTemplateTypeParmPackTypeLoc>(Result);
+        NewTL.setNameLoc(TL.getNameLoc());
+        return Result;
+      }
+      
+      assert(getSema().ArgumentPackSubstitutionIndex < (int)Arg.pack_size());
+      Arg = Arg.pack_begin()[getSema().ArgumentPackSubstitutionIndex];
+    }
+    
+    assert(Arg.getKind() == TemplateArgument::Type &&
            "Template argument kind mismatch");
 
-    QualType Replacement
-      = TemplateArgs(T->getDepth(), T->getIndex()).getAsType();
+    QualType Replacement = Arg.getAsType();
 
     // TODO: only do this uniquing once, at the start of instantiation.
     QualType Result
@@ -928,6 +1228,32 @@ TemplateInstantiator::TransformTemplateTypeParmType(TypeLocBuilder &TLB,
   return Result;
 }
 
+QualType 
+TemplateInstantiator::TransformSubstTemplateTypeParmPackType(
+                                                            TypeLocBuilder &TLB,
+                                         SubstTemplateTypeParmPackTypeLoc TL) {
+  if (getSema().ArgumentPackSubstitutionIndex == -1) {
+    // We aren't expanding the parameter pack, so just return ourselves.
+    SubstTemplateTypeParmPackTypeLoc NewTL
+      = TLB.push<SubstTemplateTypeParmPackTypeLoc>(TL.getType());
+    NewTL.setNameLoc(TL.getNameLoc());
+    return TL.getType();
+  }
+  
+  const TemplateArgument &ArgPack = TL.getTypePtr()->getArgumentPack();
+  unsigned Index = (unsigned)getSema().ArgumentPackSubstitutionIndex;
+  assert(Index < ArgPack.pack_size() && "Substitution index out-of-range");
+  
+  QualType Result = ArgPack.pack_begin()[Index].getAsType();
+  Result = getSema().Context.getSubstTemplateTypeParmType(
+                                      TL.getTypePtr()->getReplacedParameter(),
+                                                          Result);
+  SubstTemplateTypeParmTypeLoc NewTL
+    = TLB.push<SubstTemplateTypeParmTypeLoc>(Result);
+  NewTL.setNameLoc(TL.getNameLoc());
+  return Result;
+}
+
 /// \brief Perform substitution on the type T with a given set of template
 /// arguments.
 ///
@@ -971,6 +1297,36 @@ TypeSourceInfo *Sema::SubstType(TypeSourceInfo *T,
   return Instantiator.TransformType(T);
 }
 
+TypeSourceInfo *Sema::SubstType(TypeLoc TL,
+                                const MultiLevelTemplateArgumentList &Args,
+                                SourceLocation Loc,
+                                DeclarationName Entity) {
+  assert(!ActiveTemplateInstantiations.empty() &&
+         "Cannot perform an instantiation without some context on the "
+         "instantiation stack");
+  
+  if (TL.getType().isNull())
+    return 0;
+
+  if (!TL.getType()->isDependentType() && 
+      !TL.getType()->isVariablyModifiedType()) {
+    // FIXME: Make a copy of the TypeLoc data here, so that we can
+    // return a new TypeSourceInfo. Inefficient!
+    TypeLocBuilder TLB;
+    TLB.pushFullCopy(TL);
+    return TLB.getTypeSourceInfo(Context, TL.getType());
+  }
+
+  TemplateInstantiator Instantiator(*this, Args, Loc, Entity);
+  TypeLocBuilder TLB;
+  TLB.reserve(TL.getFullDataSize());
+  QualType Result = Instantiator.TransformType(TLB, TL);
+  if (Result.isNull())
+    return 0;
+
+  return TLB.getTypeSourceInfo(Context, Result);
+}
+
 /// Deprecated form of the above.
 QualType Sema::SubstType(QualType T,
                          const MultiLevelTemplateArgumentList &TemplateArgs,
@@ -992,7 +1348,7 @@ static bool NeedsInstantiationAsFunctionType(TypeSourceInfo *T) {
   if (T->getType()->isDependentType() || T->getType()->isVariablyModifiedType())
     return true;
 
-  TypeLoc TL = T->getTypeLoc();
+  TypeLoc TL = T->getTypeLoc().IgnoreParens();
   if (!isa<FunctionProtoTypeLoc>(TL))
     return false;
 
@@ -1003,7 +1359,7 @@ static bool NeedsInstantiationAsFunctionType(TypeSourceInfo *T) {
     // TODO: currently we always rebuild expressions.  When we
     // properly get lazier about this, we should use the same
     // logic to avoid rebuilding prototypes here.
-    if (P->hasInit())
+    if (P->hasDefaultArg())
       return true;
   }
 
@@ -1031,7 +1387,7 @@ TypeSourceInfo *Sema::SubstFunctionDeclType(TypeSourceInfo *T,
   TypeLoc TL = T->getTypeLoc();
   TLB.reserve(TL.getFullDataSize());
 
-  QualType Result = Instantiator.TransformType(TLB, TL, QualType());
+  QualType Result = Instantiator.TransformType(TLB, TL);
   if (Result.isNull())
     return 0;
 
@@ -1039,10 +1395,34 @@ TypeSourceInfo *Sema::SubstFunctionDeclType(TypeSourceInfo *T,
 }
 
 ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm, 
-                          const MultiLevelTemplateArgumentList &TemplateArgs) {
+                            const MultiLevelTemplateArgumentList &TemplateArgs,
+                                    llvm::Optional<unsigned> NumExpansions) {
   TypeSourceInfo *OldDI = OldParm->getTypeSourceInfo();
-  TypeSourceInfo *NewDI = SubstType(OldDI, TemplateArgs, OldParm->getLocation(),
-                                    OldParm->getDeclName());
+  TypeSourceInfo *NewDI = 0;
+  
+  TypeLoc OldTL = OldDI->getTypeLoc();
+  if (isa<PackExpansionTypeLoc>(OldTL)) {    
+    PackExpansionTypeLoc ExpansionTL = cast<PackExpansionTypeLoc>(OldTL);
+    
+    // We have a function parameter pack. Substitute into the pattern of the 
+    // expansion.
+    NewDI = SubstType(ExpansionTL.getPatternLoc(), TemplateArgs, 
+                      OldParm->getLocation(), OldParm->getDeclName());
+    if (!NewDI)
+      return 0;
+        
+    if (NewDI->getType()->containsUnexpandedParameterPack()) {
+      // We still have unexpanded parameter packs, which means that
+      // our function parameter is still a function parameter pack.
+      // Therefore, make its type a pack expansion type.
+      NewDI = CheckPackExpansion(NewDI, ExpansionTL.getEllipsisLoc(),
+                                 NumExpansions);
+    }
+  } else {
+    NewDI = SubstType(OldDI, TemplateArgs, OldParm->getLocation(), 
+                      OldParm->getDeclName());
+  }
+  
   if (!NewDI)
     return 0;
 
@@ -1064,12 +1444,24 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
   if (OldParm->hasUninstantiatedDefaultArg()) {
     Expr *Arg = OldParm->getUninstantiatedDefaultArg();
     NewParm->setUninstantiatedDefaultArg(Arg);
+  } else if (OldParm->hasUnparsedDefaultArg()) {
+    NewParm->setUnparsedDefaultArg();
+    UnparsedDefaultArgInstantiations[OldParm].push_back(NewParm);
   } else if (Expr *Arg = OldParm->getDefaultArg())
     NewParm->setUninstantiatedDefaultArg(Arg);
 
   NewParm->setHasInheritedDefaultArg(OldParm->hasInheritedDefaultArg());
 
-  CurrentInstantiationScope->InstantiatedLocal(OldParm, NewParm);
+  // FIXME: When OldParm is a parameter pack and NewParm is not a parameter
+  // pack, we actually have a set of instantiated locations. Maintain this set!
+  if (OldParm->isParameterPack() && !NewParm->isParameterPack()) {
+    // Add the new parameter to 
+    CurrentInstantiationScope->InstantiatedLocalPackArg(OldParm, NewParm);
+  } else {
+    // Introduce an Old -> New mapping
+    CurrentInstantiationScope->InstantiatedLocal(OldParm, NewParm);  
+  }
+  
   // FIXME: OldParm may come from a FunctionProtoType, in which case CurContext
   // can be anything, is this right ?
   NewParm->setDeclContext(CurContext);
@@ -1077,6 +1469,24 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
   return NewParm;  
 }
 
+/// \brief Substitute the given template arguments into the given set of
+/// parameters, producing the set of parameter types that would be generated
+/// from such a substitution.
+bool Sema::SubstParmTypes(SourceLocation Loc, 
+                          ParmVarDecl **Params, unsigned NumParams,
+                          const MultiLevelTemplateArgumentList &TemplateArgs,
+                          llvm::SmallVectorImpl<QualType> &ParamTypes,
+                          llvm::SmallVectorImpl<ParmVarDecl *> *OutParams) {
+  assert(!ActiveTemplateInstantiations.empty() &&
+         "Cannot perform an instantiation without some context on the "
+         "instantiation stack");
+  
+  TemplateInstantiator Instantiator(*this, TemplateArgs, Loc, 
+                                    DeclarationName());
+  return Instantiator.TransformFunctionTypeParams(Loc, Params, NumParams, 0,
+                                                  ParamTypes, OutParams);
+}
+
 /// \brief Perform substitution on the base class specifiers of the
 /// given class template specialization.
 ///
@@ -1093,17 +1503,64 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
          Base = Pattern->bases_begin(), BaseEnd = Pattern->bases_end();
        Base != BaseEnd; ++Base) {
     if (!Base->getType()->isDependentType()) {
-      const CXXRecordDecl *BaseDecl =
-        cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
-      
-      // Make sure to set the attributes from the base.
-      SetClassDeclAttributesFromBase(Instantiation, BaseDecl, 
-                                     Base->isVirtual());
-      
       InstantiatedBases.push_back(new (Context) CXXBaseSpecifier(*Base));
       continue;
     }
 
+    SourceLocation EllipsisLoc;
+    if (Base->isPackExpansion()) {
+      // This is a pack expansion. See whether we should expand it now, or
+      // wait until later.
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      collectUnexpandedParameterPacks(Base->getTypeSourceInfo()->getTypeLoc(),
+                                      Unexpanded);
+      bool ShouldExpand = false;
+      bool RetainExpansion = false;
+      llvm::Optional<unsigned> NumExpansions;
+      if (CheckParameterPacksForExpansion(Base->getEllipsisLoc(), 
+                                          Base->getSourceRange(),
+                                          Unexpanded.data(), Unexpanded.size(),
+                                          TemplateArgs, ShouldExpand, 
+                                          RetainExpansion,
+                                          NumExpansions)) {
+        Invalid = true;
+        continue;
+      }
+      
+      // If we should expand this pack expansion now, do so.
+      if (ShouldExpand) {
+        for (unsigned I = 0; I != *NumExpansions; ++I) {
+            Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(*this, I);
+          
+          TypeSourceInfo *BaseTypeLoc = SubstType(Base->getTypeSourceInfo(),
+                                                  TemplateArgs,
+                                              Base->getSourceRange().getBegin(),
+                                                  DeclarationName());
+          if (!BaseTypeLoc) {
+            Invalid = true;
+            continue;
+          }
+          
+          if (CXXBaseSpecifier *InstantiatedBase
+                = CheckBaseSpecifier(Instantiation,
+                                     Base->getSourceRange(),
+                                     Base->isVirtual(),
+                                     Base->getAccessSpecifierAsWritten(),
+                                     BaseTypeLoc,
+                                     SourceLocation()))
+            InstantiatedBases.push_back(InstantiatedBase);
+          else
+            Invalid = true;
+        }
+      
+        continue;
+      }
+      
+      // The resulting base specifier will (still) be a pack expansion.
+      EllipsisLoc = Base->getEllipsisLoc();
+    }
+    
+    Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(*this, -1);
     TypeSourceInfo *BaseTypeLoc = SubstType(Base->getTypeSourceInfo(),
                                             TemplateArgs,
                                             Base->getSourceRange().getBegin(),
@@ -1118,7 +1575,8 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
                                Base->getSourceRange(),
                                Base->isVirtual(),
                                Base->getAccessSpecifierAsWritten(),
-                               BaseTypeLoc))
+                               BaseTypeLoc,
+                               EllipsisLoc))
       InstantiatedBases.push_back(InstantiatedBase);
     else
       Invalid = true;
@@ -1221,11 +1679,29 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   if (SubstBaseSpecifiers(Instantiation, Pattern, TemplateArgs))
     Invalid = true;
 
+  TemplateDeclInstantiator Instantiator(*this, Instantiation, TemplateArgs);
   llvm::SmallVector<Decl*, 4> Fields;
   for (RecordDecl::decl_iterator Member = Pattern->decls_begin(),
          MemberEnd = Pattern->decls_end();
        Member != MemberEnd; ++Member) {
-    Decl *NewMember = SubstDecl(*Member, Instantiation, TemplateArgs);
+    // Don't instantiate members not belonging in this semantic context.
+    // e.g. for:
+    // @code
+    //    template <int i> class A {
+    //      class B *g;
+    //    };
+    // @endcode
+    // 'class B' has the template as lexical context but semantically it is
+    // introduced in namespace scope.
+    if ((*Member)->getDeclContext() != Pattern)
+      continue;
+
+    if ((*Member)->isInvalidDecl()) {
+      Invalid = true; 
+      continue;
+    }
+
+    Decl *NewMember = Instantiator.Visit(*Member);
     if (NewMember) {
       if (FieldDecl *Field = dyn_cast<FieldDecl>(NewMember))
         Fields.push_back(Field);
@@ -1245,7 +1721,22 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   CheckCompletedCXXClass(Instantiation);
   if (Instantiation->isInvalidDecl())
     Invalid = true;
-  
+  else {
+    // Instantiate any out-of-line class template partial
+    // specializations now.
+    for (TemplateDeclInstantiator::delayed_partial_spec_iterator 
+              P = Instantiator.delayed_partial_spec_begin(),
+           PEnd = Instantiator.delayed_partial_spec_end();
+         P != PEnd; ++P) {
+      if (!Instantiator.InstantiateClassTemplatePartialSpecialization(
+                                                                P->first,
+                                                                P->second)) {
+        Invalid = true;
+        break;
+      }
+    }
+  }
+
   // Exit the scope of this instantiation.
   SavedContext.pop();
 
@@ -1261,6 +1752,15 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   return Invalid;
 }
 
+namespace {
+  /// \brief A partial specialization whose template arguments have matched
+  /// a given template-id.
+  struct PartialSpecMatchResult {
+    ClassTemplatePartialSpecializationDecl *Partial;
+    TemplateArgumentList *Args;
+  };
+}
+
 bool
 Sema::InstantiateClassTemplateSpecialization(
                            SourceLocation PointOfInstantiation,
@@ -1310,8 +1810,7 @@ Sema::InstantiateClassTemplateSpecialization(
   //   matching the template arguments of the class template
   //   specialization with the template argument lists of the partial
   //   specializations.
-  typedef std::pair<ClassTemplatePartialSpecializationDecl *,
-                    TemplateArgumentList *> MatchResult;
+  typedef PartialSpecMatchResult MatchResult;
   llvm::SmallVector<MatchResult, 4> Matched;
   llvm::SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
   Template->getPartialSpecializations(PartialSpecs);
@@ -1326,10 +1825,17 @@ Sema::InstantiateClassTemplateSpecialization(
       // diagnostics, later.
       (void)Result;
     } else {
-      Matched.push_back(std::make_pair(Partial, Info.take()));
+      Matched.push_back(PartialSpecMatchResult());
+      Matched.back().Partial = Partial;
+      Matched.back().Args = Info.take();
     }
   }
 
+  // If we're dealing with a member template where the template parameters
+  // have been instantiated, this provides the original template parameters
+  // from which the member template's parameters were instantiated.
+  llvm::SmallVector<const NamedDecl *, 4> InstantiatedTemplateParameters;
+  
   if (Matched.size() >= 1) {
     llvm::SmallVector<MatchResult, 4>::iterator Best = Matched.begin();
     if (Matched.size() == 1) {
@@ -1347,9 +1853,9 @@ Sema::InstantiateClassTemplateSpecialization(
       for (llvm::SmallVector<MatchResult, 4>::iterator P = Best + 1,
                                                     PEnd = Matched.end();
            P != PEnd; ++P) {
-        if (getMoreSpecializedPartialSpecialization(P->first, Best->first,
+        if (getMoreSpecializedPartialSpecialization(P->Partial, Best->Partial,
                                                     PointOfInstantiation) 
-              == P->first)
+              == P->Partial)
           Best = P;
       }
       
@@ -1360,9 +1866,9 @@ Sema::InstantiateClassTemplateSpecialization(
                                                     PEnd = Matched.end();
            P != PEnd; ++P) {
         if (P != Best &&
-            getMoreSpecializedPartialSpecialization(P->first, Best->first,
+            getMoreSpecializedPartialSpecialization(P->Partial, Best->Partial,
                                                     PointOfInstantiation)
-              != Best->first) {
+              != Best->Partial) {
           Ambiguous = true;
           break;
         }
@@ -1378,16 +1884,17 @@ Sema::InstantiateClassTemplateSpecialization(
         for (llvm::SmallVector<MatchResult, 4>::iterator P = Matched.begin(),
                                                       PEnd = Matched.end();
              P != PEnd; ++P)
-          Diag(P->first->getLocation(), diag::note_partial_spec_match)
-            << getTemplateArgumentBindingsText(P->first->getTemplateParameters(),
-                                               *P->second);
+          Diag(P->Partial->getLocation(), diag::note_partial_spec_match)
+            << getTemplateArgumentBindingsText(
+                                            P->Partial->getTemplateParameters(),
+                                               *P->Args);
 
         return true;
       }
     }
     
     // Instantiate using the best class template partial specialization.
-    ClassTemplatePartialSpecializationDecl *OrigPartialSpec = Best->first;
+    ClassTemplatePartialSpecializationDecl *OrigPartialSpec = Best->Partial;
     while (OrigPartialSpec->getInstantiatedFromMember()) {
       // If we've found an explicit specialization of this class template,
       // stop here and use that as the pattern.
@@ -1398,7 +1905,7 @@ Sema::InstantiateClassTemplateSpecialization(
     }
     
     Pattern = OrigPartialSpec;
-    ClassTemplateSpec->setInstantiationOf(Best->first, Best->second);
+    ClassTemplateSpec->setInstantiationOf(Best->Partial, Best->Args);
   } else {
     //   -- If no matches are found, the instantiation is generated
     //      from the primary template.
@@ -1449,7 +1956,7 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
         if (CheckSpecializationInstantiationRedecl(PointOfInstantiation, TSK, 
                                                    Function, 
                                         MSInfo->getTemplateSpecializationKind(),
-                                              MSInfo->getPointOfInstantiation(), 
+                                              MSInfo->getPointOfInstantiation(),
                                                    SuppressNew) ||
             SuppressNew)
           continue;
@@ -1485,7 +1992,7 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
         if (CheckSpecializationInstantiationRedecl(PointOfInstantiation, TSK, 
                                                    Var, 
                                         MSInfo->getTemplateSpecializationKind(),
-                                              MSInfo->getPointOfInstantiation(), 
+                                              MSInfo->getPointOfInstantiation(),
                                                    SuppressNew) ||
             SuppressNew)
           continue;
@@ -1520,11 +2027,11 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
       if (MSInfo->getTemplateSpecializationKind()
                                                 == TSK_ExplicitSpecialization)
         continue;
-      
+
       if (CheckSpecializationInstantiationRedecl(PointOfInstantiation, TSK, 
                                                  Record, 
                                         MSInfo->getTemplateSpecializationKind(),
-                                              MSInfo->getPointOfInstantiation(), 
+                                              MSInfo->getPointOfInstantiation(),
                                                  SuppressNew) ||
           SuppressNew)
         continue;
@@ -1551,6 +2058,13 @@ Sema::InstantiateClassMembers(SourceLocation PointOfInstantiation,
         InstantiateClass(PointOfInstantiation, Record, Pattern,
                          TemplateArgs,
                          TSK);
+      } else {
+        if (TSK == TSK_ExplicitInstantiationDefinition &&
+            Record->getTemplateSpecializationKind() ==
+                TSK_ExplicitInstantiationDeclaration) {
+          Record->setTemplateSpecializationKind(TSK);
+          MarkVTableUsed(PointOfInstantiation, Record, true);
+        }
       }
       
       Pattern = cast_or_null<CXXRecordDecl>(Record->getDefinition());
@@ -1604,6 +2118,18 @@ Sema::SubstExpr(Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs) {
   return Instantiator.TransformExpr(E);
 }
 
+bool Sema::SubstExprs(Expr **Exprs, unsigned NumExprs, bool IsCall,
+                      const MultiLevelTemplateArgumentList &TemplateArgs,
+                      llvm::SmallVectorImpl<Expr *> &Outputs) {
+  if (NumExprs == 0)
+    return false;
+  
+  TemplateInstantiator Instantiator(*this, TemplateArgs,
+                                    SourceLocation(),
+                                    DeclarationName());
+  return Instantiator.TransformExprs(Exprs, NumExprs, IsCall, Outputs);
+}
+
 /// \brief Do template substitution on a nested-name-specifier.
 NestedNameSpecifier *
 Sema::SubstNestedNameSpecifier(NestedNameSpecifier *NNS,
@@ -1631,35 +2157,107 @@ Sema::SubstTemplateName(TemplateName Name, SourceLocation Loc,
   return Instantiator.TransformTemplateName(Name);
 }
 
-bool Sema::Subst(const TemplateArgumentLoc &Input, TemplateArgumentLoc &Output,
+bool Sema::Subst(const TemplateArgumentLoc *Args, unsigned NumArgs,
+                 TemplateArgumentListInfo &Result,
                  const MultiLevelTemplateArgumentList &TemplateArgs) {
   TemplateInstantiator Instantiator(*this, TemplateArgs, SourceLocation(),
                                     DeclarationName());
-
-  return Instantiator.TransformTemplateArgument(Input, Output);
+  
+  return Instantiator.TransformTemplateArguments(Args, NumArgs, Result);
 }
 
-Decl *LocalInstantiationScope::getInstantiationOf(const Decl *D) {
-  for (LocalInstantiationScope *Current = this; Current; 
+llvm::PointerUnion<Decl *, LocalInstantiationScope::DeclArgumentPack *> *
+LocalInstantiationScope::findInstantiationOf(const Decl *D) {
+  for (LocalInstantiationScope *Current = this; Current;
        Current = Current->Outer) {
+
     // Check if we found something within this scope.
-    llvm::DenseMap<const Decl *, Decl *>::iterator Found
-      = Current->LocalDecls.find(D);
-    if (Found != Current->LocalDecls.end())
-      return Found->second;
-   
+    const Decl *CheckD = D;
+    do {
+      LocalDeclsMap::iterator Found = Current->LocalDecls.find(CheckD);
+      if (Found != Current->LocalDecls.end())
+        return &Found->second;
+      
+      // If this is a tag declaration, it's possible that we need to look for
+      // a previous declaration.
+      if (const TagDecl *Tag = dyn_cast<TagDecl>(CheckD))
+        CheckD = Tag->getPreviousDeclaration();
+      else
+        CheckD = 0;
+    } while (CheckD);
+    
     // If we aren't combined with our outer scope, we're done. 
     if (!Current->CombineWithOuterScope)
       break;
   }
-  
-  assert(D->isInvalidDecl() && 
-         "declaration was not instantiated in this scope!");
+
+  // If we didn't find the decl, then we either have a sema bug, or we have a
+  // forward reference to a label declaration.  Return null to indicate that
+  // we have an uninstantiated label.
+  assert(isa<LabelDecl>(D) && "declaration not instantiated in this scope");
   return 0;
 }
 
 void LocalInstantiationScope::InstantiatedLocal(const Decl *D, Decl *Inst) {
-  Decl *&Stored = LocalDecls[D];
-  assert((!Stored || Stored == Inst)&& "Already instantiated this local");
-  Stored = Inst;
+  llvm::PointerUnion<Decl *, DeclArgumentPack *> &Stored = LocalDecls[D];
+  if (Stored.isNull())
+    Stored = Inst;
+  else if (Stored.is<Decl *>()) {
+    assert(Stored.get<Decl *>() == Inst && "Already instantiated this local");
+    Stored = Inst;
+  } else
+    LocalDecls[D].get<DeclArgumentPack *>()->push_back(Inst);
+}
+
+void LocalInstantiationScope::InstantiatedLocalPackArg(const Decl *D, 
+                                                       Decl *Inst) {
+  DeclArgumentPack *Pack = LocalDecls[D].get<DeclArgumentPack *>();
+  Pack->push_back(Inst);
+}
+
+void LocalInstantiationScope::MakeInstantiatedLocalArgPack(const Decl *D) {
+  llvm::PointerUnion<Decl *, DeclArgumentPack *> &Stored = LocalDecls[D];
+  assert(Stored.isNull() && "Already instantiated this local");
+  DeclArgumentPack *Pack = new DeclArgumentPack;
+  Stored = Pack;
+  ArgumentPacks.push_back(Pack);
+}
+
+void LocalInstantiationScope::SetPartiallySubstitutedPack(NamedDecl *Pack, 
+                                          const TemplateArgument *ExplicitArgs,
+                                                    unsigned NumExplicitArgs) {
+  assert((!PartiallySubstitutedPack || PartiallySubstitutedPack == Pack) &&
+         "Already have a partially-substituted pack");
+  assert((!PartiallySubstitutedPack 
+          || NumArgsInPartiallySubstitutedPack == NumExplicitArgs) &&
+         "Wrong number of arguments in partially-substituted pack");
+  PartiallySubstitutedPack = Pack;
+  ArgsInPartiallySubstitutedPack = ExplicitArgs;
+  NumArgsInPartiallySubstitutedPack = NumExplicitArgs;
+}
+
+NamedDecl *LocalInstantiationScope::getPartiallySubstitutedPack(
+                                         const TemplateArgument **ExplicitArgs,
+                                              unsigned *NumExplicitArgs) const {
+  if (ExplicitArgs)
+    *ExplicitArgs = 0;
+  if (NumExplicitArgs)
+    *NumExplicitArgs = 0;
+  
+  for (const LocalInstantiationScope *Current = this; Current; 
+       Current = Current->Outer) {
+    if (Current->PartiallySubstitutedPack) {
+      if (ExplicitArgs)
+        *ExplicitArgs = Current->ArgsInPartiallySubstitutedPack;
+      if (NumExplicitArgs)
+        *NumExplicitArgs = Current->NumArgsInPartiallySubstitutedPack;
+      
+      return Current->PartiallySubstitutedPack;
+    }
+
+    if (!Current->CombineWithOuterScope)
+      break;
+  }
+  
+  return 0;
 }
diff --git a/lib/Sema/SemaTemplateInstantiateDecl.cpp b/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 1c7869fecd86..c0150c07bbf1 100644
--- a/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -25,85 +25,6 @@
 
 using namespace clang;
 
-namespace {
-  class TemplateDeclInstantiator
-    : public DeclVisitor<TemplateDeclInstantiator, Decl *> {
-    Sema &SemaRef;
-    DeclContext *Owner;
-    const MultiLevelTemplateArgumentList &TemplateArgs;
-
-  public:
-    TemplateDeclInstantiator(Sema &SemaRef, DeclContext *Owner,
-                             const MultiLevelTemplateArgumentList &TemplateArgs)
-      : SemaRef(SemaRef), Owner(Owner), TemplateArgs(TemplateArgs) { }
-
-    // FIXME: Once we get closer to completion, replace these manually-written
-    // declarations with automatically-generated ones from
-    // clang/AST/DeclNodes.inc.
-    Decl *VisitTranslationUnitDecl(TranslationUnitDecl *D);
-    Decl *VisitNamespaceDecl(NamespaceDecl *D);
-    Decl *VisitNamespaceAliasDecl(NamespaceAliasDecl *D);
-    Decl *VisitTypedefDecl(TypedefDecl *D);
-    Decl *VisitVarDecl(VarDecl *D);
-    Decl *VisitAccessSpecDecl(AccessSpecDecl *D);
-    Decl *VisitFieldDecl(FieldDecl *D);
-    Decl *VisitStaticAssertDecl(StaticAssertDecl *D);
-    Decl *VisitEnumDecl(EnumDecl *D);
-    Decl *VisitEnumConstantDecl(EnumConstantDecl *D);
-    Decl *VisitFriendDecl(FriendDecl *D);
-    Decl *VisitFunctionDecl(FunctionDecl *D,
-                            TemplateParameterList *TemplateParams = 0);
-    Decl *VisitCXXRecordDecl(CXXRecordDecl *D);
-    Decl *VisitCXXMethodDecl(CXXMethodDecl *D,
-                             TemplateParameterList *TemplateParams = 0);
-    Decl *VisitCXXConstructorDecl(CXXConstructorDecl *D);
-    Decl *VisitCXXDestructorDecl(CXXDestructorDecl *D);
-    Decl *VisitCXXConversionDecl(CXXConversionDecl *D);
-    ParmVarDecl *VisitParmVarDecl(ParmVarDecl *D);
-    Decl *VisitClassTemplateDecl(ClassTemplateDecl *D);
-    Decl *VisitClassTemplatePartialSpecializationDecl(
-                                    ClassTemplatePartialSpecializationDecl *D);
-    Decl *VisitFunctionTemplateDecl(FunctionTemplateDecl *D);
-    Decl *VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D);
-    Decl *VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D);
-    Decl *VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D);
-    Decl *VisitUsingDirectiveDecl(UsingDirectiveDecl *D);
-    Decl *VisitUsingDecl(UsingDecl *D);
-    Decl *VisitUsingShadowDecl(UsingShadowDecl *D);
-    Decl *VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D);
-    Decl *VisitUnresolvedUsingTypenameDecl(UnresolvedUsingTypenameDecl *D);
-
-    // Base case. FIXME: Remove once we can instantiate everything.
-    Decl *VisitDecl(Decl *D) {
-      unsigned DiagID = SemaRef.getDiagnostics().getCustomDiagID(
-                                                            Diagnostic::Error,
-                                                   "cannot instantiate %0 yet");
-      SemaRef.Diag(D->getLocation(), DiagID)
-        << D->getDeclKindName();
-      
-      return 0;
-    }
-
-    // Helper functions for instantiating methods.
-    TypeSourceInfo *SubstFunctionType(FunctionDecl *D,
-                             llvm::SmallVectorImpl<ParmVarDecl *> &Params);
-    bool InitFunctionInstantiation(FunctionDecl *New, FunctionDecl *Tmpl);
-    bool InitMethodInstantiation(CXXMethodDecl *New, CXXMethodDecl *Tmpl);
-
-    TemplateParameterList *
-      SubstTemplateParams(TemplateParameterList *List);
-
-    bool SubstQualifier(const DeclaratorDecl *OldDecl,
-                        DeclaratorDecl *NewDecl);
-    bool SubstQualifier(const TagDecl *OldDecl,
-                        TagDecl *NewDecl);
-      
-    bool InstantiateClassTemplatePartialSpecialization(
-                                              ClassTemplateDecl *ClassTemplate,
-                           ClassTemplatePartialSpecializationDecl *PartialSpec);
-  };
-}
-
 bool TemplateDeclInstantiator::SubstQualifier(const DeclaratorDecl *OldDecl,
                                               DeclaratorDecl *NewDecl) {
   NestedNameSpecifier *OldQual = OldDecl->getQualifier();
@@ -151,15 +72,15 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
 
         if (Aligned->isAlignmentExpr()) {
           ExprResult Result = SubstExpr(Aligned->getAlignmentExpr(),
-                                              TemplateArgs);
+                                        TemplateArgs);
           if (!Result.isInvalid())
             AddAlignedAttr(Aligned->getLocation(), New, Result.takeAs<Expr>());
         }
         else {
           TypeSourceInfo *Result = SubstType(Aligned->getAlignmentType(),
-                                              TemplateArgs,
-                                              Aligned->getLocation(), 
-                                              DeclarationName());
+                                             TemplateArgs,
+                                             Aligned->getLocation(), 
+                                             DeclarationName());
           if (Result)
             AddAlignedAttr(Aligned->getLocation(), New, Result);
         }
@@ -180,6 +101,14 @@ TemplateDeclInstantiator::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
 }
 
 Decl *
+TemplateDeclInstantiator::VisitLabelDecl(LabelDecl *D) {
+  LabelDecl *Inst = LabelDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+                                      D->getIdentifier());
+  Owner->addDecl(Inst);
+  return Inst;
+}
+
+Decl *
 TemplateDeclInstantiator::VisitNamespaceDecl(NamespaceDecl *D) {
   assert(false && "Namespaces cannot be instantiated");
   return D;
@@ -222,13 +151,15 @@ Decl *TemplateDeclInstantiator::VisitTypedefDecl(TypedefDecl *D) {
   if (Invalid)
     Typedef->setInvalidDecl();
 
-  if (const TagType *TT = DI->getType()->getAs<TagType>()) {
-    TagDecl *TD = TT->getDecl();
-    
-    // If the TagDecl that the TypedefDecl points to is an anonymous decl
-    // keep track of the TypedefDecl.
-    if (!TD->getIdentifier() && !TD->getTypedefForAnonDecl())
-      TD->setTypedefForAnonDecl(Typedef);
+  // If the old typedef was the name for linkage purposes of an anonymous
+  // tag decl, re-establish that relationship for the new typedef.
+  if (const TagType *oldTagType = D->getUnderlyingType()->getAs<TagType>()) {
+    TagDecl *oldTag = oldTagType->getDecl();
+    if (oldTag->getTypedefForAnonDecl() == D) {
+      TagDecl *newTag = DI->getType()->castAs<TagType>()->getDecl();
+      assert(!newTag->getIdentifier() && !newTag->getTypedefForAnonDecl());
+      newTag->setTypedefForAnonDecl(Typedef);
+    }
   }
   
   if (TypedefDecl *Prev = D->getPreviousDeclaration()) {
@@ -245,35 +176,6 @@ Decl *TemplateDeclInstantiator::VisitTypedefDecl(TypedefDecl *D) {
   return Typedef;
 }
 
-/// \brief Instantiate the arguments provided as part of initialization.
-///
-/// \returns true if an error occurred, false otherwise.
-static bool InstantiateInitializationArguments(Sema &SemaRef,
-                                               Expr **Args, unsigned NumArgs,
-                           const MultiLevelTemplateArgumentList &TemplateArgs,
-                         llvm::SmallVectorImpl<SourceLocation> &FakeCommaLocs,
-                           ASTOwningVector<Expr*> &InitArgs) {
-  for (unsigned I = 0; I != NumArgs; ++I) {
-    // When we hit the first defaulted argument, break out of the loop:
-    // we don't pass those default arguments on.
-    if (Args[I]->isDefaultArgument())
-      break;
-  
-    ExprResult Arg = SemaRef.SubstExpr(Args[I], TemplateArgs);
-    if (Arg.isInvalid())
-      return true;
-  
-    Expr *ArgExpr = (Expr *)Arg.get();
-    InitArgs.push_back(Arg.release());
-    
-    // FIXME: We're faking all of the comma locations. Do we need them?
-    FakeCommaLocs.push_back(
-                          SemaRef.PP.getLocForEndOfToken(ArgExpr->getLocEnd()));
-  }
-  
-  return false;
-}
-
 /// \brief Instantiate an initializer, breaking it into separate
 /// initialization arguments.
 ///
@@ -290,8 +192,7 @@ static bool InstantiateInitializationArguments(Sema &SemaRef,
 static bool InstantiateInitializer(Sema &S, Expr *Init,
                             const MultiLevelTemplateArgumentList &TemplateArgs,
                                    SourceLocation &LParenLoc,
-                               llvm::SmallVector<SourceLocation, 4> &CommaLocs,
-                             ASTOwningVector<Expr*> &NewArgs,
+                                   ASTOwningVector<Expr*> &NewArgs,
                                    SourceLocation &RParenLoc) {
   NewArgs.clear();
   LParenLoc = SourceLocation();
@@ -300,7 +201,7 @@ static bool InstantiateInitializer(Sema &S, Expr *Init,
   if (!Init)
     return false;
 
-  if (CXXExprWithTemporaries *ExprTemp = dyn_cast<CXXExprWithTemporaries>(Init))
+  if (ExprWithCleanups *ExprTemp = dyn_cast<ExprWithCleanups>(Init))
     Init = ExprTemp->getSubExpr();
 
   while (CXXBindTemporaryExpr *Binder = dyn_cast<CXXBindTemporaryExpr>(Init))
@@ -312,24 +213,19 @@ static bool InstantiateInitializer(Sema &S, Expr *Init,
   if (ParenListExpr *ParenList = dyn_cast<ParenListExpr>(Init)) {
     LParenLoc = ParenList->getLParenLoc();
     RParenLoc = ParenList->getRParenLoc();
-    return InstantiateInitializationArguments(S, ParenList->getExprs(),
-                                              ParenList->getNumExprs(),
-                                              TemplateArgs, CommaLocs, 
-                                              NewArgs);
+    return S.SubstExprs(ParenList->getExprs(), ParenList->getNumExprs(),
+                        true, TemplateArgs, NewArgs);
   }
 
   if (CXXConstructExpr *Construct = dyn_cast<CXXConstructExpr>(Init)) {
     if (!isa<CXXTemporaryObjectExpr>(Construct)) {
-      if (InstantiateInitializationArguments(S,
-                                             Construct->getArgs(),
-                                             Construct->getNumArgs(),
-                                             TemplateArgs,
-                                             CommaLocs, NewArgs))
+      if (S.SubstExprs(Construct->getArgs(), Construct->getNumArgs(), true,
+                       TemplateArgs, NewArgs))
         return true;
 
       // FIXME: Fake locations!
       LParenLoc = S.PP.getLocForEndOfToken(Init->getLocStart());
-      RParenLoc = CommaLocs.empty()? LParenLoc : CommaLocs.back();
+      RParenLoc = LParenLoc;
       return false;
     }
   }
@@ -358,6 +254,12 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
   if (!DI)
     return 0;
 
+  if (DI->getType()->isFunctionType()) {
+    SemaRef.Diag(D->getLocation(), diag::err_variable_instantiates_to_function)
+      << D->isStaticDataMember() << DI->getType();
+    return 0;
+  }
+  
   // Build the instantiated declaration
   VarDecl *Var = VarDecl::Create(SemaRef.Context, Owner,
                                  D->getLocation(), D->getIdentifier(),
@@ -419,24 +321,24 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
 
     // Instantiate the initializer.
     SourceLocation LParenLoc, RParenLoc;
-    llvm::SmallVector<SourceLocation, 4> CommaLocs;
     ASTOwningVector<Expr*> InitArgs(SemaRef);
     if (!InstantiateInitializer(SemaRef, D->getInit(), TemplateArgs, LParenLoc,
-                                CommaLocs, InitArgs, RParenLoc)) {
-      // Attach the initializer to the declaration.
-      if (D->hasCXXDirectInitializer()) {
+                                InitArgs, RParenLoc)) {
+      bool TypeMayContainAuto = true;
+      // Attach the initializer to the declaration, if we have one.
+      if (InitArgs.size() == 0)
+        SemaRef.ActOnUninitializedDecl(Var, TypeMayContainAuto);
+      else if (D->hasCXXDirectInitializer()) {
         // Add the direct initializer to the declaration.
         SemaRef.AddCXXDirectInitializerToDecl(Var,
                                               LParenLoc,
                                               move_arg(InitArgs),
-                                              CommaLocs.data(),
-                                              RParenLoc);
-      } else if (InitArgs.size() == 1) {
-        Expr *Init = InitArgs.take()[0];
-        SemaRef.AddInitializerToDecl(Var, Init, false);
+                                              RParenLoc,
+                                              TypeMayContainAuto);
       } else {
-        assert(InitArgs.size() == 0);
-        SemaRef.ActOnUninitializedDecl(Var, false);    
+        assert(InitArgs.size() == 1);
+        Expr *Init = InitArgs.take()[0];
+        SemaRef.AddInitializerToDecl(Var, Init, false, TypeMayContainAuto);
       }
     } else {
       // FIXME: Not too happy about invalidating the declaration
@@ -540,6 +442,30 @@ Decl *TemplateDeclInstantiator::VisitFieldDecl(FieldDecl *D) {
   return Field;
 }
 
+Decl *TemplateDeclInstantiator::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
+  NamedDecl **NamedChain =
+    new (SemaRef.Context)NamedDecl*[D->getChainingSize()];
+
+  int i = 0;
+  for (IndirectFieldDecl::chain_iterator PI =
+       D->chain_begin(), PE = D->chain_end();
+       PI != PE; ++PI)
+    NamedChain[i++] = (SemaRef.FindInstantiatedDecl(D->getLocation(),
+                                            *PI, TemplateArgs));
+
+  QualType T = cast<FieldDecl>(NamedChain[i-1])->getType();
+  IndirectFieldDecl* IndirectField
+    = IndirectFieldDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+                                D->getIdentifier(), T,
+                                NamedChain, D->getChainingSize());
+
+
+  IndirectField->setImplicit(D->isImplicit());
+  IndirectField->setAccess(D->getAccess());
+  Owner->addDecl(IndirectField);
+  return IndirectField;
+}
+
 Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
   // Handle friend type expressions by simply substituting template
   // parameters into the pattern type and checking the result.
@@ -555,6 +481,7 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
       return 0;
     
     FD->setAccess(AS_public);
+    FD->setUnsupportedFriend(D->isUnsupportedFriend());
     Owner->addDecl(FD);
     return FD;
   } 
@@ -573,6 +500,7 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
     FriendDecl::Create(SemaRef.Context, Owner, D->getLocation(), 
                        cast<NamedDecl>(NewND), D->getFriendLoc());
   FD->setAccess(AS_public);
+  FD->setUnsupportedFriend(D->isUnsupportedFriend());
   Owner->addDecl(FD);
   return FD;
 }
@@ -589,7 +517,7 @@ Decl *TemplateDeclInstantiator::VisitStaticAssertDecl(StaticAssertDecl *D) {
     return 0;
 
   ExprResult Message(D->getMessage());
-  D->getMessage()->Retain();
+  D->getMessage();
   return SemaRef.ActOnStaticAssertDeclaration(D->getLocation(),
                                               InstantiatedAssertExpr.get(),
                                               Message.get());
@@ -599,7 +527,31 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
   EnumDecl *Enum = EnumDecl::Create(SemaRef.Context, Owner,
                                     D->getLocation(), D->getIdentifier(),
                                     D->getTagKeywordLoc(),
-                                    /*PrevDecl=*/0);
+                                    /*PrevDecl=*/0, D->isScoped(),
+                                    D->isScopedUsingClassTag(), D->isFixed());
+  if (D->isFixed()) {
+    if (TypeSourceInfo* TI = D->getIntegerTypeSourceInfo()) {
+      // If we have type source information for the underlying type, it means it
+      // has been explicitly set by the user. Perform substitution on it before
+      // moving on.
+      SourceLocation UnderlyingLoc = TI->getTypeLoc().getBeginLoc();
+      Enum->setIntegerTypeSourceInfo(SemaRef.SubstType(TI,
+                                                       TemplateArgs,
+                                                       UnderlyingLoc,
+                                                       DeclarationName()));
+      
+      if (!Enum->getIntegerTypeSourceInfo())
+        Enum->setIntegerType(SemaRef.Context.IntTy);
+    }
+    else {
+      assert(!D->getIntegerType()->isDependentType()
+             && "Dependent type without type source info");
+      Enum->setIntegerType(D->getIntegerType());
+    }
+  }
+
+  SemaRef.InstantiateAttrs(TemplateArgs, D, Enum);
+
   Enum->setInstantiationOfMemberEnum(D);
   Enum->setAccess(D->getAccess());
   if (SubstQualifier(D, Enum)) return 0;
@@ -644,6 +596,8 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
     }
 
     if (EnumConst) {
+      SemaRef.InstantiateAttrs(TemplateArgs, *EC, EnumConst);
+
       EnumConst->setAccess(Enum->getAccess());
       Enum->addDecl(EnumConst);
       Enumerators.push_back(EnumConst);
@@ -699,6 +653,15 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   CXXRecordDecl *PrevDecl = 0;
   ClassTemplateDecl *PrevClassTemplate = 0;
 
+  if (!isFriend && Pattern->getPreviousDeclaration()) {
+    DeclContext::lookup_result Found = Owner->lookup(Pattern->getDeclName());
+    if (Found.first != Found.second) {
+      PrevClassTemplate = dyn_cast<ClassTemplateDecl>(*Found.first);
+      if (PrevClassTemplate)
+        PrevDecl = PrevClassTemplate->getTemplatedDecl();
+    }
+  }
+
   // If this isn't a friend, then it's a member template, in which
   // case we just want to build the instantiation in the
   // specialization.  If it is a friend, we want to build it in
@@ -813,7 +776,8 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     // friend target decl?
   } else {
     Inst->setAccess(D->getAccess());
-    Inst->setInstantiatedFromMemberTemplate(D);
+    if (!PrevClassTemplate)
+      Inst->setInstantiatedFromMemberTemplate(D);
   }
   
   // Trigger creation of the type for the instantiation.
@@ -827,14 +791,18 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   }
   
   Owner->addDecl(Inst);
-  
-  // Instantiate all of the partial specializations of this member class 
-  // template.
-  llvm::SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
-  D->getPartialSpecializations(PartialSpecs);
-  for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I)
-    InstantiateClassTemplatePartialSpecialization(Inst, PartialSpecs[I]);
-  
+
+  if (!PrevClassTemplate) {
+    // Queue up any out-of-line partial specializations of this member
+    // class template; the client will force their instantiation once
+    // the enclosing class has been instantiated.
+    llvm::SmallVector<ClassTemplatePartialSpecializationDecl *, 4> PartialSpecs;
+    D->getPartialSpecializations(PartialSpecs);
+    for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I)
+      if (PartialSpecs[I]->isOutOfLine())
+        OutOfLinePartialSpecs.push_back(std::make_pair(Inst, PartialSpecs[I]));
+  }
+
   return Inst;
 }
 
@@ -855,7 +823,11 @@ TemplateDeclInstantiator::VisitClassTemplatePartialSpecializationDecl(
   if (!InstClassTemplate)
     return 0;
   
-  return InstClassTemplate->findPartialSpecInstantiatedFromMember(D);
+  if (ClassTemplatePartialSpecializationDecl *Result
+        = InstClassTemplate->findPartialSpecInstantiatedFromMember(D))
+    return Result;
+
+  return InstantiateClassTemplatePartialSpecialization(InstClassTemplate, D);
 }
 
 Decl *
@@ -1040,7 +1012,8 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
 
   // Attach the parameters
   for (unsigned P = 0; P < Params.size(); ++P)
-    Params[P]->setOwningFunction(Function);
+    if (Params[P])
+      Params[P]->setOwningFunction(Function);
   Function->setParams(Params.data(), Params.size());
 
   SourceLocation InstantiateAtPOI;
@@ -1078,7 +1051,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     std::pair<const TemplateArgument *, unsigned> Innermost 
       = TemplateArgs.getInnermost();
     Function->setFunctionTemplateSpecialization(FunctionTemplate,
-                  new (SemaRef.Context) TemplateArgumentList(SemaRef.Context,
+                            TemplateArgumentList::CreateCopy(SemaRef.Context,
                                                              Innermost.first,
                                                              Innermost.second),
                                                 InsertPos);
@@ -1092,7 +1065,6 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     Function->setInvalidDecl();
 
   bool Redeclaration = false;
-  bool OverloadableAttrRequired = false;
   bool isExplicitSpecialization = false;
     
   LookupResult Previous(SemaRef, Function->getDeclName(), SourceLocation(),
@@ -1108,13 +1080,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     // Instantiate the explicit template arguments.
     TemplateArgumentListInfo ExplicitArgs(Info->getLAngleLoc(),
                                           Info->getRAngleLoc());
-    for (unsigned I = 0, E = Info->getNumTemplateArgs(); I != E; ++I) {
-      TemplateArgumentLoc Loc;
-      if (SemaRef.Subst(Info->getTemplateArg(I), Loc, TemplateArgs))
-        return 0;
-
-      ExplicitArgs.addArgument(Loc);
-    }
+    if (SemaRef.Subst(Info->getTemplateArgs(), Info->getNumTemplateArgs(),
+                      ExplicitArgs, TemplateArgs))
+      return 0;
 
     // Map the candidate templates to their instantiations.
     for (unsigned I = 0, E = Info->getNumTemplates(); I != E; ++I) {
@@ -1148,8 +1116,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
   }
   
   SemaRef.CheckFunctionDeclaration(/*Scope*/ 0, Function, Previous,
-                                   isExplicitSpecialization, Redeclaration,
-                                   /*FIXME:*/OverloadableAttrRequired);
+                                   isExplicitSpecialization, Redeclaration);
 
   NamedDecl *PrincipalDecl = (TemplateParams
                               ? cast<NamedDecl>(FunctionTemplate)
@@ -1256,6 +1223,20 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
       cast<Decl>(Owner)->isDefinedOutsideFunctionOrMethod());
   LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
+  // Instantiate enclosing template arguments for friends.
+  llvm::SmallVector<TemplateParameterList *, 4> TempParamLists;
+  unsigned NumTempParamLists = 0;
+  if (isFriend && (NumTempParamLists = D->getNumTemplateParameterLists())) {
+    TempParamLists.set_size(NumTempParamLists);
+    for (unsigned I = 0; I != NumTempParamLists; ++I) {
+      TemplateParameterList *TempParams = D->getTemplateParameterList(I);
+      TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
+      if (!InstParams)
+        return NULL;
+      TempParamLists[I] = InstParams;
+    }
+  }
+
   llvm::SmallVector<ParmVarDecl *, 4> Params;
   TypeSourceInfo *TInfo = D->getTypeSourceInfo();
   TInfo = SubstFunctionType(D, Params);
@@ -1263,25 +1244,22 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     return 0;
   QualType T = TInfo->getType();
 
-  // \brief If the type of this function is not *directly* a function
-  // type, then we're instantiating the a function that was declared
-  // via a typedef, e.g.,
+  // \brief If the type of this function, after ignoring parentheses,
+  // is not *directly* a function type, then we're instantiating a function
+  // that was declared via a typedef, e.g.,
   //
   //   typedef int functype(int, int);
   //   functype func;
   //
   // In this case, we'll just go instantiate the ParmVarDecls that we
   // synthesized in the method declaration.
-  if (!isa<FunctionProtoType>(T)) {
+  if (!isa<FunctionProtoType>(T.IgnoreParens())) {
     assert(!Params.size() && "Instantiating type could not yield parameters");
-    for (unsigned I = 0, N = D->getNumParams(); I != N; ++I) {
-      ParmVarDecl *P = SemaRef.SubstParmVarDecl(D->getParamDecl(I), 
-                                                TemplateArgs);
-      if (!P)
-        return 0;
-
-      Params.push_back(P);
-    }
+    llvm::SmallVector<QualType, 4> ParamTypes;
+    if (SemaRef.SubstParmTypes(D->getLocation(), D->param_begin(), 
+                               D->getNumParams(), TemplateArgs, ParamTypes, 
+                               &Params))
+      return 0;    
   }
 
   NestedNameSpecifier *Qualifier = D->getQualifier();
@@ -1299,6 +1277,9 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
       SS.setScopeRep(Qualifier);
       SS.setRange(D->getQualifierRange());
       DC = SemaRef.computeDeclContext(SS);
+
+      if (DC && SemaRef.RequireCompleteDeclContext(SS, DC))
+        return 0;
     } else {
       DC = SemaRef.FindInstantiatedContext(D->getLocation(),
                                            D->getDeclContext(),
@@ -1321,7 +1302,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
                                         false);
   } else if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(D)) {
     Method = CXXDestructorDecl::Create(SemaRef.Context, Record,
-                                       NameInfo, T,
+                                       NameInfo, T, TInfo,
                                        Destructor->isInlineSpecified(),
                                        false);
   } else if (CXXConversionDecl *Conversion = dyn_cast<CXXConversionDecl>(D)) {
@@ -1369,9 +1350,9 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     std::pair<const TemplateArgument *, unsigned> Innermost 
       = TemplateArgs.getInnermost();
     Method->setFunctionTemplateSpecialization(FunctionTemplate,
-                    new (SemaRef.Context) TemplateArgumentList(SemaRef.Context,
-                                                              Innermost.first,
-                                                              Innermost.second),
+                         TemplateArgumentList::CreateCopy(SemaRef.Context,
+                                                          Innermost.first,
+                                                          Innermost.second),
                                               InsertPos);
   } else if (!isFriend) {
     // Record that this is an instantiation of a member function.
@@ -1382,6 +1363,11 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   // out-of-line, the instantiation will have the same lexical
   // context (which will be a namespace scope) as the template.
   if (isFriend) {
+    if (NumTempParamLists)
+      Method->setTemplateParameterListsInfo(SemaRef.Context,
+                                            NumTempParamLists,
+                                            TempParamLists.data());
+
     Method->setLexicalDeclContext(Owner);
     Method->setObjectOfFriendDecl(true);
   } else if (D->isOutOfLine())
@@ -1410,15 +1396,15 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   }
 
   bool Redeclaration = false;
-  bool OverloadableAttrRequired = false;
-  SemaRef.CheckFunctionDeclaration(0, Method, Previous, false, Redeclaration,
-                                   /*FIXME:*/OverloadableAttrRequired);
+  SemaRef.CheckFunctionDeclaration(0, Method, Previous, false, Redeclaration);
 
   if (D->isPure())
     SemaRef.CheckPureMethod(Method, SourceRange());
 
   Method->setAccess(D->getAccess());
 
+  SemaRef.CheckOverrideControl(Method);
+
   if (FunctionTemplate) {
     // If there's a function template, let our caller handle it.
   } else if (Method->isInvalidDecl() && !Previous.empty()) {
@@ -1449,7 +1435,7 @@ Decl *TemplateDeclInstantiator::VisitCXXConversionDecl(CXXConversionDecl *D) {
 }
 
 ParmVarDecl *TemplateDeclInstantiator::VisitParmVarDecl(ParmVarDecl *D) {
-  return SemaRef.SubstParmVarDecl(D, TemplateArgs);
+  return SemaRef.SubstParmVarDecl(D, TemplateArgs, llvm::Optional<unsigned>());
 }
 
 Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
@@ -1462,7 +1448,7 @@ Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
   TemplateTypeParmDecl *Inst =
     TemplateTypeParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
                                  TTPT->getDepth() - TemplateArgs.getNumLevels(),
-                                 TTPT->getIndex(),TTPT->getName(),
+                                 TTPT->getIndex(), D->getIdentifier(),
                                  D->wasDeclaredWithTypename(),
                                  D->isParameterPack());
 
@@ -1479,33 +1465,140 @@ Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
 Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
                                                  NonTypeTemplateParmDecl *D) {
   // Substitute into the type of the non-type template parameter.
+  TypeLoc TL = D->getTypeSourceInfo()->getTypeLoc();
+  llvm::SmallVector<TypeSourceInfo *, 4> ExpandedParameterPackTypesAsWritten;
+  llvm::SmallVector<QualType, 4> ExpandedParameterPackTypes;
+  bool IsExpandedParameterPack = false;
+  TypeSourceInfo *DI;  
   QualType T;
-  TypeSourceInfo *DI = D->getTypeSourceInfo();
-  if (DI) {
-    DI = SemaRef.SubstType(DI, TemplateArgs, D->getLocation(),
-                           D->getDeclName());
-    if (DI) T = DI->getType();
-  } else {
-    T = SemaRef.SubstType(D->getType(), TemplateArgs, D->getLocation(),
-                          D->getDeclName());
-    DI = 0;
-  }
-  if (T.isNull())
-    return 0;
-  
-  // Check that this type is acceptable for a non-type template parameter.
   bool Invalid = false;
-  T = SemaRef.CheckNonTypeTemplateParameterType(T, D->getLocation());
-  if (T.isNull()) {
-    T = SemaRef.Context.IntTy;
-    Invalid = true;
+
+  if (D->isExpandedParameterPack()) {
+    // The non-type template parameter pack is an already-expanded pack 
+    // expansion of types. Substitute into each of the expanded types.
+    ExpandedParameterPackTypes.reserve(D->getNumExpansionTypes());
+    ExpandedParameterPackTypesAsWritten.reserve(D->getNumExpansionTypes());
+    for (unsigned I = 0, N = D->getNumExpansionTypes(); I != N; ++I) {
+      TypeSourceInfo *NewDI =SemaRef.SubstType(D->getExpansionTypeSourceInfo(I),
+                                               TemplateArgs,
+                                               D->getLocation(), 
+                                               D->getDeclName());
+      if (!NewDI)
+        return 0;
+      
+      ExpandedParameterPackTypesAsWritten.push_back(NewDI);
+      QualType NewT =SemaRef.CheckNonTypeTemplateParameterType(NewDI->getType(),
+                                                              D->getLocation());
+      if (NewT.isNull())
+        return 0;
+      ExpandedParameterPackTypes.push_back(NewT);
+    }
+    
+    IsExpandedParameterPack = true;
+    DI = D->getTypeSourceInfo();
+    T = DI->getType();
+  } else if (isa<PackExpansionTypeLoc>(TL)) {
+    // The non-type template parameter pack's type is a pack expansion of types.
+    // Determine whether we need to expand this parameter pack into separate
+    // types.
+    PackExpansionTypeLoc Expansion = cast<PackExpansionTypeLoc>(TL);
+    TypeLoc Pattern = Expansion.getPatternLoc();
+    llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+    SemaRef.collectUnexpandedParameterPacks(Pattern, Unexpanded);
+    
+    // Determine whether the set of unexpanded parameter packs can and should
+    // be expanded.
+    bool Expand = true;
+    bool RetainExpansion = false;
+    llvm::Optional<unsigned> OrigNumExpansions
+      = Expansion.getTypePtr()->getNumExpansions();
+    llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
+    if (SemaRef.CheckParameterPacksForExpansion(Expansion.getEllipsisLoc(),
+                                                Pattern.getSourceRange(),
+                                                Unexpanded.data(),
+                                                Unexpanded.size(),
+                                                TemplateArgs,
+                                                Expand, RetainExpansion, 
+                                                NumExpansions))
+      return 0;
+    
+    if (Expand) {
+      for (unsigned I = 0; I != *NumExpansions; ++I) {
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, I);
+        TypeSourceInfo *NewDI = SemaRef.SubstType(Pattern, TemplateArgs,
+                                                  D->getLocation(), 
+                                                  D->getDeclName());
+        if (!NewDI)
+          return 0;
+        
+        ExpandedParameterPackTypesAsWritten.push_back(NewDI);
+        QualType NewT = SemaRef.CheckNonTypeTemplateParameterType(
+                                                              NewDI->getType(),
+                                                              D->getLocation());
+        if (NewT.isNull())
+          return 0;
+        ExpandedParameterPackTypes.push_back(NewT);
+      }
+      
+      // Note that we have an expanded parameter pack. The "type" of this
+      // expanded parameter pack is the original expansion type, but callers
+      // will end up using the expanded parameter pack types for type-checking.
+      IsExpandedParameterPack = true;
+      DI = D->getTypeSourceInfo();
+      T = DI->getType();
+    } else {
+      // We cannot fully expand the pack expansion now, so substitute into the
+      // pattern and create a new pack expansion type.
+      Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, -1);
+      TypeSourceInfo *NewPattern = SemaRef.SubstType(Pattern, TemplateArgs,
+                                                     D->getLocation(), 
+                                                     D->getDeclName());
+      if (!NewPattern)
+        return 0;
+      
+      DI = SemaRef.CheckPackExpansion(NewPattern, Expansion.getEllipsisLoc(),
+                                      NumExpansions);
+      if (!DI)
+        return 0;
+      
+      T = DI->getType();
+    }
+  } else {
+    // Simple case: substitution into a parameter that is not a parameter pack.
+    DI = SemaRef.SubstType(D->getTypeSourceInfo(), TemplateArgs, 
+                           D->getLocation(), D->getDeclName());
+    if (!DI)
+      return 0;
+    
+    // Check that this type is acceptable for a non-type template parameter.
+    bool Invalid = false;
+    T = SemaRef.CheckNonTypeTemplateParameterType(DI->getType(), 
+                                                  D->getLocation());
+    if (T.isNull()) {
+      T = SemaRef.Context.IntTy;
+      Invalid = true;
+    }
   }
   
-  NonTypeTemplateParmDecl *Param
-    = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+  NonTypeTemplateParmDecl *Param;
+  if (IsExpandedParameterPack)
+    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, 
+                                            D->getLocation(), 
                                     D->getDepth() - TemplateArgs.getNumLevels(), 
-                                      D->getPosition(), D->getIdentifier(), T, 
-                                      DI);
+                                            D->getPosition(), 
+                                            D->getIdentifier(), T,
+                                            DI,
+                                            ExpandedParameterPackTypes.data(),
+                                            ExpandedParameterPackTypes.size(),
+                                    ExpandedParameterPackTypesAsWritten.data());
+  else
+    Param = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, 
+                                            D->getLocation(),
+                                    D->getDepth() - TemplateArgs.getNumLevels(), 
+                                            D->getPosition(), 
+                                            D->getIdentifier(), T, 
+                                            D->isParameterPack(), DI);
+  
   if (Invalid)
     Param->setInvalidDecl();
   
@@ -1536,8 +1629,8 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
   TemplateTemplateParmDecl *Param
     = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
                                    D->getDepth() - TemplateArgs.getNumLevels(), 
-                                       D->getPosition(), D->getIdentifier(), 
-                                       InstParams);
+                                       D->getPosition(), D->isParameterPack(), 
+                                       D->getIdentifier(), InstParams);
   Param->setDefaultArgument(D->getDefaultArgument(), false);
   
   // Introduce this template parameter's instantiation into the instantiation 
@@ -1562,8 +1655,24 @@ Decl *TemplateDeclInstantiator::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
 }
 
 Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) {
-  // The nested name specifier is non-dependent, so no transformation
-  // is required. The same holds for the name info.
+
+  // The nested name specifier may be dependent, for example
+  //     template <typename T> struct t {
+  //       struct s1 { T f1(); };
+  //       struct s2 : s1 { using s1::f1; };
+  //     };
+  //     template struct t<int>;
+  // Here, in using s1::f1, s1 refers to t<T>::s1;
+  // we need to substitute for t<int>::s1.
+  NestedNameSpecifier *NNS =
+      SemaRef.SubstNestedNameSpecifier(D->getTargetNestedNameDecl(),
+      D->getNestedNameRange(),
+      TemplateArgs);
+  if (!NNS)
+      return 0;
+
+  // The name info is non-dependent, so no transformation
+  // is required.
   DeclarationNameInfo NameInfo = D->getNameInfo();
 
   // We only need to do redeclaration lookups if we're in a class
@@ -1577,12 +1686,12 @@ Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) {
   UsingDecl *NewUD = UsingDecl::Create(SemaRef.Context, Owner,
                                        D->getNestedNameRange(),
                                        D->getUsingLocation(),
-                                       D->getTargetNestedNameDecl(),
+                                       NNS,
                                        NameInfo,
                                        D->isTypeName());
 
   CXXScopeSpec SS;
-  SS.setScopeRep(D->getTargetNestedNameDecl());
+  SS.setScopeRep(NNS);
   SS.setRange(D->getNestedNameRange());
 
   if (CheckRedeclaration) {
@@ -1746,8 +1855,9 @@ TemplateDeclInstantiator::SubstTemplateParams(TemplateParameterList *L) {
 /// \param PartialSpec the (uninstantiated) class template partial 
 /// specialization that we are instantiating.
 ///
-/// \returns true if there was an error, false otherwise.
-bool 
+/// \returns The instantiated partial specialization, if successful; otherwise,
+/// NULL to indicate an error.
+ClassTemplatePartialSpecializationDecl *
 TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
                                             ClassTemplateDecl *ClassTemplate,
                           ClassTemplatePartialSpecializationDecl *PartialSpec) {
@@ -1761,47 +1871,39 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   TemplateParameterList *TempParams = PartialSpec->getTemplateParameters();
   TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
   if (!InstParams)
-    return true;
+    return 0;
   
   // Substitute into the template arguments of the class template partial
   // specialization.
-  const TemplateArgumentLoc *PartialSpecTemplateArgs
-    = PartialSpec->getTemplateArgsAsWritten();
-  unsigned N = PartialSpec->getNumTemplateArgsAsWritten();
-
   TemplateArgumentListInfo InstTemplateArgs; // no angle locations
-  for (unsigned I = 0; I != N; ++I) {
-    TemplateArgumentLoc Loc;
-    if (SemaRef.Subst(PartialSpecTemplateArgs[I], Loc, TemplateArgs))
-      return true;
-    InstTemplateArgs.addArgument(Loc);
-  }
+  if (SemaRef.Subst(PartialSpec->getTemplateArgsAsWritten(), 
+                    PartialSpec->getNumTemplateArgsAsWritten(), 
+                    InstTemplateArgs, TemplateArgs))
+    return 0;
   
-
   // Check that the template argument list is well-formed for this
   // class template.
-  TemplateArgumentListBuilder Converted(ClassTemplate->getTemplateParameters(), 
-                                        InstTemplateArgs.size());
+  llvm::SmallVector<TemplateArgument, 4> Converted;
   if (SemaRef.CheckTemplateArgumentList(ClassTemplate, 
                                         PartialSpec->getLocation(),
                                         InstTemplateArgs, 
                                         false,
                                         Converted))
-    return true;
+    return 0;
 
   // Figure out where to insert this class template partial specialization
   // in the member template's set of class template partial specializations.
   void *InsertPos = 0;
   ClassTemplateSpecializationDecl *PrevDecl
-    = ClassTemplate->findPartialSpecialization(Converted.getFlatArguments(),
-                                                Converted.flatSize(), InsertPos);
+    = ClassTemplate->findPartialSpecialization(Converted.data(),
+                                               Converted.size(), InsertPos);
   
   // Build the canonical type that describes the converted template
   // arguments of the class template partial specialization.
   QualType CanonType 
     = SemaRef.Context.getTemplateSpecializationType(TemplateName(ClassTemplate),
-                                                  Converted.getFlatArguments(),
-                                                    Converted.flatSize());
+                                                    Converted.data(),
+                                                    Converted.size());
 
   // Build the fully-sugared type for this class template
   // specialization as the user wrote in the specialization
@@ -1834,10 +1936,10 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
     //   Outer<int, int> outer; // error: the partial specializations of Inner
     //                          // have the same signature.
     SemaRef.Diag(PartialSpec->getLocation(), diag::err_partial_spec_redeclared)
-      << WrittenTy;
+      << WrittenTy->getType();
     SemaRef.Diag(PrevDecl->getLocation(), diag::note_prev_partial_spec_here)
       << SemaRef.Context.getTypeDeclType(PrevDecl);
-    return true;
+    return 0;
   }
   
   
@@ -1849,7 +1951,8 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
                                                      PartialSpec->getLocation(), 
                                                      InstParams,
                                                      ClassTemplate, 
-                                                     Converted,
+                                                     Converted.data(),
+                                                     Converted.size(),
                                                      InstTemplateArgs,
                                                      CanonType,
                                                      0,
@@ -1864,7 +1967,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   // Add this partial specialization to the set of class template partial
   // specializations.
   ClassTemplate->AddPartialSpecialization(InstPartialSpec, InsertPos);
-  return false;
+  return InstPartialSpec;
 }
 
 TypeSourceInfo*
@@ -1882,25 +1985,47 @@ TemplateDeclInstantiator::SubstFunctionType(FunctionDecl *D,
 
   if (NewTInfo != OldTInfo) {
     // Get parameters from the new type info.
-    TypeLoc OldTL = OldTInfo->getTypeLoc();
+    TypeLoc OldTL = OldTInfo->getTypeLoc().IgnoreParens();
     if (FunctionProtoTypeLoc *OldProtoLoc
                                   = dyn_cast<FunctionProtoTypeLoc>(&OldTL)) {
-      TypeLoc NewTL = NewTInfo->getTypeLoc();
+      TypeLoc NewTL = NewTInfo->getTypeLoc().IgnoreParens();
       FunctionProtoTypeLoc *NewProtoLoc = cast<FunctionProtoTypeLoc>(&NewTL);
       assert(NewProtoLoc && "Missing prototype?");
-      for (unsigned i = 0, i_end = NewProtoLoc->getNumArgs(); i != i_end; ++i) {
-        // FIXME: Variadic templates will break this.
-        Params.push_back(NewProtoLoc->getArg(i));
-        SemaRef.CurrentInstantiationScope->InstantiatedLocal(
-                                                        OldProtoLoc->getArg(i),
-                                                        NewProtoLoc->getArg(i));
+      unsigned NewIdx = 0, NumNewParams = NewProtoLoc->getNumArgs();
+      for (unsigned OldIdx = 0, NumOldParams = OldProtoLoc->getNumArgs();
+           OldIdx != NumOldParams; ++OldIdx) {
+        ParmVarDecl *OldParam = OldProtoLoc->getArg(OldIdx);
+        if (!OldParam->isParameterPack() ||
+            (NewIdx < NumNewParams &&
+             NewProtoLoc->getArg(NewIdx)->isParameterPack())) {
+          // Simple case: normal parameter, or a parameter pack that's 
+          // instantiated to a (still-dependent) parameter pack.
+          ParmVarDecl *NewParam = NewProtoLoc->getArg(NewIdx++);
+          Params.push_back(NewParam);
+          SemaRef.CurrentInstantiationScope->InstantiatedLocal(OldParam,
+                                                               NewParam);
+          continue;
+        }
+        
+        // Parameter pack: make the instantiation an argument pack.
+        SemaRef.CurrentInstantiationScope->MakeInstantiatedLocalArgPack(
+                                                                      OldParam);
+        unsigned NumArgumentsInExpansion
+          = SemaRef.getNumArgumentsInExpansion(OldParam->getType(),
+                                               TemplateArgs);
+        while (NumArgumentsInExpansion--) {
+          ParmVarDecl *NewParam = NewProtoLoc->getArg(NewIdx++);
+          Params.push_back(NewParam);
+          SemaRef.CurrentInstantiationScope->InstantiatedLocalPackArg(OldParam,
+                                                                      NewParam);
+        }
       }
     }
   } else {
     // The function type itself was not dependent and therefore no
     // substitution occurred. However, we still need to instantiate
     // the function parameters themselves.
-    TypeLoc OldTL = OldTInfo->getTypeLoc();
+    TypeLoc OldTL = OldTInfo->getTypeLoc().IgnoreParens();
     if (FunctionProtoTypeLoc *OldProtoLoc
                                     = dyn_cast<FunctionProtoTypeLoc>(&OldTL)) {
       for (unsigned i = 0, i_end = OldProtoLoc->getNumArgs(); i != i_end; ++i) {
@@ -1957,6 +2082,67 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
     llvm::SmallVector<QualType, 4> Exceptions;
     for (unsigned I = 0, N = Proto->getNumExceptions(); I != N; ++I) {
       // FIXME: Poor location information!
+      if (const PackExpansionType *PackExpansion
+            = Proto->getExceptionType(I)->getAs<PackExpansionType>()) {
+        // We have a pack expansion. Instantiate it.
+        llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+        SemaRef.collectUnexpandedParameterPacks(PackExpansion->getPattern(),
+                                                Unexpanded);
+        assert(!Unexpanded.empty() && 
+               "Pack expansion without parameter packs?");
+        
+        bool Expand = false;
+        bool RetainExpansion = false;
+        llvm::Optional<unsigned> NumExpansions
+                                          = PackExpansion->getNumExpansions();
+        if (SemaRef.CheckParameterPacksForExpansion(New->getLocation(), 
+                                                    SourceRange(),
+                                                    Unexpanded.data(), 
+                                                    Unexpanded.size(),
+                                                    TemplateArgs,
+                                                    Expand, 
+                                                    RetainExpansion,
+                                                    NumExpansions))
+          break;
+                      
+        if (!Expand) {
+          // We can't expand this pack expansion into separate arguments yet;
+          // just substitute into the pattern and create a new pack expansion 
+          // type.
+          Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, -1);
+          QualType T = SemaRef.SubstType(PackExpansion->getPattern(), 
+                                         TemplateArgs,
+                                       New->getLocation(), New->getDeclName());
+          if (T.isNull())
+            break;
+          
+          T = SemaRef.Context.getPackExpansionType(T, NumExpansions);
+          Exceptions.push_back(T);
+          continue;
+        }
+        
+        // Substitute into the pack expansion pattern for each template
+        bool Invalid = false;
+        for (unsigned ArgIdx = 0; ArgIdx != *NumExpansions; ++ArgIdx) {
+          Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, ArgIdx);
+          
+          QualType T = SemaRef.SubstType(PackExpansion->getPattern(), 
+                                         TemplateArgs,
+                                       New->getLocation(), New->getDeclName());
+          if (T.isNull()) {
+            Invalid = true;
+            break;
+          }
+          
+          Exceptions.push_back(T);
+        }
+        
+        if (Invalid)
+          break;
+        
+        continue;
+      }
+      
       QualType T
         = SemaRef.SubstType(Proto->getExceptionType(I), TemplateArgs,
                             New->getLocation(), New->getDeclName());
@@ -1969,19 +2155,20 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
 
     // Rebuild the function type 
 
+    FunctionProtoType::ExtProtoInfo EPI = Proto->getExtProtoInfo();
+    EPI.HasExceptionSpec = Proto->hasExceptionSpec();
+    EPI.HasAnyExceptionSpec = Proto->hasAnyExceptionSpec();
+    EPI.NumExceptions = Exceptions.size();
+    EPI.Exceptions = Exceptions.data();
+    EPI.ExtInfo = Proto->getExtInfo();
+
     const FunctionProtoType *NewProto
       = New->getType()->getAs<FunctionProtoType>();
     assert(NewProto && "Template instantiation without function prototype?");
     New->setType(SemaRef.Context.getFunctionType(NewProto->getResultType(),
                                                  NewProto->arg_type_begin(),
                                                  NewProto->getNumArgs(),
-                                                 NewProto->isVariadic(),
-                                                 NewProto->getTypeQuals(),
-                                                 Proto->hasExceptionSpec(),
-                                                 Proto->hasAnyExceptionSpec(),
-                                                 Exceptions.size(),
-                                                 Exceptions.data(),
-                                                 Proto->getExtInfo()));
+                                                 EPI));
   }
 
   SemaRef.InstantiateAttrs(TemplateArgs, Tmpl, New);
@@ -2000,10 +2187,9 @@ TemplateDeclInstantiator::InitMethodInstantiation(CXXMethodDecl *New,
   if (InitFunctionInstantiation(New, Tmpl))
     return true;
 
-  CXXRecordDecl *Record = cast<CXXRecordDecl>(Owner);
   New->setAccess(Tmpl->getAccess());
   if (Tmpl->isVirtualAsWritten())
-    Record->setMethodAsVirtual(New);
+    New->setVirtualAsWritten(true);
 
   // FIXME: attributes
   // FIXME: New needs a pointer to Tmpl
@@ -2084,9 +2270,12 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   // If we're performing recursive template instantiation, create our own
   // queue of pending implicit instantiations that we will instantiate later,
   // while we're still within our own instantiation context.
+  llvm::SmallVector<VTableUse, 16> SavedVTableUses;
   std::deque<PendingImplicitInstantiation> SavedPendingInstantiations;
-  if (Recursive)
+  if (Recursive) {
+    VTableUses.swap(SavedVTableUses);
     PendingInstantiations.swap(SavedPendingInstantiations);
+  }
 
   EnterExpressionEvaluationContext EvalContext(*this, 
                                                Sema::PotentiallyEvaluated);
@@ -2105,17 +2294,33 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   // Introduce the instantiated function parameters into the local
   // instantiation scope, and set the parameter names to those used
   // in the template.
+  unsigned FParamIdx = 0;
   for (unsigned I = 0, N = PatternDecl->getNumParams(); I != N; ++I) {
     const ParmVarDecl *PatternParam = PatternDecl->getParamDecl(I);
-    ParmVarDecl *FunctionParam = Function->getParamDecl(I);
-    FunctionParam->setDeclName(PatternParam->getDeclName());
-    Scope.InstantiatedLocal(PatternParam, FunctionParam);
+    if (!PatternParam->isParameterPack()) {
+      // Simple case: not a parameter pack.
+      assert(FParamIdx < Function->getNumParams());
+      ParmVarDecl *FunctionParam = Function->getParamDecl(I);
+      FunctionParam->setDeclName(PatternParam->getDeclName());
+      Scope.InstantiatedLocal(PatternParam, FunctionParam);
+      ++FParamIdx;
+      continue;
+    }
+    
+    // Expand the parameter pack.
+    Scope.MakeInstantiatedLocalArgPack(PatternParam);
+    for (unsigned NumFParams = Function->getNumParams(); 
+         FParamIdx < NumFParams; 
+         ++FParamIdx) {
+      ParmVarDecl *FunctionParam = Function->getParamDecl(FParamIdx);
+      FunctionParam->setDeclName(PatternParam->getDeclName());
+      Scope.InstantiatedLocalPackArg(PatternParam, FunctionParam);
+    }
   }
 
   // Enter the scope of this instantiation. We don't use
   // PushDeclContext because we don't have a scope.
-  DeclContext *PreviousContext = CurContext;
-  CurContext = Function;
+  Sema::ContextRAII savedContext(*this, Function);
 
   MultiLevelTemplateArgumentList TemplateArgs =
     getTemplateInstantiationArgs(Function, 0, false, PatternDecl);
@@ -2138,7 +2343,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
 
   PerformDependentDiagnostics(PatternDecl, TemplateArgs);
 
-  CurContext = PreviousContext;
+  savedContext.pop();
 
   DeclGroupRef DG(Function);
   Consumer.HandleTopLevelDecl(DG);
@@ -2149,10 +2354,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   Scope.Exit();
 
   if (Recursive) {
+    // Define any pending vtables.
+    DefineUsedVTables();
+
     // Instantiate any pending implicit instantiations found during the
     // instantiation of this template.
     PerformPendingInstantiations();
 
+    // Restore the set of pending vtables.
+    VTableUses.swap(SavedVTableUses);
+
     // Restore the set of pending implicit instantiations.
     PendingInstantiations.swap(SavedPendingInstantiations);
   }
@@ -2233,13 +2444,13 @@ void Sema::InstantiateStaticDataMemberDefinition(
 
   // Enter the scope of this instantiation. We don't use
   // PushDeclContext because we don't have a scope.
-  DeclContext *PreviousContext = CurContext;
-  CurContext = Var->getDeclContext();
+  ContextRAII previousContext(*this, Var->getDeclContext());
 
   VarDecl *OldVar = Var;
   Var = cast_or_null<VarDecl>(SubstDecl(Def, Var->getDeclContext(),
-                                          getTemplateInstantiationArgs(Var)));
-  CurContext = PreviousContext;
+                                        getTemplateInstantiationArgs(Var)));
+
+  previousContext.pop();
 
   if (Var) {
     MemberSpecializationInfo *MSInfo = OldVar->getMemberSpecializationInfo();
@@ -2272,7 +2483,7 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
   for (CXXConstructorDecl::init_const_iterator Inits = Tmpl->init_begin(),
                                             InitsEnd = Tmpl->init_end();
        Inits != InitsEnd; ++Inits) {
-    CXXBaseOrMemberInitializer *Init = *Inits;
+    CXXCtorInitializer *Init = *Inits;
 
     // Only instantiate written initializers, let Sema re-construct implicit
     // ones.
@@ -2281,11 +2492,75 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
 
     SourceLocation LParenLoc, RParenLoc;
     ASTOwningVector<Expr*> NewArgs(*this);
-    llvm::SmallVector<SourceLocation, 4> CommaLocs;
+
+    SourceLocation EllipsisLoc;
+    
+    if (Init->isPackExpansion()) {
+      // This is a pack expansion. We should expand it now.
+      TypeLoc BaseTL = Init->getBaseClassInfo()->getTypeLoc();
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      collectUnexpandedParameterPacks(BaseTL, Unexpanded);
+      bool ShouldExpand = false;
+      bool RetainExpansion = false;
+      llvm::Optional<unsigned> NumExpansions;
+      if (CheckParameterPacksForExpansion(Init->getEllipsisLoc(), 
+                                          BaseTL.getSourceRange(),
+                                          Unexpanded.data(), 
+                                          Unexpanded.size(),
+                                          TemplateArgs, ShouldExpand, 
+                                          RetainExpansion,
+                                          NumExpansions)) {
+        AnyErrors = true;
+        New->setInvalidDecl();
+        continue;
+      }
+      assert(ShouldExpand && "Partial instantiation of base initializer?");
+      
+      // Loop over all of the arguments in the argument pack(s), 
+      for (unsigned I = 0; I != *NumExpansions; ++I) {
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(*this, I);
+
+        // Instantiate the initializer.
+        if (InstantiateInitializer(*this, Init->getInit(), TemplateArgs, 
+                                   LParenLoc, NewArgs, RParenLoc)) {
+          AnyErrors = true;
+          break;
+        }
+
+        // Instantiate the base type.
+        TypeSourceInfo *BaseTInfo = SubstType(Init->getBaseClassInfo(), 
+                                              TemplateArgs, 
+                                              Init->getSourceLocation(), 
+                                              New->getDeclName());
+        if (!BaseTInfo) {
+          AnyErrors = true;
+          break;
+        }
+
+        // Build the initializer.
+        MemInitResult NewInit = BuildBaseInitializer(BaseTInfo->getType(), 
+                                                     BaseTInfo,
+                                                     (Expr **)NewArgs.data(),
+                                                     NewArgs.size(),
+                                                     Init->getLParenLoc(),
+                                                     Init->getRParenLoc(),
+                                                     New->getParent(),
+                                                     SourceLocation());
+        if (NewInit.isInvalid()) {
+          AnyErrors = true;
+          break;
+        }
+        
+        NewInits.push_back(NewInit.get());
+        NewArgs.clear();
+      }
+      
+      continue;
+    }
 
     // Instantiate the initializer.
     if (InstantiateInitializer(*this, Init->getInit(), TemplateArgs, 
-                               LParenLoc, CommaLocs, NewArgs, RParenLoc)) {
+                               LParenLoc, NewArgs, RParenLoc)) {
       AnyErrors = true;
       continue;
     }
@@ -2307,24 +2582,30 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
                                      NewArgs.size(),
                                      Init->getLParenLoc(),
                                      Init->getRParenLoc(),
-                                     New->getParent());
+                                     New->getParent(),
+                                     EllipsisLoc);
     } else if (Init->isMemberInitializer()) {
-      FieldDecl *Member;
-
-      // Is this an anonymous union?
-      if (FieldDecl *UnionInit = Init->getAnonUnionMember())
-        Member = cast<FieldDecl>(FindInstantiatedDecl(Init->getMemberLocation(),
-                                                      UnionInit, TemplateArgs));
-      else
-        Member = cast<FieldDecl>(FindInstantiatedDecl(Init->getMemberLocation(),
-                                                      Init->getMember(),
-                                                      TemplateArgs));
+      FieldDecl *Member = cast<FieldDecl>(FindInstantiatedDecl(
+                                                     Init->getMemberLocation(),
+                                                     Init->getMember(),
+                                                     TemplateArgs));
 
       NewInit = BuildMemberInitializer(Member, (Expr **)NewArgs.data(),
                                        NewArgs.size(),
                                        Init->getSourceLocation(),
                                        Init->getLParenLoc(),
                                        Init->getRParenLoc());
+    } else if (Init->isIndirectMemberInitializer()) {
+      IndirectFieldDecl *IndirectMember =
+         cast<IndirectFieldDecl>(FindInstantiatedDecl(
+                                 Init->getMemberLocation(),
+                                 Init->getIndirectMember(), TemplateArgs));
+
+      NewInit = BuildMemberInitializer(IndirectMember, (Expr **)NewArgs.data(),
+                                       NewArgs.size(),
+                                       Init->getSourceLocation(),
+                                       Init->getLParenLoc(),
+                                       Init->getRParenLoc());
     }
 
     if (NewInit.isInvalid()) {
@@ -2589,7 +2870,27 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
       (ParentDC->isFunctionOrMethod() && ParentDC->isDependentContext())) {
     // D is a local of some kind. Look into the map of local
     // declarations to their instantiations.
-    return cast<NamedDecl>(CurrentInstantiationScope->getInstantiationOf(D));
+    typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
+    llvm::PointerUnion<Decl *, DeclArgumentPack *> *Found
+      = CurrentInstantiationScope->findInstantiationOf(D);
+    
+    if (Found) {
+      if (Decl *FD = Found->dyn_cast<Decl *>())
+        return cast<NamedDecl>(FD);
+      
+      unsigned PackIdx = ArgumentPackSubstitutionIndex;
+      return cast<NamedDecl>((*Found->get<DeclArgumentPack *>())[PackIdx]);
+    }
+
+    // If we didn't find the decl, then we must have a label decl that hasn't
+    // been found yet.  Lazily instantiate it and return it now.
+    assert(isa<LabelDecl>(D));
+    
+    Decl *Inst = SubstDecl(D, CurContext, TemplateArgs);
+    assert(Inst && "Failed to instantiate label??");
+    
+    CurrentInstantiationScope->InstantiatedLocal(D, Inst);
+    return cast<LabelDecl>(Inst);
   }
 
   if (CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(D)) {
@@ -2693,6 +2994,8 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
         if (!Tag->isBeingDefined() &&
             RequireCompleteType(Loc, T, diag::err_incomplete_type))
           return 0;
+
+        ParentDC = Tag->getDecl();
       }
     }
 
@@ -2800,4 +3103,3 @@ void Sema::PerformDependentDiagnostics(const DeclContext *Pattern,
     }
   }
 }
-
diff --git a/lib/Sema/SemaTemplateVariadic.cpp b/lib/Sema/SemaTemplateVariadic.cpp
new file mode 100644
index 000000000000..0da801c7e343
--- /dev/null
+++ b/lib/Sema/SemaTemplateVariadic.cpp
@@ -0,0 +1,748 @@
+//===------- SemaTemplateVariadic.cpp - C++ Variadic Templates ------------===/
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//===----------------------------------------------------------------------===/
+//
+//  This file implements semantic analysis for C++0x variadic templates.
+//===----------------------------------------------------------------------===/
+
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Template.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/TypeLoc.h"
+
+using namespace clang;
+
+//----------------------------------------------------------------------------
+// Visitor that collects unexpanded parameter packs
+//----------------------------------------------------------------------------
+
+namespace {
+  /// \brief A class that collects unexpanded parameter packs.
+  class CollectUnexpandedParameterPacksVisitor :
+    public RecursiveASTVisitor<CollectUnexpandedParameterPacksVisitor> 
+  {
+    typedef RecursiveASTVisitor<CollectUnexpandedParameterPacksVisitor>
+      inherited;
+
+    llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded;
+
+  public:
+    explicit CollectUnexpandedParameterPacksVisitor(
+                  llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded)
+      : Unexpanded(Unexpanded) { }
+
+    bool shouldWalkTypesOfTypeLocs() const { return false; }
+    
+    //------------------------------------------------------------------------
+    // Recording occurrences of (unexpanded) parameter packs.
+    //------------------------------------------------------------------------
+
+    /// \brief Record occurrences of template type parameter packs.
+    bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) {
+      if (TL.getTypePtr()->isParameterPack())
+        Unexpanded.push_back(std::make_pair(TL.getTypePtr(), TL.getNameLoc()));
+      return true;
+    }
+
+    /// \brief Record occurrences of template type parameter packs
+    /// when we don't have proper source-location information for
+    /// them.
+    ///
+    /// Ideally, this routine would never be used.
+    bool VisitTemplateTypeParmType(TemplateTypeParmType *T) {
+      if (T->isParameterPack())
+        Unexpanded.push_back(std::make_pair(T, SourceLocation()));
+
+      return true;
+    }
+
+    /// \brief Record occurrences of function and non-type template
+    /// parameter packs in an expression.
+    bool VisitDeclRefExpr(DeclRefExpr *E) {
+      if (E->getDecl()->isParameterPack())
+        Unexpanded.push_back(std::make_pair(E->getDecl(), E->getLocation()));
+      
+      return true;
+    }
+    
+    // \brief Record occurrences of function and non-type template parameter
+    // packs in a block-captured expression.
+    bool VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
+      if (E->getDecl()->isParameterPack())
+        Unexpanded.push_back(std::make_pair(E->getDecl(), E->getLocation()));
+      
+      return true;
+    }
+    
+    /// \brief Record occurrences of template template parameter packs.
+    bool TraverseTemplateName(TemplateName Template) {
+      if (TemplateTemplateParmDecl *TTP 
+            = dyn_cast_or_null<TemplateTemplateParmDecl>(
+                                                  Template.getAsTemplateDecl()))
+        if (TTP->isParameterPack())
+          Unexpanded.push_back(std::make_pair(TTP, SourceLocation()));
+      
+      return inherited::TraverseTemplateName(Template);
+    }
+
+    //------------------------------------------------------------------------
+    // Pruning the search for unexpanded parameter packs.
+    //------------------------------------------------------------------------
+
+    /// \brief Suppress traversal into statements and expressions that
+    /// do not contain unexpanded parameter packs.
+    bool TraverseStmt(Stmt *S) { 
+      if (Expr *E = dyn_cast_or_null<Expr>(S))
+        if (E->containsUnexpandedParameterPack())
+          return inherited::TraverseStmt(E);
+
+      return true; 
+    }
+
+    /// \brief Suppress traversal into types that do not contain
+    /// unexpanded parameter packs.
+    bool TraverseType(QualType T) {
+      if (!T.isNull() && T->containsUnexpandedParameterPack())
+        return inherited::TraverseType(T);
+
+      return true;
+    }
+
+    /// \brief Suppress traversel into types with location information
+    /// that do not contain unexpanded parameter packs.
+    bool TraverseTypeLoc(TypeLoc TL) {
+      if (!TL.getType().isNull() && 
+          TL.getType()->containsUnexpandedParameterPack())
+        return inherited::TraverseTypeLoc(TL);
+
+      return true;
+    }
+
+    /// \brief Suppress traversal of non-parameter declarations, since
+    /// they cannot contain unexpanded parameter packs.
+    bool TraverseDecl(Decl *D) { 
+      if (D && isa<ParmVarDecl>(D))
+        return inherited::TraverseDecl(D);
+
+      return true; 
+    }
+
+    /// \brief Suppress traversal of template argument pack expansions.
+    bool TraverseTemplateArgument(const TemplateArgument &Arg) {
+      if (Arg.isPackExpansion())
+        return true;
+
+      return inherited::TraverseTemplateArgument(Arg);
+    }
+
+    /// \brief Suppress traversal of template argument pack expansions.
+    bool TraverseTemplateArgumentLoc(const TemplateArgumentLoc &ArgLoc) {
+      if (ArgLoc.getArgument().isPackExpansion())
+        return true;
+      
+      return inherited::TraverseTemplateArgumentLoc(ArgLoc);
+    }
+  };
+}
+
+/// \brief Diagnose all of the unexpanded parameter packs in the given
+/// vector.
+static void 
+DiagnoseUnexpandedParameterPacks(Sema &S, SourceLocation Loc,
+                                 Sema::UnexpandedParameterPackContext UPPC,
+             const llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  llvm::SmallVector<SourceLocation, 4> Locations;
+  llvm::SmallVector<IdentifierInfo *, 4> Names;
+  llvm::SmallPtrSet<IdentifierInfo *, 4> NamesKnown;
+
+  for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
+    IdentifierInfo *Name = 0;
+    if (const TemplateTypeParmType *TTP
+          = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>())
+      Name = TTP->getName();
+    else
+      Name = Unexpanded[I].first.get<NamedDecl *>()->getIdentifier();
+
+    if (Name && NamesKnown.insert(Name))
+      Names.push_back(Name);
+
+    if (Unexpanded[I].second.isValid())
+      Locations.push_back(Unexpanded[I].second);
+  }
+
+  DiagnosticBuilder DB
+    = Names.size() == 0? S.Diag(Loc, diag::err_unexpanded_parameter_pack_0)
+                           << (int)UPPC
+    : Names.size() == 1? S.Diag(Loc, diag::err_unexpanded_parameter_pack_1)
+                           << (int)UPPC << Names[0]
+    : Names.size() == 2? S.Diag(Loc, diag::err_unexpanded_parameter_pack_2)
+                           << (int)UPPC << Names[0] << Names[1]
+    : S.Diag(Loc, diag::err_unexpanded_parameter_pack_3_or_more)
+        << (int)UPPC << Names[0] << Names[1];
+
+  for (unsigned I = 0, N = Locations.size(); I != N; ++I)
+    DB << SourceRange(Locations[I]);
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(SourceLocation Loc, 
+                                           TypeSourceInfo *T,
+                                         UnexpandedParameterPackContext UPPC) {
+  // C++0x [temp.variadic]p5:
+  //   An appearance of a name of a parameter pack that is not expanded is 
+  //   ill-formed.
+  if (!T->getType()->containsUnexpandedParameterPack())
+    return false;
+
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseTypeLoc(
+                                                              T->getTypeLoc());
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, Loc, UPPC, Unexpanded);
+  return true;
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(Expr *E,
+                                        UnexpandedParameterPackContext UPPC) {
+  // C++0x [temp.variadic]p5:
+  //   An appearance of a name of a parameter pack that is not expanded is 
+  //   ill-formed.
+  if (!E->containsUnexpandedParameterPack())
+    return false;
+
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseStmt(E);
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, E->getLocStart(), UPPC, Unexpanded);
+  return true;
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(const CXXScopeSpec &SS,
+                                        UnexpandedParameterPackContext UPPC) {
+  // C++0x [temp.variadic]p5:
+  //   An appearance of a name of a parameter pack that is not expanded is 
+  //   ill-formed.
+  if (!SS.getScopeRep() || 
+      !SS.getScopeRep()->containsUnexpandedParameterPack())
+    return false;
+
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseNestedNameSpecifier(SS.getScopeRep());
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, SS.getRange().getBegin(), 
+                                   UPPC, Unexpanded);
+  return true;
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(const DeclarationNameInfo &NameInfo,
+                                         UnexpandedParameterPackContext UPPC) {
+  // C++0x [temp.variadic]p5:
+  //   An appearance of a name of a parameter pack that is not expanded is 
+  //   ill-formed.
+  switch (NameInfo.getName().getNameKind()) {
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXOperatorName:
+  case DeclarationName::CXXLiteralOperatorName:
+  case DeclarationName::CXXUsingDirective:
+    return false;
+
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    // FIXME: We shouldn't need this null check!
+    if (TypeSourceInfo *TSInfo = NameInfo.getNamedTypeInfo())
+      return DiagnoseUnexpandedParameterPack(NameInfo.getLoc(), TSInfo, UPPC);
+
+    if (!NameInfo.getName().getCXXNameType()->containsUnexpandedParameterPack())
+      return false;
+
+    break;
+  }
+
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseType(NameInfo.getName().getCXXNameType());
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, NameInfo.getLoc(), UPPC, Unexpanded);
+  return true;
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(SourceLocation Loc,
+                                           TemplateName Template,
+                                       UnexpandedParameterPackContext UPPC) {
+  
+  if (Template.isNull() || !Template.containsUnexpandedParameterPack())
+    return false;
+
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseTemplateName(Template);
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, Loc, UPPC, Unexpanded);
+  return true;
+}
+
+bool Sema::DiagnoseUnexpandedParameterPack(TemplateArgumentLoc Arg,
+                                         UnexpandedParameterPackContext UPPC) {
+  if (Arg.getArgument().isNull() || 
+      !Arg.getArgument().containsUnexpandedParameterPack())
+    return false;
+  
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseTemplateArgumentLoc(Arg);
+  assert(!Unexpanded.empty() && "Unable to find unexpanded parameter packs");
+  DiagnoseUnexpandedParameterPacks(*this, Arg.getLocation(), UPPC, Unexpanded);
+  return true;  
+}
+
+void Sema::collectUnexpandedParameterPacks(TemplateArgument Arg,
+                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseTemplateArgument(Arg);
+}
+
+void Sema::collectUnexpandedParameterPacks(TemplateArgumentLoc Arg,
+                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  CollectUnexpandedParameterPacksVisitor(Unexpanded)
+    .TraverseTemplateArgumentLoc(Arg);
+}
+
+void Sema::collectUnexpandedParameterPacks(QualType T,
+                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseType(T);  
+}  
+
+void Sema::collectUnexpandedParameterPacks(TypeLoc TL,
+                   llvm::SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseTypeLoc(TL);  
+}  
+
+ParsedTemplateArgument 
+Sema::ActOnPackExpansion(const ParsedTemplateArgument &Arg,
+                         SourceLocation EllipsisLoc) {
+  if (Arg.isInvalid())
+    return Arg;
+
+  switch (Arg.getKind()) {
+  case ParsedTemplateArgument::Type: {
+    TypeResult Result = ActOnPackExpansion(Arg.getAsType(), EllipsisLoc);
+    if (Result.isInvalid())
+      return ParsedTemplateArgument();
+
+    return ParsedTemplateArgument(Arg.getKind(), Result.get().getAsOpaquePtr(), 
+                                  Arg.getLocation());
+  }
+
+  case ParsedTemplateArgument::NonType: {
+    ExprResult Result = ActOnPackExpansion(Arg.getAsExpr(), EllipsisLoc);
+    if (Result.isInvalid())
+      return ParsedTemplateArgument();
+    
+    return ParsedTemplateArgument(Arg.getKind(), Result.get(), 
+                                  Arg.getLocation());
+  }
+    
+  case ParsedTemplateArgument::Template:
+    if (!Arg.getAsTemplate().get().containsUnexpandedParameterPack()) {
+      SourceRange R(Arg.getLocation());
+      if (Arg.getScopeSpec().isValid())
+        R.setBegin(Arg.getScopeSpec().getBeginLoc());
+      Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+        << R;
+      return ParsedTemplateArgument();
+    }
+      
+    return Arg.getTemplatePackExpansion(EllipsisLoc);
+  }
+  llvm_unreachable("Unhandled template argument kind?");
+  return ParsedTemplateArgument();
+}
+
+TypeResult Sema::ActOnPackExpansion(ParsedType Type, 
+                                    SourceLocation EllipsisLoc) {
+  TypeSourceInfo *TSInfo;
+  GetTypeFromParser(Type, &TSInfo);
+  if (!TSInfo)
+    return true;
+
+  TypeSourceInfo *TSResult = CheckPackExpansion(TSInfo, EllipsisLoc,
+                                                llvm::Optional<unsigned>());
+  if (!TSResult)
+    return true;
+  
+  return CreateParsedType(TSResult->getType(), TSResult);
+}
+
+TypeSourceInfo *Sema::CheckPackExpansion(TypeSourceInfo *Pattern,
+                                         SourceLocation EllipsisLoc,
+                                       llvm::Optional<unsigned> NumExpansions) {
+  // Create the pack expansion type and source-location information.
+  QualType Result = CheckPackExpansion(Pattern->getType(), 
+                                       Pattern->getTypeLoc().getSourceRange(),
+                                       EllipsisLoc, NumExpansions);
+  if (Result.isNull())
+    return 0;
+  
+  TypeSourceInfo *TSResult = Context.CreateTypeSourceInfo(Result);
+  PackExpansionTypeLoc TL = cast<PackExpansionTypeLoc>(TSResult->getTypeLoc());
+  TL.setEllipsisLoc(EllipsisLoc);
+  
+  // Copy over the source-location information from the type.
+  memcpy(TL.getNextTypeLoc().getOpaqueData(),
+         Pattern->getTypeLoc().getOpaqueData(),
+         Pattern->getTypeLoc().getFullDataSize());
+  return TSResult;
+}
+
+QualType Sema::CheckPackExpansion(QualType Pattern,
+                                  SourceRange PatternRange,
+                                  SourceLocation EllipsisLoc,
+                                  llvm::Optional<unsigned> NumExpansions) {
+  // C++0x [temp.variadic]p5:
+  //   The pattern of a pack expansion shall name one or more
+  //   parameter packs that are not expanded by a nested pack
+  //   expansion.
+  if (!Pattern->containsUnexpandedParameterPack()) {
+    Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+      << PatternRange;
+    return QualType();
+  }
+
+  return Context.getPackExpansionType(Pattern, NumExpansions);
+}
+
+ExprResult Sema::ActOnPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc) {
+  return CheckPackExpansion(Pattern, EllipsisLoc, llvm::Optional<unsigned>());
+}
+
+ExprResult Sema::CheckPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc,
+                                    llvm::Optional<unsigned> NumExpansions) {
+  if (!Pattern)
+    return ExprError();
+  
+  // C++0x [temp.variadic]p5:
+  //   The pattern of a pack expansion shall name one or more
+  //   parameter packs that are not expanded by a nested pack
+  //   expansion.
+  if (!Pattern->containsUnexpandedParameterPack()) {
+    Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+    << Pattern->getSourceRange();
+    return ExprError();
+  }
+  
+  // Create the pack expansion expression and source-location information.
+  return Owned(new (Context) PackExpansionExpr(Context.DependentTy, Pattern,
+                                               EllipsisLoc, NumExpansions));
+}
+
+/// \brief Retrieve the depth and index of a parameter pack.
+static std::pair<unsigned, unsigned> 
+getDepthAndIndex(NamedDecl *ND) {
+  if (TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(ND))
+    return std::make_pair(TTP->getDepth(), TTP->getIndex());
+  
+  if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(ND))
+    return std::make_pair(NTTP->getDepth(), NTTP->getIndex());
+  
+  TemplateTemplateParmDecl *TTP = cast<TemplateTemplateParmDecl>(ND);
+  return std::make_pair(TTP->getDepth(), TTP->getIndex());
+}
+
+bool Sema::CheckParameterPacksForExpansion(SourceLocation EllipsisLoc,
+                                           SourceRange PatternRange,
+                                     const UnexpandedParameterPack *Unexpanded,
+                                           unsigned NumUnexpanded,
+                             const MultiLevelTemplateArgumentList &TemplateArgs,
+                                           bool &ShouldExpand,
+                                           bool &RetainExpansion,
+                                     llvm::Optional<unsigned> &NumExpansions) {                                        
+  ShouldExpand = true;
+  RetainExpansion = false;
+  std::pair<IdentifierInfo *, SourceLocation> FirstPack;
+  bool HaveFirstPack = false;
+  
+  for (unsigned I = 0; I != NumUnexpanded; ++I) {
+    // Compute the depth and index for this parameter pack.
+    unsigned Depth = 0, Index = 0;
+    IdentifierInfo *Name;
+    bool IsFunctionParameterPack = false;
+    
+    if (const TemplateTypeParmType *TTP
+        = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>()) {
+      Depth = TTP->getDepth();
+      Index = TTP->getIndex();
+      Name = TTP->getName();
+    } else {
+      NamedDecl *ND = Unexpanded[I].first.get<NamedDecl *>();
+      if (isa<ParmVarDecl>(ND))
+        IsFunctionParameterPack = true;
+      else
+        llvm::tie(Depth, Index) = getDepthAndIndex(ND);        
+      
+      Name = ND->getIdentifier();
+    }
+    
+    // Determine the size of this argument pack.
+    unsigned NewPackSize;    
+    if (IsFunctionParameterPack) {
+      // Figure out whether we're instantiating to an argument pack or not.
+      typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
+      
+      llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation
+        = CurrentInstantiationScope->findInstantiationOf(
+                                        Unexpanded[I].first.get<NamedDecl *>());
+      if (Instantiation->is<DeclArgumentPack *>()) {
+        // We could expand this function parameter pack.
+        NewPackSize = Instantiation->get<DeclArgumentPack *>()->size();
+      } else {
+        // We can't expand this function parameter pack, so we can't expand
+        // the pack expansion.
+        ShouldExpand = false;
+        continue;
+      }
+    } else {
+      // If we don't have a template argument at this depth/index, then we 
+      // cannot expand the pack expansion. Make a note of this, but we still 
+      // want to check any parameter packs we *do* have arguments for.
+      if (Depth >= TemplateArgs.getNumLevels() ||
+          !TemplateArgs.hasTemplateArgument(Depth, Index)) {
+        ShouldExpand = false;
+        continue;
+      }
+      
+      // Determine the size of the argument pack.
+      NewPackSize = TemplateArgs(Depth, Index).pack_size();
+    }
+    
+    // C++0x [temp.arg.explicit]p9:
+    //   Template argument deduction can extend the sequence of template 
+    //   arguments corresponding to a template parameter pack, even when the
+    //   sequence contains explicitly specified template arguments.
+    if (!IsFunctionParameterPack) {
+      if (NamedDecl *PartialPack 
+                    = CurrentInstantiationScope->getPartiallySubstitutedPack()){
+        unsigned PartialDepth, PartialIndex;
+        llvm::tie(PartialDepth, PartialIndex) = getDepthAndIndex(PartialPack);
+        if (PartialDepth == Depth && PartialIndex == Index)
+          RetainExpansion = true;
+      }
+    }
+    
+    if (!NumExpansions) {
+      // The is the first pack we've seen for which we have an argument. 
+      // Record it.
+      NumExpansions = NewPackSize;
+      FirstPack.first = Name;
+      FirstPack.second = Unexpanded[I].second;
+      HaveFirstPack = true;
+      continue;
+    }
+    
+    if (NewPackSize != *NumExpansions) {
+      // C++0x [temp.variadic]p5:
+      //   All of the parameter packs expanded by a pack expansion shall have 
+      //   the same number of arguments specified.
+      if (HaveFirstPack)
+        Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict)
+          << FirstPack.first << Name << *NumExpansions << NewPackSize
+          << SourceRange(FirstPack.second) << SourceRange(Unexpanded[I].second);
+      else
+        Diag(EllipsisLoc, diag::err_pack_expansion_length_conflict_multilevel)
+          << Name << *NumExpansions << NewPackSize
+          << SourceRange(Unexpanded[I].second);
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+unsigned Sema::getNumArgumentsInExpansion(QualType T, 
+                          const MultiLevelTemplateArgumentList &TemplateArgs) {
+  QualType Pattern = cast<PackExpansionType>(T)->getPattern();
+  llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+  CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseType(Pattern);
+
+  for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
+    // Compute the depth and index for this parameter pack.
+    unsigned Depth;
+    unsigned Index;
+    
+    if (const TemplateTypeParmType *TTP
+          = Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>()) {
+      Depth = TTP->getDepth();
+      Index = TTP->getIndex();
+    } else {      
+      NamedDecl *ND = Unexpanded[I].first.get<NamedDecl *>();
+      if (isa<ParmVarDecl>(ND)) {
+        // Function parameter pack.
+        typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
+        
+        llvm::PointerUnion<Decl *, DeclArgumentPack *> *Instantiation
+          = CurrentInstantiationScope->findInstantiationOf(
+                                        Unexpanded[I].first.get<NamedDecl *>());
+        if (Instantiation->is<DeclArgumentPack *>())
+          return Instantiation->get<DeclArgumentPack *>()->size();
+        
+        continue;
+      }
+      
+      llvm::tie(Depth, Index) = getDepthAndIndex(ND);        
+    }
+    if (Depth >= TemplateArgs.getNumLevels() ||
+        !TemplateArgs.hasTemplateArgument(Depth, Index))
+      continue;
+    
+    // Determine the size of the argument pack.
+    return TemplateArgs(Depth, Index).pack_size();
+  }
+  
+  llvm_unreachable("No unexpanded parameter packs in type expansion.");
+  return 0;
+}
+
+bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
+  const DeclSpec &DS = D.getDeclSpec();
+  switch (DS.getTypeSpecType()) {
+  case TST_typename:
+  case TST_typeofType: {
+    QualType T = DS.getRepAsType().get();
+    if (!T.isNull() && T->containsUnexpandedParameterPack())
+      return true;
+    break;
+  }
+      
+  case TST_typeofExpr:
+  case TST_decltype:
+    if (DS.getRepAsExpr() && 
+        DS.getRepAsExpr()->containsUnexpandedParameterPack())
+      return true;
+    break;
+      
+  case TST_unspecified:
+  case TST_void:
+  case TST_char:
+  case TST_wchar:
+  case TST_char16:
+  case TST_char32:
+  case TST_int:
+  case TST_float:
+  case TST_double:
+  case TST_bool:
+  case TST_decimal32:
+  case TST_decimal64:
+  case TST_decimal128:
+  case TST_enum:
+  case TST_union:
+  case TST_struct:
+  case TST_class:
+  case TST_auto:
+  case TST_error:
+    break;
+  }
+  
+  for (unsigned I = 0, N = D.getNumTypeObjects(); I != N; ++I) {
+    const DeclaratorChunk &Chunk = D.getTypeObject(I);
+    switch (Chunk.Kind) {
+    case DeclaratorChunk::Pointer:
+    case DeclaratorChunk::Reference:
+    case DeclaratorChunk::Paren:
+      // These declarator chunks cannot contain any parameter packs.
+      break;
+        
+    case DeclaratorChunk::Array:
+    case DeclaratorChunk::Function:
+    case DeclaratorChunk::BlockPointer:
+      // Syntactically, these kinds of declarator chunks all come after the
+      // declarator-id (conceptually), so the parser should not invoke this
+      // routine at this time.
+      llvm_unreachable("Could not have seen this kind of declarator chunk");
+      break;
+        
+    case DeclaratorChunk::MemberPointer:
+      if (Chunk.Mem.Scope().getScopeRep() &&
+          Chunk.Mem.Scope().getScopeRep()->containsUnexpandedParameterPack())
+        return true;
+      break;
+    }
+  }
+  
+  return false;
+}
+
+/// \brief Called when an expression computing the size of a parameter pack
+/// is parsed.
+///
+/// \code
+/// template<typename ...Types> struct count {
+///   static const unsigned value = sizeof...(Types);
+/// };
+/// \endcode
+///
+//
+/// \param OpLoc The location of the "sizeof" keyword.
+/// \param Name The name of the parameter pack whose size will be determined.
+/// \param NameLoc The source location of the name of the parameter pack.
+/// \param RParenLoc The location of the closing parentheses.
+ExprResult Sema::ActOnSizeofParameterPackExpr(Scope *S,
+                                              SourceLocation OpLoc,
+                                              IdentifierInfo &Name,
+                                              SourceLocation NameLoc,
+                                              SourceLocation RParenLoc) {
+  // C++0x [expr.sizeof]p5:
+  //   The identifier in a sizeof... expression shall name a parameter pack.
+  LookupResult R(*this, &Name, NameLoc, LookupOrdinaryName);
+  LookupName(R, S);
+  
+  NamedDecl *ParameterPack = 0;
+  switch (R.getResultKind()) {
+  case LookupResult::Found:
+    ParameterPack = R.getFoundDecl();
+    break;
+    
+  case LookupResult::NotFound:
+  case LookupResult::NotFoundInCurrentInstantiation:
+    if (DeclarationName CorrectedName = CorrectTypo(R, S, 0, 0, false, 
+                                                    CTC_NoKeywords)) {
+      if (NamedDecl *CorrectedResult = R.getAsSingle<NamedDecl>())
+        if (CorrectedResult->isParameterPack()) {
+          ParameterPack = CorrectedResult;
+          Diag(NameLoc, diag::err_sizeof_pack_no_pack_name_suggest)
+            << &Name << CorrectedName
+            << FixItHint::CreateReplacement(NameLoc, 
+                                            CorrectedName.getAsString());
+          Diag(ParameterPack->getLocation(), diag::note_parameter_pack_here)
+            << CorrectedName;
+        }
+    }
+      
+  case LookupResult::FoundOverloaded:
+  case LookupResult::FoundUnresolvedValue:
+    break;
+    
+  case LookupResult::Ambiguous:
+    DiagnoseAmbiguousLookup(R);
+    return ExprError();
+  }
+  
+  if (!ParameterPack || !ParameterPack->isParameterPack()) {
+    Diag(NameLoc, diag::err_sizeof_pack_no_pack_name)
+      << &Name;
+    return ExprError();
+  }
+
+  return new (Context) SizeOfPackExpr(Context.getSizeType(), OpLoc, 
+                                      ParameterPack, NameLoc, RParenLoc);
+}
diff --git a/lib/Sema/SemaType.cpp b/lib/Sema/SemaType.cpp
index aa30b5c2da30..c88baa540f52 100644
--- a/lib/Sema/SemaType.cpp
+++ b/lib/Sema/SemaType.cpp
@@ -70,37 +70,455 @@ static bool isOmittedBlockReturnType(const Declarator &D) {
   return false;
 }
 
-typedef std::pair<const AttributeList*,QualType> DelayedAttribute;
-typedef llvm::SmallVectorImpl<DelayedAttribute> DelayedAttributeSet;
-
-static void ProcessTypeAttributeList(Sema &S, QualType &Type,
-                                     bool IsDeclSpec,
-                                     const AttributeList *Attrs,
-                                     DelayedAttributeSet &DelayedFnAttrs);
-static bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr);
-
-static void ProcessDelayedFnAttrs(Sema &S, QualType &Type,
-                                  DelayedAttributeSet &Attrs) {
-  for (DelayedAttributeSet::iterator I = Attrs.begin(),
-         E = Attrs.end(); I != E; ++I)
-    if (ProcessFnAttr(S, Type, *I->first)) {
-      S.Diag(I->first->getLoc(), diag::warn_function_attribute_wrong_type)
-        << I->first->getName() << I->second;
-      // Avoid any further processing of this attribute.
-      I->first->setInvalid();
-    }
-  Attrs.clear();
+// objc_gc applies to Objective-C pointers or, otherwise, to the
+// smallest available pointer type (i.e. 'void*' in 'void**').
+#define OBJC_POINTER_TYPE_ATTRS_CASELIST \
+    case AttributeList::AT_objc_gc
+
+// Function type attributes.
+#define FUNCTION_TYPE_ATTRS_CASELIST \
+    case AttributeList::AT_noreturn: \
+    case AttributeList::AT_cdecl: \
+    case AttributeList::AT_fastcall: \
+    case AttributeList::AT_stdcall: \
+    case AttributeList::AT_thiscall: \
+    case AttributeList::AT_pascal: \
+    case AttributeList::AT_regparm
+
+namespace {
+  /// An object which stores processing state for the entire
+  /// GetTypeForDeclarator process.
+  class TypeProcessingState {
+    Sema &sema;
+
+    /// The declarator being processed.
+    Declarator &declarator;
+
+    /// The index of the declarator chunk we're currently processing.
+    /// May be the total number of valid chunks, indicating the
+    /// DeclSpec.
+    unsigned chunkIndex;
+
+    /// Whether there are non-trivial modifications to the decl spec.
+    bool trivial;
+
+    /// The original set of attributes on the DeclSpec.
+    llvm::SmallVector<AttributeList*, 2> savedAttrs;
+
+    /// A list of attributes to diagnose the uselessness of when the
+    /// processing is complete.
+    llvm::SmallVector<AttributeList*, 2> ignoredTypeAttrs;
+
+  public:
+    TypeProcessingState(Sema &sema, Declarator &declarator)
+      : sema(sema), declarator(declarator),
+        chunkIndex(declarator.getNumTypeObjects()),
+        trivial(true) {}
+
+    Sema &getSema() const {
+      return sema;
+    }
+
+    Declarator &getDeclarator() const {
+      return declarator;
+    }
+
+    unsigned getCurrentChunkIndex() const {
+      return chunkIndex;
+    }
+
+    void setCurrentChunkIndex(unsigned idx) {
+      assert(idx <= declarator.getNumTypeObjects());
+      chunkIndex = idx;
+    }
+
+    AttributeList *&getCurrentAttrListRef() const {
+      assert(chunkIndex <= declarator.getNumTypeObjects());
+      if (chunkIndex == declarator.getNumTypeObjects())
+        return getMutableDeclSpec().getAttributes().getListRef();
+      return declarator.getTypeObject(chunkIndex).getAttrListRef();
+    }
+
+    /// Save the current set of attributes on the DeclSpec.
+    void saveDeclSpecAttrs() {
+      // Don't try to save them multiple times.
+      if (!savedAttrs.empty()) return;
+
+      DeclSpec &spec = getMutableDeclSpec();
+      for (AttributeList *attr = spec.getAttributes().getList(); attr;
+             attr = attr->getNext())
+        savedAttrs.push_back(attr);
+      trivial &= savedAttrs.empty();
+    }
+
+    /// Record that we had nowhere to put the given type attribute.
+    /// We will diagnose such attributes later.
+    void addIgnoredTypeAttr(AttributeList &attr) {
+      ignoredTypeAttrs.push_back(&attr);
+    }
+
+    /// Diagnose all the ignored type attributes, given that the
+    /// declarator worked out to the given type.
+    void diagnoseIgnoredTypeAttrs(QualType type) const {
+      for (llvm::SmallVectorImpl<AttributeList*>::const_iterator
+             i = ignoredTypeAttrs.begin(), e = ignoredTypeAttrs.end();
+           i != e; ++i) {
+        AttributeList &attr = **i;
+        getSema().Diag(attr.getLoc(), diag::warn_function_attribute_wrong_type)
+          << attr.getName() << type;
+      }
+    }
+
+    ~TypeProcessingState() {
+      if (trivial) return;
+
+      restoreDeclSpecAttrs();
+    }
+
+  private:
+    DeclSpec &getMutableDeclSpec() const {
+      return const_cast<DeclSpec&>(declarator.getDeclSpec());
+    }
+
+    void restoreDeclSpecAttrs() {
+      assert(!savedAttrs.empty());
+      getMutableDeclSpec().getAttributes().set(savedAttrs[0]);
+      for (unsigned i = 0, e = savedAttrs.size() - 1; i != e; ++i)
+        savedAttrs[i]->setNext(savedAttrs[i+1]);
+      savedAttrs.back()->setNext(0);
+    }
+  };
+
+  /// Basically std::pair except that we really want to avoid an
+  /// implicit operator= for safety concerns.  It's also a minor
+  /// link-time optimization for this to be a private type.
+  struct AttrAndList {
+    /// The attribute.
+    AttributeList &first;
+
+    /// The head of the list the attribute is currently in.
+    AttributeList *&second;
+
+    AttrAndList(AttributeList &attr, AttributeList *&head)
+      : first(attr), second(head) {}
+  };
+}
+
+namespace llvm {
+  template <> struct isPodLike<AttrAndList> {
+    static const bool value = true;
+  };
+}
+
+static void spliceAttrIntoList(AttributeList &attr, AttributeList *&head) {
+  attr.setNext(head);
+  head = &attr;
+}
+
+static void spliceAttrOutOfList(AttributeList &attr, AttributeList *&head) {
+  if (head == &attr) {
+    head = attr.getNext();
+    return;
+  }
+
+  AttributeList *cur = head;
+  while (true) {
+    assert(cur && cur->getNext() && "ran out of attrs?");
+    if (cur->getNext() == &attr) {
+      cur->setNext(attr.getNext());
+      return;
+    }
+    cur = cur->getNext();
+  }
+}
+
+static void moveAttrFromListToList(AttributeList &attr,
+                                   AttributeList *&fromList,
+                                   AttributeList *&toList) {
+  spliceAttrOutOfList(attr, fromList);
+  spliceAttrIntoList(attr, toList);
+}
+
+static void processTypeAttrs(TypeProcessingState &state,
+                             QualType &type, bool isDeclSpec,
+                             AttributeList *attrs);
+
+static bool handleFunctionTypeAttr(TypeProcessingState &state,
+                                   AttributeList &attr,
+                                   QualType &type);
+
+static bool handleObjCGCTypeAttr(TypeProcessingState &state,
+                                 AttributeList &attr, QualType &type);
+
+static bool handleObjCPointerTypeAttr(TypeProcessingState &state,
+                                      AttributeList &attr, QualType &type) {
+  // Right now, we have exactly one of these attributes: objc_gc.
+  assert(attr.getKind() == AttributeList::AT_objc_gc);
+  return handleObjCGCTypeAttr(state, attr, type);
+}
+
+/// Given that an objc_gc attribute was written somewhere on a
+/// declaration *other* than on the declarator itself (for which, use
+/// distributeObjCPointerTypeAttrFromDeclarator), and given that it
+/// didn't apply in whatever position it was written in, try to move
+/// it to a more appropriate position.
+static void distributeObjCPointerTypeAttr(TypeProcessingState &state,
+                                          AttributeList &attr,
+                                          QualType type) {
+  Declarator &declarator = state.getDeclarator();
+  for (unsigned i = state.getCurrentChunkIndex(); i != 0; --i) {
+    DeclaratorChunk &chunk = declarator.getTypeObject(i-1);
+    switch (chunk.Kind) {
+    case DeclaratorChunk::Pointer:
+    case DeclaratorChunk::BlockPointer:
+      moveAttrFromListToList(attr, state.getCurrentAttrListRef(),
+                             chunk.getAttrListRef());
+      return;
+
+    case DeclaratorChunk::Paren:
+    case DeclaratorChunk::Array:
+      continue;
+
+    // Don't walk through these.
+    case DeclaratorChunk::Reference:
+    case DeclaratorChunk::Function:
+    case DeclaratorChunk::MemberPointer:
+      goto error;
+    }
+  }
+ error:
+  
+  state.getSema().Diag(attr.getLoc(), diag::warn_function_attribute_wrong_type)
+    << attr.getName() << type;
+}
+
+/// Distribute an objc_gc type attribute that was written on the
+/// declarator.
+static void
+distributeObjCPointerTypeAttrFromDeclarator(TypeProcessingState &state,
+                                            AttributeList &attr,
+                                            QualType &declSpecType) {
+  Declarator &declarator = state.getDeclarator();
+
+  // objc_gc goes on the innermost pointer to something that's not a
+  // pointer.
+  unsigned innermost = -1U;
+  bool considerDeclSpec = true;
+  for (unsigned i = 0, e = declarator.getNumTypeObjects(); i != e; ++i) {
+    DeclaratorChunk &chunk = declarator.getTypeObject(i);
+    switch (chunk.Kind) {
+    case DeclaratorChunk::Pointer:
+    case DeclaratorChunk::BlockPointer:
+      innermost = i;
+      continue;
+
+    case DeclaratorChunk::Reference:
+    case DeclaratorChunk::MemberPointer:
+    case DeclaratorChunk::Paren:
+    case DeclaratorChunk::Array:
+      continue;
+
+    case DeclaratorChunk::Function:
+      considerDeclSpec = false;
+      goto done;
+    }
+  }
+ done:
+
+  // That might actually be the decl spec if we weren't blocked by
+  // anything in the declarator.
+  if (considerDeclSpec) {
+    if (handleObjCPointerTypeAttr(state, attr, declSpecType))
+      return;
+  }
+
+  // Otherwise, if we found an appropriate chunk, splice the attribute
+  // into it.
+  if (innermost != -1U) {
+    moveAttrFromListToList(attr, declarator.getAttrListRef(),
+                       declarator.getTypeObject(innermost).getAttrListRef());
+    return;
+  }
+
+  // Otherwise, diagnose when we're done building the type.
+  spliceAttrOutOfList(attr, declarator.getAttrListRef());
+  state.addIgnoredTypeAttr(attr);
+}
+
+/// A function type attribute was written somewhere in a declaration
+/// *other* than on the declarator itself or in the decl spec.  Given
+/// that it didn't apply in whatever position it was written in, try
+/// to move it to a more appropriate position.
+static void distributeFunctionTypeAttr(TypeProcessingState &state,
+                                       AttributeList &attr,
+                                       QualType type) {
+  Declarator &declarator = state.getDeclarator();
+
+  // Try to push the attribute from the return type of a function to
+  // the function itself.
+  for (unsigned i = state.getCurrentChunkIndex(); i != 0; --i) {
+    DeclaratorChunk &chunk = declarator.getTypeObject(i-1);
+    switch (chunk.Kind) {
+    case DeclaratorChunk::Function:
+      moveAttrFromListToList(attr, state.getCurrentAttrListRef(),
+                             chunk.getAttrListRef());
+      return;
+
+    case DeclaratorChunk::Paren:
+    case DeclaratorChunk::Pointer:
+    case DeclaratorChunk::BlockPointer:
+    case DeclaratorChunk::Array:
+    case DeclaratorChunk::Reference:
+    case DeclaratorChunk::MemberPointer:
+      continue;
+    }
+  }
+  
+  state.getSema().Diag(attr.getLoc(), diag::warn_function_attribute_wrong_type)
+    << attr.getName() << type;
+}
+
+/// Try to distribute a function type attribute to the innermost
+/// function chunk or type.  Returns true if the attribute was
+/// distributed, false if no location was found.
+static bool
+distributeFunctionTypeAttrToInnermost(TypeProcessingState &state,
+                                      AttributeList &attr,
+                                      AttributeList *&attrList,
+                                      QualType &declSpecType) {
+  Declarator &declarator = state.getDeclarator();
+
+  // Put it on the innermost function chunk, if there is one.
+  for (unsigned i = 0, e = declarator.getNumTypeObjects(); i != e; ++i) {
+    DeclaratorChunk &chunk = declarator.getTypeObject(i);
+    if (chunk.Kind != DeclaratorChunk::Function) continue;
+
+    moveAttrFromListToList(attr, attrList, chunk.getAttrListRef());
+    return true;
+  }
+
+  return handleFunctionTypeAttr(state, attr, declSpecType);
+}
+
+/// A function type attribute was written in the decl spec.  Try to
+/// apply it somewhere.
+static void
+distributeFunctionTypeAttrFromDeclSpec(TypeProcessingState &state,
+                                       AttributeList &attr,
+                                       QualType &declSpecType) {
+  state.saveDeclSpecAttrs();
+
+  // Try to distribute to the innermost.
+  if (distributeFunctionTypeAttrToInnermost(state, attr,
+                                            state.getCurrentAttrListRef(),
+                                            declSpecType))
+    return;
+
+  // If that failed, diagnose the bad attribute when the declarator is
+  // fully built.
+  state.addIgnoredTypeAttr(attr);
 }
 
-static void DiagnoseDelayedFnAttrs(Sema &S, DelayedAttributeSet &Attrs) {
-  for (DelayedAttributeSet::iterator I = Attrs.begin(),
-         E = Attrs.end(); I != E; ++I) {
-    S.Diag(I->first->getLoc(), diag::warn_function_attribute_wrong_type)
-      << I->first->getName() << I->second;
-    // Avoid any further processing of this attribute.
-    I->first->setInvalid();
+/// A function type attribute was written on the declarator.  Try to
+/// apply it somewhere.
+static void
+distributeFunctionTypeAttrFromDeclarator(TypeProcessingState &state,
+                                         AttributeList &attr,
+                                         QualType &declSpecType) {
+  Declarator &declarator = state.getDeclarator();
+
+  // Try to distribute to the innermost.
+  if (distributeFunctionTypeAttrToInnermost(state, attr,
+                                            declarator.getAttrListRef(),
+                                            declSpecType))
+    return;
+
+  // If that failed, diagnose the bad attribute when the declarator is
+  // fully built.
+  spliceAttrOutOfList(attr, declarator.getAttrListRef());
+  state.addIgnoredTypeAttr(attr);
+}
+
+/// \brief Given that there are attributes written on the declarator
+/// itself, try to distribute any type attributes to the appropriate
+/// declarator chunk.
+///
+/// These are attributes like the following:
+///   int f ATTR;
+///   int (f ATTR)();
+/// but not necessarily this:
+///   int f() ATTR;
+static void distributeTypeAttrsFromDeclarator(TypeProcessingState &state,
+                                              QualType &declSpecType) {
+  // Collect all the type attributes from the declarator itself.
+  assert(state.getDeclarator().getAttributes() && "declarator has no attrs!");
+  AttributeList *attr = state.getDeclarator().getAttributes();
+  AttributeList *next;
+  do {
+    next = attr->getNext();
+
+    switch (attr->getKind()) {
+    OBJC_POINTER_TYPE_ATTRS_CASELIST:
+      distributeObjCPointerTypeAttrFromDeclarator(state, *attr, declSpecType);
+      break;
+
+    FUNCTION_TYPE_ATTRS_CASELIST:
+      distributeFunctionTypeAttrFromDeclarator(state, *attr, declSpecType);
+      break;
+
+    default:
+      break;
+    }
+  } while ((attr = next));
+}
+
+/// Add a synthetic '()' to a block-literal declarator if it is
+/// required, given the return type.
+static void maybeSynthesizeBlockSignature(TypeProcessingState &state,
+                                          QualType declSpecType) {
+  Declarator &declarator = state.getDeclarator();
+
+  // First, check whether the declarator would produce a function,
+  // i.e. whether the innermost semantic chunk is a function.
+  if (declarator.isFunctionDeclarator()) {
+    // If so, make that declarator a prototyped declarator.
+    declarator.getFunctionTypeInfo().hasPrototype = true;
+    return;
   }
-  Attrs.clear();
+
+  // If there are any type objects, the type as written won't name a
+  // function, regardless of the decl spec type.  This is because a
+  // block signature declarator is always an abstract-declarator, and
+  // abstract-declarators can't just be parentheses chunks.  Therefore
+  // we need to build a function chunk unless there are no type
+  // objects and the decl spec type is a function.
+  if (!declarator.getNumTypeObjects() && declSpecType->isFunctionType())
+    return;
+
+  // Note that there *are* cases with invalid declarators where
+  // declarators consist solely of parentheses.  In general, these
+  // occur only in failed efforts to make function declarators, so
+  // faking up the function chunk is still the right thing to do.
+
+  // Otherwise, we need to fake up a function declarator.
+  SourceLocation loc = declarator.getSourceRange().getBegin();
+
+  // ...and *prepend* it to the declarator.
+  declarator.AddInnermostTypeInfo(DeclaratorChunk::getFunction(
+                             ParsedAttributes(),
+                             /*proto*/ true,
+                             /*variadic*/ false, SourceLocation(),
+                             /*args*/ 0, 0,
+                             /*type quals*/ 0,
+                             /*ref-qualifier*/true, SourceLocation(),
+                             /*EH*/ false, SourceLocation(), false, 0, 0, 0,
+                             /*parens*/ loc, loc,
+                             declarator));
+
+  // For consistency, make sure the state still has us as processing
+  // the decl spec.
+  assert(state.getCurrentChunkIndex() == declarator.getNumTypeObjects() - 1);
+  state.setCurrentChunkIndex(declarator.getNumTypeObjects());
 }
 
 /// \brief Convert the specified declspec to the appropriate type
@@ -108,17 +526,17 @@ static void DiagnoseDelayedFnAttrs(Sema &S, DelayedAttributeSet &Attrs) {
 /// \param D  the declarator containing the declaration specifier.
 /// \returns The type described by the declaration specifiers.  This function
 /// never returns null.
-static QualType ConvertDeclSpecToType(Sema &TheSema,
-                                      Declarator &TheDeclarator,
-                                      DelayedAttributeSet &Delayed) {
+static QualType ConvertDeclSpecToType(Sema &S, TypeProcessingState &state) {
   // FIXME: Should move the logic from DeclSpec::Finish to here for validity
   // checking.
-  const DeclSpec &DS = TheDeclarator.getDeclSpec();
-  SourceLocation DeclLoc = TheDeclarator.getIdentifierLoc();
+
+  Declarator &declarator = state.getDeclarator();
+  const DeclSpec &DS = declarator.getDeclSpec();
+  SourceLocation DeclLoc = declarator.getIdentifierLoc();
   if (DeclLoc.isInvalid())
     DeclLoc = DS.getSourceRange().getBegin();
   
-  ASTContext &Context = TheSema.Context;
+  ASTContext &Context = S.Context;
 
   QualType Result;
   switch (DS.getTypeSpecType()) {
@@ -140,13 +558,13 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     if (DS.getTypeSpecSign() == DeclSpec::TSS_unspecified)
       Result = Context.WCharTy;
     else if (DS.getTypeSpecSign() == DeclSpec::TSS_signed) {
-      TheSema.Diag(DS.getTypeSpecSignLoc(), diag::ext_invalid_sign_spec)
+      S.Diag(DS.getTypeSpecSignLoc(), diag::ext_invalid_sign_spec)
         << DS.getSpecifierName(DS.getTypeSpecType());
       Result = Context.getSignedWCharType();
     } else {
       assert(DS.getTypeSpecSign() == DeclSpec::TSS_unsigned &&
         "Unknown TSS value");
-      TheSema.Diag(DS.getTypeSpecSignLoc(), diag::ext_invalid_sign_spec)
+      S.Diag(DS.getTypeSpecSignLoc(), diag::ext_invalid_sign_spec)
         << DS.getSpecifierName(DS.getTypeSpecType());
       Result = Context.getUnsignedWCharType();
     }
@@ -173,7 +591,7 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     
     // If this is a missing declspec in a block literal return context, then it
     // is inferred from the return statements inside the block.
-    if (isOmittedBlockReturnType(TheDeclarator)) {
+    if (isOmittedBlockReturnType(declarator)) {
       Result = Context.DependentTy;
       break;
     }
@@ -185,11 +603,11 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     // allowed to be completely missing a declspec.  This is handled in the
     // parser already though by it pretending to have seen an 'int' in this
     // case.
-    if (TheSema.getLangOptions().ImplicitInt) {
+    if (S.getLangOptions().ImplicitInt) {
       // In C89 mode, we only warn if there is a completely missing declspec
       // when one is not allowed.
       if (DS.isEmpty()) {
-        TheSema.Diag(DeclLoc, diag::ext_missing_declspec)
+        S.Diag(DeclLoc, diag::ext_missing_declspec)
           << DS.getSourceRange()
         << FixItHint::CreateInsertion(DS.getSourceRange().getBegin(), "int");
       }
@@ -199,17 +617,17 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
       // specifiers in each declaration, and in the specifier-qualifier list in
       // each struct declaration and type name."
       // FIXME: Does Microsoft really have the implicit int extension in C++?
-      if (TheSema.getLangOptions().CPlusPlus &&
-          !TheSema.getLangOptions().Microsoft) {
-        TheSema.Diag(DeclLoc, diag::err_missing_type_specifier)
+      if (S.getLangOptions().CPlusPlus &&
+          !S.getLangOptions().Microsoft) {
+        S.Diag(DeclLoc, diag::err_missing_type_specifier)
           << DS.getSourceRange();
 
         // When this occurs in C++ code, often something is very broken with the
         // value being declared, poison it as invalid so we don't get chains of
         // errors.
-        TheDeclarator.setInvalidType(true);
+        declarator.setInvalidType(true);
       } else {
-        TheSema.Diag(DeclLoc, diag::ext_missing_type_specifier)
+        S.Diag(DeclLoc, diag::ext_missing_type_specifier)
           << DS.getSourceRange();
       }
     }
@@ -225,9 +643,9 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
         Result = Context.LongLongTy;
           
         // long long is a C99 feature.
-        if (!TheSema.getLangOptions().C99 &&
-            !TheSema.getLangOptions().CPlusPlus0x)
-          TheSema.Diag(DS.getTypeSpecWidthLoc(), diag::ext_longlong);
+        if (!S.getLangOptions().C99 &&
+            !S.getLangOptions().CPlusPlus0x)
+          S.Diag(DS.getTypeSpecWidthLoc(), diag::ext_longlong);
         break;
       }
     } else {
@@ -239,9 +657,9 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
         Result = Context.UnsignedLongLongTy;
           
         // long long is a C99 feature.
-        if (!TheSema.getLangOptions().C99 &&
-            !TheSema.getLangOptions().CPlusPlus0x)
-          TheSema.Diag(DS.getTypeSpecWidthLoc(), diag::ext_longlong);
+        if (!S.getLangOptions().C99 &&
+            !S.getLangOptions().CPlusPlus0x)
+          S.Diag(DS.getTypeSpecWidthLoc(), diag::ext_longlong);
         break;
       }
     }
@@ -253,14 +671,19 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
       Result = Context.LongDoubleTy;
     else
       Result = Context.DoubleTy;
+
+    if (S.getLangOptions().OpenCL && !S.getOpenCLOptions().cl_khr_fp64) {
+      S.Diag(DS.getTypeSpecTypeLoc(), diag::err_double_requires_fp64);
+      declarator.setInvalidType(true);
+    }
     break;
   case DeclSpec::TST_bool: Result = Context.BoolTy; break; // _Bool or bool
   case DeclSpec::TST_decimal32:    // _Decimal32
   case DeclSpec::TST_decimal64:    // _Decimal64
   case DeclSpec::TST_decimal128:   // _Decimal128
-    TheSema.Diag(DS.getTypeSpecTypeLoc(), diag::err_decimal_unsupported);
+    S.Diag(DS.getTypeSpecTypeLoc(), diag::err_decimal_unsupported);
     Result = Context.IntTy;
-    TheDeclarator.setInvalidType(true);
+    declarator.setInvalidType(true);
     break;
   case DeclSpec::TST_class:
   case DeclSpec::TST_enum:
@@ -270,12 +693,12 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     if (!D) {
       // This can happen in C++ with ambiguous lookups.
       Result = Context.IntTy;
-      TheDeclarator.setInvalidType(true);
+      declarator.setInvalidType(true);
       break;
     }
 
     // If the type is deprecated or unavailable, diagnose it.
-    TheSema.DiagnoseUseOfDecl(D, DS.getTypeSpecTypeLoc());
+    S.DiagnoseUseOfDecl(D, DS.getTypeSpecTypeLoc());
     
     assert(DS.getTypeSpecWidth() == 0 && DS.getTypeSpecComplex() == 0 &&
            DS.getTypeSpecSign() == 0 && "No qualifiers on tag names!");
@@ -284,23 +707,22 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     Result = Context.getTypeDeclType(D);
 
     // In C++, make an ElaboratedType.
-    if (TheSema.getLangOptions().CPlusPlus) {
+    if (S.getLangOptions().CPlusPlus) {
       ElaboratedTypeKeyword Keyword
         = ElaboratedType::getKeywordForTypeSpec(DS.getTypeSpecType());
-      Result = TheSema.getElaboratedType(Keyword, DS.getTypeSpecScope(),
-                                         Result);
+      Result = S.getElaboratedType(Keyword, DS.getTypeSpecScope(), Result);
     }
     if (D->isInvalidDecl())
-      TheDeclarator.setInvalidType(true);
+      declarator.setInvalidType(true);
     break;
   }
   case DeclSpec::TST_typename: {
     assert(DS.getTypeSpecWidth() == 0 && DS.getTypeSpecComplex() == 0 &&
            DS.getTypeSpecSign() == 0 &&
            "Can't handle qualifiers on typedef names yet!");
-    Result = TheSema.GetTypeFromParser(DS.getRepAsType());
+    Result = S.GetTypeFromParser(DS.getRepAsType());
     if (Result.isNull())
-      TheDeclarator.setInvalidType(true);
+      declarator.setInvalidType(true);
     else if (DeclSpec::ProtocolQualifierListTy PQ
                = DS.getProtocolQualifiers()) {
       if (const ObjCObjectType *ObjT = Result->getAs<ObjCObjectType>()) {
@@ -326,9 +748,9 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
                                            DS.getNumProtocolQualifiers());
         Result = Context.getObjCObjectPointerType(Result);
       } else {
-        TheSema.Diag(DeclLoc, diag::err_invalid_protocol_qualifiers)
+        S.Diag(DeclLoc, diag::err_invalid_protocol_qualifiers)
           << DS.getSourceRange();
-        TheDeclarator.setInvalidType(true);
+        declarator.setInvalidType(true);
       }
     }
 
@@ -337,8 +759,11 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
   }
   case DeclSpec::TST_typeofType:
     // FIXME: Preserve type source info.
-    Result = TheSema.GetTypeFromParser(DS.getRepAsType());
+    Result = S.GetTypeFromParser(DS.getRepAsType());
     assert(!Result.isNull() && "Didn't get a type for typeof?");
+    if (!Result->isDependentType())
+      if (const TagType *TT = Result->getAs<TagType>())
+        S.DiagnoseUseOfDecl(TT->getDecl(), DS.getTypeSpecTypeLoc());
     // TypeQuals handled by caller.
     Result = Context.getTypeOfType(Result);
     break;
@@ -346,10 +771,10 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     Expr *E = DS.getRepAsExpr();
     assert(E && "Didn't get an expression for typeof?");
     // TypeQuals handled by caller.
-    Result = TheSema.BuildTypeofExprType(E);
+    Result = S.BuildTypeofExprType(E, DS.getTypeSpecTypeLoc());
     if (Result.isNull()) {
       Result = Context.IntTy;
-      TheDeclarator.setInvalidType(true);
+      declarator.setInvalidType(true);
     }
     break;
   }
@@ -357,48 +782,55 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     Expr *E = DS.getRepAsExpr();
     assert(E && "Didn't get an expression for decltype?");
     // TypeQuals handled by caller.
-    Result = TheSema.BuildDecltypeType(E);
+    Result = S.BuildDecltypeType(E, DS.getTypeSpecTypeLoc());
     if (Result.isNull()) {
       Result = Context.IntTy;
-      TheDeclarator.setInvalidType(true);
+      declarator.setInvalidType(true);
     }
     break;
   }
   case DeclSpec::TST_auto: {
     // TypeQuals handled by caller.
-    Result = Context.UndeducedAutoTy;
+    Result = Context.getAutoType(QualType());
     break;
   }
 
   case DeclSpec::TST_error:
     Result = Context.IntTy;
-    TheDeclarator.setInvalidType(true);
+    declarator.setInvalidType(true);
     break;
   }
 
   // Handle complex types.
   if (DS.getTypeSpecComplex() == DeclSpec::TSC_complex) {
-    if (TheSema.getLangOptions().Freestanding)
-      TheSema.Diag(DS.getTypeSpecComplexLoc(), diag::ext_freestanding_complex);
+    if (S.getLangOptions().Freestanding)
+      S.Diag(DS.getTypeSpecComplexLoc(), diag::ext_freestanding_complex);
     Result = Context.getComplexType(Result);
   } else if (DS.isTypeAltiVecVector()) {
     unsigned typeSize = static_cast<unsigned>(Context.getTypeSize(Result));
     assert(typeSize > 0 && "type size for vector must be greater than 0 bits");
-    VectorType::AltiVecSpecific AltiVecSpec = VectorType::AltiVec;
+    VectorType::VectorKind VecKind = VectorType::AltiVecVector;
     if (DS.isTypeAltiVecPixel())
-      AltiVecSpec = VectorType::Pixel;
+      VecKind = VectorType::AltiVecPixel;
     else if (DS.isTypeAltiVecBool())
-      AltiVecSpec = VectorType::Bool;
-    Result = Context.getVectorType(Result, 128/typeSize, AltiVecSpec);
+      VecKind = VectorType::AltiVecBool;
+    Result = Context.getVectorType(Result, 128/typeSize, VecKind);
   }
 
-  assert(DS.getTypeSpecComplex() != DeclSpec::TSC_imaginary &&
-         "FIXME: imaginary types not supported yet!");
+  // FIXME: Imaginary.
+  if (DS.getTypeSpecComplex() == DeclSpec::TSC_imaginary)
+    S.Diag(DS.getTypeSpecComplexLoc(), diag::err_imaginary_not_supported);
 
-  // See if there are any attributes on the declspec that apply to the type (as
-  // opposed to the decl).
-  if (const AttributeList *AL = DS.getAttributes())
-    ProcessTypeAttributeList(TheSema, Result, true, AL, Delayed);
+  // Before we process any type attributes, synthesize a block literal
+  // function declarator if necessary.
+  if (declarator.getContext() == Declarator::BlockLiteralContext)
+    maybeSynthesizeBlockSignature(state, Result);
+
+  // Apply any type attributes from the decl spec.  This may cause the
+  // list of type attributes to be temporarily saved while the type
+  // attributes are pushed around.
+  if (AttributeList *attrs = DS.getAttributes().getList())
+    processTypeAttrs(state, Result, true, attrs);
 
   // Apply const/volatile/restrict qualifiers to T.
   if (unsigned TypeQuals = DS.getTypeQualifiers()) {
@@ -419,14 +851,14 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
         // If we have a pointer or reference, the pointee must have an object
         // incomplete type.
         if (!EltTy->isIncompleteOrObjectType()) {
-          TheSema.Diag(DS.getRestrictSpecLoc(),
+          S.Diag(DS.getRestrictSpecLoc(),
                diag::err_typecheck_invalid_restrict_invalid_pointee)
             << EltTy << DS.getSourceRange();
           TypeQuals &= ~DeclSpec::TQ_restrict; // Remove the restrict qualifier.
         }
       } else {
-        TheSema.Diag(DS.getRestrictSpecLoc(),
-             diag::err_typecheck_invalid_restrict_not_pointer)
+        S.Diag(DS.getRestrictSpecLoc(),
+               diag::err_typecheck_invalid_restrict_not_pointer)
           << Result << DS.getSourceRange();
         TypeQuals &= ~DeclSpec::TQ_restrict; // Remove the restrict qualifier.
       }
@@ -447,7 +879,7 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
                "Has CVR quals but not C, V, or R?");
         Loc = DS.getRestrictSpecLoc();
       }
-      TheSema.Diag(Loc, diag::warn_typecheck_function_qualifiers)
+      S.Diag(Loc, diag::warn_typecheck_function_qualifiers)
         << Result << DS.getSourceRange();
     }
 
@@ -516,6 +948,11 @@ QualType Sema::BuildQualifiedType(QualType T, SourceLocation Loc,
   return Context.getQualifiedType(T, Qs);
 }
 
+/// \brief Build a paren type including \p T.
+QualType Sema::BuildParenType(QualType T) {
+  return Context.getParenType(T);
+}
+
 /// \brief Build a pointer type.
 ///
 /// \param T The type to which we'll be building a pointer.
@@ -560,14 +997,14 @@ QualType Sema::BuildPointerType(QualType T,
 QualType Sema::BuildReferenceType(QualType T, bool SpelledAsLValue,
                                   SourceLocation Loc,
                                   DeclarationName Entity) {
+  // C++0x [dcl.ref]p6:
+  //   If a typedef (7.1.3), a type template-parameter (14.3.1), or a 
+  //   decltype-specifier (7.1.6.2) denotes a type TR that is a reference to a 
+  //   type T, an attempt to create the type "lvalue reference to cv TR" creates 
+  //   the type "lvalue reference to T", while an attempt to create the type 
+  //   "rvalue reference to cv TR" creates the type TR.
   bool LValueRef = SpelledAsLValue || T->getAs<LValueReferenceType>();
 
-  // C++0x [dcl.typedef]p9: If a typedef TD names a type that is a
-  //   reference to a type T, and attempt to create the type "lvalue
-  //   reference to cv TD" creates the type "lvalue reference to T".
-  // We use the qualifiers (restrict or none) of the original reference,
-  // not the new ones. This is consistent with GCC.
-
   // C++ [dcl.ref]p4: There shall be no references to references.
   //
   // According to C++ DR 106, references to references are only
@@ -579,8 +1016,8 @@ QualType Sema::BuildReferenceType(QualType T, bool SpelledAsLValue,
   //
   // Parser::ParseDeclaratorInternal diagnoses the case where
   // references are written directly; here, we handle the
-  // collapsing of references-to-references as described in C++
-  // DR 106 and amended by C++ DR 540.
+  // collapsing of references-to-references as described in C++0x.
+  // DR 106 and 540 introduce reference-collapsing into C++98/03.
 
   // C++ [dcl.ref]p1:
   //   A declarator that specifies the type "reference to cv void"
@@ -654,9 +1091,9 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
     return QualType();
   }
 
-  if (Context.getCanonicalType(T) == Context.UndeducedAutoTy) {
-    Diag(Loc,  diag::err_illegal_decl_array_of_auto)
-      << getPrintableNameForEntity(Entity);
+  if (T->getContainedAutoType()) {
+    Diag(Loc, diag::err_illegal_decl_array_of_auto)
+      << getPrintableNameForEntity(Entity) << T;
     return QualType();
   }
 
@@ -670,9 +1107,15 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
     return QualType();
   }
 
+  // Do lvalue-to-rvalue conversions on the array size expression.
+  if (ArraySize && !ArraySize->isRValue())
+    DefaultLvalueConversion(ArraySize);
+
   // C99 6.7.5.2p1: The size expression shall have integer type.
+  // TODO: in theory, if we were insane, we could allow contextual
+  // conversions to integer type here.
   if (ArraySize && !ArraySize->isTypeDependent() &&
-      !ArraySize->getType()->isIntegerType()) {
+      !ArraySize->getType()->isIntegralOrUnscopedEnumerationType()) {
     Diag(ArraySize->getLocStart(), diag::err_array_size_non_int)
       << ArraySize->getType() << ArraySize->getSourceRange();
     return QualType();
@@ -695,9 +1138,12 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
     // C99 6.7.5.2p1: If the expression is a constant expression, it shall
     // have a value greater than zero.
     if (ConstVal.isSigned() && ConstVal.isNegative()) {
-      Diag(ArraySize->getLocStart(),
-           diag::err_typecheck_negative_array_size)
-        << ArraySize->getSourceRange();
+      if (Entity)
+        Diag(ArraySize->getLocStart(), diag::err_decl_negative_array_size)
+          << getPrintableNameForEntity(Entity) << ArraySize->getSourceRange();
+      else
+        Diag(ArraySize->getLocStart(), diag::err_typecheck_negative_array_size)
+          << ArraySize->getSourceRange();
       return QualType();
     }
     if (ConstVal == 0) {
@@ -818,8 +1264,9 @@ QualType Sema::BuildFunctionType(QualType T,
                                  QualType *ParamTypes,
                                  unsigned NumParamTypes,
                                  bool Variadic, unsigned Quals,
+                                 RefQualifierKind RefQualifier,
                                  SourceLocation Loc, DeclarationName Entity,
-                                 const FunctionType::ExtInfo &Info) {
+                                 FunctionType::ExtInfo Info) {
   if (T->isArrayType() || T->isFunctionType()) {
     Diag(Loc, diag::err_func_returning_array_function) 
       << T->isFunctionType() << T;
@@ -840,8 +1287,13 @@ QualType Sema::BuildFunctionType(QualType T,
   if (Invalid)
     return QualType();
 
-  return Context.getFunctionType(T, ParamTypes, NumParamTypes, Variadic,
-                                 Quals, false, false, 0, 0, Info);
+  FunctionProtoType::ExtProtoInfo EPI;
+  EPI.Variadic = Variadic;
+  EPI.TypeQuals = Quals;
+  EPI.RefQualifier = RefQualifier;
+  EPI.ExtInfo = Info;
+
+  return Context.getFunctionType(T, ParamTypes, NumParamTypes, EPI);
 }
 
 /// \brief Build a member pointer type \c T Class::*.
@@ -934,7 +1386,7 @@ QualType Sema::GetTypeFromParser(ParsedType Ty, TypeSourceInfo **TInfo) {
   }
 
   TypeSourceInfo *DI = 0;
-  if (LocInfoType *LIT = dyn_cast<LocInfoType>(QT)) {
+  if (const LocInfoType *LIT = dyn_cast<LocInfoType>(QT)) {
     QT = LIT->getType();
     DI = LIT->getTypeSourceInfo();
   }
@@ -953,20 +1405,21 @@ QualType Sema::GetTypeFromParser(ParsedType Ty, TypeSourceInfo **TInfo) {
 /// The result of this call will never be null, but the associated
 /// type may be a null type if there's an unrecoverable error.
 TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
-                                           TagDecl **OwnedDecl) {
+                                           TagDecl **OwnedDecl,
+                                           bool AutoAllowedInTypeName) {
   // Determine the type of the declarator. Not all forms of declarator
   // have a type.
   QualType T;
   TypeSourceInfo *ReturnTypeInfo = 0;
-  
-  llvm::SmallVector<DelayedAttribute,4> FnAttrsFromDeclSpec;
+
+  TypeProcessingState state(*this, D);
 
   switch (D.getName().getKind()) {
   case UnqualifiedId::IK_Identifier:
   case UnqualifiedId::IK_OperatorFunctionId:
   case UnqualifiedId::IK_LiteralOperatorId:
   case UnqualifiedId::IK_TemplateId:
-    T = ConvertDeclSpecToType(*this, D, FnAttrsFromDeclSpec);
+    T = ConvertDeclSpecToType(*this, state);
     
     if (!D.isInvalidType() && D.getDeclSpec().isTypeSpecOwned()) {
       TagDecl* Owned = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
@@ -993,11 +1446,12 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
                           &ReturnTypeInfo);
     break;
   }
-  
-  if (T.isNull())
-    return Context.getNullTypeSourceInfo();
 
-  if (T == Context.UndeducedAutoTy) {
+  if (D.getAttributes())
+    distributeTypeAttrsFromDeclarator(state, T);
+
+  if (D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto &&
+      !D.isFunctionDeclarator()) {
     int Error = -1;
 
     switch (D.getContext()) {
@@ -1022,13 +1476,19 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
       Error = 5; // Template parameter
       break;
     case Declarator::BlockLiteralContext:
-      Error = 6;  // Block literal
+      Error = 6; // Block literal
+      break;
+    case Declarator::TemplateTypeArgContext:
+      Error = 7; // Template type argument
+      break;
+    case Declarator::TypeNameContext:
+      if (!AutoAllowedInTypeName)
+        Error = 8; // Generic
       break;
     case Declarator::FileContext:
     case Declarator::BlockContext:
     case Declarator::ForContext:
     case Declarator::ConditionContext:
-    case Declarator::TypeNameContext:
       break;
     }
 
@@ -1040,20 +1500,26 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
     }
   }
 
+  if (T.isNull())
+    return Context.getNullTypeSourceInfo();
+
   // The name we're declaring, if any.
   DeclarationName Name;
   if (D.getIdentifier())
     Name = D.getIdentifier();
 
-  llvm::SmallVector<DelayedAttribute,4> FnAttrsFromPreviousChunk;
-
   // Walk the DeclTypeInfo, building the recursive type as we go.
   // DeclTypeInfos are ordered from the identifier out, which is
   // opposite of what we want :).
   for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i) {
-    DeclaratorChunk &DeclType = D.getTypeObject(e-i-1);
+    unsigned chunkIndex = e - i - 1;
+    state.setCurrentChunkIndex(chunkIndex);
+    DeclaratorChunk &DeclType = D.getTypeObject(chunkIndex);
     switch (DeclType.Kind) {
     default: assert(0 && "Unknown decltype!");
+    case DeclaratorChunk::Paren:
+      T = BuildParenType(T);
+      break;
     case DeclaratorChunk::BlockPointer:
       // If blocks are disabled, emit an error.
       if (!LangOpts.Blocks)
@@ -1080,6 +1546,7 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
       T = BuildPointerType(T, DeclType.Loc, Name);
       if (DeclType.Ptr.TypeQuals)
         T = BuildQualifiedType(T, DeclType.Loc, DeclType.Ptr.TypeQuals);
+
       break;
     case DeclaratorChunk::Reference: {
       // Verify that we're not building a reference to pointer to function with
@@ -1137,12 +1604,43 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
       // For conversion functions, we'll diagnose this particular error later.
       if ((T->isArrayType() || T->isFunctionType()) &&
           (D.getName().getKind() != UnqualifiedId::IK_ConversionFunctionId)) {
-        Diag(DeclType.Loc, diag::err_func_returning_array_function) 
-          << T->isFunctionType() << T;
+        unsigned diagID = diag::err_func_returning_array_function;
+        // Last processing chunk in block context means this function chunk
+        // represents the block.
+        if (chunkIndex == 0 &&
+            D.getContext() == Declarator::BlockLiteralContext)
+          diagID = diag::err_block_returning_array_function;
+        Diag(DeclType.Loc, diagID) << T->isFunctionType() << T;
         T = Context.IntTy;
         D.setInvalidType(true);
       }
 
+      // Check for auto functions and trailing return type and adjust the
+      // return type accordingly.
+      if (!D.isInvalidType()) {
+        // trailing-return-type is only required if we're declaring a function,
+        // and not, for instance, a pointer to a function.
+        if (D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_auto &&
+            !FTI.TrailingReturnType && chunkIndex == 0) {
+          Diag(D.getDeclSpec().getTypeSpecTypeLoc(),
+               diag::err_auto_missing_trailing_return);
+          T = Context.IntTy;
+          D.setInvalidType(true);
+        } else if (FTI.TrailingReturnType) {
+          if (T.hasQualifiers() || !isa<AutoType>(T)) {
+            // T must be exactly 'auto' at this point. See CWG issue 681.
+            Diag(D.getDeclSpec().getTypeSpecTypeLoc(),
+                 diag::err_trailing_return_without_auto)
+              << T << D.getDeclSpec().getSourceRange();
+            D.setInvalidType(true);
+          }
+
+          T = GetTypeFromParser(
+            ParsedType::getFromOpaquePtr(FTI.TrailingReturnType),
+            &ReturnTypeInfo);
+        }
+      }
+
       // cv-qualifiers on return types are pointless except when the type is a
       // class type in C++.
       if (T.getCVRQualifiers() && D.getDeclSpec().getTypeQualifiers() &&
@@ -1229,6 +1727,13 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
           break;
         }
 
+        FunctionProtoType::ExtProtoInfo EPI;
+        EPI.Variadic = FTI.isVariadic;
+        EPI.TypeQuals = FTI.TypeQuals;
+        EPI.RefQualifier = !FTI.hasRefQualifier()? RQ_None
+                    : FTI.RefQualifierIsLValueRef? RQ_LValue
+                    : RQ_RValue;
+        
         // Otherwise, we have a function with an argument list that is
         // potentially variadic.
         llvm::SmallVector<QualType, 16> ArgTys;
@@ -1280,30 +1785,25 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
         }
 
         llvm::SmallVector<QualType, 4> Exceptions;
-        Exceptions.reserve(FTI.NumExceptions);
-        for (unsigned ei = 0, ee = FTI.NumExceptions; ei != ee; ++ei) {
-          // FIXME: Preserve type source info.
-          QualType ET = GetTypeFromParser(FTI.Exceptions[ei].Ty);
-          // Check that the type is valid for an exception spec, and drop it if
-          // not.
-          if (!CheckSpecifiedExceptionType(ET, FTI.Exceptions[ei].Range))
-            Exceptions.push_back(ET);
+        if (FTI.hasExceptionSpec) {
+          EPI.HasExceptionSpec = FTI.hasExceptionSpec;
+          EPI.HasAnyExceptionSpec = FTI.hasAnyExceptionSpec;
+          Exceptions.reserve(FTI.NumExceptions);
+          for (unsigned ei = 0, ee = FTI.NumExceptions; ei != ee; ++ei) {
+            // FIXME: Preserve type source info.
+            QualType ET = GetTypeFromParser(FTI.Exceptions[ei].Ty);
+            // Check that the type is valid for an exception spec, and
+            // drop it if not.
+            if (!CheckSpecifiedExceptionType(ET, FTI.Exceptions[ei].Range))
+              Exceptions.push_back(ET);
+          }
+          EPI.NumExceptions = Exceptions.size();
+          EPI.Exceptions = Exceptions.data();
         }
 
-        T = Context.getFunctionType(T, ArgTys.data(), ArgTys.size(),
-                                    FTI.isVariadic, FTI.TypeQuals,
-                                    FTI.hasExceptionSpec,
-                                    FTI.hasAnyExceptionSpec,
-                                    Exceptions.size(), Exceptions.data(),
-                                    FunctionType::ExtInfo());
+        T = Context.getFunctionType(T, ArgTys.data(), ArgTys.size(), EPI);
       }
 
-      // For GCC compatibility, we allow attributes that apply only to
-      // function types to be placed on a function's return type
-      // instead (as long as that type doesn't happen to be function
-      // or function-pointer itself).
-      ProcessDelayedFnAttrs(*this, T, FnAttrsFromPreviousChunk);
-
       break;
     }
     case DeclaratorChunk::MemberPointer:
@@ -1364,61 +1864,190 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
       T = Context.IntTy;
     }
 
-    DiagnoseDelayedFnAttrs(*this, FnAttrsFromPreviousChunk);
-
     // See if there are any attributes on this declarator chunk.
-    if (const AttributeList *AL = DeclType.getAttrs())
-      ProcessTypeAttributeList(*this, T, false, AL, FnAttrsFromPreviousChunk);
+    if (AttributeList *attrs = const_cast<AttributeList*>(DeclType.getAttrs()))
+      processTypeAttrs(state, T, false, attrs);
   }
 
   if (getLangOptions().CPlusPlus && T->isFunctionType()) {
     const FunctionProtoType *FnTy = T->getAs<FunctionProtoType>();
     assert(FnTy && "Why oh why is there not a FunctionProtoType here?");
 
-    // C++ 8.3.5p4: A cv-qualifier-seq shall only be part of the function type
-    // for a nonstatic member function, the function type to which a pointer
-    // to member refers, or the top-level function type of a function typedef
-    // declaration.
-    bool FreeFunction = (D.getContext() != Declarator::MemberContext &&
-        (!D.getCXXScopeSpec().isSet() ||
-         !computeDeclContext(D.getCXXScopeSpec(), /*FIXME:*/true)->isRecord()));
-    if (FnTy->getTypeQuals() != 0 &&
+    // C++ 8.3.5p4: 
+    //   A cv-qualifier-seq shall only be part of the function type
+    //   for a nonstatic member function, the function type to which a pointer
+    //   to member refers, or the top-level function type of a function typedef
+    //   declaration.
+    //
+    // Core issue 547 also allows cv-qualifiers on function types that are
+    // top-level template type arguments.
+    bool FreeFunction;
+    if (!D.getCXXScopeSpec().isSet()) {
+      FreeFunction = (D.getContext() != Declarator::MemberContext ||
+                      D.getDeclSpec().isFriendSpecified());
+    } else {
+      DeclContext *DC = computeDeclContext(D.getCXXScopeSpec());
+      FreeFunction = (DC && !DC->isRecord());
+    }
+
+    // C++0x [dcl.fct]p6:
+    //   A ref-qualifier shall only be part of the function type for a
+    //   non-static member function, the function type to which a pointer to
+    //   member refers, or the top-level function type of a function typedef 
+    //   declaration.
+    if ((FnTy->getTypeQuals() != 0 || FnTy->getRefQualifier()) &&
+        !(D.getContext() == Declarator::TemplateTypeArgContext &&
+          !D.isFunctionDeclarator()) &&
         D.getDeclSpec().getStorageClassSpec() != DeclSpec::SCS_typedef &&
         (FreeFunction ||
          D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_static)) {
-      if (D.isFunctionDeclarator())
-        Diag(D.getIdentifierLoc(), diag::err_invalid_qualified_function_type);
-      else
-        Diag(D.getIdentifierLoc(),
-             diag::err_invalid_qualified_typedef_function_type_use)
-          << FreeFunction;
+      if (D.getContext() == Declarator::TemplateTypeArgContext) {
+        // Accept qualified function types as template type arguments as a GNU
+        // extension. This is also the subject of C++ core issue 547.
+        std::string Quals;
+        if (FnTy->getTypeQuals() != 0)
+          Quals = Qualifiers::fromCVRMask(FnTy->getTypeQuals()).getAsString();
+        
+        switch (FnTy->getRefQualifier()) {
+        case RQ_None:
+          break;
+            
+        case RQ_LValue:
+          if (!Quals.empty())
+            Quals += ' ';
+          Quals += '&';
+          break;
+          
+        case RQ_RValue:
+          if (!Quals.empty())
+            Quals += ' ';
+          Quals += "&&";
+          break;
+        }
+        
+        Diag(D.getIdentifierLoc(), 
+             diag::ext_qualified_function_type_template_arg)
+          << Quals;
+      } else {
+        if (FnTy->getTypeQuals() != 0) {
+          if (D.isFunctionDeclarator())
+            Diag(D.getIdentifierLoc(), 
+                 diag::err_invalid_qualified_function_type);
+          else
+            Diag(D.getIdentifierLoc(),
+                 diag::err_invalid_qualified_typedef_function_type_use)
+              << FreeFunction;
+        }
+          
+        if (FnTy->getRefQualifier()) {
+          if (D.isFunctionDeclarator()) {
+            SourceLocation Loc = D.getIdentifierLoc();
+            for (unsigned I = 0, N = D.getNumTypeObjects(); I != N; ++I) {
+              const DeclaratorChunk &Chunk = D.getTypeObject(N-I-1);
+              if (Chunk.Kind == DeclaratorChunk::Function &&
+                  Chunk.Fun.hasRefQualifier()) {
+                Loc = Chunk.Fun.getRefQualifierLoc();
+                break;
+              }
+            }
 
-      // Strip the cv-quals from the type.
-      T = Context.getFunctionType(FnTy->getResultType(), FnTy->arg_type_begin(),
-                                  FnTy->getNumArgs(), FnTy->isVariadic(), 0, 
-                                  false, false, 0, 0, FunctionType::ExtInfo());
+            Diag(Loc, diag::err_invalid_ref_qualifier_function_type)
+              << (FnTy->getRefQualifier() == RQ_LValue)
+              << FixItHint::CreateRemoval(Loc);
+          } else {
+            Diag(D.getIdentifierLoc(), 
+                 diag::err_invalid_ref_qualifier_typedef_function_type_use)
+              << FreeFunction
+              << (FnTy->getRefQualifier() == RQ_LValue);
+          }
+        }
+          
+        // Strip the cv-qualifiers and ref-qualifiers from the type.
+        FunctionProtoType::ExtProtoInfo EPI = FnTy->getExtProtoInfo();
+        EPI.TypeQuals = 0;
+        EPI.RefQualifier = RQ_None;
+          
+        T = Context.getFunctionType(FnTy->getResultType(), 
+                                    FnTy->arg_type_begin(),
+                                    FnTy->getNumArgs(), EPI);
+      }
     }
   }
 
+  // Apply any undistributed attributes from the declarator.
+  if (!T.isNull())
+    if (AttributeList *attrs = D.getAttributes())
+      processTypeAttrs(state, T, false, attrs);
+
+  // Diagnose any ignored type attributes.
+  if (!T.isNull()) state.diagnoseIgnoredTypeAttrs(T);
+
   // If there's a constexpr specifier, treat it as a top-level const.
   if (D.getDeclSpec().isConstexprSpecified()) {
     T.addConst();
   }
 
-  // Process any function attributes we might have delayed from the
-  // declaration-specifiers.
-  ProcessDelayedFnAttrs(*this, T, FnAttrsFromDeclSpec);
-
-  // If there were any type attributes applied to the decl itself, not
-  // the type, apply them to the result type.  But don't do this for
-  // block-literal expressions, which are parsed wierdly.
-  if (D.getContext() != Declarator::BlockLiteralContext)
-    if (const AttributeList *Attrs = D.getAttributes())
-      ProcessTypeAttributeList(*this, T, false, Attrs,
-                               FnAttrsFromPreviousChunk);
-
-  DiagnoseDelayedFnAttrs(*this, FnAttrsFromPreviousChunk);
-
+  // If there was an ellipsis in the declarator, the declaration declares a 
+  // parameter pack whose type may be a pack expansion type.
+  if (D.hasEllipsis() && !T.isNull()) {
+    // C++0x [dcl.fct]p13:
+    //   A declarator-id or abstract-declarator containing an ellipsis shall 
+    //   only be used in a parameter-declaration. Such a parameter-declaration
+    //   is a parameter pack (14.5.3). [...]
+    switch (D.getContext()) {
+    case Declarator::PrototypeContext:
+      // C++0x [dcl.fct]p13:
+      //   [...] When it is part of a parameter-declaration-clause, the 
+      //   parameter pack is a function parameter pack (14.5.3). The type T 
+      //   of the declarator-id of the function parameter pack shall contain
+      //   a template parameter pack; each template parameter pack in T is 
+      //   expanded by the function parameter pack.
+      //
+      // We represent function parameter packs as function parameters whose
+      // type is a pack expansion.
+      if (!T->containsUnexpandedParameterPack()) {
+        Diag(D.getEllipsisLoc(), 
+             diag::err_function_parameter_pack_without_parameter_packs)
+          << T <<  D.getSourceRange();
+        D.setEllipsisLoc(SourceLocation());
+      } else {
+        T = Context.getPackExpansionType(T, llvm::Optional<unsigned>());
+      }
+      break;
+        
+    case Declarator::TemplateParamContext:
+      // C++0x [temp.param]p15:
+      //   If a template-parameter is a [...] is a parameter-declaration that 
+      //   declares a parameter pack (8.3.5), then the template-parameter is a
+      //   template parameter pack (14.5.3).
+      //
+      // Note: core issue 778 clarifies that, if there are any unexpanded
+      // parameter packs in the type of the non-type template parameter, then
+      // it expands those parameter packs.
+      if (T->containsUnexpandedParameterPack())
+        T = Context.getPackExpansionType(T, llvm::Optional<unsigned>());
+      else if (!getLangOptions().CPlusPlus0x)
+        Diag(D.getEllipsisLoc(), diag::ext_variadic_templates);
+      break;
+    
+    case Declarator::FileContext:
+    case Declarator::KNRTypeListContext:
+    case Declarator::TypeNameContext:
+    case Declarator::MemberContext:
+    case Declarator::BlockContext:
+    case Declarator::ForContext:
+    case Declarator::ConditionContext:
+    case Declarator::CXXCatchContext:
+    case Declarator::BlockLiteralContext:
+    case Declarator::TemplateTypeArgContext:
+      // FIXME: We may want to allow parameter packs in block-literal contexts
+      // in the future.
+      Diag(D.getEllipsisLoc(), diag::err_ellipsis_in_declarator_not_parameter);
+      D.setEllipsisLoc(SourceLocation());
+      break;
+    }
+  }
+  
   if (T.isNull())
     return Context.getNullTypeSourceInfo();
   else if (D.isInvalidType())
@@ -1428,10 +2057,12 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
 
 namespace {
   class TypeSpecLocFiller : public TypeLocVisitor<TypeSpecLocFiller> {
+    ASTContext &Context;
     const DeclSpec &DS;
 
   public:
-    TypeSpecLocFiller(const DeclSpec &DS) : DS(DS) {}
+    TypeSpecLocFiller(ASTContext &Context, const DeclSpec &DS) 
+      : Context(Context), DS(DS) {}
 
     void VisitQualifiedTypeLoc(QualifiedTypeLoc TL) {
       Visit(TL.getUnqualifiedLoc());
@@ -1446,7 +2077,7 @@ namespace {
       // Handle the base type, which might not have been written explicitly.
       if (DS.getTypeSpecType() == DeclSpec::TST_unspecified) {
         TL.setHasBaseTypeAsWritten(false);
-        TL.getBaseLoc().initialize(SourceLocation());
+        TL.getBaseLoc().initialize(Context, SourceLocation());
       } else {
         TL.setHasBaseTypeAsWritten(true);
         Visit(TL.getBaseLoc());
@@ -1477,7 +2108,7 @@ namespace {
       // If we got no declarator info from previous Sema routines,
       // just fill with the typespec loc.
       if (!TInfo) {
-        TL.initialize(DS.getTypeSpecTypeLoc());
+        TL.initialize(Context, DS.getTypeSpecTypeLoc());
         return;
       }
 
@@ -1523,7 +2154,7 @@ namespace {
     void VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
       ElaboratedTypeKeyword Keyword
         = TypeWithKeyword::getKeywordForTypeSpec(DS.getTypeSpecType());
-      if (Keyword == ETK_Typename) {
+      if (DS.getTypeSpecType() == TST_typename) {
         TypeSourceInfo *TInfo = 0;
         Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
         if (TInfo) {
@@ -1541,7 +2172,7 @@ namespace {
     void VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
       ElaboratedTypeKeyword Keyword
         = TypeWithKeyword::getKeywordForTypeSpec(DS.getTypeSpecType());
-      if (Keyword == ETK_Typename) {
+      if (DS.getTypeSpecType() == TST_typename) {
         TypeSourceInfo *TInfo = 0;
         Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
         if (TInfo) {
@@ -1570,7 +2201,7 @@ namespace {
           return;
         }
       }
-      TL.initializeLocal(SourceLocation());
+      TL.initializeLocal(Context, SourceLocation());
       TL.setKeywordLoc(Keyword != ETK_None
                        ? DS.getTypeSpecTypeLoc()
                        : SourceLocation());
@@ -1582,7 +2213,7 @@ namespace {
 
     void VisitTypeLoc(TypeLoc TL) {
       // FIXME: add other typespec types and change this to an assert.
-      TL.initialize(DS.getTypeSpecTypeLoc());
+      TL.initialize(Context, DS.getTypeSpecTypeLoc());
     }
   };
 
@@ -1634,6 +2265,7 @@ namespace {
       assert(Chunk.Kind == DeclaratorChunk::Function);
       TL.setLParenLoc(Chunk.Loc);
       TL.setRParenLoc(Chunk.EndLoc);
+      TL.setTrailingReturn(!!Chunk.Fun.TrailingReturnType);
 
       const DeclaratorChunk::FunctionTypeInfo &FTI = Chunk.Fun;
       for (unsigned i = 0, e = TL.getNumArgs(), tpi = 0; i != e; ++i) {
@@ -1642,6 +2274,11 @@ namespace {
       }
       // FIXME: exception specs
     }
+    void VisitParenTypeLoc(ParenTypeLoc TL) {
+      assert(Chunk.Kind == DeclaratorChunk::Paren);
+      TL.setLParenLoc(Chunk.Loc);
+      TL.setRParenLoc(Chunk.EndLoc);
+    }
 
     void VisitTypeLoc(TypeLoc TL) {
       llvm_unreachable("unsupported TypeLoc kind in declarator!");
@@ -1663,6 +2300,12 @@ Sema::GetTypeSourceInfoForDeclarator(Declarator &D, QualType T,
   TypeSourceInfo *TInfo = Context.CreateTypeSourceInfo(T);
   UnqualTypeLoc CurrTL = TInfo->getTypeLoc().getUnqualifiedLoc();
 
+  // Handle parameter packs whose type is a pack expansion.
+  if (isa<PackExpansionType>(T)) {
+    cast<PackExpansionTypeLoc>(CurrTL).setEllipsisLoc(D.getEllipsisLoc());
+    CurrTL = CurrTL.getNextTypeLoc().getUnqualifiedLoc();    
+  }
+  
   for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i) {
     DeclaratorLocFiller(D.getTypeObject(i)).Visit(CurrTL);
     CurrTL = CurrTL.getNextTypeLoc().getUnqualifiedLoc();
@@ -1675,7 +2318,7 @@ Sema::GetTypeSourceInfoForDeclarator(Declarator &D, QualType T,
     assert(TL.getFullDataSize() == CurrTL.getFullDataSize());
     memcpy(CurrTL.getOpaqueData(), TL.getOpaqueData(), TL.getFullDataSize());
   } else {
-    TypeSpecLocFiller(D.getDeclSpec()).Visit(CurrTL);
+    TypeSpecLocFiller(Context, D.getDeclSpec()).Visit(CurrTL);
   }
       
   return TInfo;
@@ -1686,7 +2329,8 @@ ParsedType Sema::CreateParsedType(QualType T, TypeSourceInfo *TInfo) {
   // FIXME: LocInfoTypes are "transient", only needed for passing to/from Parser
   // and Sema during declaration parsing. Try deallocating/caching them when
   // it's appropriate, instead of allocating them and keeping them around.
-  LocInfoType *LocT = (LocInfoType*)BumpAlloc.Allocate(sizeof(LocInfoType), 8);
+  LocInfoType *LocT = (LocInfoType*)BumpAlloc.Allocate(sizeof(LocInfoType), 
+                                                       TypeAlignment);
   new (LocT) LocInfoType(T, TInfo);
   assert(LocT->getTypeClass() != T->getTypeClass() &&
          "LocInfoType's TypeClass conflicts with an existing Type class");
@@ -1787,160 +2431,291 @@ static void HandleAddressSpaceTypeAttribute(QualType &Type,
   Type = S.Context.getAddrSpaceQualType(Type, ASIdx);
 }
 
-/// HandleObjCGCTypeAttribute - Process an objc's gc attribute on the
-/// specified type.  The attribute contains 1 argument, weak or strong.
-static void HandleObjCGCTypeAttribute(QualType &Type,
-                                      const AttributeList &Attr, Sema &S) {
-  if (Type.getObjCGCAttr() != Qualifiers::GCNone) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_multiple_objc_gc);
-    Attr.setInvalid();
-    return;
+/// handleObjCGCTypeAttr - Process the __attribute__((objc_gc)) type
+/// attribute on the specified type.  Returns true to indicate that
+/// the attribute was handled, false to indicate that the type does
+/// not permit the attribute.
+static bool handleObjCGCTypeAttr(TypeProcessingState &state,
+                                 AttributeList &attr,
+                                 QualType &type) {
+  Sema &S = state.getSema();
+
+  // Delay if this isn't some kind of pointer.
+  if (!type->isPointerType() &&
+      !type->isObjCObjectPointerType() &&
+      !type->isBlockPointerType())
+    return false;
+
+  if (type.getObjCGCAttr() != Qualifiers::GCNone) {
+    S.Diag(attr.getLoc(), diag::err_attribute_multiple_objc_gc);
+    attr.setInvalid();
+    return true;
   }
 
   // Check the attribute arguments.
-  if (!Attr.getParameterName()) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
+  if (!attr.getParameterName()) {
+    S.Diag(attr.getLoc(), diag::err_attribute_argument_n_not_string)
       << "objc_gc" << 1;
-    Attr.setInvalid();
-    return;
+    attr.setInvalid();
+    return true;
   }
   Qualifiers::GC GCAttr;
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
-    Attr.setInvalid();
-    return;
+  if (attr.getNumArgs() != 0) {
+    S.Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+    attr.setInvalid();
+    return true;
   }
-  if (Attr.getParameterName()->isStr("weak"))
+  if (attr.getParameterName()->isStr("weak"))
     GCAttr = Qualifiers::Weak;
-  else if (Attr.getParameterName()->isStr("strong"))
+  else if (attr.getParameterName()->isStr("strong"))
     GCAttr = Qualifiers::Strong;
   else {
-    S.Diag(Attr.getLoc(), diag::warn_attribute_type_not_supported)
-      << "objc_gc" << Attr.getParameterName();
-    Attr.setInvalid();
-    return;
+    S.Diag(attr.getLoc(), diag::warn_attribute_type_not_supported)
+      << "objc_gc" << attr.getParameterName();
+    attr.setInvalid();
+    return true;
   }
 
-  Type = S.Context.getObjCGCQualType(Type, GCAttr);
+  type = S.Context.getObjCGCQualType(type, GCAttr);
+  return true;
 }
 
-/// Process an individual function attribute.  Returns true if the
-/// attribute does not make sense to apply to this type.
-bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
-  if (Attr.getKind() == AttributeList::AT_noreturn) {
-    // Complain immediately if the arg count is wrong.
-    if (Attr.getNumArgs() != 0) {
-      S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-      Attr.setInvalid();
-      return false;
+namespace {
+  /// A helper class to unwrap a type down to a function for the
+  /// purposes of applying attributes there.
+  ///
+  /// Use:
+  ///   FunctionTypeUnwrapper unwrapped(SemaRef, T);
+  ///   if (unwrapped.isFunctionType()) {
+  ///     const FunctionType *fn = unwrapped.get();
+  ///     // change fn somehow
+  ///     T = unwrapped.wrap(fn);
+  ///   }
+  struct FunctionTypeUnwrapper {
+    enum WrapKind {
+      Desugar,
+      Parens,
+      Pointer,
+      BlockPointer,
+      Reference,
+      MemberPointer
+    };
+
+    QualType Original;
+    const FunctionType *Fn;
+    llvm::SmallVector<unsigned char /*WrapKind*/, 8> Stack;
+
+    FunctionTypeUnwrapper(Sema &S, QualType T) : Original(T) {
+      while (true) {
+        const Type *Ty = T.getTypePtr();
+        if (isa<FunctionType>(Ty)) {
+          Fn = cast<FunctionType>(Ty);
+          return;
+        } else if (isa<ParenType>(Ty)) {
+          T = cast<ParenType>(Ty)->getInnerType();
+          Stack.push_back(Parens);
+        } else if (isa<PointerType>(Ty)) {
+          T = cast<PointerType>(Ty)->getPointeeType();
+          Stack.push_back(Pointer);
+        } else if (isa<BlockPointerType>(Ty)) {
+          T = cast<BlockPointerType>(Ty)->getPointeeType();
+          Stack.push_back(BlockPointer);
+        } else if (isa<MemberPointerType>(Ty)) {
+          T = cast<MemberPointerType>(Ty)->getPointeeType();
+          Stack.push_back(MemberPointer);
+        } else if (isa<ReferenceType>(Ty)) {
+          T = cast<ReferenceType>(Ty)->getPointeeType();
+          Stack.push_back(Reference);
+        } else {
+          const Type *DTy = Ty->getUnqualifiedDesugaredType();
+          if (Ty == DTy) {
+            Fn = 0;
+            return;
+          }
+
+          T = QualType(DTy, 0);
+          Stack.push_back(Desugar);
+        }
+      }
     }
 
-    // Delay if this is not a function or pointer to block.
-    if (!Type->isFunctionPointerType()
-        && !Type->isBlockPointerType()
-        && !Type->isFunctionType()
-        && !Type->isMemberFunctionPointerType())
-      return true;
-    
-    // Otherwise we can process right away.
-    Type = S.Context.getNoReturnType(Type);
-    return false;
-  }
+    bool isFunctionType() const { return (Fn != 0); }
+    const FunctionType *get() const { return Fn; }
 
-  if (Attr.getKind() == AttributeList::AT_regparm) {
-    // The warning is emitted elsewhere
-    if (Attr.getNumArgs() != 1) {
-      return false;
+    QualType wrap(Sema &S, const FunctionType *New) {
+      // If T wasn't modified from the unwrapped type, do nothing.
+      if (New == get()) return Original;
+
+      Fn = New;
+      return wrap(S.Context, Original, 0);
+    }
+
+  private:
+    QualType wrap(ASTContext &C, QualType Old, unsigned I) {
+      if (I == Stack.size())
+        return C.getQualifiedType(Fn, Old.getQualifiers());
+
+      // Build up the inner type, applying the qualifiers from the old
+      // type to the new type.
+      SplitQualType SplitOld = Old.split();
+
+      // As a special case, tail-recurse if there are no qualifiers.
+      if (SplitOld.second.empty())
+        return wrap(C, SplitOld.first, I);
+      return C.getQualifiedType(wrap(C, SplitOld.first, I), SplitOld.second);
+    }
+
+    QualType wrap(ASTContext &C, const Type *Old, unsigned I) {
+      if (I == Stack.size()) return QualType(Fn, 0);
+
+      switch (static_cast<WrapKind>(Stack[I++])) {
+      case Desugar:
+        // This is the point at which we potentially lose source
+        // information.
+        return wrap(C, Old->getUnqualifiedDesugaredType(), I);
+
+      case Parens: {
+        QualType New = wrap(C, cast<ParenType>(Old)->getInnerType(), I);
+        return C.getParenType(New);
+      }
+
+      case Pointer: {
+        QualType New = wrap(C, cast<PointerType>(Old)->getPointeeType(), I);
+        return C.getPointerType(New);
+      }
+
+      case BlockPointer: {
+        QualType New = wrap(C, cast<BlockPointerType>(Old)->getPointeeType(),I);
+        return C.getBlockPointerType(New);
+      }
+
+      case MemberPointer: {
+        const MemberPointerType *OldMPT = cast<MemberPointerType>(Old);
+        QualType New = wrap(C, OldMPT->getPointeeType(), I);
+        return C.getMemberPointerType(New, OldMPT->getClass());
+      }
+
+      case Reference: {
+        const ReferenceType *OldRef = cast<ReferenceType>(Old);
+        QualType New = wrap(C, OldRef->getPointeeType(), I);
+        if (isa<LValueReferenceType>(OldRef))
+          return C.getLValueReferenceType(New, OldRef->isSpelledAsLValue());
+        else
+          return C.getRValueReferenceType(New);
+      }
+      }
+
+      llvm_unreachable("unknown wrapping kind");
+      return QualType();
     }
+  };
+}
+
+/// Process an individual function attribute.  Returns true to
+/// indicate that the attribute was handled, false if it wasn't.
+static bool handleFunctionTypeAttr(TypeProcessingState &state,
+                                   AttributeList &attr,
+                                   QualType &type) {
+  Sema &S = state.getSema();
 
-    // Delay if this is not a function or pointer to block.
-    if (!Type->isFunctionPointerType()
-        && !Type->isBlockPointerType()
-        && !Type->isFunctionType()
-        && !Type->isMemberFunctionPointerType())
+  FunctionTypeUnwrapper unwrapped(S, type);
+
+  if (attr.getKind() == AttributeList::AT_noreturn) {
+    if (S.CheckNoReturnAttr(attr))
       return true;
 
+    // Delay if this is not a function type.
+    if (!unwrapped.isFunctionType())
+      return false;
+
     // Otherwise we can process right away.
-    Expr *NumParamsExpr = static_cast<Expr *>(Attr.getArg(0));
-    llvm::APSInt NumParams(32);
+    FunctionType::ExtInfo EI = unwrapped.get()->getExtInfo().withNoReturn(true);
+    type = unwrapped.wrap(S, S.Context.adjustFunctionType(unwrapped.get(), EI));
+    return true;
+  }
 
-    // The warning is emitted elsewhere
-    if (NumParamsExpr->isTypeDependent() || NumParamsExpr->isValueDependent() ||
-        !NumParamsExpr->isIntegerConstantExpr(NumParams, S.Context))
+  if (attr.getKind() == AttributeList::AT_regparm) {
+    unsigned value;
+    if (S.CheckRegparmAttr(attr, value))
+      return true;
+
+    // Delay if this is not a function type.
+    if (!unwrapped.isFunctionType())
       return false;
 
-    Type = S.Context.getRegParmType(Type, NumParams.getZExtValue());
-    return false;
-  }
+    // Diagnose regparm with fastcall.
+    const FunctionType *fn = unwrapped.get();
+    CallingConv CC = fn->getCallConv();
+    if (CC == CC_X86FastCall) {
+      S.Diag(attr.getLoc(), diag::err_attributes_are_not_compatible)
+        << FunctionType::getNameForCallConv(CC)
+        << "regparm";
+      attr.setInvalid();
+      return true;
+    }
 
-  // Otherwise, a calling convention.
-  if (Attr.getNumArgs() != 0) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    Attr.setInvalid();
-    return false;
+    FunctionType::ExtInfo EI = 
+      unwrapped.get()->getExtInfo().withRegParm(value);
+    type = unwrapped.wrap(S, S.Context.adjustFunctionType(unwrapped.get(), EI));
+    return true;
   }
 
-  QualType T = Type;
-  if (const PointerType *PT = Type->getAs<PointerType>())
-    T = PT->getPointeeType();
-  else if (const BlockPointerType *BPT = Type->getAs<BlockPointerType>())
-    T = BPT->getPointeeType();
-  else if (const MemberPointerType *MPT = Type->getAs<MemberPointerType>())
-    T = MPT->getPointeeType();
-  else if (const ReferenceType *RT = Type->getAs<ReferenceType>())
-    T = RT->getPointeeType();
-  const FunctionType *Fn = T->getAs<FunctionType>();
+  // Otherwise, a calling convention.
+  CallingConv CC;
+  if (S.CheckCallingConvAttr(attr, CC))
+    return true;
 
   // Delay if the type didn't work out to a function.
-  if (!Fn) return true;
+  if (!unwrapped.isFunctionType()) return false;
 
-  // TODO: diagnose uses of these conventions on the wrong target.
-  CallingConv CC;
-  switch (Attr.getKind()) {
-  case AttributeList::AT_cdecl: CC = CC_C; break;
-  case AttributeList::AT_fastcall: CC = CC_X86FastCall; break;
-  case AttributeList::AT_stdcall: CC = CC_X86StdCall; break;
-  case AttributeList::AT_thiscall: CC = CC_X86ThisCall; break;
-  case AttributeList::AT_pascal: CC = CC_X86Pascal; break;
-  default: llvm_unreachable("unexpected attribute kind"); return false;
-  }
-
-  CallingConv CCOld = Fn->getCallConv();
+  const FunctionType *fn = unwrapped.get();
+  CallingConv CCOld = fn->getCallConv();
   if (S.Context.getCanonicalCallConv(CC) ==
       S.Context.getCanonicalCallConv(CCOld)) {
-    Attr.setInvalid();
-    return false;
+    FunctionType::ExtInfo EI= unwrapped.get()->getExtInfo().withCallingConv(CC);
+    type = unwrapped.wrap(S, S.Context.adjustFunctionType(unwrapped.get(), EI));
+    return true;
   }
 
   if (CCOld != CC_Default) {
     // Should we diagnose reapplications of the same convention?
-    S.Diag(Attr.getLoc(), diag::err_attributes_are_not_compatible)
+    S.Diag(attr.getLoc(), diag::err_attributes_are_not_compatible)
       << FunctionType::getNameForCallConv(CC)
       << FunctionType::getNameForCallConv(CCOld);
-    Attr.setInvalid();
-    return false;
+    attr.setInvalid();
+    return true;
   }
 
   // Diagnose the use of X86 fastcall on varargs or unprototyped functions.
   if (CC == CC_X86FastCall) {
-    if (isa<FunctionNoProtoType>(Fn)) {
-      S.Diag(Attr.getLoc(), diag::err_cconv_knr)
+    if (isa<FunctionNoProtoType>(fn)) {
+      S.Diag(attr.getLoc(), diag::err_cconv_knr)
         << FunctionType::getNameForCallConv(CC);
-      Attr.setInvalid();
-      return false;
+      attr.setInvalid();
+      return true;
     }
 
-    const FunctionProtoType *FnP = cast<FunctionProtoType>(Fn);
+    const FunctionProtoType *FnP = cast<FunctionProtoType>(fn);
     if (FnP->isVariadic()) {
-      S.Diag(Attr.getLoc(), diag::err_cconv_varargs)
+      S.Diag(attr.getLoc(), diag::err_cconv_varargs)
         << FunctionType::getNameForCallConv(CC);
-      Attr.setInvalid();
-      return false;
+      attr.setInvalid();
+      return true;
+    }
+
+    // Also diagnose fastcall with regparm.
+    if (fn->getRegParmType()) {
+      S.Diag(attr.getLoc(), diag::err_attributes_are_not_compatible)
+        << "regparm"
+        << FunctionType::getNameForCallConv(CC);
+      attr.setInvalid();
+      return true;
     }
   }
 
-  Type = S.Context.getCallConvType(Type, CC);
-  return false;
+  FunctionType::ExtInfo EI = unwrapped.get()->getExtInfo().withCallingConv(CC);
+  type = unwrapped.wrap(S, S.Context.adjustFunctionType(unwrapped.get(), EI));
+  return true;
 }
 
 /// HandleVectorSizeAttribute - this attribute is only applicable to integral
@@ -1952,7 +2727,7 @@ bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
 /// this routine will return a new vector type.
 static void HandleVectorSizeAttr(QualType& CurType, const AttributeList &Attr,
                                  Sema &S) {
-  // Check the attribute arugments.
+  // Check the attribute arguments.
   if (Attr.getNumArgs() != 1) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
     Attr.setInvalid();
@@ -1994,50 +2769,121 @@ static void HandleVectorSizeAttr(QualType& CurType, const AttributeList &Attr,
   // Success! Instantiate the vector type, the number of elements is > 0, and
   // not required to be a power of 2, unlike GCC.
   CurType = S.Context.getVectorType(CurType, vectorSize/typeSize,
-                                    VectorType::NotAltiVec);
+                                    VectorType::GenericVector);
 }
 
-void ProcessTypeAttributeList(Sema &S, QualType &Result,
-                              bool IsDeclSpec, const AttributeList *AL,
-                              DelayedAttributeSet &FnAttrs) {
+/// HandleNeonVectorTypeAttr - The "neon_vector_type" and
+/// "neon_polyvector_type" attributes are used to create vector types that
+/// are mangled according to ARM's ABI.  Otherwise, these types are identical
+/// to those created with the "vector_size" attribute.  Unlike "vector_size"
+/// the argument to these Neon attributes is the number of vector elements,
+/// not the vector size in bytes.  The vector width and element type must
+/// match one of the standard Neon vector types.
+static void HandleNeonVectorTypeAttr(QualType& CurType,
+                                     const AttributeList &Attr, Sema &S,
+                                     VectorType::VectorKind VecKind,
+                                     const char *AttrName) {
+  // Check the attribute arguments.
+  if (Attr.getNumArgs() != 1) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+    Attr.setInvalid();
+    return;
+  }
+  // The number of elements must be an ICE.
+  Expr *numEltsExpr = static_cast<Expr *>(Attr.getArg(0));
+  llvm::APSInt numEltsInt(32);
+  if (numEltsExpr->isTypeDependent() || numEltsExpr->isValueDependent() ||
+      !numEltsExpr->isIntegerConstantExpr(numEltsInt, S.Context)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_argument_not_int)
+      << AttrName << numEltsExpr->getSourceRange();
+    Attr.setInvalid();
+    return;
+  }
+  // Only certain element types are supported for Neon vectors.
+  const BuiltinType* BTy = CurType->getAs<BuiltinType>();
+  if (!BTy ||
+      (VecKind == VectorType::NeonPolyVector &&
+       BTy->getKind() != BuiltinType::SChar &&
+       BTy->getKind() != BuiltinType::Short) ||
+      (BTy->getKind() != BuiltinType::SChar &&
+       BTy->getKind() != BuiltinType::UChar &&
+       BTy->getKind() != BuiltinType::Short &&
+       BTy->getKind() != BuiltinType::UShort &&
+       BTy->getKind() != BuiltinType::Int &&
+       BTy->getKind() != BuiltinType::UInt &&
+       BTy->getKind() != BuiltinType::LongLong &&
+       BTy->getKind() != BuiltinType::ULongLong &&
+       BTy->getKind() != BuiltinType::Float)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_invalid_vector_type) <<CurType;
+    Attr.setInvalid();
+    return;
+  }
+  // The total size of the vector must be 64 or 128 bits.
+  unsigned typeSize = static_cast<unsigned>(S.Context.getTypeSize(CurType));
+  unsigned numElts = static_cast<unsigned>(numEltsInt.getZExtValue());
+  unsigned vecSize = typeSize * numElts;
+  if (vecSize != 64 && vecSize != 128) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_bad_neon_vector_size) << CurType;
+    Attr.setInvalid();
+    return;
+  }
+
+  CurType = S.Context.getVectorType(CurType, numElts, VecKind);
+}
+
+static void processTypeAttrs(TypeProcessingState &state, QualType &type,
+                             bool isDeclSpec, AttributeList *attrs) {
   // Scan through and apply attributes to this type where it makes sense.  Some
   // attributes (such as __address_space__, __vector_size__, etc) apply to the
   // type, but others can be present in the type specifiers even though they
   // apply to the decl.  Here we apply type attributes and ignore the rest.
-  for (; AL; AL = AL->getNext()) {
+
+  AttributeList *next;
+  do {
+    AttributeList &attr = *attrs;
+    next = attr.getNext();
+
     // Skip attributes that were marked to be invalid.
-    if (AL->isInvalid())
+    if (attr.isInvalid())
       continue;
 
     // If this is an attribute we can handle, do so now,
     // otherwise, add it to the FnAttrs list for rechaining.
-    switch (AL->getKind()) {
+    switch (attr.getKind()) {
     default: break;
 
     case AttributeList::AT_address_space:
-      HandleAddressSpaceTypeAttribute(Result, *AL, S);
+      HandleAddressSpaceTypeAttribute(type, attr, state.getSema());
       break;
-    case AttributeList::AT_objc_gc:
-      HandleObjCGCTypeAttribute(Result, *AL, S);
+    OBJC_POINTER_TYPE_ATTRS_CASELIST:
+      if (!handleObjCPointerTypeAttr(state, attr, type))
+        distributeObjCPointerTypeAttr(state, attr, type);
       break;
     case AttributeList::AT_vector_size:
-      HandleVectorSizeAttr(Result, *AL, S);
+      HandleVectorSizeAttr(type, attr, state.getSema());
+      break;
+    case AttributeList::AT_neon_vector_type:
+      HandleNeonVectorTypeAttr(type, attr, state.getSema(),
+                               VectorType::NeonVector, "neon_vector_type");
       break;
+    case AttributeList::AT_neon_polyvector_type:
+      HandleNeonVectorTypeAttr(type, attr, state.getSema(),
+                               VectorType::NeonPolyVector,
+                               "neon_polyvector_type");
+      break;
+
+    FUNCTION_TYPE_ATTRS_CASELIST:
+      // Never process function type attributes as part of the
+      // declaration-specifiers.
+      if (isDeclSpec)
+        distributeFunctionTypeAttrFromDeclSpec(state, attr, type);
 
-    case AttributeList::AT_noreturn:
-    case AttributeList::AT_cdecl:
-    case AttributeList::AT_fastcall:
-    case AttributeList::AT_stdcall:
-    case AttributeList::AT_thiscall:
-    case AttributeList::AT_pascal:
-    case AttributeList::AT_regparm:
-      // Don't process these on the DeclSpec.
-      if (IsDeclSpec ||
-          ProcessFnAttr(S, Result, *AL))
-        FnAttrs.push_back(DelayedAttribute(AL, Result));
+      // Otherwise, handle the possible delays.
+      else if (!handleFunctionTypeAttr(state, attr, type))
+        distributeFunctionTypeAttr(state, attr, type);
       break;
     }
-  }
+  } while ((attrs = next));
 }
 
 /// @brief Ensure that the type T is a complete type.
@@ -2110,16 +2956,19 @@ bool Sema::RequireCompleteType(SourceLocation Loc, QualType T,
   if (diag == 0)
     return true;
 
-  const TagType *Tag = 0;
-  if (const RecordType *Record = T->getAs<RecordType>())
-    Tag = Record;
-  else if (const EnumType *Enum = T->getAs<EnumType>())
-    Tag = Enum;
+  const TagType *Tag = T->getAs<TagType>();
 
   // Avoid diagnosing invalid decls as incomplete.
   if (Tag && Tag->getDecl()->isInvalidDecl())
     return true;
 
+  // Give the external AST source a chance to complete the type.
+  if (Tag && Tag->getDecl()->hasExternalLexicalStorage()) {
+    Context.getExternalSource()->CompleteType(Tag->getDecl());
+    if (!Tag->isIncompleteType())
+      return false;
+  }
+
   // We have an incomplete type. Produce a diagnostic.
   Diag(Loc, PD) << T;
 
@@ -2167,48 +3016,23 @@ QualType Sema::getElaboratedType(ElaboratedTypeKeyword Keyword,
   return Context.getElaboratedType(Keyword, NNS, T);
 }
 
-QualType Sema::BuildTypeofExprType(Expr *E) {
-  if (E->getType() == Context.OverloadTy) {
-    // C++ [temp.arg.explicit]p3 allows us to resolve a template-id to a 
-    // function template specialization wherever deduction cannot occur.
-    if (FunctionDecl *Specialization
-        = ResolveSingleFunctionTemplateSpecialization(E)) {
-      // The access doesn't really matter in this case.
-      DeclAccessPair Found = DeclAccessPair::make(Specialization,
-                                                  Specialization->getAccess());
-      E = FixOverloadedFunctionReference(E, Found, Specialization);
-      if (!E)
-        return QualType();      
-    } else {
-      Diag(E->getLocStart(),
-           diag::err_cannot_determine_declared_type_of_overloaded_function)
-        << false << E->getSourceRange();
-      return QualType();
-    }
+QualType Sema::BuildTypeofExprType(Expr *E, SourceLocation Loc) {
+  ExprResult ER = CheckPlaceholderExpr(E, Loc);
+  if (ER.isInvalid()) return QualType();
+  E = ER.take();
+
+  if (!E->isTypeDependent()) {
+    QualType T = E->getType();
+    if (const TagType *TT = T->getAs<TagType>())
+      DiagnoseUseOfDecl(TT->getDecl(), E->getExprLoc());
   }
-  
   return Context.getTypeOfExprType(E);
 }
 
-QualType Sema::BuildDecltypeType(Expr *E) {
-  if (E->getType() == Context.OverloadTy) {
-    // C++ [temp.arg.explicit]p3 allows us to resolve a template-id to a 
-    // function template specialization wherever deduction cannot occur.
-    if (FunctionDecl *Specialization
-          = ResolveSingleFunctionTemplateSpecialization(E)) {
-      // The access doesn't really matter in this case.
-      DeclAccessPair Found = DeclAccessPair::make(Specialization,
-                                                  Specialization->getAccess());
-      E = FixOverloadedFunctionReference(E, Found, Specialization);
-      if (!E)
-        return QualType();      
-    } else {
-      Diag(E->getLocStart(),
-           diag::err_cannot_determine_declared_type_of_overloaded_function)
-        << true << E->getSourceRange();
-      return QualType();
-    }
-  }
+QualType Sema::BuildDecltypeType(Expr *E, SourceLocation Loc) {
+  ExprResult ER = CheckPlaceholderExpr(E, Loc);
+  if (ER.isInvalid()) return QualType();
+  E = ER.take();
   
   return Context.getDecltypeType(E);
 }
diff --git a/lib/Sema/TargetAttributesSema.cpp b/lib/Sema/TargetAttributesSema.cpp
index 1854e7430853..c3415cb847ab 100644
--- a/lib/Sema/TargetAttributesSema.cpp
+++ b/lib/Sema/TargetAttributesSema.cpp
@@ -71,6 +71,55 @@ namespace {
   };
 }
 
+static void HandleMBlazeInterruptHandlerAttr(Decl *d, const AttributeList &Attr,
+                                             Sema &S) {
+  // Check the attribute arguments.
+  if (Attr.getNumArgs() != 0) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+    return;
+  }
+
+  // FIXME: Check for decl - it should be void ()(void).
+
+  d->addAttr(::new (S.Context) MBlazeInterruptHandlerAttr(Attr.getLoc(),
+                                                          S.Context));
+  d->addAttr(::new (S.Context) UsedAttr(Attr.getLoc(), S.Context));
+}
+
+static void HandleMBlazeSaveVolatilesAttr(Decl *d, const AttributeList &Attr,
+                                          Sema &S) {
+  // Check the attribute arguments.
+  if (Attr.getNumArgs() != 0) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
+    return;
+  }
+
+  // FIXME: Check for decl - it should be void ()(void).
+
+  d->addAttr(::new (S.Context) MBlazeSaveVolatilesAttr(Attr.getLoc(),
+                                                       S.Context));
+  d->addAttr(::new (S.Context) UsedAttr(Attr.getLoc(), S.Context));
+}
+
+
+namespace {
+  class MBlazeAttributesSema : public TargetAttributesSema {
+  public:
+    MBlazeAttributesSema() { }
+    bool ProcessDeclAttribute(Scope *scope, Decl *D, const AttributeList &Attr,
+                              Sema &S) const {
+      if (Attr.getName()->getName() == "interrupt_handler") {
+        HandleMBlazeInterruptHandlerAttr(D, Attr, S);
+        return true;
+      } else if (Attr.getName()->getName() == "save_volatiles") {
+        HandleMBlazeSaveVolatilesAttr(D, Attr, S);
+        return true;
+      }
+      return false;
+    }
+  };
+}
+
 static void HandleX86ForceAlignArgPointerAttr(Decl *D,
                                               const AttributeList& Attr,
                                               Sema &S) {
@@ -189,8 +238,7 @@ namespace {
                               const AttributeList &Attr, Sema &S) const {
       const llvm::Triple &Triple(S.Context.Target.getTriple());
       if (Triple.getOS() == llvm::Triple::Win32 ||
-          Triple.getOS() == llvm::Triple::MinGW32 ||
-          Triple.getOS() == llvm::Triple::MinGW64) {
+          Triple.getOS() == llvm::Triple::MinGW32) {
         switch (Attr.getKind()) {
         case AttributeList::AT_dllimport: HandleDLLImportAttr(D, Attr, S);
                                           return true;
@@ -220,8 +268,9 @@ const TargetAttributesSema &Sema::getTargetAttributesSema() const {
 
   case llvm::Triple::msp430:
     return *(TheTargetAttributesSema = new MSP430AttributesSema);
+  case llvm::Triple::mblaze:
+    return *(TheTargetAttributesSema = new MBlazeAttributesSema);
   case llvm::Triple::x86:
     return *(TheTargetAttributesSema = new X86AttributesSema);
   }
 }
-
diff --git a/lib/Sema/TreeTransform.h b/lib/Sema/TreeTransform.h
index e7bfbe6fe955..944e6a13e113 100644
--- a/lib/Sema/TreeTransform.h
+++ b/lib/Sema/TreeTransform.h
@@ -1,20 +1,22 @@
-//===------- TreeTransform.h - Semantic Tree Transformation -----*- C++ -*-===/
+//===------- TreeTransform.h - Semantic Tree Transformation -----*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
-//===----------------------------------------------------------------------===/
+//===----------------------------------------------------------------------===//
 //
 //  This file implements a semantic tree transformation that takes a given
 //  AST and rebuilds it, possibly transforming some nodes in the process.
 //
-//===----------------------------------------------------------------------===/
+//===----------------------------------------------------------------------===//
+
 #ifndef LLVM_CLANG_SEMA_TREETRANSFORM_H
 #define LLVM_CLANG_SEMA_TREETRANSFORM_H
 
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/AST/Decl.h"
@@ -25,11 +27,11 @@
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtObjC.h"
-#include "clang/AST/TypeLocBuilder.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "TypeLocBuilder.h"
 #include <algorithm>
 
 namespace clang {
@@ -88,9 +90,26 @@ using namespace sema;
 /// (\c getBaseLocation(), \c getBaseEntity()).
 template<typename Derived>
 class TreeTransform {
+  /// \brief Private RAII object that helps us forget and then re-remember
+  /// the template argument corresponding to a partially-substituted parameter
+  /// pack.
+  class ForgetPartiallySubstitutedPackRAII {
+    Derived &Self;
+    TemplateArgument Old;
+    
+  public:
+    ForgetPartiallySubstitutedPackRAII(Derived &Self) : Self(Self) {
+      Old = Self.ForgetPartiallySubstitutedPack();
+    }
+    
+    ~ForgetPartiallySubstitutedPackRAII() {
+      Self.RememberPartiallySubstitutedPack(Old);
+    }
+  };
+  
 protected:
   Sema &SemaRef;
-
+  
 public:
   /// \brief Initializes a new tree transformer.
   TreeTransform(Sema &SemaRef) : SemaRef(SemaRef) { }
@@ -151,7 +170,9 @@ public:
                   DeclarationName Entity) : Self(Self) {
       OldLocation = Self.getDerived().getBaseLocation();
       OldEntity = Self.getDerived().getBaseEntity();
-      Self.getDerived().setBase(Location, Entity);
+      
+      if (Location.isValid())
+        Self.getDerived().setBase(Location, Entity);
     }
 
     ~TemporaryBase() {
@@ -180,6 +201,77 @@ public:
     return E->isDefaultArgument();
   }
   
+  /// \brief Determine whether we should expand a pack expansion with the
+  /// given set of parameter packs into separate arguments by repeatedly
+  /// transforming the pattern.
+  ///
+  /// By default, the transformer never tries to expand pack expansions.
+  /// Subclasses can override this routine to provide different behavior.
+  ///
+  /// \param EllipsisLoc The location of the ellipsis that identifies the
+  /// pack expansion.
+  ///
+  /// \param PatternRange The source range that covers the entire pattern of
+  /// the pack expansion.
+  ///
+  /// \param Unexpanded The set of unexpanded parameter packs within the 
+  /// pattern.
+  ///
+  /// \param NumUnexpanded The number of unexpanded parameter packs in
+  /// \p Unexpanded.
+  ///
+  /// \param ShouldExpand Will be set to \c true if the transformer should
+  /// expand the corresponding pack expansions into separate arguments. When
+  /// set, \c NumExpansions must also be set.
+  ///
+  /// \param RetainExpansion Whether the caller should add an unexpanded
+  /// pack expansion after all of the expanded arguments. This is used
+  /// when extending explicitly-specified template argument packs per
+  /// C++0x [temp.arg.explicit]p9.
+  ///
+  /// \param NumExpansions The number of separate arguments that will be in
+  /// the expanded form of the corresponding pack expansion. This is both an
+  /// input and an output parameter, which can be set by the caller if the
+  /// number of expansions is known a priori (e.g., due to a prior substitution)
+  /// and will be set by the callee when the number of expansions is known.
+  /// The callee must set this value when \c ShouldExpand is \c true; it may
+  /// set this value in other cases.
+  ///
+  /// \returns true if an error occurred (e.g., because the parameter packs 
+  /// are to be instantiated with arguments of different lengths), false 
+  /// otherwise. If false, \c ShouldExpand (and possibly \c NumExpansions) 
+  /// must be set.
+  bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
+                               SourceRange PatternRange,
+                               const UnexpandedParameterPack *Unexpanded,
+                               unsigned NumUnexpanded,
+                               bool &ShouldExpand,
+                               bool &RetainExpansion,
+                               llvm::Optional<unsigned> &NumExpansions) {
+    ShouldExpand = false;
+    return false;
+  }
+  
+  /// \brief "Forget" about the partially-substituted pack template argument,
+  /// when performing an instantiation that must preserve the parameter pack
+  /// use.
+  ///
+  /// This routine is meant to be overridden by the template instantiator.
+  TemplateArgument ForgetPartiallySubstitutedPack() {
+    return TemplateArgument();
+  }
+  
+  /// \brief "Remember" the partially-substituted pack template argument
+  /// after performing an instantiation that must preserve the parameter pack
+  /// use.
+  ///
+  /// This routine is meant to be overridden by the template instantiator.
+  void RememberPartiallySubstitutedPack(TemplateArgument Arg) { }
+  
+  /// \brief Note to the derived class when a function parameter pack is
+  /// being expanded.
+  void ExpandingFunctionParameterPack(ParmVarDecl *Pack) { }
+                                      
   /// \brief Transforms the given type into another type.
   ///
   /// By default, this routine transforms a type by creating a
@@ -189,7 +281,7 @@ public:
   /// switched to storing TypeSourceInfos.
   ///
   /// \returns the transformed type.
-  QualType TransformType(QualType T, QualType ObjectType = QualType());
+  QualType TransformType(QualType T);
 
   /// \brief Transforms the given type-with-location into a new
   /// type-with-location.
@@ -199,15 +291,13 @@ public:
   /// may override this function (to take over all type
   /// transformations) or some set of the TransformXXXType functions
   /// to alter the transformation.
-  TypeSourceInfo *TransformType(TypeSourceInfo *DI, 
-                                QualType ObjectType = QualType());
+  TypeSourceInfo *TransformType(TypeSourceInfo *DI);
 
   /// \brief Transform the given type-with-location into a new
   /// type, collecting location information in the given builder
   /// as necessary.
   ///
-  QualType TransformType(TypeLocBuilder &TLB, TypeLoc TL, 
-                         QualType ObjectType = QualType());
+  QualType TransformType(TypeLocBuilder &TLB, TypeLoc TL);
 
   /// \brief Transform the given statement.
   ///
@@ -230,6 +320,33 @@ public:
   /// \returns the transformed expression.
   ExprResult TransformExpr(Expr *E);
 
+  /// \brief Transform the given list of expressions.
+  ///
+  /// This routine transforms a list of expressions by invoking 
+  /// \c TransformExpr() for each subexpression. However, it also provides 
+  /// support for variadic templates by expanding any pack expansions (if the
+  /// derived class permits such expansion) along the way. When pack expansions
+  /// are present, the number of outputs may not equal the number of inputs.
+  ///
+  /// \param Inputs The set of expressions to be transformed.
+  ///
+  /// \param NumInputs The number of expressions in \c Inputs.
+  ///
+  /// \param IsCall If \c true, then this transform is being performed on
+  /// function-call arguments, and any arguments that should be dropped, will 
+  /// be.
+  ///
+  /// \param Outputs The transformed input expressions will be added to this
+  /// vector.
+  ///
+  /// \param ArgChanged If non-NULL, will be set \c true if any argument changed
+  /// due to transformation.
+  ///
+  /// \returns true if an error occurred, false otherwise.
+  bool TransformExprs(Expr **Inputs, unsigned NumInputs, bool IsCall,
+                      llvm::SmallVectorImpl<Expr *> &Outputs,
+                      bool *ArgChanged = 0);
+  
   /// \brief Transform the given declaration, which is referenced from a type
   /// or expression.
   ///
@@ -275,8 +392,7 @@ public:
   /// Identifiers and selectors are returned unmodified. Sublcasses may
   /// override this function to provide alternate behavior.
   DeclarationNameInfo
-  TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
-                               QualType ObjectType = QualType());
+  TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo);
 
   /// \brief Transform the given template name.
   ///
@@ -284,7 +400,8 @@ public:
   /// and nested-name-specifiers that occur within the template name.
   /// Subclasses may override this function to provide alternate behavior.
   TemplateName TransformTemplateName(TemplateName Name,
-                                     QualType ObjectType = QualType());
+                                     QualType ObjectType = QualType(),
+                                     NamedDecl *FirstQualifierInScope = 0);
 
   /// \brief Transform the given template argument.
   ///
@@ -297,6 +414,49 @@ public:
   bool TransformTemplateArgument(const TemplateArgumentLoc &Input,
                                  TemplateArgumentLoc &Output);
 
+  /// \brief Transform the given set of template arguments.
+  ///
+  /// By default, this operation transforms all of the template arguments 
+  /// in the input set using \c TransformTemplateArgument(), and appends
+  /// the transformed arguments to the output list.
+  ///
+  /// Note that this overload of \c TransformTemplateArguments() is merely
+  /// a convenience function. Subclasses that wish to override this behavior
+  /// should override the iterator-based member template version.
+  ///
+  /// \param Inputs The set of template arguments to be transformed.
+  ///
+  /// \param NumInputs The number of template arguments in \p Inputs.
+  ///
+  /// \param Outputs The set of transformed template arguments output by this
+  /// routine.
+  ///
+  /// Returns true if an error occurred.
+  bool TransformTemplateArguments(const TemplateArgumentLoc *Inputs,
+                                  unsigned NumInputs,
+                                  TemplateArgumentListInfo &Outputs) {
+    return TransformTemplateArguments(Inputs, Inputs + NumInputs, Outputs);
+  }
+
+  /// \brief Transform the given set of template arguments.
+  ///
+  /// By default, this operation transforms all of the template arguments 
+  /// in the input set using \c TransformTemplateArgument(), and appends
+  /// the transformed arguments to the output list. 
+  ///
+  /// \param First An iterator to the first template argument.
+  ///
+  /// \param Last An iterator one step past the last template argument.
+  ///
+  /// \param Outputs The set of transformed template arguments output by this
+  /// routine.
+  ///
+  /// Returns true if an error occurred.
+  template<typename InputIterator>
+  bool TransformTemplateArguments(InputIterator First,
+                                  InputIterator Last,
+                                  TemplateArgumentListInfo &Outputs);
+
   /// \brief Fakes up a TemplateArgumentLoc for a given TemplateArgument.
   void InventTemplateArgumentLoc(const TemplateArgument &Arg,
                                  TemplateArgumentLoc &ArgLoc);
@@ -309,10 +469,19 @@ public:
 
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
 #define TYPELOC(CLASS, PARENT)                                   \
-  QualType Transform##CLASS##Type(TypeLocBuilder &TLB, CLASS##TypeLoc T, \
-                                  QualType ObjectType = QualType());
+  QualType Transform##CLASS##Type(TypeLocBuilder &TLB, CLASS##TypeLoc T);
 #include "clang/AST/TypeLocNodes.def"
 
+  QualType 
+  TransformTemplateSpecializationType(TypeLocBuilder &TLB,
+                                      TemplateSpecializationTypeLoc TL,
+                                      TemplateName Template);
+
+  QualType 
+  TransformDependentTemplateSpecializationType(TypeLocBuilder &TLB,
+                                      DependentTemplateSpecializationTypeLoc TL,
+                                               NestedNameSpecifier *Prefix);
+
   /// \brief Transforms the parameters of a function type into the
   /// given vectors.
   ///
@@ -320,20 +489,18 @@ public:
   /// variables vector are acceptable.
   ///
   /// Return true on error.
-  bool TransformFunctionTypeParams(FunctionProtoTypeLoc TL,
+  bool TransformFunctionTypeParams(SourceLocation Loc,
+                                   ParmVarDecl **Params, unsigned NumParams,
+                                   const QualType *ParamTypes,
                                    llvm::SmallVectorImpl<QualType> &PTypes,
-                                   llvm::SmallVectorImpl<ParmVarDecl*> &PVars);
+                                   llvm::SmallVectorImpl<ParmVarDecl*> *PVars);
 
   /// \brief Transforms a single function-type parameter.  Return null
   /// on error.
-  ParmVarDecl *TransformFunctionTypeParam(ParmVarDecl *OldParm);
+  ParmVarDecl *TransformFunctionTypeParam(ParmVarDecl *OldParm,
+                                        llvm::Optional<unsigned> NumExpansions);
 
-  QualType TransformReferenceType(TypeLocBuilder &TLB, ReferenceTypeLoc TL, 
-                                  QualType ObjectType);
-
-  QualType 
-  TransformTemplateSpecializationType(const TemplateSpecializationType *T,
-                                      QualType ObjectType);
+  QualType TransformReferenceType(TypeLocBuilder &TLB, ReferenceTypeLoc TL);
 
   StmtResult TransformCompoundStmt(CompoundStmt *S, bool IsStmtExpr);
   ExprResult TransformCXXNamedCastExpr(CXXNamedCastExpr *E);
@@ -440,7 +607,7 @@ public:
   /// By default, performs semantic analysis when building the vector type.
   /// Subclasses may override this routine to provide different behavior.
   QualType RebuildVectorType(QualType ElementType, unsigned NumElements,
-    VectorType::AltiVecSpecific AltiVecSpec);
+                             VectorType::VectorKind VecKind);
 
   /// \brief Build a new extended vector type given the element type and
   /// number of elements.
@@ -467,6 +634,7 @@ public:
                                     QualType *ParamTypes,
                                     unsigned NumParamTypes,
                                     bool Variadic, unsigned Quals,
+                                    RefQualifierKind RefQualifier,
                                     const FunctionType::ExtInfo &Info);
 
   /// \brief Build a new unprototyped function type.
@@ -495,7 +663,7 @@ public:
   ///
   /// By default, performs semantic analysis when building the typeof type.
   /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildTypeOfExprType(Expr *Underlying);
+  QualType RebuildTypeOfExprType(Expr *Underlying, SourceLocation Loc);
 
   /// \brief Build a new typeof(type) type.
   ///
@@ -506,7 +674,14 @@ public:
   ///
   /// By default, performs semantic analysis when building the decltype type.
   /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildDecltypeType(Expr *Underlying);
+  QualType RebuildDecltypeType(Expr *Underlying, SourceLocation Loc);
+
+  /// \brief Build a new C++0x auto type.
+  ///
+  /// By default, builds a new AutoType with the given deduced type.
+  QualType RebuildAutoType(QualType Deduced) {
+    return SemaRef.Context.getAutoType(Deduced);
+  }
 
   /// \brief Build a new template specialization type.
   ///
@@ -517,12 +692,21 @@ public:
                                              SourceLocation TemplateLoc,
                                        const TemplateArgumentListInfo &Args);
 
+  /// \brief Build a new parenthesized type.
+  ///
+  /// By default, builds a new ParenType type from the inner type.
+  /// Subclasses may override this routine to provide different behavior.
+  QualType RebuildParenType(QualType InnerType) {
+    return SemaRef.Context.getParenType(InnerType);
+  }
+
   /// \brief Build a new qualified name type.
   ///
   /// By default, builds a new ElaboratedType type from the keyword,
   /// the nested-name-specifier and the named type.
   /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
+  QualType RebuildElaboratedType(SourceLocation KeywordLoc,
+                                 ElaboratedTypeKeyword Keyword,
                                  NestedNameSpecifier *NNS, QualType Named) {
     return SemaRef.Context.getElaboratedType(Keyword, NNS, Named);
   }
@@ -534,14 +718,16 @@ public:
   /// this routine to provide different behavior.
   QualType RebuildDependentTemplateSpecializationType(
                                     ElaboratedTypeKeyword Keyword,
-                                    NestedNameSpecifier *NNS,
+                                    NestedNameSpecifier *Qualifier,
+                                    SourceRange QualifierRange,
                                     const IdentifierInfo *Name,
                                     SourceLocation NameLoc,
                                     const TemplateArgumentListInfo &Args) {
     // Rebuild the template name.
     // TODO: avoid TemplateName abstraction
     TemplateName InstName =
-      getDerived().RebuildTemplateName(NNS, *Name, QualType());
+      getDerived().RebuildTemplateName(Qualifier, QualifierRange, *Name, 
+                                       QualType(), 0);
     
     if (InstName.isNull())
       return QualType();
@@ -549,7 +735,7 @@ public:
     // If it's still dependent, make a dependent specialization.
     if (InstName.getAsDependentTemplateName())
       return SemaRef.Context.getDependentTemplateSpecializationType(
-                                          Keyword, NNS, Name, Args);
+                                          Keyword, Qualifier, Name, Args);
 
     // Otherwise, make an elaborated type wrapping a non-dependent
     // specialization.
@@ -621,9 +807,28 @@ public:
     }
 
     if (!Tag) {
-      // FIXME: Would be nice to highlight just the source range.
-      SemaRef.Diag(IdLoc, diag::err_not_tag_in_scope)
-        << Kind << Id << DC;
+      // Check where the name exists but isn't a tag type and use that to emit
+      // better diagnostics.
+      LookupResult Result(SemaRef, Id, IdLoc, Sema::LookupTagName);
+      SemaRef.LookupQualifiedName(Result, DC);
+      switch (Result.getResultKind()) {
+        case LookupResult::Found:
+        case LookupResult::FoundOverloaded:
+        case LookupResult::FoundUnresolvedValue: {
+	  NamedDecl *SomeDecl = Result.getRepresentativeDecl();
+          unsigned Kind = 0;
+          if (isa<TypedefDecl>(SomeDecl)) Kind = 1;
+          else if (isa<ClassTemplateDecl>(SomeDecl)) Kind = 2;
+          SemaRef.Diag(IdLoc, diag::err_tag_reference_non_tag) << Kind;
+          SemaRef.Diag(SomeDecl->getLocation(), diag::note_declared_at);
+          break;
+	}
+        default:
+          // FIXME: Would be nice to highlight just the source range.
+          SemaRef.Diag(IdLoc, diag::err_not_tag_in_scope)
+            << Kind << Id << DC;
+          break;
+      }
       return QualType();
     }
 
@@ -638,6 +843,18 @@ public:
     return SemaRef.Context.getElaboratedType(Keyword, NNS, T);
   }
 
+  /// \brief Build a new pack expansion type.
+  ///
+  /// By default, builds a new PackExpansionType type from the given pattern.
+  /// Subclasses may override this routine to provide different behavior.
+  QualType RebuildPackExpansionType(QualType Pattern, 
+                                    SourceRange PatternRange,
+                                    SourceLocation EllipsisLoc,
+                                    llvm::Optional<unsigned> NumExpansions) {
+    return getSema().CheckPackExpansion(Pattern, PatternRange, EllipsisLoc,
+                                        NumExpansions);
+  }
+
   /// \brief Build a new nested-name-specifier given the prefix and an
   /// identifier that names the next step in the nested-name-specifier.
   ///
@@ -689,8 +906,10 @@ public:
   /// template name. Subclasses may override this routine to provide different
   /// behavior.
   TemplateName RebuildTemplateName(NestedNameSpecifier *Qualifier,
+                                   SourceRange QualifierRange,
                                    const IdentifierInfo &II,
-                                   QualType ObjectType);
+                                   QualType ObjectType,
+                                   NamedDecl *FirstQualifierInScope);
 
   /// \brief Build a new template name given a nested name specifier and the
   /// overloaded operator name that is referred to as a template.
@@ -702,7 +921,19 @@ public:
   TemplateName RebuildTemplateName(NestedNameSpecifier *Qualifier,
                                    OverloadedOperatorKind Operator,
                                    QualType ObjectType);
-  
+
+  /// \brief Build a new template name given a template template parameter pack
+  /// and the 
+  ///
+  /// By default, performs semantic analysis to determine whether the name can
+  /// be resolved to a specific template, then builds the appropriate kind of
+  /// template name. Subclasses may override this routine to provide different
+  /// behavior.
+  TemplateName RebuildTemplateName(TemplateTemplateParmDecl *Param,
+                                   const TemplateArgument &ArgPack) {
+    return getSema().Context.getSubstTemplateTemplateParmPack(Param, ArgPack);
+  }
+
   /// \brief Build a new compound statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
@@ -752,11 +983,9 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildLabelStmt(SourceLocation IdentLoc,
-                                    IdentifierInfo *Id,
-                                    SourceLocation ColonLoc,
-                                    Stmt *SubStmt) {
-    return SemaRef.ActOnLabelStmt(IdentLoc, Id, ColonLoc, SubStmt);
+  StmtResult RebuildLabelStmt(SourceLocation IdentLoc, LabelDecl *L,
+                              SourceLocation ColonLoc, Stmt *SubStmt) {
+    return SemaRef.ActOnLabelStmt(IdentLoc, L, ColonLoc, SubStmt);
   }
 
   /// \brief Build a new "if" statement.
@@ -764,8 +993,8 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildIfStmt(SourceLocation IfLoc, Sema::FullExprArg Cond,
-                                 VarDecl *CondVar, Stmt *Then, 
-                                 SourceLocation ElseLoc, Stmt *Else) {
+                           VarDecl *CondVar, Stmt *Then, 
+                           SourceLocation ElseLoc, Stmt *Else) {
     return getSema().ActOnIfStmt(IfLoc, Cond, CondVar, Then, ElseLoc, Else);
   }
 
@@ -774,7 +1003,7 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildSwitchStmtStart(SourceLocation SwitchLoc,
-                                          Expr *Cond, VarDecl *CondVar) {
+                                    Expr *Cond, VarDecl *CondVar) {
     return getSema().ActOnStartOfSwitchStmt(SwitchLoc, Cond, 
                                             CondVar);
   }
@@ -784,7 +1013,7 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildSwitchStmtBody(SourceLocation SwitchLoc,
-                                         Stmt *Switch, Stmt *Body) {
+                                   Stmt *Switch, Stmt *Body) {
     return getSema().ActOnFinishSwitchStmt(SwitchLoc, Switch, Body);
   }
 
@@ -792,10 +1021,8 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildWhileStmt(SourceLocation WhileLoc,
-                                    Sema::FullExprArg Cond,
-                                    VarDecl *CondVar,
-                                    Stmt *Body) {
+  StmtResult RebuildWhileStmt(SourceLocation WhileLoc, Sema::FullExprArg Cond,
+                              VarDecl *CondVar, Stmt *Body) {
     return getSema().ActOnWhileStmt(WhileLoc, Cond, CondVar, Body);
   }
 
@@ -804,10 +1031,8 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildDoStmt(SourceLocation DoLoc, Stmt *Body,
-                                 SourceLocation WhileLoc,
-                                 SourceLocation LParenLoc,
-                                 Expr *Cond,
-                                 SourceLocation RParenLoc) {
+                           SourceLocation WhileLoc, SourceLocation LParenLoc,
+                           Expr *Cond, SourceLocation RParenLoc) {
     return getSema().ActOnDoStmt(DoLoc, Body, WhileLoc, LParenLoc,
                                  Cond, RParenLoc);
   }
@@ -816,24 +1041,21 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildForStmt(SourceLocation ForLoc,
-                                  SourceLocation LParenLoc,
-                                  Stmt *Init, Sema::FullExprArg Cond, 
-                                  VarDecl *CondVar, Sema::FullExprArg Inc,
-                                  SourceLocation RParenLoc, Stmt *Body) {
+  StmtResult RebuildForStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
+                            Stmt *Init, Sema::FullExprArg Cond, 
+                            VarDecl *CondVar, Sema::FullExprArg Inc,
+                            SourceLocation RParenLoc, Stmt *Body) {
     return getSema().ActOnForStmt(ForLoc, LParenLoc, Init, Cond, 
-                                  CondVar,
-                                  Inc, RParenLoc, Body);
+                                  CondVar, Inc, RParenLoc, Body);
   }
 
   /// \brief Build a new goto statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildGotoStmt(SourceLocation GotoLoc,
-                                   SourceLocation LabelLoc,
-                                   LabelStmt *Label) {
-    return getSema().ActOnGotoStmt(GotoLoc, LabelLoc, Label->getID());
+  StmtResult RebuildGotoStmt(SourceLocation GotoLoc, SourceLocation LabelLoc,
+                             LabelDecl *Label) {
+    return getSema().ActOnGotoStmt(GotoLoc, LabelLoc, Label);
   }
 
   /// \brief Build a new indirect goto statement.
@@ -841,8 +1063,8 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildIndirectGotoStmt(SourceLocation GotoLoc,
-                                           SourceLocation StarLoc,
-                                           Expr *Target) {
+                                     SourceLocation StarLoc,
+                                     Expr *Target) {
     return getSema().ActOnIndirectGotoStmt(GotoLoc, StarLoc, Target);
   }
 
@@ -850,9 +1072,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildReturnStmt(SourceLocation ReturnLoc,
-                                     Expr *Result) {
-
+  StmtResult RebuildReturnStmt(SourceLocation ReturnLoc, Expr *Result) {
     return getSema().ActOnReturnStmt(ReturnLoc, Result);
   }
 
@@ -977,13 +1197,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new decaration.
   /// Subclasses may override this routine to provide different behavior.
-  VarDecl *RebuildExceptionDecl(VarDecl *ExceptionDecl, QualType T,
+  VarDecl *RebuildExceptionDecl(VarDecl *ExceptionDecl, 
                                 TypeSourceInfo *Declarator,
                                 IdentifierInfo *Name,
-                                SourceLocation Loc,
-                                SourceRange TypeRange) {
-    return getSema().BuildExceptionDeclaration(0, T, Declarator, Name, Loc,
-                                               TypeRange);
+                                SourceLocation Loc) {
+    return getSema().BuildExceptionDeclaration(0, Declarator, Name, Loc);
   }
 
   /// \brief Build a new C++ catch statement.
@@ -1126,10 +1344,10 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildCallExpr(Expr *Callee, SourceLocation LParenLoc,
                                    MultiExprArg Args,
-                                   SourceLocation *CommaLocs,
-                                   SourceLocation RParenLoc) {
+                                   SourceLocation RParenLoc,
+                                   Expr *ExecConfig = 0) {
     return getSema().ActOnCallExpr(/*Scope=*/0, Callee, LParenLoc,
-                                   move(Args), CommaLocs, RParenLoc);
+                                   move(Args), RParenLoc, ExecConfig);
   }
 
   /// \brief Build a new member access expression.
@@ -1137,26 +1355,32 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildMemberExpr(Expr *Base, SourceLocation OpLoc,
-                                     bool isArrow,
-                                     NestedNameSpecifier *Qualifier,
-                                     SourceRange QualifierRange,
-                                     const DeclarationNameInfo &MemberNameInfo,
-                                     ValueDecl *Member,
-                                     NamedDecl *FoundDecl,
+                               bool isArrow,
+                               NestedNameSpecifier *Qualifier,
+                               SourceRange QualifierRange,
+                               const DeclarationNameInfo &MemberNameInfo,
+                               ValueDecl *Member,
+                               NamedDecl *FoundDecl,
                         const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                                     NamedDecl *FirstQualifierInScope) {
+                               NamedDecl *FirstQualifierInScope) {
     if (!Member->getDeclName()) {
-      // We have a reference to an unnamed field.
+      // We have a reference to an unnamed field.  This is always the
+      // base of an anonymous struct/union member access, i.e. the
+      // field is always of record type.
       assert(!Qualifier && "Can't have an unnamed field with a qualifier!");
+      assert(Member->getType()->isRecordType() &&
+             "unnamed member not of record type?");
 
       if (getSema().PerformObjectMemberConversion(Base, Qualifier,
                                                   FoundDecl, Member))
         return ExprError();
 
+      ExprValueKind VK = isArrow ? VK_LValue : Base->getValueKind();
       MemberExpr *ME =
         new (getSema().Context) MemberExpr(Base, isArrow,
                                            Member, MemberNameInfo,
-                                           cast<FieldDecl>(Member)->getType());
+                                           cast<FieldDecl>(Member)->getType(),
+                                           VK, OK_Ordinary);
       return getSema().Owned(ME);
     }
 
@@ -1195,10 +1419,10 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildConditionalOperator(Expr *Cond,
-                                              SourceLocation QuestionLoc,
-                                              Expr *LHS,
-                                              SourceLocation ColonLoc,
-                                              Expr *RHS) {
+                                        SourceLocation QuestionLoc,
+                                        Expr *LHS,
+                                        SourceLocation ColonLoc,
+                                        Expr *RHS) {
     return getSema().ActOnConditionalOp(QuestionLoc, ColonLoc, Cond,
                                         LHS, RHS);
   }
@@ -1322,9 +1546,8 @@ public:
   /// rather than attempting to map the label statement itself.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildAddrLabelExpr(SourceLocation AmpAmpLoc,
-                                        SourceLocation LabelLoc,
-                                        LabelStmt *Label) {
-    return getSema().ActOnAddrLabel(AmpAmpLoc, LabelLoc, Label->getID());
+                                  SourceLocation LabelLoc, LabelDecl *Label) {
+    return getSema().ActOnAddrLabel(AmpAmpLoc, LabelLoc, Label);
   }
 
   /// \brief Build a new GNU statement expression.
@@ -1337,19 +1560,6 @@ public:
     return getSema().ActOnStmtExpr(LParenLoc, SubStmt, RParenLoc);
   }
 
-  /// \brief Build a new __builtin_types_compatible_p expression.
-  ///
-  /// By default, performs semantic analysis to build the new expression.
-  /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                              TypeSourceInfo *TInfo1,
-                                              TypeSourceInfo *TInfo2,
-                                              SourceLocation RParenLoc) {
-    return getSema().BuildTypesCompatibleExpr(BuiltinLoc,
-                                              TInfo1, TInfo2,
-                                              RParenLoc);
-  }
-
   /// \brief Build a new __builtin_choose_expr expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
@@ -1492,16 +1702,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildCXXFunctionalCastExpr(SourceRange TypeRange,
-                                                TypeSourceInfo *TInfo,
-                                                SourceLocation LParenLoc,
-                                                Expr *Sub,
-                                                SourceLocation RParenLoc) {
-    return getSema().ActOnCXXTypeConstructExpr(TypeRange,
-                                             ParsedType::make(TInfo->getType()),
-                                               LParenLoc,
+  ExprResult RebuildCXXFunctionalCastExpr(TypeSourceInfo *TInfo,
+                                          SourceLocation LParenLoc,
+                                          Expr *Sub,
+                                          SourceLocation RParenLoc) {
+    return getSema().BuildCXXTypeConstructExpr(TInfo, LParenLoc,
                                                MultiExprArg(&Sub, 1),
-                                               /*CommaLocs=*/0,
                                                RParenLoc);
   }
 
@@ -1517,6 +1723,7 @@ public:
                                     RParenLoc);
   }
 
+
   /// \brief Build a new C++ typeid(expr) expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
@@ -1529,14 +1736,38 @@ public:
                                     RParenLoc);
   }
 
+  /// \brief Build a new C++ __uuidof(type) expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildCXXUuidofExpr(QualType TypeInfoType,
+                                        SourceLocation TypeidLoc,
+                                        TypeSourceInfo *Operand,
+                                        SourceLocation RParenLoc) {
+    return getSema().BuildCXXUuidof(TypeInfoType, TypeidLoc, Operand, 
+                                    RParenLoc);
+  }
+
+  /// \brief Build a new C++ __uuidof(expr) expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildCXXUuidofExpr(QualType TypeInfoType,
+                                        SourceLocation TypeidLoc,
+                                        Expr *Operand,
+                                        SourceLocation RParenLoc) {
+    return getSema().BuildCXXUuidof(TypeInfoType, TypeidLoc, Operand,
+                                    RParenLoc);
+  }
+
   /// \brief Build a new C++ "this" expression.
   ///
   /// By default, builds a new "this" expression without performing any
   /// semantic analysis. Subclasses may override this routine to provide
   /// different behavior.
   ExprResult RebuildCXXThisExpr(SourceLocation ThisLoc,
-                                      QualType ThisType,
-                                      bool isImplicit) {
+                                QualType ThisType,
+                                bool isImplicit) {
     return getSema().Owned(
                       new (getSema().Context) CXXThisExpr(ThisLoc, ThisType,
                                                           isImplicit));
@@ -1565,14 +1796,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildCXXScalarValueInitExpr(SourceLocation TypeStartLoc,
-                                               SourceLocation LParenLoc,
-                                               QualType T,
-                                               SourceLocation RParenLoc) {
-    return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeStartLoc),
-                                               ParsedType::make(T), LParenLoc,
+  ExprResult RebuildCXXScalarValueInitExpr(TypeSourceInfo *TSInfo,
+                                           SourceLocation LParenLoc,
+                                           SourceLocation RParenLoc) {
+    return getSema().BuildCXXTypeConstructExpr(TSInfo, LParenLoc,
                                                MultiExprArg(getSema(), 0, 0),
-                                               0, RParenLoc);
+                                               RParenLoc);
   }
 
   /// \brief Build a new C++ "new" expression.
@@ -1580,26 +1809,24 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildCXXNewExpr(SourceLocation StartLoc,
-                                     bool UseGlobal,
-                                     SourceLocation PlacementLParen,
-                                     MultiExprArg PlacementArgs,
-                                     SourceLocation PlacementRParen,
-                                     SourceRange TypeIdParens,
-                                     QualType AllocType,
-                                     SourceLocation TypeLoc,
-                                     SourceRange TypeRange,
-                                     Expr *ArraySize,
-                                     SourceLocation ConstructorLParen,
-                                     MultiExprArg ConstructorArgs,
-                                     SourceLocation ConstructorRParen) {
+                               bool UseGlobal,
+                               SourceLocation PlacementLParen,
+                               MultiExprArg PlacementArgs,
+                               SourceLocation PlacementRParen,
+                               SourceRange TypeIdParens,
+                               QualType AllocatedType,
+                               TypeSourceInfo *AllocatedTypeInfo,
+                               Expr *ArraySize,
+                               SourceLocation ConstructorLParen,
+                               MultiExprArg ConstructorArgs,
+                               SourceLocation ConstructorRParen) {
     return getSema().BuildCXXNew(StartLoc, UseGlobal,
                                  PlacementLParen,
                                  move(PlacementArgs),
                                  PlacementRParen,
                                  TypeIdParens,
-                                 AllocType,
-                                 TypeLoc,
-                                 TypeRange,
+                                 AllocatedType,
+                                 AllocatedTypeInfo,
                                  ArraySize,
                                  ConstructorLParen,
                                  move(ConstructorArgs),
@@ -1623,12 +1850,22 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildUnaryTypeTrait(UnaryTypeTrait Trait,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         QualType T,
-                                         SourceLocation RParenLoc) {
-    return getSema().ActOnUnaryTypeTrait(Trait, StartLoc, LParenLoc,
-                                         ParsedType::make(T), RParenLoc);
+                                   SourceLocation StartLoc,
+                                   TypeSourceInfo *T,
+                                   SourceLocation RParenLoc) {
+    return getSema().BuildUnaryTypeTrait(Trait, StartLoc, T, RParenLoc);
+  }
+
+  /// \brief Build a new binary type trait expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildBinaryTypeTrait(BinaryTypeTrait Trait,
+                                    SourceLocation StartLoc,
+                                    TypeSourceInfo *LhsT,
+                                    TypeSourceInfo *RhsT,
+                                    SourceLocation RParenLoc) {
+    return getSema().BuildBinaryTypeTrait(Trait, StartLoc, LhsT, RhsT, RParenLoc);
   }
 
   /// \brief Build a new (previously unresolved) declaration reference
@@ -1672,7 +1909,8 @@ public:
                                            bool IsElidable,
                                            MultiExprArg Args,
                                            bool RequiresZeroInit,
-                             CXXConstructExpr::ConstructionKind ConstructKind) {
+                             CXXConstructExpr::ConstructionKind ConstructKind,
+                                           SourceRange ParenRange) {
     ASTOwningVector<Expr*> ConvertedArgs(SemaRef);
     if (getSema().CompleteConstructorCall(Constructor, move(Args), Loc, 
                                           ConvertedArgs))
@@ -1680,24 +1918,21 @@ public:
     
     return getSema().BuildCXXConstructExpr(Loc, T, Constructor, IsElidable,
                                            move_arg(ConvertedArgs),
-                                           RequiresZeroInit, ConstructKind);
+                                           RequiresZeroInit, ConstructKind,
+                                           ParenRange);
   }
 
   /// \brief Build a new object-construction expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildCXXTemporaryObjectExpr(SourceLocation TypeBeginLoc,
-                                                 QualType T,
-                                                 SourceLocation LParenLoc,
-                                                 MultiExprArg Args,
-                                                 SourceLocation *Commas,
-                                                 SourceLocation RParenLoc) {
-    return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeBeginLoc),
-                                               ParsedType::make(T),
+  ExprResult RebuildCXXTemporaryObjectExpr(TypeSourceInfo *TSInfo,
+                                           SourceLocation LParenLoc,
+                                           MultiExprArg Args,
+                                           SourceLocation RParenLoc) {
+    return getSema().BuildCXXTypeConstructExpr(TSInfo,
                                                LParenLoc,
                                                move(Args),
-                                               Commas,
                                                RParenLoc);
   }
 
@@ -1705,18 +1940,13 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildCXXUnresolvedConstructExpr(SourceLocation TypeBeginLoc,
-                                                     QualType T,
-                                                     SourceLocation LParenLoc,
-                                                     MultiExprArg Args,
-                                                     SourceLocation *Commas,
-                                                     SourceLocation RParenLoc) {
-    return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeBeginLoc,
-                                                           /*FIXME*/LParenLoc),
-                                               ParsedType::make(T),
+  ExprResult RebuildCXXUnresolvedConstructExpr(TypeSourceInfo *TSInfo,
+                                               SourceLocation LParenLoc,
+                                               MultiExprArg Args,
+                                               SourceLocation RParenLoc) {
+    return getSema().BuildCXXTypeConstructExpr(TSInfo,
                                                LParenLoc,
                                                move(Args),
-                                               Commas,
                                                RParenLoc);
   }
 
@@ -1767,6 +1997,24 @@ public:
                                             R, TemplateArgs);
   }
 
+  /// \brief Build a new noexcept expression.
+  ///
+  /// By default, performs semantic analysis to build the new expression.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildCXXNoexceptExpr(SourceRange Range, Expr *Arg) {
+    return SemaRef.BuildCXXNoexceptExpr(Range.getBegin(), Arg, Range.getEnd());
+  }
+
+  /// \brief Build a new expression to compute the length of a parameter pack.
+  ExprResult RebuildSizeOfPackExpr(SourceLocation OperatorLoc, NamedDecl *Pack, 
+                                   SourceLocation PackLoc, 
+                                   SourceLocation RParenLoc,
+                                   unsigned Length) {
+    return new (SemaRef.Context) SizeOfPackExpr(SemaRef.Context.getSizeType(), 
+                                                OperatorLoc, Pack, PackLoc, 
+                                                RParenLoc, Length);
+  }
+                                   
   /// \brief Build a new Objective-C @encode expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
@@ -1781,6 +2029,7 @@ public:
   /// \brief Build a new Objective-C class message.
   ExprResult RebuildObjCMessageExpr(TypeSourceInfo *ReceiverTypeInfo,
                                           Selector Sel,
+                                          SourceLocation SelectorLoc,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
                                           MultiExprArg Args,
@@ -1788,13 +2037,14 @@ public:
     return SemaRef.BuildClassMessage(ReceiverTypeInfo,
                                      ReceiverTypeInfo->getType(),
                                      /*SuperLoc=*/SourceLocation(),
-                                     Sel, Method, LBracLoc, RBracLoc,
-                                     move(Args));
+                                     Sel, Method, LBracLoc, SelectorLoc,
+                                     RBracLoc, move(Args));
   }
 
   /// \brief Build a new Objective-C instance message.
   ExprResult RebuildObjCMessageExpr(Expr *Receiver,
                                           Selector Sel,
+                                          SourceLocation SelectorLoc,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
                                           MultiExprArg Args,
@@ -1802,8 +2052,8 @@ public:
     return SemaRef.BuildInstanceMessage(Receiver,
                                         Receiver->getType(),
                                         /*SuperLoc=*/SourceLocation(),
-                                        Sel, Method, LBracLoc, RBracLoc,
-                                        move(Args));
+                                        Sel, Method, LBracLoc, SelectorLoc,
+                                        RBracLoc, move(Args));
   }
 
   /// \brief Build a new Objective-C ivar reference expression.
@@ -1864,24 +2114,20 @@ public:
                                               /*TemplateArgs=*/0);
   }
   
-  /// \brief Build a new Objective-C implicit setter/getter reference 
-  /// expression.
+  /// \brief Build a new Objective-C property reference expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
-  /// Subclasses may override this routine to provide different behavior.  
-  ExprResult RebuildObjCImplicitSetterGetterRefExpr(
-                                                        ObjCMethodDecl *Getter,
-                                                          QualType T,
-                                                        ObjCMethodDecl *Setter,
-                                                        SourceLocation NameLoc,
-                                                          Expr *Base) {
-    // Since these expressions can only be value-dependent, we do not need to
-    // perform semantic analysis again.
+  /// Subclasses may override this routine to provide different behavior.
+  ExprResult RebuildObjCPropertyRefExpr(Expr *Base, QualType T,
+                                        ObjCMethodDecl *Getter,
+                                        ObjCMethodDecl *Setter,
+                                        SourceLocation PropertyLoc) {
+    // Since these expressions can only be value-dependent, we do not
+    // need to perform semantic analysis again.
     return Owned(
-             new (getSema().Context) ObjCImplicitSetterGetterRefExpr(Getter, T,
-                                                                     Setter,
-                                                                     NameLoc,
-                                                                     Base));
+      new (getSema().Context) ObjCPropertyRefExpr(Getter, Setter, T,
+                                                  VK_LValue, OK_ObjCProperty,
+                                                  PropertyLoc, Base));
   }
 
   /// \brief Build a new Objective-C "isa" expression.
@@ -1915,8 +2161,8 @@ public:
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildShuffleVectorExpr(SourceLocation BuiltinLoc,
-                                            MultiExprArg SubExprs,
-                                            SourceLocation RParenLoc) {
+                                      MultiExprArg SubExprs,
+                                      SourceLocation RParenLoc) {
     // Find the declaration for __builtin_shufflevector
     const IdentifierInfo &Name
       = SemaRef.Context.Idents.get("__builtin_shufflevector");
@@ -1928,7 +2174,7 @@ public:
     FunctionDecl *Builtin = cast<FunctionDecl>(*Lookup.first);
     Expr *Callee
       = new (SemaRef.Context) DeclRefExpr(Builtin, Builtin->getType(),
-                                          BuiltinLoc);
+                                          VK_LValue, BuiltinLoc);
     SemaRef.UsualUnaryConversions(Callee);
 
     // Build the CallExpr
@@ -1937,6 +2183,7 @@ public:
     CallExpr *TheCall = new (SemaRef.Context) CallExpr(SemaRef.Context, Callee,
                                                        Subs, NumSubExprs,
                                                    Builtin->getCallResultType(),
+                            Expr::getValueKindForType(Builtin->getResultType()),
                                                        RParenLoc);
     ExprResult OwnedCall(SemaRef.Owned(TheCall));
 
@@ -1948,6 +2195,74 @@ public:
     OwnedCall.release();
     return move(Result);
   }
+
+  /// \brief Build a new template argument pack expansion.
+  ///
+  /// By default, performs semantic analysis to build a new pack expansion
+  /// for a template argument. Subclasses may override this routine to provide 
+  /// different behavior.
+  TemplateArgumentLoc RebuildPackExpansion(TemplateArgumentLoc Pattern,
+                                           SourceLocation EllipsisLoc,
+                                       llvm::Optional<unsigned> NumExpansions) {
+    switch (Pattern.getArgument().getKind()) {
+    case TemplateArgument::Expression: {
+      ExprResult Result
+        = getSema().CheckPackExpansion(Pattern.getSourceExpression(),
+                                       EllipsisLoc, NumExpansions);
+      if (Result.isInvalid())
+        return TemplateArgumentLoc();
+          
+      return TemplateArgumentLoc(Result.get(), Result.get());
+    }
+        
+    case TemplateArgument::Template:
+      return TemplateArgumentLoc(TemplateArgument(
+                                          Pattern.getArgument().getAsTemplate(),
+                                                  NumExpansions),
+                                 Pattern.getTemplateQualifierRange(),
+                                 Pattern.getTemplateNameLoc(),
+                                 EllipsisLoc);
+        
+    case TemplateArgument::Null:
+    case TemplateArgument::Integral:
+    case TemplateArgument::Declaration:
+    case TemplateArgument::Pack:
+    case TemplateArgument::TemplateExpansion:
+      llvm_unreachable("Pack expansion pattern has no parameter packs");
+        
+    case TemplateArgument::Type:
+      if (TypeSourceInfo *Expansion 
+            = getSema().CheckPackExpansion(Pattern.getTypeSourceInfo(),
+                                           EllipsisLoc,
+                                           NumExpansions))
+        return TemplateArgumentLoc(TemplateArgument(Expansion->getType()),
+                                   Expansion);
+      break;
+    }
+    
+    return TemplateArgumentLoc();
+  }
+  
+  /// \brief Build a new expression pack expansion.
+  ///
+  /// By default, performs semantic analysis to build a new pack expansion
+  /// for an expression. Subclasses may override this routine to provide 
+  /// different behavior.
+  ExprResult RebuildPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc,
+                                  llvm::Optional<unsigned> NumExpansions) {
+    return getSema().CheckPackExpansion(Pattern, EllipsisLoc, NumExpansions);
+  }
+  
+private:
+  QualType TransformTypeInObjectScope(QualType T,
+                                      QualType ObjectType,
+                                      NamedDecl *FirstQualifierInScope,
+                                      NestedNameSpecifier *Prefix);
+
+  TypeSourceInfo *TransformTypeInObjectScope(TypeSourceInfo *T,
+                                             QualType ObjectType,
+                                             NamedDecl *FirstQualifierInScope,
+                                             NestedNameSpecifier *Prefix);
 };
 
 template<typename Derived>
@@ -1961,6 +2276,7 @@ StmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
   // Transform individual statement nodes
 #define STMT(Node, Parent)                                              \
   case Stmt::Node##Class: return getDerived().Transform##Node(cast<Node>(S));
+#define ABSTRACT_STMT(Node)
 #define EXPR(Node, Parent)
 #include "clang/AST/StmtNodes.inc"
 
@@ -1978,7 +2294,7 @@ StmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
     }
   }
 
-  return SemaRef.Owned(S->Retain());
+  return SemaRef.Owned(S);
 }
 
 
@@ -1996,7 +2312,101 @@ ExprResult TreeTransform<Derived>::TransformExpr(Expr *E) {
 #include "clang/AST/StmtNodes.inc"
   }
 
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
+}
+
+template<typename Derived>
+bool TreeTransform<Derived>::TransformExprs(Expr **Inputs, 
+                                            unsigned NumInputs, 
+                                            bool IsCall,
+                                      llvm::SmallVectorImpl<Expr *> &Outputs,
+                                            bool *ArgChanged) {
+  for (unsigned I = 0; I != NumInputs; ++I) {
+    // If requested, drop call arguments that need to be dropped.
+    if (IsCall && getDerived().DropCallArgument(Inputs[I])) {
+      if (ArgChanged)
+        *ArgChanged = true;
+      
+      break;
+    }
+    
+    if (PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(Inputs[I])) {
+      Expr *Pattern = Expansion->getPattern();
+        
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
+      assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
+      
+      // Determine whether the set of unexpanded parameter packs can and should
+      // be expanded.
+      bool Expand = true;
+      bool RetainExpansion = false;
+      llvm::Optional<unsigned> OrigNumExpansions
+        = Expansion->getNumExpansions();
+      llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
+      if (getDerived().TryExpandParameterPacks(Expansion->getEllipsisLoc(),
+                                               Pattern->getSourceRange(),
+                                               Unexpanded.data(),
+                                               Unexpanded.size(),
+                                               Expand, RetainExpansion,
+                                               NumExpansions))
+        return true;
+        
+      if (!Expand) {
+        // The transform has determined that we should perform a simple
+        // transformation on the pack expansion, producing another pack 
+        // expansion.
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), -1);
+        ExprResult OutPattern = getDerived().TransformExpr(Pattern);
+        if (OutPattern.isInvalid())
+          return true;
+        
+        ExprResult Out = getDerived().RebuildPackExpansion(OutPattern.get(), 
+                                                Expansion->getEllipsisLoc(),
+                                                           NumExpansions);
+        if (Out.isInvalid())
+          return true;
+        
+        if (ArgChanged)
+          *ArgChanged = true;
+        Outputs.push_back(Out.get());
+        continue;
+      }
+      
+      // The transform has determined that we should perform an elementwise
+      // expansion of the pattern. Do so.
+      for (unsigned I = 0; I != *NumExpansions; ++I) {
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
+        ExprResult Out = getDerived().TransformExpr(Pattern);
+        if (Out.isInvalid())
+          return true;
+
+        if (Out.get()->containsUnexpandedParameterPack()) {
+          Out = RebuildPackExpansion(Out.get(), Expansion->getEllipsisLoc(),
+                                     OrigNumExpansions);
+          if (Out.isInvalid())
+            return true;
+        }
+        
+        if (ArgChanged)
+          *ArgChanged = true;  
+        Outputs.push_back(Out.get());
+      }
+        
+      continue;
+    }
+    
+    ExprResult Result = getDerived().TransformExpr(Inputs[I]);
+    if (Result.isInvalid())
+      return true;
+    
+    if (Result.get() != Inputs[I] && ArgChanged)
+      *ArgChanged = true;
+    
+    Outputs.push_back(Result.get());    
+  }
+  
+  return false;
 }
 
 template<typename Derived>
@@ -2005,26 +2415,26 @@ TreeTransform<Derived>::TransformNestedNameSpecifier(NestedNameSpecifier *NNS,
                                                      SourceRange Range,
                                                      QualType ObjectType,
                                              NamedDecl *FirstQualifierInScope) {
-  if (!NNS)
-    return 0;
+  NestedNameSpecifier *Prefix = NNS->getPrefix();
 
   // Transform the prefix of this nested name specifier.
-  NestedNameSpecifier *Prefix = NNS->getPrefix();
   if (Prefix) {
     Prefix = getDerived().TransformNestedNameSpecifier(Prefix, Range,
                                                        ObjectType,
                                                        FirstQualifierInScope);
     if (!Prefix)
       return 0;
-
-    // Clear out the object type and the first qualifier in scope; they only
-    // apply to the first element in the nested-name-specifier.
-    ObjectType = QualType();
-    FirstQualifierInScope = 0;
   }
 
   switch (NNS->getKind()) {
   case NestedNameSpecifier::Identifier:
+    if (Prefix) {
+      // The object type and qualifier-in-scope really apply to the
+      // leftmost entity.
+      ObjectType = QualType();
+      FirstQualifierInScope = 0;
+    }
+
     assert((Prefix || !ObjectType.isNull()) &&
             "Identifier nested-name-specifier with no prefix or object type");
     if (!getDerived().AlwaysRebuild() && Prefix == NNS->getPrefix() &&
@@ -2057,8 +2467,10 @@ TreeTransform<Derived>::TransformNestedNameSpecifier(NestedNameSpecifier *NNS,
   case NestedNameSpecifier::TypeSpecWithTemplate:
   case NestedNameSpecifier::TypeSpec: {
     TemporaryBase Rebase(*this, Range.getBegin(), DeclarationName());
-    QualType T = getDerived().TransformType(QualType(NNS->getAsType(), 0),
-                                            ObjectType);
+    QualType T = TransformTypeInObjectScope(QualType(NNS->getAsType(), 0),
+                                            ObjectType,
+                                            FirstQualifierInScope,
+                                            Prefix);
     if (T.isNull())
       return 0;
 
@@ -2080,8 +2492,7 @@ TreeTransform<Derived>::TransformNestedNameSpecifier(NestedNameSpecifier *NNS,
 template<typename Derived>
 DeclarationNameInfo
 TreeTransform<Derived>
-::TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
-                               QualType ObjectType) {
+::TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo) {
   DeclarationName Name = NameInfo.getName();
   if (!Name)
     return DeclarationNameInfo();
@@ -2102,16 +2513,15 @@ TreeTransform<Derived>
     TypeSourceInfo *NewTInfo;
     CanQualType NewCanTy;
     if (TypeSourceInfo *OldTInfo = NameInfo.getNamedTypeInfo()) {
-       NewTInfo = getDerived().TransformType(OldTInfo, ObjectType);
-       if (!NewTInfo)
-         return DeclarationNameInfo();
-       NewCanTy = SemaRef.Context.getCanonicalType(NewTInfo->getType());
+      NewTInfo = getDerived().TransformType(OldTInfo);
+      if (!NewTInfo)
+        return DeclarationNameInfo();
+      NewCanTy = SemaRef.Context.getCanonicalType(NewTInfo->getType());
     }
     else {
       NewTInfo = 0;
       TemporaryBase Rebase(*this, NameInfo.getLoc(), Name);
-      QualType NewT = getDerived().TransformType(Name.getCXXNameType(),
-                                                 ObjectType);
+      QualType NewT = getDerived().TransformType(Name.getCXXNameType());
       if (NewT.isNull())
         return DeclarationNameInfo();
       NewCanTy = SemaRef.Context.getCanonicalType(NewT);
@@ -2134,14 +2544,16 @@ TreeTransform<Derived>
 template<typename Derived>
 TemplateName
 TreeTransform<Derived>::TransformTemplateName(TemplateName Name,
-                                              QualType ObjectType) {
+                                              QualType ObjectType,
+                                              NamedDecl *FirstQualifierInScope) {
   SourceLocation Loc = getDerived().getBaseLocation();
 
   if (QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName()) {
     NestedNameSpecifier *NNS
       = getDerived().TransformNestedNameSpecifier(QTN->getQualifier(),
-                        /*FIXME:*/SourceRange(getDerived().getBaseLocation()),
-                                                  ObjectType);
+                                                  /*FIXME*/ SourceRange(Loc),
+                                                  ObjectType,
+                                                  FirstQualifierInScope);
     if (!NNS)
       return TemplateName();
 
@@ -2161,25 +2573,36 @@ TreeTransform<Derived>::TransformTemplateName(TemplateName Name,
     }
 
     // These should be getting filtered out before they make it into the AST.
-    assert(false && "overloaded template name survived to here");
+    llvm_unreachable("overloaded template name survived to here");
   }
 
   if (DependentTemplateName *DTN = Name.getAsDependentTemplateName()) {
-    NestedNameSpecifier *NNS
-      = getDerived().TransformNestedNameSpecifier(DTN->getQualifier(),
-                        /*FIXME:*/SourceRange(getDerived().getBaseLocation()),
-                                                  ObjectType);
-    if (!NNS && DTN->getQualifier())
-      return TemplateName();
+    NestedNameSpecifier *NNS = DTN->getQualifier();
+    if (NNS) {
+      NNS = getDerived().TransformNestedNameSpecifier(NNS,
+                                                  /*FIXME:*/SourceRange(Loc),
+                                                      ObjectType,
+                                                      FirstQualifierInScope);
+      if (!NNS) return TemplateName();
+
+      // These apply to the scope specifier, not the template.
+      ObjectType = QualType();
+      FirstQualifierInScope = 0;
+    }
 
     if (!getDerived().AlwaysRebuild() &&
         NNS == DTN->getQualifier() &&
         ObjectType.isNull())
       return Name;
 
-    if (DTN->isIdentifier())
-      return getDerived().RebuildTemplateName(NNS, *DTN->getIdentifier(), 
-                                              ObjectType);
+    if (DTN->isIdentifier()) {
+      // FIXME: Bad range
+      SourceRange QualifierRange(getDerived().getBaseLocation());
+      return getDerived().RebuildTemplateName(NNS, QualifierRange,
+                                              *DTN->getIdentifier(), 
+                                              ObjectType,
+                                              FirstQualifierInScope);
+    }
     
     return getDerived().RebuildTemplateName(NNS, DTN->getOperator(), 
                                             ObjectType);
@@ -2198,8 +2621,24 @@ TreeTransform<Derived>::TransformTemplateName(TemplateName Name,
     return TemplateName(TransTemplate);
   }
 
+  if (SubstTemplateTemplateParmPackStorage *SubstPack
+                                 = Name.getAsSubstTemplateTemplateParmPack()) {
+    TemplateTemplateParmDecl *TransParam
+      = cast_or_null<TemplateTemplateParmDecl>(
+               getDerived().TransformDecl(Loc, SubstPack->getParameterPack()));
+    if (!TransParam)
+      return TemplateName();
+    
+    if (!getDerived().AlwaysRebuild() &&
+        TransParam == SubstPack->getParameterPack())
+      return Name;
+    
+    return getDerived().RebuildTemplateName(TransParam, 
+                                            SubstPack->getArgumentPack());
+  }
+  
   // These should be getting filtered out before they reach the AST.
-  assert(false && "overloaded function decl survived to here");
+  llvm_unreachable("overloaded function decl survived to here");
   return TemplateName();
 }
 
@@ -2222,7 +2661,11 @@ void TreeTransform<Derived>::InventTemplateArgumentLoc(
   case TemplateArgument::Template:
     Output = TemplateArgumentLoc(Arg, SourceRange(), Loc);
     break;
-      
+
+  case TemplateArgument::TemplateExpansion:
+    Output = TemplateArgumentLoc(Arg, SourceRange(), Loc, Loc);
+    break;
+
   case TemplateArgument::Expression:
     Output = TemplateArgumentLoc(Arg, Arg.getAsExpr());
     break;
@@ -2291,7 +2734,10 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
                                  Input.getTemplateNameLoc());
     return false;
   }
-      
+
+  case TemplateArgument::TemplateExpansion:
+    llvm_unreachable("Caller should expand pack expansions");
+
   case TemplateArgument::Expression: {
     // Template argument expressions are not potentially evaluated.
     EnterExpressionEvaluationContext Unevaluated(getSema(),
@@ -2325,10 +2771,14 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
 
       TransformedArgs.push_back(OutputArg.getArgument());
     }
-    TemplateArgument Result;
-    Result.setArgumentPack(TransformedArgs.data(), TransformedArgs.size(),
-                           true);
-    Output = TemplateArgumentLoc(Result, Input.getLocInfo());
+
+    TemplateArgument *TransformedArgsPtr
+      = new (getSema().Context) TemplateArgument[TransformedArgs.size()];
+    std::copy(TransformedArgs.begin(), TransformedArgs.end(),
+              TransformedArgsPtr);
+    Output = TemplateArgumentLoc(TemplateArgument(TransformedArgsPtr, 
+                                                  TransformedArgs.size()), 
+                                 Input.getLocInfo());
     return false;
   }
   }
@@ -2337,22 +2787,201 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
   return true;
 }
 
+/// \brief Iterator adaptor that invents template argument location information
+/// for each of the template arguments in its underlying iterator.
+template<typename Derived, typename InputIterator>
+class TemplateArgumentLocInventIterator {
+  TreeTransform<Derived> &Self;
+  InputIterator Iter;
+  
+public:
+  typedef TemplateArgumentLoc value_type;
+  typedef TemplateArgumentLoc reference;
+  typedef typename std::iterator_traits<InputIterator>::difference_type
+    difference_type;
+  typedef std::input_iterator_tag iterator_category;
+  
+  class pointer {
+    TemplateArgumentLoc Arg;
+    
+  public:
+    explicit pointer(TemplateArgumentLoc Arg) : Arg(Arg) { }
+    
+    const TemplateArgumentLoc *operator->() const { return &Arg; }
+  };
+  
+  TemplateArgumentLocInventIterator() { }
+  
+  explicit TemplateArgumentLocInventIterator(TreeTransform<Derived> &Self,
+                                             InputIterator Iter)
+    : Self(Self), Iter(Iter) { }
+  
+  TemplateArgumentLocInventIterator &operator++() {
+    ++Iter;
+    return *this;
+  }
+  
+  TemplateArgumentLocInventIterator operator++(int) {
+    TemplateArgumentLocInventIterator Old(*this);
+    ++(*this);
+    return Old;
+  }
+  
+  reference operator*() const {
+    TemplateArgumentLoc Result;
+    Self.InventTemplateArgumentLoc(*Iter, Result);
+    return Result;
+  }
+  
+  pointer operator->() const { return pointer(**this); }
+  
+  friend bool operator==(const TemplateArgumentLocInventIterator &X,
+                         const TemplateArgumentLocInventIterator &Y) {
+    return X.Iter == Y.Iter;
+  }
+
+  friend bool operator!=(const TemplateArgumentLocInventIterator &X,
+                         const TemplateArgumentLocInventIterator &Y) {
+    return X.Iter != Y.Iter;
+  }
+};
+  
+template<typename Derived>
+template<typename InputIterator>
+bool TreeTransform<Derived>::TransformTemplateArguments(InputIterator First,
+                                                        InputIterator Last,
+                                            TemplateArgumentListInfo &Outputs) {
+  for (; First != Last; ++First) {
+    TemplateArgumentLoc Out;
+    TemplateArgumentLoc In = *First;
+    
+    if (In.getArgument().getKind() == TemplateArgument::Pack) {
+      // Unpack argument packs, which we translate them into separate
+      // arguments.
+      // FIXME: We could do much better if we could guarantee that the
+      // TemplateArgumentLocInfo for the pack expansion would be usable for
+      // all of the template arguments in the argument pack.
+      typedef TemplateArgumentLocInventIterator<Derived, 
+                                                TemplateArgument::pack_iterator>
+        PackLocIterator;
+      if (TransformTemplateArguments(PackLocIterator(*this, 
+                                                 In.getArgument().pack_begin()),
+                                     PackLocIterator(*this,
+                                                   In.getArgument().pack_end()),
+                                     Outputs))
+        return true;
+      
+      continue;
+    }
+    
+    if (In.getArgument().isPackExpansion()) {
+      // We have a pack expansion, for which we will be substituting into
+      // the pattern.
+      SourceLocation Ellipsis;
+      llvm::Optional<unsigned> OrigNumExpansions;
+      TemplateArgumentLoc Pattern
+        = In.getPackExpansionPattern(Ellipsis, OrigNumExpansions, 
+                                     getSema().Context);
+      
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
+      assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
+      
+      // Determine whether the set of unexpanded parameter packs can and should
+      // be expanded.
+      bool Expand = true;
+      bool RetainExpansion = false;
+      llvm::Optional<unsigned> NumExpansions = OrigNumExpansions;
+      if (getDerived().TryExpandParameterPacks(Ellipsis,
+                                               Pattern.getSourceRange(),
+                                               Unexpanded.data(),
+                                               Unexpanded.size(),
+                                               Expand, 
+                                               RetainExpansion,
+                                               NumExpansions))
+        return true;
+      
+      if (!Expand) {
+        // The transform has determined that we should perform a simple
+        // transformation on the pack expansion, producing another pack 
+        // expansion.
+        TemplateArgumentLoc OutPattern;
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), -1);
+        if (getDerived().TransformTemplateArgument(Pattern, OutPattern))
+          return true;
+                
+        Out = getDerived().RebuildPackExpansion(OutPattern, Ellipsis,
+                                                NumExpansions);
+        if (Out.getArgument().isNull())
+          return true;
+        
+        Outputs.addArgument(Out);
+        continue;
+      }
+      
+      // The transform has determined that we should perform an elementwise
+      // expansion of the pattern. Do so.
+      for (unsigned I = 0; I != *NumExpansions; ++I) {
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
+
+        if (getDerived().TransformTemplateArgument(Pattern, Out))
+          return true;
+        
+        if (Out.getArgument().containsUnexpandedParameterPack()) {
+          Out = getDerived().RebuildPackExpansion(Out, Ellipsis,
+                                                  OrigNumExpansions);
+          if (Out.getArgument().isNull())
+            return true;
+        }
+          
+        Outputs.addArgument(Out);
+      }
+      
+      // If we're supposed to retain a pack expansion, do so by temporarily
+      // forgetting the partially-substituted parameter pack.
+      if (RetainExpansion) {
+        ForgetPartiallySubstitutedPackRAII Forget(getDerived());
+        
+        if (getDerived().TransformTemplateArgument(Pattern, Out))
+          return true;
+        
+        Out = getDerived().RebuildPackExpansion(Out, Ellipsis,
+                                                OrigNumExpansions);
+        if (Out.getArgument().isNull())
+          return true;
+        
+        Outputs.addArgument(Out);
+      }
+      
+      continue;
+    }
+    
+    // The simple case: 
+    if (getDerived().TransformTemplateArgument(In, Out))
+      return true;
+    
+    Outputs.addArgument(Out);
+  }
+  
+  return false;
+
+}
+
 //===----------------------------------------------------------------------===//
 // Type transformation
 //===----------------------------------------------------------------------===//
 
 template<typename Derived>
-QualType TreeTransform<Derived>::TransformType(QualType T, 
-                                               QualType ObjectType) {
+QualType TreeTransform<Derived>::TransformType(QualType T) {
   if (getDerived().AlreadyTransformed(T))
     return T;
 
   // Temporary workaround.  All of these transformations should
   // eventually turn into transformations on TypeLocs.
-  TypeSourceInfo *DI = getSema().Context.CreateTypeSourceInfo(T);
-  DI->getTypeLoc().initialize(getDerived().getBaseLocation());
+  TypeSourceInfo *DI = getSema().Context.getTrivialTypeSourceInfo(T,
+                                                getDerived().getBaseLocation());
   
-  TypeSourceInfo *NewDI = getDerived().TransformType(DI, ObjectType);
+  TypeSourceInfo *NewDI = getDerived().TransformType(DI);
 
   if (!NewDI)
     return QualType();
@@ -2361,8 +2990,7 @@ QualType TreeTransform<Derived>::TransformType(QualType T,
 }
 
 template<typename Derived>
-TypeSourceInfo *TreeTransform<Derived>::TransformType(TypeSourceInfo *DI,
-                                                      QualType ObjectType) {
+TypeSourceInfo *TreeTransform<Derived>::TransformType(TypeSourceInfo *DI) {
   if (getDerived().AlreadyTransformed(DI->getType()))
     return DI;
 
@@ -2371,7 +2999,7 @@ TypeSourceInfo *TreeTransform<Derived>::TransformType(TypeSourceInfo *DI,
   TypeLoc TL = DI->getTypeLoc();
   TLB.reserve(TL.getFullDataSize());
 
-  QualType Result = getDerived().TransformType(TLB, TL, ObjectType);
+  QualType Result = getDerived().TransformType(TLB, TL);
   if (Result.isNull())
     return 0;
 
@@ -2380,14 +3008,12 @@ TypeSourceInfo *TreeTransform<Derived>::TransformType(TypeSourceInfo *DI,
 
 template<typename Derived>
 QualType
-TreeTransform<Derived>::TransformType(TypeLocBuilder &TLB, TypeLoc T,
-                                      QualType ObjectType) {
+TreeTransform<Derived>::TransformType(TypeLocBuilder &TLB, TypeLoc T) {
   switch (T.getTypeLocClass()) {
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
 #define TYPELOC(CLASS, PARENT) \
   case TypeLoc::CLASS: \
-    return getDerived().Transform##CLASS##Type(TLB, cast<CLASS##TypeLoc>(T), \
-                                               ObjectType);
+    return getDerived().Transform##CLASS##Type(TLB, cast<CLASS##TypeLoc>(T));
 #include "clang/AST/TypeLocNodes.def"
   }
 
@@ -2403,12 +3029,10 @@ TreeTransform<Derived>::TransformType(TypeLocBuilder &TLB, TypeLoc T,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformQualifiedType(TypeLocBuilder &TLB,
-                                               QualifiedTypeLoc T,
-                                               QualType ObjectType) {
+                                               QualifiedTypeLoc T) {
   Qualifiers Quals = T.getType().getLocalQualifiers();
 
-  QualType Result = getDerived().TransformType(TLB, T.getUnqualifiedLoc(),
-                                               ObjectType);
+  QualType Result = getDerived().TransformType(TLB, T.getUnqualifiedLoc());
   if (Result.isNull())
     return QualType();
 
@@ -2427,6 +3051,77 @@ TreeTransform<Derived>::TransformQualifiedType(TypeLocBuilder &TLB,
   return Result;
 }
 
+/// \brief Transforms a type that was written in a scope specifier,
+/// given an object type, the results of unqualified lookup, and
+/// an already-instantiated prefix.
+///
+/// The object type is provided iff the scope specifier qualifies the
+/// member of a dependent member-access expression.  The prefix is
+/// provided iff the the scope specifier in which this appears has a
+/// prefix.
+///
+/// This is private to TreeTransform.
+template<typename Derived>
+QualType
+TreeTransform<Derived>::TransformTypeInObjectScope(QualType T,
+                                                   QualType ObjectType,
+                                                   NamedDecl *UnqualLookup,
+                                                  NestedNameSpecifier *Prefix) {
+  if (getDerived().AlreadyTransformed(T))
+    return T;
+
+  TypeSourceInfo *TSI =
+    SemaRef.Context.getTrivialTypeSourceInfo(T, getDerived().getBaseLocation());
+
+  TSI = getDerived().TransformTypeInObjectScope(TSI, ObjectType,
+                                                UnqualLookup, Prefix);
+  if (!TSI) return QualType();
+  return TSI->getType();
+}
+
+template<typename Derived>
+TypeSourceInfo *
+TreeTransform<Derived>::TransformTypeInObjectScope(TypeSourceInfo *TSI,
+                                                   QualType ObjectType,
+                                                   NamedDecl *UnqualLookup,
+                                                  NestedNameSpecifier *Prefix) {
+  // TODO: in some cases, we might be some verification to do here.
+  if (ObjectType.isNull())
+    return getDerived().TransformType(TSI);
+
+  QualType T = TSI->getType();
+  if (getDerived().AlreadyTransformed(T))
+    return TSI;
+
+  TypeLocBuilder TLB;
+  QualType Result;
+
+  if (isa<TemplateSpecializationType>(T)) {
+    TemplateSpecializationTypeLoc TL
+      = cast<TemplateSpecializationTypeLoc>(TSI->getTypeLoc());
+
+    TemplateName Template =
+      getDerived().TransformTemplateName(TL.getTypePtr()->getTemplateName(),
+                                         ObjectType, UnqualLookup);
+    if (Template.isNull()) return 0;
+
+    Result = getDerived()
+      .TransformTemplateSpecializationType(TLB, TL, Template);
+  } else if (isa<DependentTemplateSpecializationType>(T)) {
+    DependentTemplateSpecializationTypeLoc TL
+      = cast<DependentTemplateSpecializationTypeLoc>(TSI->getTypeLoc());
+
+    Result = getDerived()
+      .TransformDependentTemplateSpecializationType(TLB, TL, Prefix);
+  } else {
+    // Nothing special needs to be done for these.
+    Result = getDerived().TransformType(TLB, TSI->getTypeLoc());
+  }
+
+  if (Result.isNull()) return 0;
+  return TLB.getTypeSourceInfo(SemaRef.Context, Result);
+}
+
 template <class TyLoc> static inline
 QualType TransformTypeSpecType(TypeLocBuilder &TLB, TyLoc T) {
   TyLoc NewT = TLB.push<TyLoc>(T.getType());
@@ -2436,8 +3131,7 @@ QualType TransformTypeSpecType(TypeLocBuilder &TLB, TyLoc T) {
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformBuiltinType(TypeLocBuilder &TLB,
-                                                      BuiltinTypeLoc T,
-                                                      QualType ObjectType) {
+                                                      BuiltinTypeLoc T) {
   BuiltinTypeLoc NewT = TLB.push<BuiltinTypeLoc>(T.getType());
   NewT.setBuiltinLoc(T.getBuiltinLoc());
   if (T.needsExtraLocalData())
@@ -2447,16 +3141,14 @@ QualType TreeTransform<Derived>::TransformBuiltinType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformComplexType(TypeLocBuilder &TLB,
-                                                      ComplexTypeLoc T,
-                                                      QualType ObjectType) {
+                                                      ComplexTypeLoc T) {
   // FIXME: recurse?
   return TransformTypeSpecType(TLB, T);
 }
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformPointerType(TypeLocBuilder &TLB,
-                                                      PointerTypeLoc TL, 
-                                                      QualType ObjectType) {
+                                                      PointerTypeLoc TL) {
   QualType PointeeType                                      
     = getDerived().TransformType(TLB, TL.getPointeeLoc());  
   if (PointeeType.isNull())
@@ -2474,7 +3166,7 @@ QualType TreeTransform<Derived>::TransformPointerType(TypeLocBuilder &TLB,
     NewT.setStarLoc(TL.getStarLoc());
     return Result;
   }
-                                                            
+
   if (getDerived().AlwaysRebuild() ||
       PointeeType != TL.getPointeeLoc().getType()) {
     Result = getDerived().RebuildPointerType(PointeeType, TL.getSigilLoc());
@@ -2490,8 +3182,7 @@ QualType TreeTransform<Derived>::TransformPointerType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformBlockPointerType(TypeLocBuilder &TLB,
-                                                  BlockPointerTypeLoc TL,
-                                                  QualType ObjectType) {
+                                                  BlockPointerTypeLoc TL) {
   QualType PointeeType
     = getDerived().TransformType(TLB, TL.getPointeeLoc());  
   if (PointeeType.isNull())                                 
@@ -2518,8 +3209,7 @@ TreeTransform<Derived>::TransformBlockPointerType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformReferenceType(TypeLocBuilder &TLB,
-                                               ReferenceTypeLoc TL,
-                                               QualType ObjectType) {
+                                               ReferenceTypeLoc TL) {
   const ReferenceType *T = TL.getTypePtr();
 
   // Note that this works with the pointee-as-written.
@@ -2551,25 +3241,22 @@ TreeTransform<Derived>::TransformReferenceType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformLValueReferenceType(TypeLocBuilder &TLB,
-                                                 LValueReferenceTypeLoc TL,
-                                                     QualType ObjectType) {
-  return TransformReferenceType(TLB, TL, ObjectType);
+                                                 LValueReferenceTypeLoc TL) {
+  return TransformReferenceType(TLB, TL);
 }
 
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformRValueReferenceType(TypeLocBuilder &TLB,
-                                                 RValueReferenceTypeLoc TL,
-                                                     QualType ObjectType) {
-  return TransformReferenceType(TLB, TL, ObjectType);
+                                                 RValueReferenceTypeLoc TL) {
+  return TransformReferenceType(TLB, TL);
 }
 
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformMemberPointerType(TypeLocBuilder &TLB,
-                                                   MemberPointerTypeLoc TL,
-                                                   QualType ObjectType) {
-  MemberPointerType *T = TL.getTypePtr();
+                                                   MemberPointerTypeLoc TL) {
+  const MemberPointerType *T = TL.getTypePtr();
 
   QualType PointeeType = getDerived().TransformType(TLB, TL.getPointeeLoc());
   if (PointeeType.isNull())
@@ -2600,9 +3287,8 @@ TreeTransform<Derived>::TransformMemberPointerType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformConstantArrayType(TypeLocBuilder &TLB,
-                                                   ConstantArrayTypeLoc TL,
-                                                   QualType ObjectType) {
-  ConstantArrayType *T = TL.getTypePtr();
+                                                   ConstantArrayTypeLoc TL) {
+  const ConstantArrayType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(TLB, TL.getElementLoc());
   if (ElementType.isNull())
     return QualType();
@@ -2636,9 +3322,8 @@ TreeTransform<Derived>::TransformConstantArrayType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformIncompleteArrayType(
                                               TypeLocBuilder &TLB,
-                                              IncompleteArrayTypeLoc TL,
-                                              QualType ObjectType) {
-  IncompleteArrayType *T = TL.getTypePtr();
+                                              IncompleteArrayTypeLoc TL) {
+  const IncompleteArrayType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(TLB, TL.getElementLoc());
   if (ElementType.isNull())
     return QualType();
@@ -2665,9 +3350,8 @@ QualType TreeTransform<Derived>::TransformIncompleteArrayType(
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformVariableArrayType(TypeLocBuilder &TLB,
-                                                   VariableArrayTypeLoc TL,
-                                                   QualType ObjectType) {
-  VariableArrayType *T = TL.getTypePtr();
+                                                   VariableArrayTypeLoc TL) {
+  const VariableArrayType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(TLB, TL.getElementLoc());
   if (ElementType.isNull())
     return QualType();
@@ -2706,9 +3390,8 @@ TreeTransform<Derived>::TransformVariableArrayType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformDependentSizedArrayType(TypeLocBuilder &TLB,
-                                             DependentSizedArrayTypeLoc TL,
-                                                        QualType ObjectType) {
-  DependentSizedArrayType *T = TL.getTypePtr();
+                                             DependentSizedArrayTypeLoc TL) {
+  const DependentSizedArrayType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(TLB, TL.getElementLoc());
   if (ElementType.isNull())
     return QualType();
@@ -2716,33 +3399,36 @@ TreeTransform<Derived>::TransformDependentSizedArrayType(TypeLocBuilder &TLB,
   // Array bounds are not potentially evaluated contexts
   EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  ExprResult SizeResult
-    = getDerived().TransformExpr(T->getSizeExpr());
-  if (SizeResult.isInvalid())
+  // Prefer the expression from the TypeLoc;  the other may have been uniqued.
+  Expr *origSize = TL.getSizeExpr();
+  if (!origSize) origSize = T->getSizeExpr();
+
+  ExprResult sizeResult
+    = getDerived().TransformExpr(origSize);
+  if (sizeResult.isInvalid())
     return QualType();
 
-  Expr *Size = static_cast<Expr*>(SizeResult.get());
+  Expr *size = sizeResult.get();
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       ElementType != T->getElementType() ||
-      Size != T->getSizeExpr()) {
+      size != origSize) {
     Result = getDerived().RebuildDependentSizedArrayType(ElementType,
                                                          T->getSizeModifier(),
-                                                         Size,
+                                                         size,
                                                 T->getIndexTypeCVRQualifiers(),
                                                         TL.getBracketsRange());
     if (Result.isNull())
       return QualType();
   }
-  else SizeResult.take();
 
   // We might have any sort of array type now, but fortunately they
   // all have the same location layout.
   ArrayTypeLoc NewTL = TLB.push<ArrayTypeLoc>(Result);
   NewTL.setLBracketLoc(TL.getLBracketLoc());
   NewTL.setRBracketLoc(TL.getRBracketLoc());
-  NewTL.setSizeExpr(Size);
+  NewTL.setSizeExpr(size);
 
   return Result;
 }
@@ -2750,9 +3436,8 @@ TreeTransform<Derived>::TransformDependentSizedArrayType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformDependentSizedExtVectorType(
                                       TypeLocBuilder &TLB,
-                                      DependentSizedExtVectorTypeLoc TL,
-                                      QualType ObjectType) {
-  DependentSizedExtVectorType *T = TL.getTypePtr();
+                                      DependentSizedExtVectorTypeLoc TL) {
+  const DependentSizedExtVectorType *T = TL.getTypePtr();
 
   // FIXME: ext vector locs should be nested
   QualType ElementType = getDerived().TransformType(T->getElementType());
@@ -2792,9 +3477,8 @@ QualType TreeTransform<Derived>::TransformDependentSizedExtVectorType(
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformVectorType(TypeLocBuilder &TLB,
-                                                     VectorTypeLoc TL,
-                                                     QualType ObjectType) {
-  VectorType *T = TL.getTypePtr();
+                                                     VectorTypeLoc TL) {
+  const VectorType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(T->getElementType());
   if (ElementType.isNull())
     return QualType();
@@ -2803,7 +3487,7 @@ QualType TreeTransform<Derived>::TransformVectorType(TypeLocBuilder &TLB,
   if (getDerived().AlwaysRebuild() ||
       ElementType != T->getElementType()) {
     Result = getDerived().RebuildVectorType(ElementType, T->getNumElements(),
-      T->getAltiVecSpecific());
+                                            T->getVectorKind());
     if (Result.isNull())
       return QualType();
   }
@@ -2816,9 +3500,8 @@ QualType TreeTransform<Derived>::TransformVectorType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformExtVectorType(TypeLocBuilder &TLB,
-                                                        ExtVectorTypeLoc TL,
-                                                        QualType ObjectType) {
-  VectorType *T = TL.getTypePtr();
+                                                        ExtVectorTypeLoc TL) {
+  const VectorType *T = TL.getTypePtr();
   QualType ElementType = getDerived().TransformType(T->getElementType());
   if (ElementType.isNull())
     return QualType();
@@ -2841,9 +3524,38 @@ QualType TreeTransform<Derived>::TransformExtVectorType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 ParmVarDecl *
-TreeTransform<Derived>::TransformFunctionTypeParam(ParmVarDecl *OldParm) {
+TreeTransform<Derived>::TransformFunctionTypeParam(ParmVarDecl *OldParm,
+                                       llvm::Optional<unsigned> NumExpansions) {
   TypeSourceInfo *OldDI = OldParm->getTypeSourceInfo();
-  TypeSourceInfo *NewDI = getDerived().TransformType(OldDI);
+  TypeSourceInfo *NewDI = 0;
+  
+  if (NumExpansions && isa<PackExpansionType>(OldDI->getType())) {
+    // If we're substituting into a pack expansion type and we know the 
+    TypeLoc OldTL = OldDI->getTypeLoc();
+    PackExpansionTypeLoc OldExpansionTL = cast<PackExpansionTypeLoc>(OldTL);
+    
+    TypeLocBuilder TLB;
+    TypeLoc NewTL = OldDI->getTypeLoc();
+    TLB.reserve(NewTL.getFullDataSize());
+    
+    QualType Result = getDerived().TransformType(TLB, 
+                                               OldExpansionTL.getPatternLoc());
+    if (Result.isNull())
+      return 0;
+   
+    Result = RebuildPackExpansionType(Result, 
+                                OldExpansionTL.getPatternLoc().getSourceRange(), 
+                                      OldExpansionTL.getEllipsisLoc(),
+                                      NumExpansions);
+    if (Result.isNull())
+      return 0;
+    
+    PackExpansionTypeLoc NewExpansionTL
+      = TLB.push<PackExpansionTypeLoc>(Result);
+    NewExpansionTL.setEllipsisLoc(OldExpansionTL.getEllipsisLoc());
+    NewDI = TLB.getTypeSourceInfo(SemaRef.Context, Result);
+  } else
+    NewDI = getDerived().TransformType(OldDI);
   if (!NewDI)
     return 0;
 
@@ -2863,75 +3575,226 @@ TreeTransform<Derived>::TransformFunctionTypeParam(ParmVarDecl *OldParm) {
 
 template<typename Derived>
 bool TreeTransform<Derived>::
-  TransformFunctionTypeParams(FunctionProtoTypeLoc TL,
-                              llvm::SmallVectorImpl<QualType> &PTypes,
-                              llvm::SmallVectorImpl<ParmVarDecl*> &PVars) {
-  FunctionProtoType *T = TL.getTypePtr();
-
-  for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
-    ParmVarDecl *OldParm = TL.getArg(i);
-
-    QualType NewType;
-    ParmVarDecl *NewParm;
-
-    if (OldParm) {
-      NewParm = getDerived().TransformFunctionTypeParam(OldParm);
+  TransformFunctionTypeParams(SourceLocation Loc,
+                              ParmVarDecl **Params, unsigned NumParams,
+                              const QualType *ParamTypes,
+                              llvm::SmallVectorImpl<QualType> &OutParamTypes,
+                              llvm::SmallVectorImpl<ParmVarDecl*> *PVars) {
+  for (unsigned i = 0; i != NumParams; ++i) {
+    if (ParmVarDecl *OldParm = Params[i]) {
+      llvm::Optional<unsigned> NumExpansions;
+      if (OldParm->isParameterPack()) {
+        // We have a function parameter pack that may need to be expanded.
+        llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+
+        // Find the parameter packs that could be expanded.
+        TypeLoc TL = OldParm->getTypeSourceInfo()->getTypeLoc();
+        PackExpansionTypeLoc ExpansionTL = cast<PackExpansionTypeLoc>(TL);
+        TypeLoc Pattern = ExpansionTL.getPatternLoc();
+        SemaRef.collectUnexpandedParameterPacks(Pattern, Unexpanded);
+        
+        // Determine whether we should expand the parameter packs.
+        bool ShouldExpand = false;
+        bool RetainExpansion = false;
+        llvm::Optional<unsigned> OrigNumExpansions
+          = ExpansionTL.getTypePtr()->getNumExpansions();
+        NumExpansions = OrigNumExpansions;
+        if (getDerived().TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
+                                                 Pattern.getSourceRange(),
+                                                 Unexpanded.data(), 
+                                                 Unexpanded.size(),
+                                                 ShouldExpand, 
+                                                 RetainExpansion,
+                                                 NumExpansions)) {
+          return true;
+        }
+        
+        if (ShouldExpand) {
+          // Expand the function parameter pack into multiple, separate
+          // parameters.
+          getDerived().ExpandingFunctionParameterPack(OldParm);
+          for (unsigned I = 0; I != *NumExpansions; ++I) {
+            Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
+            ParmVarDecl *NewParm 
+              = getDerived().TransformFunctionTypeParam(OldParm,
+                                                        OrigNumExpansions);
+            if (!NewParm)
+              return true;
+            
+            OutParamTypes.push_back(NewParm->getType());
+            if (PVars)
+              PVars->push_back(NewParm);
+          }
+
+          // If we're supposed to retain a pack expansion, do so by temporarily
+          // forgetting the partially-substituted parameter pack.
+          if (RetainExpansion) {
+            ForgetPartiallySubstitutedPackRAII Forget(getDerived());
+            ParmVarDecl *NewParm 
+              = getDerived().TransformFunctionTypeParam(OldParm,
+                                                        OrigNumExpansions);
+            if (!NewParm)
+              return true;
+            
+            OutParamTypes.push_back(NewParm->getType());
+            if (PVars)
+              PVars->push_back(NewParm);
+          }
+
+          // We're done with the pack expansion.
+          continue;
+        }
+        
+        // We'll substitute the parameter now without expanding the pack 
+        // expansion.
+      }
+      
+      Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), -1);
+      ParmVarDecl *NewParm = getDerived().TransformFunctionTypeParam(OldParm,
+                                                                 NumExpansions);
       if (!NewParm)
         return true;
-      NewType = NewParm->getType();
+      
+      OutParamTypes.push_back(NewParm->getType());
+      if (PVars)
+        PVars->push_back(NewParm);
+      continue;
+    }
 
     // Deal with the possibility that we don't have a parameter
     // declaration for this parameter.
-    } else {
-      NewParm = 0;
-
-      QualType OldType = T->getArgType(i);
-      NewType = getDerived().TransformType(OldType);
-      if (NewType.isNull())
+    QualType OldType = ParamTypes[i];
+    bool IsPackExpansion = false;
+    llvm::Optional<unsigned> NumExpansions;
+    if (const PackExpansionType *Expansion 
+                                       = dyn_cast<PackExpansionType>(OldType)) {
+      // We have a function parameter pack that may need to be expanded.
+      QualType Pattern = Expansion->getPattern();
+      llvm::SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+      getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
+      
+      // Determine whether we should expand the parameter packs.
+      bool ShouldExpand = false;
+      bool RetainExpansion = false;
+      if (getDerived().TryExpandParameterPacks(Loc, SourceRange(),
+                                               Unexpanded.data(), 
+                                               Unexpanded.size(),
+                                               ShouldExpand, 
+                                               RetainExpansion,
+                                               NumExpansions)) {
         return true;
+      }
+      
+      if (ShouldExpand) {
+        // Expand the function parameter pack into multiple, separate 
+        // parameters.
+        for (unsigned I = 0; I != *NumExpansions; ++I) {
+          Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), I);
+          QualType NewType = getDerived().TransformType(Pattern);
+          if (NewType.isNull())
+            return true;
+
+          OutParamTypes.push_back(NewType);
+          if (PVars)
+            PVars->push_back(0);
+        }
+        
+        // We're done with the pack expansion.
+        continue;
+      }
+      
+      // If we're supposed to retain a pack expansion, do so by temporarily
+      // forgetting the partially-substituted parameter pack.
+      if (RetainExpansion) {
+        ForgetPartiallySubstitutedPackRAII Forget(getDerived());
+        QualType NewType = getDerived().TransformType(Pattern);
+        if (NewType.isNull())
+          return true;
+        
+        OutParamTypes.push_back(NewType);
+        if (PVars)
+          PVars->push_back(0);
+      }
+
+      // We'll substitute the parameter now without expanding the pack 
+      // expansion.
+      OldType = Expansion->getPattern();
+      IsPackExpansion = true;
     }
+    
+    Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(getSema(), -1);
+    QualType NewType = getDerived().TransformType(OldType);
+    if (NewType.isNull())
+      return true;
 
-    PTypes.push_back(NewType);
-    PVars.push_back(NewParm);
+    if (IsPackExpansion)
+      NewType = getSema().Context.getPackExpansionType(NewType,
+                                                       NumExpansions);
+      
+    OutParamTypes.push_back(NewType);
+    if (PVars)
+      PVars->push_back(0);
   }
 
   return false;
-}
+  }
 
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
-                                                   FunctionProtoTypeLoc TL,
-                                                   QualType ObjectType) {
+                                                   FunctionProtoTypeLoc TL) {
   // Transform the parameters and return type.
   //
   // We instantiate in source order, with the return type first followed by
   // the parameters, because users tend to expect this (even if they shouldn't
   // rely on it!).
   //
-  // FIXME: When we implement late-specified return types, we'll need to
-  // instantiate the return tpe *after* the parameter types in that case,
-  // since the return type can then refer to the parameters themselves (via
-  // decltype, sizeof, etc.).
+  // When the function has a trailing return type, we instantiate the
+  // parameters before the return type,  since the return type can then refer
+  // to the parameters themselves (via decltype, sizeof, etc.).
+  //
   llvm::SmallVector<QualType, 4> ParamTypes;
   llvm::SmallVector<ParmVarDecl*, 4> ParamDecls;
-  FunctionProtoType *T = TL.getTypePtr();
-  QualType ResultType = getDerived().TransformType(TLB, TL.getResultLoc());
-  if (ResultType.isNull())
-    return QualType();
+  const FunctionProtoType *T = TL.getTypePtr();
+
+  QualType ResultType;
+
+  if (TL.getTrailingReturn()) {
+    if (getDerived().TransformFunctionTypeParams(TL.getBeginLoc(), 
+                                                 TL.getParmArray(),
+                                                 TL.getNumArgs(),
+                                             TL.getTypePtr()->arg_type_begin(),                                                
+                                                 ParamTypes, &ParamDecls))
+      return QualType();
+
+    ResultType = getDerived().TransformType(TLB, TL.getResultLoc());
+    if (ResultType.isNull())
+      return QualType();
+  }
+  else {
+    ResultType = getDerived().TransformType(TLB, TL.getResultLoc());
+    if (ResultType.isNull())
+      return QualType();
+
+    if (getDerived().TransformFunctionTypeParams(TL.getBeginLoc(), 
+                                                 TL.getParmArray(),
+                                                 TL.getNumArgs(),
+                                             TL.getTypePtr()->arg_type_begin(),                                                
+                                                 ParamTypes, &ParamDecls))
+      return QualType();
+  }
 
-  if (getDerived().TransformFunctionTypeParams(TL, ParamTypes, ParamDecls))
-    return QualType();  
-  
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       ResultType != T->getResultType() ||
+      T->getNumArgs() != ParamTypes.size() ||
       !std::equal(T->arg_type_begin(), T->arg_type_end(), ParamTypes.begin())) {
     Result = getDerived().RebuildFunctionProtoType(ResultType,
                                                    ParamTypes.data(),
                                                    ParamTypes.size(),
                                                    T->isVariadic(),
                                                    T->getTypeQuals(),
+                                                   T->getRefQualifier(),
                                                    T->getExtInfo());
     if (Result.isNull())
       return QualType();
@@ -2940,6 +3803,7 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
   FunctionProtoTypeLoc NewTL = TLB.push<FunctionProtoTypeLoc>(Result);
   NewTL.setLParenLoc(TL.getLParenLoc());
   NewTL.setRParenLoc(TL.getRParenLoc());
+  NewTL.setTrailingReturn(TL.getTrailingReturn());
   for (unsigned i = 0, e = NewTL.getNumArgs(); i != e; ++i)
     NewTL.setArg(i, ParamDecls[i]);
 
@@ -2949,9 +3813,8 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
                                                  TypeLocBuilder &TLB,
-                                                 FunctionNoProtoTypeLoc TL,
-                                                 QualType ObjectType) {
-  FunctionNoProtoType *T = TL.getTypePtr();
+                                                 FunctionNoProtoTypeLoc TL) {
+  const FunctionNoProtoType *T = TL.getTypePtr();
   QualType ResultType = getDerived().TransformType(TLB, TL.getResultLoc());
   if (ResultType.isNull())
     return QualType();
@@ -2964,15 +3827,15 @@ QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
   FunctionNoProtoTypeLoc NewTL = TLB.push<FunctionNoProtoTypeLoc>(Result);
   NewTL.setLParenLoc(TL.getLParenLoc());
   NewTL.setRParenLoc(TL.getRParenLoc());
+  NewTL.setTrailingReturn(false);
 
   return Result;
 }
 
 template<typename Derived> QualType
 TreeTransform<Derived>::TransformUnresolvedUsingType(TypeLocBuilder &TLB,
-                                                 UnresolvedUsingTypeLoc TL,
-                                                     QualType ObjectType) {
-  UnresolvedUsingType *T = TL.getTypePtr();
+                                                 UnresolvedUsingTypeLoc TL) {
+  const UnresolvedUsingType *T = TL.getTypePtr();
   Decl *D = getDerived().TransformDecl(TL.getNameLoc(), T->getDecl());
   if (!D)
     return QualType();
@@ -2994,9 +3857,8 @@ TreeTransform<Derived>::TransformUnresolvedUsingType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypedefType(TypeLocBuilder &TLB,
-                                                      TypedefTypeLoc TL,
-                                                      QualType ObjectType) {
-  TypedefType *T = TL.getTypePtr();
+                                                      TypedefTypeLoc TL) {
+  const TypedefType *T = TL.getTypePtr();
   TypedefDecl *Typedef
     = cast_or_null<TypedefDecl>(getDerived().TransformDecl(TL.getNameLoc(),
                                                            T->getDecl()));
@@ -3019,8 +3881,7 @@ QualType TreeTransform<Derived>::TransformTypedefType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypeOfExprType(TypeLocBuilder &TLB,
-                                                      TypeOfExprTypeLoc TL,
-                                                       QualType ObjectType) {
+                                                      TypeOfExprTypeLoc TL) {
   // typeof expressions are not potentially evaluated contexts
   EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
@@ -3031,7 +3892,7 @@ QualType TreeTransform<Derived>::TransformTypeOfExprType(TypeLocBuilder &TLB,
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       E.get() != TL.getUnderlyingExpr()) {
-    Result = getDerived().RebuildTypeOfExprType(E.get());
+    Result = getDerived().RebuildTypeOfExprType(E.get(), TL.getTypeofLoc());
     if (Result.isNull())
       return QualType();
   }
@@ -3047,8 +3908,7 @@ QualType TreeTransform<Derived>::TransformTypeOfExprType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypeOfType(TypeLocBuilder &TLB,
-                                                     TypeOfTypeLoc TL,
-                                                     QualType ObjectType) {
+                                                     TypeOfTypeLoc TL) {
   TypeSourceInfo* Old_Under_TI = TL.getUnderlyingTInfo();
   TypeSourceInfo* New_Under_TI = getDerived().TransformType(Old_Under_TI);
   if (!New_Under_TI)
@@ -3072,9 +3932,8 @@ QualType TreeTransform<Derived>::TransformTypeOfType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformDecltypeType(TypeLocBuilder &TLB,
-                                                       DecltypeTypeLoc TL,
-                                                       QualType ObjectType) {
-  DecltypeType *T = TL.getTypePtr();
+                                                       DecltypeTypeLoc TL) {
+  const DecltypeType *T = TL.getTypePtr();
 
   // decltype expressions are not potentially evaluated contexts
   EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
@@ -3086,7 +3945,7 @@ QualType TreeTransform<Derived>::TransformDecltypeType(TypeLocBuilder &TLB,
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       E.get() != T->getUnderlyingExpr()) {
-    Result = getDerived().RebuildDecltypeType(E.get());
+    Result = getDerived().RebuildDecltypeType(E.get(), TL.getNameLoc());
     if (Result.isNull())
       return QualType();
   }
@@ -3099,10 +3958,34 @@ QualType TreeTransform<Derived>::TransformDecltypeType(TypeLocBuilder &TLB,
 }
 
 template<typename Derived>
+QualType TreeTransform<Derived>::TransformAutoType(TypeLocBuilder &TLB,
+                                                   AutoTypeLoc TL) {
+  const AutoType *T = TL.getTypePtr();
+  QualType OldDeduced = T->getDeducedType();
+  QualType NewDeduced;
+  if (!OldDeduced.isNull()) {
+    NewDeduced = getDerived().TransformType(OldDeduced);
+    if (NewDeduced.isNull())
+      return QualType();
+  }
+
+  QualType Result = TL.getType();
+  if (getDerived().AlwaysRebuild() || NewDeduced != OldDeduced) {
+    Result = getDerived().RebuildAutoType(NewDeduced);
+    if (Result.isNull())
+      return QualType();
+  }
+
+  AutoTypeLoc NewTL = TLB.push<AutoTypeLoc>(Result);
+  NewTL.setNameLoc(TL.getNameLoc());
+
+  return Result;
+}
+
+template<typename Derived>
 QualType TreeTransform<Derived>::TransformRecordType(TypeLocBuilder &TLB,
-                                                     RecordTypeLoc TL,
-                                                     QualType ObjectType) {
-  RecordType *T = TL.getTypePtr();
+                                                     RecordTypeLoc TL) {
+  const RecordType *T = TL.getTypePtr();
   RecordDecl *Record
     = cast_or_null<RecordDecl>(getDerived().TransformDecl(TL.getNameLoc(),
                                                           T->getDecl()));
@@ -3125,9 +4008,8 @@ QualType TreeTransform<Derived>::TransformRecordType(TypeLocBuilder &TLB,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformEnumType(TypeLocBuilder &TLB,
-                                                   EnumTypeLoc TL,
-                                                   QualType ObjectType) {
-  EnumType *T = TL.getTypePtr();
+                                                   EnumTypeLoc TL) {
+  const EnumType *T = TL.getTypePtr();
   EnumDecl *Enum
     = cast_or_null<EnumDecl>(getDerived().TransformDecl(TL.getNameLoc(),
                                                         T->getDecl()));
@@ -3151,8 +4033,7 @@ QualType TreeTransform<Derived>::TransformEnumType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformInjectedClassNameType(
                                          TypeLocBuilder &TLB,
-                                         InjectedClassNameTypeLoc TL,
-                                         QualType ObjectType) {
+                                         InjectedClassNameTypeLoc TL) {
   Decl *D = getDerived().TransformDecl(TL.getNameLoc(),
                                        TL.getTypePtr()->getDecl());
   if (!D) return QualType();
@@ -3162,73 +4043,122 @@ QualType TreeTransform<Derived>::TransformInjectedClassNameType(
   return T;
 }
 
-
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTemplateTypeParmType(
                                                 TypeLocBuilder &TLB,
-                                                TemplateTypeParmTypeLoc TL,
-                                                QualType ObjectType) {
+                                                TemplateTypeParmTypeLoc TL) {
   return TransformTypeSpecType(TLB, TL);
 }
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformSubstTemplateTypeParmType(
                                          TypeLocBuilder &TLB,
-                                         SubstTemplateTypeParmTypeLoc TL,
-                                         QualType ObjectType) {
+                                         SubstTemplateTypeParmTypeLoc TL) {
   return TransformTypeSpecType(TLB, TL);
 }
 
 template<typename Derived>
-QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
-                                      const TemplateSpecializationType *TST,
-                                                        QualType ObjectType) {
-  // FIXME: this entire method is a temporary workaround; callers
-  // should be rewritten to provide real type locs.
-
-  // Fake up a TemplateSpecializationTypeLoc.
-  TypeLocBuilder TLB;
-  TemplateSpecializationTypeLoc TL
-    = TLB.push<TemplateSpecializationTypeLoc>(QualType(TST, 0));
-
-  SourceLocation BaseLoc = getDerived().getBaseLocation();
-
-  TL.setTemplateNameLoc(BaseLoc);
-  TL.setLAngleLoc(BaseLoc);
-  TL.setRAngleLoc(BaseLoc);
-  for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
-    const TemplateArgument &TA = TST->getArg(i);
-    TemplateArgumentLoc TAL;
-    getDerived().InventTemplateArgumentLoc(TA, TAL);
-    TL.setArgLocInfo(i, TAL.getLocInfo());
-  }
-
-  TypeLocBuilder IgnoredTLB;
-  return TransformTemplateSpecializationType(IgnoredTLB, TL, ObjectType);
+QualType TreeTransform<Derived>::TransformSubstTemplateTypeParmPackType(
+                                          TypeLocBuilder &TLB,
+                                          SubstTemplateTypeParmPackTypeLoc TL) {
+  return TransformTypeSpecType(TLB, TL);
 }
-  
+
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
                                                         TypeLocBuilder &TLB,
-                                           TemplateSpecializationTypeLoc TL,
-                                                        QualType ObjectType) {
+                                           TemplateSpecializationTypeLoc TL) {
   const TemplateSpecializationType *T = TL.getTypePtr();
 
   TemplateName Template
-    = getDerived().TransformTemplateName(T->getTemplateName(), ObjectType);
+    = getDerived().TransformTemplateName(T->getTemplateName());
   if (Template.isNull())
     return QualType();
 
+  return getDerived().TransformTemplateSpecializationType(TLB, TL, Template);
+}
+
+namespace {
+  /// \brief Simple iterator that traverses the template arguments in a 
+  /// container that provides a \c getArgLoc() member function.
+  ///
+  /// This iterator is intended to be used with the iterator form of
+  /// \c TreeTransform<Derived>::TransformTemplateArguments().
+  template<typename ArgLocContainer>
+  class TemplateArgumentLocContainerIterator {
+    ArgLocContainer *Container;
+    unsigned Index;
+    
+  public:
+    typedef TemplateArgumentLoc value_type;
+    typedef TemplateArgumentLoc reference;
+    typedef int difference_type;
+    typedef std::input_iterator_tag iterator_category;
+    
+    class pointer {
+      TemplateArgumentLoc Arg;
+      
+    public:
+      explicit pointer(TemplateArgumentLoc Arg) : Arg(Arg) { }
+      
+      const TemplateArgumentLoc *operator->() const {
+        return &Arg;
+      }
+    };
+    
+    
+    TemplateArgumentLocContainerIterator() {}
+    
+    TemplateArgumentLocContainerIterator(ArgLocContainer &Container,
+                                 unsigned Index)
+      : Container(&Container), Index(Index) { }
+    
+    TemplateArgumentLocContainerIterator &operator++() {
+      ++Index;
+      return *this;
+    }
+    
+    TemplateArgumentLocContainerIterator operator++(int) {
+      TemplateArgumentLocContainerIterator Old(*this);
+      ++(*this);
+      return Old;
+    }
+    
+    TemplateArgumentLoc operator*() const {
+      return Container->getArgLoc(Index);
+    }
+    
+    pointer operator->() const {
+      return pointer(Container->getArgLoc(Index));
+    }
+    
+    friend bool operator==(const TemplateArgumentLocContainerIterator &X,
+                           const TemplateArgumentLocContainerIterator &Y) {
+      return X.Container == Y.Container && X.Index == Y.Index;
+    }
+    
+    friend bool operator!=(const TemplateArgumentLocContainerIterator &X,
+                           const TemplateArgumentLocContainerIterator &Y) {
+      return !(X == Y);
+    }
+  };
+}
+  
+  
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
+                                                        TypeLocBuilder &TLB,
+                                           TemplateSpecializationTypeLoc TL,
+                                                      TemplateName Template) {
   TemplateArgumentListInfo NewTemplateArgs;
   NewTemplateArgs.setLAngleLoc(TL.getLAngleLoc());
   NewTemplateArgs.setRAngleLoc(TL.getRAngleLoc());
-
-  for (unsigned i = 0, e = T->getNumArgs(); i != e; ++i) {
-    TemplateArgumentLoc Loc;
-    if (getDerived().TransformTemplateArgument(TL.getArgLoc(i), Loc))
-      return QualType();
-    NewTemplateArgs.addArgument(Loc);
-  }
+  typedef TemplateArgumentLocContainerIterator<TemplateSpecializationTypeLoc>
+    ArgIterator;
+  if (getDerived().TransformTemplateArguments(ArgIterator(TL, 0), 
+                                              ArgIterator(TL, TL.getNumArgs()),
+                                              NewTemplateArgs))
+    return QualType();
 
   // FIXME: maybe don't rebuild if all the template arguments are the same.
 
@@ -3253,46 +4183,28 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformElaboratedType(TypeLocBuilder &TLB,
-                                                ElaboratedTypeLoc TL,
-                                                QualType ObjectType) {
-  ElaboratedType *T = TL.getTypePtr();
+                                                ElaboratedTypeLoc TL) {
+  const ElaboratedType *T = TL.getTypePtr();
 
   NestedNameSpecifier *NNS = 0;
   // NOTE: the qualifier in an ElaboratedType is optional.
   if (T->getQualifier() != 0) {
     NNS = getDerived().TransformNestedNameSpecifier(T->getQualifier(),
-                                                    TL.getQualifierRange(),
-                                                    ObjectType);
+                                                    TL.getQualifierRange());
     if (!NNS)
       return QualType();
   }
 
-  QualType NamedT;
-  // FIXME: this test is meant to workaround a problem (failing assertion)
-  // occurring if directly executing the code in the else branch.
-  if (isa<TemplateSpecializationTypeLoc>(TL.getNamedTypeLoc())) {
-    TemplateSpecializationTypeLoc OldNamedTL
-      = cast<TemplateSpecializationTypeLoc>(TL.getNamedTypeLoc());
-    const TemplateSpecializationType* OldTST
-      = OldNamedTL.getType()->template getAs<TemplateSpecializationType>();
-    NamedT = TransformTemplateSpecializationType(OldTST, ObjectType);
-    if (NamedT.isNull())
-      return QualType();
-    TemplateSpecializationTypeLoc NewNamedTL
-      = TLB.push<TemplateSpecializationTypeLoc>(NamedT);
-    NewNamedTL.copy(OldNamedTL);
-  }
-  else {
-    NamedT = getDerived().TransformType(TLB, TL.getNamedTypeLoc());
-    if (NamedT.isNull())
-      return QualType();
-  }
+  QualType NamedT = getDerived().TransformType(TLB, TL.getNamedTypeLoc());
+  if (NamedT.isNull())
+    return QualType();
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       NNS != T->getQualifier() ||
       NamedT != T->getNamedType()) {
-    Result = getDerived().RebuildElaboratedType(T->getKeyword(), NNS, NamedT);
+    Result = getDerived().RebuildElaboratedType(TL.getKeywordLoc(),
+                                                T->getKeyword(), NNS, NamedT);
     if (Result.isNull())
       return QualType();
   }
@@ -3305,15 +4217,72 @@ TreeTransform<Derived>::TransformElaboratedType(TypeLocBuilder &TLB,
 }
 
 template<typename Derived>
+QualType TreeTransform<Derived>::TransformAttributedType(
+                                                TypeLocBuilder &TLB,
+                                                AttributedTypeLoc TL) {
+  const AttributedType *oldType = TL.getTypePtr();
+  QualType modifiedType = getDerived().TransformType(TLB, TL.getModifiedLoc());
+  if (modifiedType.isNull())
+    return QualType();
+
+  QualType result = TL.getType();
+
+  // FIXME: dependent operand expressions?
+  if (getDerived().AlwaysRebuild() ||
+      modifiedType != oldType->getModifiedType()) {
+    // TODO: this is really lame; we should really be rebuilding the
+    // equivalent type from first principles.
+    QualType equivalentType
+      = getDerived().TransformType(oldType->getEquivalentType());
+    if (equivalentType.isNull())
+      return QualType();
+    result = SemaRef.Context.getAttributedType(oldType->getAttrKind(),
+                                               modifiedType,
+                                               equivalentType);
+  }
+
+  AttributedTypeLoc newTL = TLB.push<AttributedTypeLoc>(result);
+  newTL.setAttrNameLoc(TL.getAttrNameLoc());
+  if (TL.hasAttrOperand())
+    newTL.setAttrOperandParensRange(TL.getAttrOperandParensRange());
+  if (TL.hasAttrExprOperand())
+    newTL.setAttrExprOperand(TL.getAttrExprOperand());
+  else if (TL.hasAttrEnumOperand())
+    newTL.setAttrEnumOperandLoc(TL.getAttrEnumOperandLoc());
+
+  return result;
+}
+
+template<typename Derived>
+QualType
+TreeTransform<Derived>::TransformParenType(TypeLocBuilder &TLB,
+                                           ParenTypeLoc TL) {
+  QualType Inner = getDerived().TransformType(TLB, TL.getInnerLoc());
+  if (Inner.isNull())
+    return QualType();
+
+  QualType Result = TL.getType();
+  if (getDerived().AlwaysRebuild() ||
+      Inner != TL.getInnerLoc().getType()) {
+    Result = getDerived().RebuildParenType(Inner);
+    if (Result.isNull())
+      return QualType();
+  }
+
+  ParenTypeLoc NewTL = TLB.push<ParenTypeLoc>(Result);
+  NewTL.setLParenLoc(TL.getLParenLoc());
+  NewTL.setRParenLoc(TL.getRParenLoc());
+  return Result;
+}
+
+template<typename Derived>
 QualType TreeTransform<Derived>::TransformDependentNameType(TypeLocBuilder &TLB,
-                                                       DependentNameTypeLoc TL,
-                                                       QualType ObjectType) {
-  DependentNameType *T = TL.getTypePtr();
+                                                      DependentNameTypeLoc TL) {
+  const DependentNameType *T = TL.getTypePtr();
 
   NestedNameSpecifier *NNS
     = getDerived().TransformNestedNameSpecifier(T->getQualifier(),
-                                                TL.getQualifierRange(),
-                                                ObjectType);
+                                                TL.getQualifierRange());
   if (!NNS)
     return QualType();
 
@@ -3345,34 +4314,44 @@ QualType TreeTransform<Derived>::TransformDependentNameType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType TreeTransform<Derived>::
           TransformDependentTemplateSpecializationType(TypeLocBuilder &TLB,
-                                 DependentTemplateSpecializationTypeLoc TL,
-                                                       QualType ObjectType) {
-  DependentTemplateSpecializationType *T = TL.getTypePtr();
+                                 DependentTemplateSpecializationTypeLoc TL) {
+  const DependentTemplateSpecializationType *T = TL.getTypePtr();
 
   NestedNameSpecifier *NNS
     = getDerived().TransformNestedNameSpecifier(T->getQualifier(),
-                                                TL.getQualifierRange(),
-                                                ObjectType);
+                                                TL.getQualifierRange());
   if (!NNS)
     return QualType();
 
+  return getDerived()
+           .TransformDependentTemplateSpecializationType(TLB, TL, NNS);
+}
+
+template<typename Derived>
+QualType TreeTransform<Derived>::
+          TransformDependentTemplateSpecializationType(TypeLocBuilder &TLB,
+                                 DependentTemplateSpecializationTypeLoc TL,
+                                                  NestedNameSpecifier *NNS) {
+  const DependentTemplateSpecializationType *T = TL.getTypePtr();
+
   TemplateArgumentListInfo NewTemplateArgs;
   NewTemplateArgs.setLAngleLoc(TL.getLAngleLoc());
   NewTemplateArgs.setRAngleLoc(TL.getRAngleLoc());
+            
+  typedef TemplateArgumentLocContainerIterator<
+                            DependentTemplateSpecializationTypeLoc> ArgIterator;
+  if (getDerived().TransformTemplateArguments(ArgIterator(TL, 0),
+                                              ArgIterator(TL, TL.getNumArgs()),
+                                              NewTemplateArgs))
+    return QualType();
 
-  for (unsigned I = 0, E = T->getNumArgs(); I != E; ++I) {
-    TemplateArgumentLoc Loc;
-    if (getDerived().TransformTemplateArgument(TL.getArgLoc(I), Loc))
-      return QualType();
-    NewTemplateArgs.addArgument(Loc);
-  }
-
-  QualType Result = getDerived().RebuildDependentTemplateSpecializationType(
-                                                     T->getKeyword(),
-                                                     NNS,
-                                                     T->getIdentifier(),
-                                                     TL.getNameLoc(),
-                                                     NewTemplateArgs);
+  QualType Result
+    = getDerived().RebuildDependentTemplateSpecializationType(T->getKeyword(),
+                                                              NNS,
+                                                        TL.getQualifierRange(),
+                                                            T->getIdentifier(),
+                                                              TL.getNameLoc(),
+                                                              NewTemplateArgs);
   if (Result.isNull())
     return QualType();
 
@@ -3399,10 +4378,33 @@ QualType TreeTransform<Derived>::
 }
 
 template<typename Derived>
+QualType TreeTransform<Derived>::TransformPackExpansionType(TypeLocBuilder &TLB,
+                                                      PackExpansionTypeLoc TL) {
+  QualType Pattern                                      
+    = getDerived().TransformType(TLB, TL.getPatternLoc());  
+  if (Pattern.isNull())
+    return QualType();
+  
+  QualType Result = TL.getType();  
+  if (getDerived().AlwaysRebuild() ||
+      Pattern != TL.getPatternLoc().getType()) {
+    Result = getDerived().RebuildPackExpansionType(Pattern, 
+                                           TL.getPatternLoc().getSourceRange(),
+                                                   TL.getEllipsisLoc(),
+                                           TL.getTypePtr()->getNumExpansions());
+    if (Result.isNull())
+      return QualType();
+  }
+  
+  PackExpansionTypeLoc NewT = TLB.push<PackExpansionTypeLoc>(Result);
+  NewT.setEllipsisLoc(TL.getEllipsisLoc());
+  return Result;
+}
+
+template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformObjCInterfaceType(TypeLocBuilder &TLB,
-                                                   ObjCInterfaceTypeLoc TL,
-                                                   QualType ObjectType) {
+                                                   ObjCInterfaceTypeLoc TL) {
   // ObjCInterfaceType is never dependent.
   TLB.pushFullCopy(TL);
   return TL.getType();
@@ -3411,8 +4413,7 @@ TreeTransform<Derived>::TransformObjCInterfaceType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformObjCObjectType(TypeLocBuilder &TLB,
-                                                ObjCObjectTypeLoc TL,
-                                                QualType ObjectType) {
+                                                ObjCObjectTypeLoc TL) {
   // ObjCObjectType is never dependent.
   TLB.pushFullCopy(TL);
   return TL.getType();
@@ -3421,8 +4422,7 @@ TreeTransform<Derived>::TransformObjCObjectType(TypeLocBuilder &TLB,
 template<typename Derived>
 QualType
 TreeTransform<Derived>::TransformObjCObjectPointerType(TypeLocBuilder &TLB,
-                                               ObjCObjectPointerTypeLoc TL,
-                                                       QualType ObjectType) {
+                                               ObjCObjectPointerTypeLoc TL) {
   // ObjCObjectPointerType is never dependent.
   TLB.pushFullCopy(TL);
   return TL.getType();
@@ -3434,7 +4434,7 @@ TreeTransform<Derived>::TransformObjCObjectPointerType(TypeLocBuilder &TLB,
 template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformNullStmt(NullStmt *S) {
-  return SemaRef.Owned(S->Retain());
+  return SemaRef.Owned(S);
 }
 
 template<typename Derived>
@@ -3473,7 +4473,7 @@ TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S,
 
   if (!getDerived().AlwaysRebuild() &&
       !SubStmtChanged)
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildCompoundStmt(S->getLBracLoc(),
                                           move_arg(Statements),
@@ -3540,9 +4540,15 @@ TreeTransform<Derived>::TransformLabelStmt(LabelStmt *S) {
   if (SubStmt.isInvalid())
     return StmtError();
 
+  Decl *LD = getDerived().TransformDecl(S->getDecl()->getLocation(),
+                                        S->getDecl());
+  if (!LD)
+    return StmtError();
+  
+  
   // FIXME: Pass the real colon location in.
-  SourceLocation ColonLoc = SemaRef.PP.getLocForEndOfToken(S->getIdentLoc());
-  return getDerived().RebuildLabelStmt(S->getIdentLoc(), S->getID(), ColonLoc,
+  return getDerived().RebuildLabelStmt(S->getIdentLoc(),
+                                       cast<LabelDecl>(LD), SourceLocation(),
                                        SubStmt.get());
 }
 
@@ -3568,9 +4574,8 @@ TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
     
     // Convert the condition to a boolean value.
     if (S->getCond()) {
-      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
-                                                               S->getIfLoc(), 
-                                                               Cond.get());
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, S->getIfLoc(), 
+                                                         Cond.get());
       if (CondE.isInvalid())
         return StmtError();
     
@@ -3597,7 +4602,7 @@ TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
       ConditionVar == S->getConditionVariable() &&
       Then.get() == S->getThen() &&
       Else.get() == S->getElse())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildIfStmt(S->getIfLoc(), FullCond, ConditionVar,
                                     Then.get(),
@@ -3664,9 +4669,8 @@ TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
 
     if (S->getCond()) {
       // Convert the condition to a boolean value.
-      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
-                                                             S->getWhileLoc(), 
-                                                               Cond.get());
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, S->getWhileLoc(), 
+                                                         Cond.get());
       if (CondE.isInvalid())
         return StmtError();
       Cond = CondE;
@@ -3708,7 +4712,7 @@ TreeTransform<Derived>::TransformDoStmt(DoStmt *S) {
   if (!getDerived().AlwaysRebuild() &&
       Cond.get() == S->getCond() &&
       Body.get() == S->getBody())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildDoStmt(S->getDoLoc(), Body.get(), S->getWhileLoc(),
                                     /*FIXME:*/S->getWhileLoc(), Cond.get(),
@@ -3742,9 +4746,8 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
 
     if (S->getCond()) {
       // Convert the condition to a boolean value.
-      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
-                                                               S->getForLoc(), 
-                                                               Cond.get());
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, S->getForLoc(), 
+                                                         Cond.get());
       if (CondE.isInvalid())
         return StmtError();
 
@@ -3775,7 +4778,7 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
       FullCond.get() == S->getCond() &&
       Inc.get() == S->getInc() &&
       Body.get() == S->getBody())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildForStmt(S->getForLoc(), S->getLParenLoc(),
                                      Init.get(), FullCond, ConditionVar,
@@ -3785,9 +4788,14 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
 template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformGotoStmt(GotoStmt *S) {
+  Decl *LD = getDerived().TransformDecl(S->getLabel()->getLocation(),
+                                        S->getLabel());
+  if (!LD)
+    return StmtError();
+  
   // Goto statements must always be rebuilt, to resolve the label.
   return getDerived().RebuildGotoStmt(S->getGotoLoc(), S->getLabelLoc(),
-                                      S->getLabel());
+                                      cast<LabelDecl>(LD));
 }
 
 template<typename Derived>
@@ -3799,7 +4807,7 @@ TreeTransform<Derived>::TransformIndirectGotoStmt(IndirectGotoStmt *S) {
 
   if (!getDerived().AlwaysRebuild() &&
       Target.get() == S->getTarget())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildIndirectGotoStmt(S->getGotoLoc(), S->getStarLoc(),
                                               Target.get());
@@ -3808,13 +4816,13 @@ TreeTransform<Derived>::TransformIndirectGotoStmt(IndirectGotoStmt *S) {
 template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformContinueStmt(ContinueStmt *S) {
-  return SemaRef.Owned(S->Retain());
+  return SemaRef.Owned(S);
 }
 
 template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformBreakStmt(BreakStmt *S) {
-  return SemaRef.Owned(S->Retain());
+  return SemaRef.Owned(S);
 }
 
 template<typename Derived>
@@ -3848,7 +4856,7 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
   }
 
   if (!getDerived().AlwaysRebuild() && !DeclChanged)
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildDeclStmt(Decls.data(), Decls.size(),
                                       S->getStartLoc(), S->getEndLoc());
@@ -3856,13 +4864,6 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
 
 template<typename Derived>
 StmtResult
-TreeTransform<Derived>::TransformSwitchCase(SwitchCase *S) {
-  assert(false && "SwitchCase is abstract and cannot be transformed");
-  return SemaRef.Owned(S->Retain());
-}
-
-template<typename Derived>
-StmtResult
 TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
   
   ASTOwningVector<Expr*> Constraints(getSema());
@@ -3879,7 +4880,7 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
     Names.push_back(S->getOutputIdentifier(I));
     
     // No need to transform the constraint literal.
-    Constraints.push_back(S->getOutputConstraintLiteral(I)->Retain());
+    Constraints.push_back(S->getOutputConstraintLiteral(I));
     
     // Transform the output expr.
     Expr *OutputExpr = S->getOutputExpr(I);
@@ -3897,7 +4898,7 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
     Names.push_back(S->getInputIdentifier(I));
     
     // No need to transform the constraint literal.
-    Constraints.push_back(S->getInputConstraintLiteral(I)->Retain());
+    Constraints.push_back(S->getInputConstraintLiteral(I));
     
     // Transform the input expr.
     Expr *InputExpr = S->getInputExpr(I);
@@ -3911,11 +4912,11 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
   }
   
   if (!getDerived().AlwaysRebuild() && !ExprsChanged)
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   // Go through the clobbers.
   for (unsigned I = 0, E = S->getNumClobbers(); I != E; ++I)
-    Clobbers.push_back(S->getClobber(I)->Retain());
+    Clobbers.push_back(S->getClobber(I));
 
   // No need to transform the asm string literal.
   AsmString = SemaRef.Owned(S->getAsmString());
@@ -3968,7 +4969,7 @@ TreeTransform<Derived>::TransformObjCAtTryStmt(ObjCAtTryStmt *S) {
       TryBody.get() == S->getTryBody() &&
       !AnyCatchChanged &&
       Finally.get() == S->getFinallyStmt())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
   
   // Build a new statement.
   return getDerived().RebuildObjCAtTryStmt(S->getAtTryLoc(), TryBody.get(),
@@ -4022,7 +5023,7 @@ TreeTransform<Derived>::TransformObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
   // If nothing changed, just retain this statement.
   if (!getDerived().AlwaysRebuild() &&
       Body.get() == S->getFinallyBody())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   // Build a new statement.
   return getDerived().RebuildObjCAtFinallyStmt(S->getAtFinallyLoc(),
@@ -4041,7 +5042,7 @@ TreeTransform<Derived>::TransformObjCAtThrowStmt(ObjCAtThrowStmt *S) {
   
   if (!getDerived().AlwaysRebuild() &&
       Operand.get() == S->getThrowExpr())
-    return getSema().Owned(S->Retain());
+    return getSema().Owned(S);
     
   return getDerived().RebuildObjCAtThrowStmt(S->getThrowLoc(), Operand.get());
 }
@@ -4064,7 +5065,7 @@ TreeTransform<Derived>::TransformObjCAtSynchronizedStmt(
   if (!getDerived().AlwaysRebuild() &&
       Object.get() == S->getSynchExpr() &&
       Body.get() == S->getSynchBody())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   // Build a new statement.
   return getDerived().RebuildObjCAtSynchronizedStmt(S->getAtSynchronizedLoc(),
@@ -4095,7 +5096,7 @@ TreeTransform<Derived>::TransformObjCForCollectionStmt(
       Element.get() == S->getElement() &&
       Collection.get() == S->getCollection() &&
       Body.get() == S->getBody())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
   
   // Build a new statement.
   return getDerived().RebuildObjCForCollectionStmt(S->getForLoc(),
@@ -4114,20 +5115,14 @@ TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
   VarDecl *Var = 0;
   if (S->getExceptionDecl()) {
     VarDecl *ExceptionDecl = S->getExceptionDecl();
-    TemporaryBase Rebase(*this, ExceptionDecl->getLocation(),
-                         ExceptionDecl->getDeclName());
-
-    QualType T = getDerived().TransformType(ExceptionDecl->getType());
-    if (T.isNull())
+    TypeSourceInfo *T = getDerived().TransformType(
+                                            ExceptionDecl->getTypeSourceInfo());
+    if (!T)
       return StmtError();
 
-    Var = getDerived().RebuildExceptionDecl(ExceptionDecl,
-                                            T,
-                                            ExceptionDecl->getTypeSourceInfo(),
+    Var = getDerived().RebuildExceptionDecl(ExceptionDecl, T,
                                             ExceptionDecl->getIdentifier(),
-                                            ExceptionDecl->getLocation(),
-                                            /*FIXME: Inaccurate*/
-                                    SourceRange(ExceptionDecl->getLocation()));
+                                            ExceptionDecl->getLocation());
     if (!Var || Var->isInvalidDecl())
       return StmtError();
   }
@@ -4140,7 +5135,7 @@ TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
   if (!getDerived().AlwaysRebuild() &&
       !Var &&
       Handler.get() == S->getHandlerBlock())
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildCXXCatchStmt(S->getCatchLoc(),
                                           Var,
@@ -4172,7 +5167,7 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
   if (!getDerived().AlwaysRebuild() &&
       TryBlock.get() == S->getTryBlock() &&
       !HandlerChanged)
-    return SemaRef.Owned(S->Retain());
+    return SemaRef.Owned(S);
 
   return getDerived().RebuildCXXTryStmt(S->getTryLoc(), TryBlock.get(),
                                         move_arg(Handlers));
@@ -4184,7 +5179,7 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformPredefinedExpr(PredefinedExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
@@ -4221,7 +5216,7 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
     // FIXME: this is a bit instantiation-specific.
     SemaRef.MarkDeclarationReferenced(E->getLocation(), ND);
 
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   }
 
   TemplateArgumentListInfo TransArgs, *TemplateArgs = 0;
@@ -4229,12 +5224,10 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
     TemplateArgs = &TransArgs;
     TransArgs.setLAngleLoc(E->getLAngleLoc());
     TransArgs.setRAngleLoc(E->getRAngleLoc());
-    for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
-      TemplateArgumentLoc Loc;
-      if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-        return ExprError();
-      TransArgs.addArgument(Loc);
-    }
+    if (getDerived().TransformTemplateArguments(E->getTemplateArgs(),
+                                                E->getNumTemplateArgs(),
+                                                TransArgs))
+      return ExprError();
   }
 
   return getDerived().RebuildDeclRefExpr(Qualifier, E->getQualifierRange(),
@@ -4244,31 +5237,31 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformIntegerLiteral(IntegerLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformFloatingLiteral(FloatingLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformImaginaryLiteral(ImaginaryLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformStringLiteral(StringLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCharacterLiteral(CharacterLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
@@ -4279,7 +5272,7 @@ TreeTransform<Derived>::TransformParenExpr(ParenExpr *E) {
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildParenExpr(SubExpr.get(), E->getLParen(),
                                        E->getRParen());
@@ -4293,7 +5286,7 @@ TreeTransform<Derived>::TransformUnaryOperator(UnaryOperator *E) {
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildUnaryOperator(E->getOperatorLoc(),
                                            E->getOpcode(),
@@ -4358,7 +5351,7 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       Type == E->getTypeSourceInfo() &&
       !ExprChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   
   // Build a new offsetof expression.
   return getDerived().RebuildOffsetOfExpr(E->getOperatorLoc(), Type,
@@ -4368,6 +5361,14 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
 
 template<typename Derived>
 ExprResult
+TreeTransform<Derived>::TransformOpaqueValueExpr(OpaqueValueExpr *E) {
+  assert(getDerived().AlreadyTransformed(E->getType()) &&
+         "opaque value expression requires transformation");
+  return SemaRef.Owned(E);
+}
+
+template<typename Derived>
+ExprResult
 TreeTransform<Derived>::TransformSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   if (E->isArgumentType()) {
     TypeSourceInfo *OldT = E->getArgumentTypeInfo();
@@ -4377,7 +5378,7 @@ TreeTransform<Derived>::TransformSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
       return ExprError();
 
     if (!getDerived().AlwaysRebuild() && OldT == NewT)
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
 
     return getDerived().RebuildSizeOfAlignOf(NewT, E->getOperatorLoc(),
                                              E->isSizeOf(),
@@ -4396,7 +5397,7 @@ TreeTransform<Derived>::TransformSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
       return ExprError();
 
     if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getArgumentExpr())
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
   }
 
   return getDerived().RebuildSizeOfAlignOf(SubExpr.get(), E->getOperatorLoc(),
@@ -4419,7 +5420,7 @@ TreeTransform<Derived>::TransformArraySubscriptExpr(ArraySubscriptExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       LHS.get() == E->getLHS() &&
       RHS.get() == E->getRHS())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildArraySubscriptExpr(LHS.get(),
                                            /*FIXME:*/E->getLHS()->getLocStart(),
@@ -4438,31 +5439,20 @@ TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
   // Transform arguments.
   bool ArgChanged = false;
   ASTOwningVector<Expr*> Args(SemaRef);
-  llvm::SmallVector<SourceLocation, 4> FakeCommaLocs;
-  for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I) {
-    ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
-    if (Arg.isInvalid())
-      return ExprError();
-
-    // FIXME: Wrong source location information for the ','.
-    FakeCommaLocs.push_back(
-       SemaRef.PP.getLocForEndOfToken(E->getArg(I)->getSourceRange().getEnd()));
-
-    ArgChanged = ArgChanged || Arg.get() != E->getArg(I);
-    Args.push_back(Arg.get());
-  }
-
+  if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args, 
+                                  &ArgChanged))
+    return ExprError();
+  
   if (!getDerived().AlwaysRebuild() &&
       Callee.get() == E->getCallee() &&
       !ArgChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   // FIXME: Wrong source location information for the '('.
   SourceLocation FakeLParenLoc
     = ((Expr *)Callee.get())->getSourceRange().getBegin();
   return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
                                       move_arg(Args),
-                                      FakeCommaLocs.data(),
                                       E->getRParenLoc());
 }
 
@@ -4508,19 +5498,17 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
     // Mark it referenced in the new context regardless.
     // FIXME: this is a bit instantiation-specific.
     SemaRef.MarkDeclarationReferenced(E->getMemberLoc(), Member);
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   }
 
   TemplateArgumentListInfo TransArgs;
   if (E->hasExplicitTemplateArgs()) {
     TransArgs.setLAngleLoc(E->getLAngleLoc());
     TransArgs.setRAngleLoc(E->getRAngleLoc());
-    for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
-      TemplateArgumentLoc Loc;
-      if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-        return ExprError();
-      TransArgs.addArgument(Loc);
-    }
+    if (getDerived().TransformTemplateArguments(E->getTemplateArgs(),
+                                                E->getNumTemplateArgs(),
+                                                TransArgs))
+      return ExprError();
   }
   
   // FIXME: Bogus source location for the operator
@@ -4559,7 +5547,7 @@ TreeTransform<Derived>::TransformBinaryOperator(BinaryOperator *E) {
   if (!getDerived().AlwaysRebuild() &&
       LHS.get() == E->getLHS() &&
       RHS.get() == E->getRHS())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildBinaryOperator(E->getOperatorLoc(), E->getOpcode(),
                                             LHS.get(), RHS.get());
@@ -4573,6 +5561,32 @@ TreeTransform<Derived>::TransformCompoundAssignOperator(
 }
 
 template<typename Derived>
+ExprResult TreeTransform<Derived>::
+TransformBinaryConditionalOperator(BinaryConditionalOperator *e) {
+  // Just rebuild the common and RHS expressions and see whether we
+  // get any changes.
+
+  ExprResult commonExpr = getDerived().TransformExpr(e->getCommon());
+  if (commonExpr.isInvalid())
+    return ExprError();
+
+  ExprResult rhs = getDerived().TransformExpr(e->getFalseExpr());
+  if (rhs.isInvalid())
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() &&
+      commonExpr.get() == e->getCommon() &&
+      rhs.get() == e->getFalseExpr())
+    return SemaRef.Owned(e);
+
+  return getDerived().RebuildConditionalOperator(commonExpr.take(),
+                                                 e->getQuestionLoc(),
+                                                 0,
+                                                 e->getColonLoc(),
+                                                 rhs.get());
+}
+
+template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformConditionalOperator(ConditionalOperator *E) {
   ExprResult Cond = getDerived().TransformExpr(E->getCond());
@@ -4591,7 +5605,7 @@ TreeTransform<Derived>::TransformConditionalOperator(ConditionalOperator *E) {
       Cond.get() == E->getCond() &&
       LHS.get() == E->getLHS() &&
       RHS.get() == E->getRHS())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildConditionalOperator(Cond.get(),
                                                  E->getQuestionLoc(),
@@ -4611,32 +5625,22 @@ TreeTransform<Derived>::TransformImplicitCastExpr(ImplicitCastExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCStyleCastExpr(CStyleCastExpr *E) {
-  TypeSourceInfo *OldT;
-  TypeSourceInfo *NewT;
-  {
-    // FIXME: Source location isn't quite accurate.
-    SourceLocation TypeStartLoc
-      = SemaRef.PP.getLocForEndOfToken(E->getLParenLoc());
-    TemporaryBase Rebase(*this, TypeStartLoc, DeclarationName());
-
-    OldT = E->getTypeInfoAsWritten();
-    NewT = getDerived().TransformType(OldT);
-    if (!NewT)
-      return ExprError();
-  }
-
+  TypeSourceInfo *Type = getDerived().TransformType(E->getTypeInfoAsWritten());
+  if (!Type)
+    return ExprError();
+  
   ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
   if (SubExpr.isInvalid())
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      OldT == NewT &&
+      Type == E->getTypeInfoAsWritten() &&
       SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCStyleCastExpr(E->getLParenLoc(),
-                                            NewT,
+                                            Type,
                                             E->getRParenLoc(),
                                             SubExpr.get());
 }
@@ -4656,7 +5660,7 @@ TreeTransform<Derived>::TransformCompoundLiteralExpr(CompoundLiteralExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       OldT == NewT &&
       Init.get() == E->getInitializer())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   // Note: the expression type doesn't necessarily match the
   // type-as-written, but that's okay, because it should always be
@@ -4676,7 +5680,7 @@ TreeTransform<Derived>::TransformExtVectorElementExpr(ExtVectorElementExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   // FIXME: Bad source location
   SourceLocation FakeOperatorLoc
@@ -4692,17 +5696,12 @@ TreeTransform<Derived>::TransformInitListExpr(InitListExpr *E) {
   bool InitChanged = false;
 
   ASTOwningVector<Expr*, 4> Inits(SemaRef);
-  for (unsigned I = 0, N = E->getNumInits(); I != N; ++I) {
-    ExprResult Init = getDerived().TransformExpr(E->getInit(I));
-    if (Init.isInvalid())
-      return ExprError();
-
-    InitChanged = InitChanged || Init.get() != E->getInit(I);
-    Inits.push_back(Init.get());
-  }
-
+  if (getDerived().TransformExprs(E->getInits(), E->getNumInits(), false, 
+                                  Inits, &InitChanged))
+    return ExprError();
+  
   if (!getDerived().AlwaysRebuild() && !InitChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildInitList(E->getLBraceLoc(), move_arg(Inits),
                                       E->getRBraceLoc(), E->getType());
@@ -4769,7 +5768,7 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       Init.get() == E->getInit() &&
       !ExprChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildDesignatedInitExpr(Desig, move_arg(ArrayExprs),
                                                 E->getEqualOrColonLoc(),
@@ -4790,7 +5789,7 @@ TreeTransform<Derived>::TransformImplicitValueInitExpr(
 
   if (!getDerived().AlwaysRebuild() &&
       T == E->getType())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildImplicitValueInitExpr(T);
 }
@@ -4809,7 +5808,7 @@ TreeTransform<Derived>::TransformVAArgExpr(VAArgExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       TInfo == E->getWrittenTypeInfo() &&
       SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildVAArgExpr(E->getBuiltinLoc(), SubExpr.get(),
                                        TInfo, E->getRParenLoc());
@@ -4820,15 +5819,10 @@ ExprResult
 TreeTransform<Derived>::TransformParenListExpr(ParenListExpr *E) {
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*, 4> Inits(SemaRef);
-  for (unsigned I = 0, N = E->getNumExprs(); I != N; ++I) {
-    ExprResult Init = getDerived().TransformExpr(E->getExpr(I));
-    if (Init.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || Init.get() != E->getExpr(I);
-    Inits.push_back(Init.get());
-  }
-
+  if (TransformExprs(E->getExprs(), E->getNumExprs(), true, Inits,
+                     &ArgumentChanged))
+    return ExprError();
+  
   return getDerived().RebuildParenListExpr(E->getLParenLoc(),
                                            move_arg(Inits),
                                            E->getRParenLoc());
@@ -4842,8 +5836,13 @@ TreeTransform<Derived>::TransformParenListExpr(ParenListExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAddrLabelExpr(AddrLabelExpr *E) {
+  Decl *LD = getDerived().TransformDecl(E->getLabel()->getLocation(),
+                                        E->getLabel());
+  if (!LD)
+    return ExprError();
+  
   return getDerived().RebuildAddrLabelExpr(E->getAmpAmpLoc(), E->getLabelLoc(),
-                                           E->getLabel());
+                                           cast<LabelDecl>(LD));
 }
 
 template<typename Derived>
@@ -4856,7 +5855,7 @@ TreeTransform<Derived>::TransformStmtExpr(StmtExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       SubStmt.get() == E->getSubStmt())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildStmtExpr(E->getLParenLoc(),
                                       SubStmt.get(),
@@ -4865,30 +5864,6 @@ TreeTransform<Derived>::TransformStmtExpr(StmtExpr *E) {
 
 template<typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformTypesCompatibleExpr(TypesCompatibleExpr *E) {
-  TypeSourceInfo *TInfo1;
-  TypeSourceInfo *TInfo2;
-  
-  TInfo1 = getDerived().TransformType(E->getArgTInfo1());
-  if (!TInfo1)
-    return ExprError();
-
-  TInfo2 = getDerived().TransformType(E->getArgTInfo2());
-  if (!TInfo2)
-    return ExprError();
-
-  if (!getDerived().AlwaysRebuild() &&
-      TInfo1 == E->getArgTInfo1() &&
-      TInfo2 == E->getArgTInfo2())
-    return SemaRef.Owned(E->Retain());
-
-  return getDerived().RebuildTypesCompatibleExpr(E->getBuiltinLoc(),
-                                                 TInfo1, TInfo2,
-                                                 E->getRParenLoc());
-}
-
-template<typename Derived>
-ExprResult
 TreeTransform<Derived>::TransformChooseExpr(ChooseExpr *E) {
   ExprResult Cond = getDerived().TransformExpr(E->getCond());
   if (Cond.isInvalid())
@@ -4906,7 +5881,7 @@ TreeTransform<Derived>::TransformChooseExpr(ChooseExpr *E) {
       Cond.get() == E->getCond() &&
       LHS.get() == E->getLHS() &&
       RHS.get() == E->getRHS())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildChooseExpr(E->getBuiltinLoc(),
                                         Cond.get(), LHS.get(), RHS.get(),
@@ -4916,7 +5891,7 @@ TreeTransform<Derived>::TransformChooseExpr(ChooseExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformGNUNullExpr(GNUNullExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
@@ -4946,26 +5921,12 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
 
     // Transform the call arguments.
     ASTOwningVector<Expr*> Args(SemaRef);
-    llvm::SmallVector<SourceLocation, 4> FakeCommaLocs;
-    for (unsigned I = 1, N = E->getNumArgs(); I != N; ++I) {
-      if (getDerived().DropCallArgument(E->getArg(I)))
-        break;
-      
-      ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
-      if (Arg.isInvalid())
-        return ExprError();
-
-      // FIXME: Poor source location information.
-      SourceLocation FakeCommaLoc
-        = SemaRef.PP.getLocForEndOfToken(
-                                 static_cast<Expr *>(Arg.get())->getLocEnd());
-      FakeCommaLocs.push_back(FakeCommaLoc);
-      Args.push_back(Arg.release());
-    }
+    if (getDerived().TransformExprs(E->getArgs() + 1, E->getNumArgs() - 1, true, 
+                                    Args))
+      return ExprError();
 
     return getDerived().RebuildCallExpr(Object.get(), FakeLParenLoc,
                                         move_arg(Args),
-                                        FakeCommaLocs.data(),
                                         E->getLocEnd());
   }
 
@@ -5006,7 +5967,7 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
       Callee.get() == E->getCallee() &&
       First.get() == E->getArg(0) &&
       (E->getNumArgs() != 2 || Second.get() == E->getArg(1)))
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCXXOperatorCallExpr(E->getOperator(),
                                                  E->getOperatorLoc(),
@@ -5023,30 +5984,53 @@ TreeTransform<Derived>::TransformCXXMemberCallExpr(CXXMemberCallExpr *E) {
 
 template<typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
-  TypeSourceInfo *OldT;
-  TypeSourceInfo *NewT;
-  {
-    // FIXME: Source location isn't quite accurate.
-    SourceLocation TypeStartLoc
-      = SemaRef.PP.getLocForEndOfToken(E->getOperatorLoc());
-    TemporaryBase Rebase(*this, TypeStartLoc, DeclarationName());
+TreeTransform<Derived>::TransformCUDAKernelCallExpr(CUDAKernelCallExpr *E) {
+  // Transform the callee.
+  ExprResult Callee = getDerived().TransformExpr(E->getCallee());
+  if (Callee.isInvalid())
+    return ExprError();
 
-    OldT = E->getTypeInfoAsWritten();
-    NewT = getDerived().TransformType(OldT);
-    if (!NewT)
-      return ExprError();
-  }
+  // Transform exec config.
+  ExprResult EC = getDerived().TransformCallExpr(E->getConfig());
+  if (EC.isInvalid())
+    return ExprError();
 
+  // Transform arguments.
+  bool ArgChanged = false;
+  ASTOwningVector<Expr*> Args(SemaRef);
+  if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args, 
+                                  &ArgChanged))
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() &&
+      Callee.get() == E->getCallee() &&
+      !ArgChanged)
+    return SemaRef.Owned(E);
+
+  // FIXME: Wrong source location information for the '('.
+  SourceLocation FakeLParenLoc
+    = ((Expr *)Callee.get())->getSourceRange().getBegin();
+  return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
+                                      move_arg(Args),
+                                      E->getRParenLoc(), EC.get());
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
+  TypeSourceInfo *Type = getDerived().TransformType(E->getTypeInfoAsWritten());
+  if (!Type)
+    return ExprError();
+  
   ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
   if (SubExpr.isInvalid())
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      OldT == NewT &&
+      Type == E->getTypeInfoAsWritten() &&
       SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   // FIXME: Poor source location information here.
   SourceLocation FakeLAngleLoc
@@ -5058,7 +6042,7 @@ TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
   return getDerived().RebuildCXXNamedCastExpr(E->getOperatorLoc(),
                                               E->getStmtClass(),
                                               FakeLAngleLoc,
-                                              NewT,
+                                              Type,
                                               FakeRAngleLoc,
                                               FakeRAngleLoc,
                                               SubExpr.get(),
@@ -5094,16 +6078,9 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXFunctionalCastExpr(
                                                      CXXFunctionalCastExpr *E) {
-  TypeSourceInfo *OldT;
-  TypeSourceInfo *NewT;
-  {
-    TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
-
-    OldT = E->getTypeInfoAsWritten();
-    NewT = getDerived().TransformType(OldT);
-    if (!NewT)
-      return ExprError();
-  }
+  TypeSourceInfo *Type = getDerived().TransformType(E->getTypeInfoAsWritten());
+  if (!Type)
+    return ExprError();
 
   ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
@@ -5111,14 +6088,11 @@ TreeTransform<Derived>::TransformCXXFunctionalCastExpr(
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      OldT == NewT &&
+      Type == E->getTypeInfoAsWritten() &&
       SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
-  // FIXME: The end of the type's source range is wrong
-  return getDerived().RebuildCXXFunctionalCastExpr(
-                                  /*FIXME:*/SourceRange(E->getTypeBeginLoc()),
-                                                   NewT,
+  return getDerived().RebuildCXXFunctionalCastExpr(Type,
                                       /*FIXME:*/E->getSubExpr()->getLocStart(),
                                                    SubExpr.get(),
                                                    E->getRParenLoc());
@@ -5135,7 +6109,7 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
 
     if (!getDerived().AlwaysRebuild() &&
         TInfo == E->getTypeOperandSourceInfo())
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
 
     return getDerived().RebuildCXXTypeidExpr(E->getType(),
                                              E->getLocStart(),
@@ -5155,7 +6129,7 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       SubExpr.get() == E->getExprOperand())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCXXTypeidExpr(E->getType(),
                                            E->getLocStart(),
@@ -5165,29 +6139,64 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
 
 template<typename Derived>
 ExprResult
+TreeTransform<Derived>::TransformCXXUuidofExpr(CXXUuidofExpr *E) {
+  if (E->isTypeOperand()) {
+    TypeSourceInfo *TInfo
+      = getDerived().TransformType(E->getTypeOperandSourceInfo());
+    if (!TInfo)
+      return ExprError();
+
+    if (!getDerived().AlwaysRebuild() &&
+        TInfo == E->getTypeOperandSourceInfo())
+      return SemaRef.Owned(E);
+
+    return getDerived().RebuildCXXTypeidExpr(E->getType(),
+                                             E->getLocStart(),
+                                             TInfo,
+                                             E->getLocEnd());
+  }
+
+  // We don't know whether the expression is potentially evaluated until
+  // after we perform semantic analysis, so the expression is potentially
+  // potentially evaluated.
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
+
+  ExprResult SubExpr = getDerived().TransformExpr(E->getExprOperand());
+  if (SubExpr.isInvalid())
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() &&
+      SubExpr.get() == E->getExprOperand())
+    return SemaRef.Owned(E);
+
+  return getDerived().RebuildCXXUuidofExpr(E->getType(),
+                                           E->getLocStart(),
+                                           SubExpr.get(),
+                                           E->getLocEnd());
+}
+
+template<typename Derived>
+ExprResult
 TreeTransform<Derived>::TransformCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXNullPtrLiteralExpr(
                                                      CXXNullPtrLiteralExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXThisExpr(CXXThisExpr *E) {
-  TemporaryBase Rebase(*this, E->getLocStart(), DeclarationName());
+  DeclContext *DC = getSema().getFunctionLevelDeclContext();
+  CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(DC);
+  QualType T = MD->getThisType(getSema().Context);
 
-  QualType T = getDerived().TransformType(E->getType());
-  if (T.isNull())
-    return ExprError();
-
-  if (!getDerived().AlwaysRebuild() &&
-      T == E->getType())
-    return SemaRef.Owned(E->Retain());
+  if (!getDerived().AlwaysRebuild() && T == E->getType())
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCXXThisExpr(E->getLocStart(), T, E->isImplicit());
 }
@@ -5201,7 +6210,7 @@ TreeTransform<Derived>::TransformCXXThrowExpr(CXXThrowExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       SubExpr.get() == E->getSubExpr())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCXXThrowExpr(E->getThrowLoc(), SubExpr.get());
 }
@@ -5217,27 +6226,25 @@ TreeTransform<Derived>::TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       Param == E->getParam())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildCXXDefaultArgExpr(E->getUsedLocation(), Param);
 }
 
 template<typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
-  TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
-
-  QualType T = getDerived().TransformType(E->getType());
-  if (T.isNull())
+TreeTransform<Derived>::TransformCXXScalarValueInitExpr(
+                                                    CXXScalarValueInitExpr *E) {
+  TypeSourceInfo *T = getDerived().TransformType(E->getTypeSourceInfo());
+  if (!T)
     return ExprError();
-
+  
   if (!getDerived().AlwaysRebuild() &&
-      T == E->getType())
-    return SemaRef.Owned(E->Retain());
+      T == E->getTypeSourceInfo())
+    return SemaRef.Owned(E);
 
-  return getDerived().RebuildCXXScalarValueInitExpr(E->getTypeBeginLoc(),
-                                                 /*FIXME:*/E->getTypeBeginLoc(),
-                                                    T,
+  return getDerived().RebuildCXXScalarValueInitExpr(T, 
+                                          /*FIXME:*/T->getTypeLoc().getEndLoc(),
                                                     E->getRParenLoc());
 }
 
@@ -5245,9 +6252,9 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
   // Transform the type that we're allocating
-  TemporaryBase Rebase(*this, E->getLocStart(), DeclarationName());
-  QualType AllocType = getDerived().TransformType(E->getAllocatedType());
-  if (AllocType.isNull())
+  TypeSourceInfo *AllocTypeInfo
+    = getDerived().TransformType(E->getAllocatedTypeSourceInfo());
+  if (!AllocTypeInfo)
     return ExprError();
 
   // Transform the size of the array we're allocating (if any).
@@ -5258,28 +6265,16 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
   // Transform the placement arguments (if any).
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*> PlacementArgs(SemaRef);
-  for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
-    ExprResult Arg = getDerived().TransformExpr(E->getPlacementArg(I));
-    if (Arg.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || Arg.get() != E->getPlacementArg(I);
-    PlacementArgs.push_back(Arg.take());
-  }
+  if (getDerived().TransformExprs(E->getPlacementArgs(), 
+                                  E->getNumPlacementArgs(), true,
+                                  PlacementArgs, &ArgumentChanged))
+    return ExprError();  
 
   // transform the constructor arguments (if any).
   ASTOwningVector<Expr*> ConstructorArgs(SemaRef);
-  for (unsigned I = 0, N = E->getNumConstructorArgs(); I != N; ++I) {
-    if (getDerived().DropCallArgument(E->getConstructorArg(I)))
-      break;
-    
-    ExprResult Arg = getDerived().TransformExpr(E->getConstructorArg(I));
-    if (Arg.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || Arg.get() != E->getConstructorArg(I);
-    ConstructorArgs.push_back(Arg.take());
-  }
+  if (TransformExprs(E->getConstructorArgs(), E->getNumConstructorArgs(), true,
+                     ConstructorArgs, &ArgumentChanged))
+    return ExprError();  
 
   // Transform constructor, new operator, and delete operator.
   CXXConstructorDecl *Constructor = 0;
@@ -5310,7 +6305,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
   }
   
   if (!getDerived().AlwaysRebuild() &&
-      AllocType == E->getAllocatedType() &&
+      AllocTypeInfo == E->getAllocatedTypeSourceInfo() &&
       ArraySize.get() == E->getArraySize() &&
       Constructor == E->getConstructor() &&
       OperatorNew == E->getOperatorNew() &&
@@ -5324,9 +6319,10 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
       SemaRef.MarkDeclarationReferenced(E->getLocStart(), OperatorNew);
     if (OperatorDelete)
       SemaRef.MarkDeclarationReferenced(E->getLocStart(), OperatorDelete);
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   }
 
+  QualType AllocType = AllocTypeInfo->getType();
   if (!ArraySize.get()) {
     // If no array size was specified, but the new expression was
     // instantiated with an array type (e.g., "new T" where T is
@@ -5347,11 +6343,12 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
     } else if (const DependentSizedArrayType *DepArrayT
                               = dyn_cast<DependentSizedArrayType>(ArrayT)) {
       if (DepArrayT->getSizeExpr()) {
-        ArraySize = SemaRef.Owned(DepArrayT->getSizeExpr()->Retain());
+        ArraySize = SemaRef.Owned(DepArrayT->getSizeExpr());
         AllocType = DepArrayT->getElementType();
       }
     }
   }
+  
   return getDerived().RebuildCXXNewExpr(E->getLocStart(),
                                         E->isGlobalNew(),
                                         /*FIXME:*/E->getLocStart(),
@@ -5359,8 +6356,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
                                         /*FIXME:*/E->getLocStart(),
                                         E->getTypeIdParens(),
                                         AllocType,
-                                        /*FIXME:*/E->getLocStart(),
-                                        /*FIXME:*/SourceRange(),
+                                        AllocTypeInfo,
                                         ArraySize.get(),
                                         /*FIXME:*/E->getLocStart(),
                                         move_arg(ConstructorArgs),
@@ -5391,7 +6387,18 @@ TreeTransform<Derived>::TransformCXXDeleteExpr(CXXDeleteExpr *E) {
     // FIXME: instantiation-specific.
     if (OperatorDelete)
       SemaRef.MarkDeclarationReferenced(E->getLocStart(), OperatorDelete);
-    return SemaRef.Owned(E->Retain());
+    
+    if (!E->getArgument()->isTypeDependent()) {
+      QualType Destroyed = SemaRef.Context.getBaseElementType(
+                                                         E->getDestroyedType());
+      if (const RecordType *DestroyedRec = Destroyed->getAs<RecordType>()) {
+        CXXRecordDecl *Record = cast<CXXRecordDecl>(DestroyedRec->getDecl());
+        SemaRef.MarkDeclarationReferenced(E->getLocStart(), 
+                                          SemaRef.LookupDestructor(Record));
+      }
+    }
+    
+    return SemaRef.Owned(E);
   }
 
   return getDerived().RebuildCXXDeleteExpr(E->getLocStart(),
@@ -5419,17 +6426,21 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
     return ExprError();
                                               
   QualType ObjectType = ObjectTypePtr.get();
-  NestedNameSpecifier *Qualifier
-    = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
-                                                E->getQualifierRange(),
-                                                ObjectType);
-  if (E->getQualifier() && !Qualifier)
-    return ExprError();
+  NestedNameSpecifier *Qualifier = E->getQualifier();
+  if (Qualifier) {
+    Qualifier
+      = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
+                                                  E->getQualifierRange(),
+                                                  ObjectType);
+    if (!Qualifier)
+      return ExprError();
+  }
 
   PseudoDestructorTypeStorage Destroyed;
   if (E->getDestroyedTypeInfo()) {
     TypeSourceInfo *DestroyedTypeInfo
-      = getDerived().TransformType(E->getDestroyedTypeInfo(), ObjectType);
+      = getDerived().TransformTypeInObjectScope(E->getDestroyedTypeInfo(),
+                                                ObjectType, 0, Qualifier);
     if (!DestroyedTypeInfo)
       return ExprError();
     Destroyed = DestroyedTypeInfo;
@@ -5462,8 +6473,7 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
 
   TypeSourceInfo *ScopeTypeInfo = 0;
   if (E->getScopeTypeInfo()) {
-    ScopeTypeInfo = getDerived().TransformType(E->getScopeTypeInfo(), 
-                                               ObjectType);
+    ScopeTypeInfo = getDerived().TransformType(E->getScopeTypeInfo());
     if (!ScopeTypeInfo)
       return ExprError();
   }
@@ -5550,12 +6560,10 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
   // If we have template arguments, rebuild them, then rebuild the
   // templateid expression.
   TemplateArgumentListInfo TransArgs(Old->getLAngleLoc(), Old->getRAngleLoc());
-  for (unsigned I = 0, N = Old->getNumTemplateArgs(); I != N; ++I) {
-    TemplateArgumentLoc Loc;
-    if (getDerived().TransformTemplateArgument(Old->getTemplateArgs()[I], Loc))
-      return ExprError();
-    TransArgs.addArgument(Loc);
-  }
+  if (getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
+                                              Old->getNumTemplateArgs(),
+                                              TransArgs))
+    return ExprError();
 
   return getDerived().RebuildTemplateIdExpr(SS, R, Old->requiresADL(),
                                             TransArgs);
@@ -5564,29 +6572,43 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
-  TemporaryBase Rebase(*this, /*FIXME*/E->getLocStart(), DeclarationName());
-
-  QualType T = getDerived().TransformType(E->getQueriedType());
-  if (T.isNull())
+  TypeSourceInfo *T = getDerived().TransformType(E->getQueriedTypeSourceInfo());
+  if (!T)
     return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      T == E->getQueriedType())
-    return SemaRef.Owned(E->Retain());
-
-  // FIXME: Bad location information
-  SourceLocation FakeLParenLoc
-    = SemaRef.PP.getLocForEndOfToken(E->getLocStart());
+      T == E->getQueriedTypeSourceInfo())
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildUnaryTypeTrait(E->getTrait(),
                                             E->getLocStart(),
-                                            /*FIXME:*/FakeLParenLoc,
                                             T,
                                             E->getLocEnd());
 }
 
 template<typename Derived>
 ExprResult
+TreeTransform<Derived>::TransformBinaryTypeTraitExpr(BinaryTypeTraitExpr *E) {
+  TypeSourceInfo *LhsT = getDerived().TransformType(E->getLhsTypeSourceInfo());
+  if (!LhsT)
+    return ExprError();
+
+  TypeSourceInfo *RhsT = getDerived().TransformType(E->getRhsTypeSourceInfo());
+  if (!RhsT)
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() &&
+      LhsT == E->getLhsTypeSourceInfo() && RhsT == E->getRhsTypeSourceInfo())
+    return SemaRef.Owned(E);
+
+  return getDerived().RebuildBinaryTypeTrait(E->getTrait(),
+                                            E->getLocStart(),
+                                            LhsT, RhsT,
+                                            E->getLocEnd());
+}
+
+template<typename Derived>
+ExprResult
 TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
                                                DependentScopeDeclRefExpr *E) {
   NestedNameSpecifier *NNS
@@ -5595,6 +6617,10 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
   if (!NNS)
     return ExprError();
 
+  // TODO: If this is a conversion-function-id, verify that the
+  // destination type name (if present) resolves the same way after
+  // instantiation as it did in the local scope.
+
   DeclarationNameInfo NameInfo
     = getDerived().TransformDeclarationNameInfo(E->getNameInfo());
   if (!NameInfo.getName())
@@ -5606,7 +6632,7 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
         // Note: it is sufficient to compare the Name component of NameInfo:
         // if name has not changed, DNLoc has not changed either.
         NameInfo.getName() == E->getDeclName())
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
 
     return getDerived().RebuildDependentScopeDeclRefExpr(NNS,
                                                          E->getQualifierRange(),
@@ -5615,12 +6641,10 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
   }
 
   TemplateArgumentListInfo TransArgs(E->getLAngleLoc(), E->getRAngleLoc());
-  for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
-    TemplateArgumentLoc Loc;
-    if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-      return ExprError();
-    TransArgs.addArgument(Loc);
-  }
+  if (getDerived().TransformTemplateArguments(E->getTemplateArgs(),
+                                              E->getNumTemplateArgs(),
+                                              TransArgs))
+    return ExprError();
 
   return getDerived().RebuildDependentScopeDeclRefExpr(NNS,
                                                        E->getQualifierRange(),
@@ -5652,22 +6676,10 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
 
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*> Args(SemaRef);
-  for (CXXConstructExpr::arg_iterator Arg = E->arg_begin(),
-       ArgEnd = E->arg_end();
-       Arg != ArgEnd; ++Arg) {
-    if (getDerived().DropCallArgument(*Arg)) {
-      ArgumentChanged = true;
-      break;
-    }
-
-    ExprResult TransArg = getDerived().TransformExpr(*Arg);
-    if (TransArg.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
-    Args.push_back(TransArg.get());
-  }
-
+  if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args, 
+                                  &ArgumentChanged))
+    return ExprError();
+  
   if (!getDerived().AlwaysRebuild() &&
       T == E->getType() &&
       Constructor == E->getConstructor() &&
@@ -5675,14 +6687,15 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
     // Mark the constructor as referenced.
     // FIXME: Instantiation-specific
     SemaRef.MarkDeclarationReferenced(E->getLocStart(), Constructor);
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   }
 
   return getDerived().RebuildCXXConstructExpr(T, /*FIXME:*/E->getLocStart(),
                                               Constructor, E->isElidable(),
                                               move_arg(Args),
                                               E->requiresZeroInitialization(),
-                                              E->getConstructionKind());
+                                              E->getConstructionKind(),
+                                              E->getParenRange());
 }
 
 /// \brief Transform a C++ temporary-binding expression.
@@ -5695,25 +6708,23 @@ TreeTransform<Derived>::TransformCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   return getDerived().TransformExpr(E->getSubExpr());
 }
 
-/// \brief Transform a C++ expression that contains temporaries that should
-/// be destroyed after the expression is evaluated.
+/// \brief Transform a C++ expression that contains cleanups that should
+/// be run after the expression is evaluated.
 ///
-/// Since CXXExprWithTemporaries nodes are implicitly generated, we
+/// Since ExprWithCleanups nodes are implicitly generated, we
 /// just transform the subexpression and return that.
 template<typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformCXXExprWithTemporaries(
-                                                    CXXExprWithTemporaries *E) {
+TreeTransform<Derived>::TransformExprWithCleanups(ExprWithCleanups *E) {
   return getDerived().TransformExpr(E->getSubExpr());
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
-                                                      CXXTemporaryObjectExpr *E) {
-  TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
-  QualType T = getDerived().TransformType(E->getType());
-  if (T.isNull())
+                                                    CXXTemporaryObjectExpr *E) {
+  TypeSourceInfo *T = getDerived().TransformType(E->getTypeSourceInfo());
+  if (!T)
     return ExprError();
 
   CXXConstructorDecl *Constructor
@@ -5726,43 +6737,22 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*> Args(SemaRef);
   Args.reserve(E->getNumArgs());
-  for (CXXTemporaryObjectExpr::arg_iterator Arg = E->arg_begin(),
-                                         ArgEnd = E->arg_end();
-       Arg != ArgEnd; ++Arg) {
-    if (getDerived().DropCallArgument(*Arg)) {
-      ArgumentChanged = true;
-      break;
-    }
-
-    ExprResult TransArg = getDerived().TransformExpr(*Arg);
-    if (TransArg.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
-    Args.push_back((Expr *)TransArg.release());
-  }
+  if (TransformExprs(E->getArgs(), E->getNumArgs(), true, Args, 
+                     &ArgumentChanged))
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      T == E->getType() &&
+      T == E->getTypeSourceInfo() &&
       Constructor == E->getConstructor() &&
       !ArgumentChanged) {
     // FIXME: Instantiation-specific
-    SemaRef.MarkDeclarationReferenced(E->getTypeBeginLoc(), Constructor);
-    return SemaRef.MaybeBindToTemporary(E->Retain());
-  }
-
-  // FIXME: Bogus location information
-  SourceLocation CommaLoc;
-  if (Args.size() > 1) {
-    Expr *First = (Expr *)Args[0];
-    CommaLoc
-      = SemaRef.PP.getLocForEndOfToken(First->getSourceRange().getEnd());
+    SemaRef.MarkDeclarationReferenced(E->getLocStart(), Constructor);
+    return SemaRef.MaybeBindToTemporary(E);
   }
-  return getDerived().RebuildCXXTemporaryObjectExpr(E->getTypeBeginLoc(),
-                                                    T,
-                                                /*FIXME:*/E->getTypeBeginLoc(),
+  
+  return getDerived().RebuildCXXTemporaryObjectExpr(T,
+                                          /*FIXME:*/T->getTypeLoc().getEndLoc(),
                                                     move_arg(Args),
-                                                    &CommaLoc,
                                                     E->getLocEnd());
 }
 
@@ -5770,38 +6760,26 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformCXXUnresolvedConstructExpr(
                                                   CXXUnresolvedConstructExpr *E) {
-  TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
-  QualType T = getDerived().TransformType(E->getTypeAsWritten());
-  if (T.isNull())
+  TypeSourceInfo *T = getDerived().TransformType(E->getTypeSourceInfo());
+  if (!T)
     return ExprError();
 
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*> Args(SemaRef);
-  llvm::SmallVector<SourceLocation, 8> FakeCommaLocs;
-  for (CXXUnresolvedConstructExpr::arg_iterator Arg = E->arg_begin(),
-                                             ArgEnd = E->arg_end();
-       Arg != ArgEnd; ++Arg) {
-    ExprResult TransArg = getDerived().TransformExpr(*Arg);
-    if (TransArg.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
-    FakeCommaLocs.push_back(
-                        SemaRef.PP.getLocForEndOfToken((*Arg)->getLocEnd()));
-    Args.push_back(TransArg.get());
-  }
-
+  Args.reserve(E->arg_size());
+  if (getDerived().TransformExprs(E->arg_begin(), E->arg_size(), true, Args, 
+                                  &ArgumentChanged))
+    return ExprError();
+  
   if (!getDerived().AlwaysRebuild() &&
-      T == E->getTypeAsWritten() &&
+      T == E->getTypeSourceInfo() &&
       !ArgumentChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   // FIXME: we're faking the locations of the commas
-  return getDerived().RebuildCXXUnresolvedConstructExpr(E->getTypeBeginLoc(),
-                                                        T,
+  return getDerived().RebuildCXXUnresolvedConstructExpr(T,
                                                         E->getLParenLoc(),
                                                         move_arg(Args),
-                                                        FakeCommaLocs.data(),
                                                         E->getRParenLoc());
 }
 
@@ -5856,9 +6834,12 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
       return ExprError();
   }
 
+  // TODO: If this is a conversion-function-id, verify that the
+  // destination type name (if present) resolves the same way after
+  // instantiation as it did in the local scope.
+
   DeclarationNameInfo NameInfo
-    = getDerived().TransformDeclarationNameInfo(E->getMemberNameInfo(),
-                                                ObjectType);
+    = getDerived().TransformDeclarationNameInfo(E->getMemberNameInfo());
   if (!NameInfo.getName())
     return ExprError();
 
@@ -5871,7 +6852,7 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
         Qualifier == E->getQualifier() &&
         NameInfo.getName() == E->getMember() &&
         FirstQualifierInScope == E->getFirstQualifierFoundInScope())
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
 
     return getDerived().RebuildCXXDependentScopeMemberExpr(Base.get(),
                                                        BaseType,
@@ -5885,12 +6866,10 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
   }
 
   TemplateArgumentListInfo TransArgs(E->getLAngleLoc(), E->getRAngleLoc());
-  for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
-    TemplateArgumentLoc Loc;
-    if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-      return ExprError();
-    TransArgs.addArgument(Loc);
-  }
+  if (getDerived().TransformTemplateArguments(E->getTemplateArgs(),
+                                              E->getNumTemplateArgs(),
+                                              TransArgs))
+    return ExprError();
 
   return getDerived().RebuildCXXDependentScopeMemberExpr(Base.get(),
                                                      BaseType,
@@ -5975,13 +6954,10 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
   if (Old->hasExplicitTemplateArgs()) {
     TransArgs.setLAngleLoc(Old->getLAngleLoc());
     TransArgs.setRAngleLoc(Old->getRAngleLoc());
-    for (unsigned I = 0, N = Old->getNumTemplateArgs(); I != N; ++I) {
-      TemplateArgumentLoc Loc;
-      if (getDerived().TransformTemplateArgument(Old->getTemplateArgs()[I],
-                                                 Loc))
-        return ExprError();
-      TransArgs.addArgument(Loc);
-    }
+    if (getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
+                                                Old->getNumTemplateArgs(),
+                                                TransArgs))
+      return ExprError();
   }
 
   // FIXME: to do this check properly, we will need to preserve the
@@ -6004,8 +6980,74 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
 
 template<typename Derived>
 ExprResult
+TreeTransform<Derived>::TransformCXXNoexceptExpr(CXXNoexceptExpr *E) {
+  ExprResult SubExpr = getDerived().TransformExpr(E->getOperand());
+  if (SubExpr.isInvalid())
+    return ExprError();
+
+  if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getOperand())
+    return SemaRef.Owned(E);
+
+  return getDerived().RebuildCXXNoexceptExpr(E->getSourceRange(),SubExpr.get());
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformPackExpansionExpr(PackExpansionExpr *E) {
+  ExprResult Pattern = getDerived().TransformExpr(E->getPattern());
+  if (Pattern.isInvalid())
+    return ExprError();
+  
+  if (!getDerived().AlwaysRebuild() && Pattern.get() == E->getPattern())
+    return SemaRef.Owned(E);
+
+  return getDerived().RebuildPackExpansion(Pattern.get(), E->getEllipsisLoc(),
+                                           E->getNumExpansions());
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
+  // If E is not value-dependent, then nothing will change when we transform it.
+  // Note: This is an instantiation-centric view.
+  if (!E->isValueDependent())
+    return SemaRef.Owned(E);
+
+  // Note: None of the implementations of TryExpandParameterPacks can ever
+  // produce a diagnostic when given only a single unexpanded parameter pack,
+  // so 
+  UnexpandedParameterPack Unexpanded(E->getPack(), E->getPackLoc());
+  bool ShouldExpand = false;
+  bool RetainExpansion = false;
+  llvm::Optional<unsigned> NumExpansions;
+  if (getDerived().TryExpandParameterPacks(E->getOperatorLoc(), E->getPackLoc(), 
+                                           &Unexpanded, 1, 
+                                           ShouldExpand, RetainExpansion,
+                                           NumExpansions))
+    return ExprError();
+  
+  if (!ShouldExpand || RetainExpansion)
+    return SemaRef.Owned(E);
+  
+  // We now know the length of the parameter pack, so build a new expression
+  // that stores that length.
+  return getDerived().RebuildSizeOfPackExpr(E->getOperatorLoc(), E->getPack(), 
+                                            E->getPackLoc(), E->getRParenLoc(), 
+                                            *NumExpansions);
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformSubstNonTypeTemplateParmPackExpr(
+                                          SubstNonTypeTemplateParmPackExpr *E) {
+  // Default behavior is to do nothing with this transformation.
+  return SemaRef.Owned(E);
+}
+
+template<typename Derived>
+ExprResult
 TreeTransform<Derived>::TransformObjCStringLiteral(ObjCStringLiteral *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
@@ -6018,7 +7060,7 @@ TreeTransform<Derived>::TransformObjCEncodeExpr(ObjCEncodeExpr *E) {
 
   if (!getDerived().AlwaysRebuild() &&
       EncodedTypeInfo == E->getEncodedTypeSourceInfo())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildObjCEncodeExpr(E->getAtLoc(),
                                             EncodedTypeInfo,
@@ -6031,15 +7073,11 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
   // Transform arguments.
   bool ArgChanged = false;
   ASTOwningVector<Expr*> Args(SemaRef);
-  for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I) {
-    ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
-    if (Arg.isInvalid())
-      return ExprError();
-    
-    ArgChanged = ArgChanged || Arg.get() != E->getArg(I);
-    Args.push_back(Arg.get());
-  }
-
+  Args.reserve(E->getNumArgs());
+  if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), false, Args, 
+                                  &ArgChanged))
+    return ExprError();
+  
   if (E->getReceiverKind() == ObjCMessageExpr::Class) {
     // Class message: transform the receiver type.
     TypeSourceInfo *ReceiverTypeInfo
@@ -6050,11 +7088,12 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
     // If nothing changed, just retain the existing message send.
     if (!getDerived().AlwaysRebuild() &&
         ReceiverTypeInfo == E->getClassReceiverTypeInfo() && !ArgChanged)
-      return SemaRef.Owned(E->Retain());
+      return SemaRef.Owned(E);
 
     // Build a new class message send.
     return getDerived().RebuildObjCMessageExpr(ReceiverTypeInfo,
                                                E->getSelector(),
+                                               E->getSelectorLoc(),
                                                E->getMethodDecl(),
                                                E->getLeftLoc(),
                                                move_arg(Args),
@@ -6072,11 +7111,12 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
   // If nothing changed, just retain the existing message send.
   if (!getDerived().AlwaysRebuild() &&
       Receiver.get() == E->getInstanceReceiver() && !ArgChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   
   // Build a new instance message send.
   return getDerived().RebuildObjCMessageExpr(Receiver.get(),
                                              E->getSelector(),
+                                             E->getSelectorLoc(),
                                              E->getMethodDecl(),
                                              E->getLeftLoc(),
                                              move_arg(Args),
@@ -6086,13 +7126,13 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformObjCSelectorExpr(ObjCSelectorExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformObjCProtocolExpr(ObjCProtocolExpr *E) {
-  return SemaRef.Owned(E->Retain());
+  return SemaRef.Owned(E);
 }
 
 template<typename Derived>
@@ -6108,7 +7148,7 @@ TreeTransform<Derived>::TransformObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   // If nothing changed, just retain the existing expression.
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   
   return getDerived().RebuildObjCIvarRefExpr(Base.get(), E->getDecl(),
                                              E->getLocation(),
@@ -6118,6 +7158,11 @@ TreeTransform<Derived>::TransformObjCIvarRefExpr(ObjCIvarRefExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
+  // 'super' and types never change. Property never changes. Just
+  // retain the existing expression.
+  if (!E->isObjectReceiver())
+    return SemaRef.Owned(E);
+  
   // Transform the base expression.
   ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
@@ -6128,47 +7173,18 @@ TreeTransform<Derived>::TransformObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   // If nothing changed, just retain the existing expression.
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
-    return SemaRef.Owned(E->Retain());
-  
-  return getDerived().RebuildObjCPropertyRefExpr(Base.get(), E->getProperty(),
-                                                 E->getLocation());
-}
+    return SemaRef.Owned(E);
 
-template<typename Derived>
-ExprResult
-TreeTransform<Derived>::TransformObjCImplicitSetterGetterRefExpr(
-                                          ObjCImplicitSetterGetterRefExpr *E) {
-  // If this implicit setter/getter refers to class methods, it cannot have any
-  // dependent parts. Just retain the existing declaration.
-  if (E->getInterfaceDecl())
-    return SemaRef.Owned(E->Retain());
-  
-  // Transform the base expression.
-  ExprResult Base = getDerived().TransformExpr(E->getBase());
-  if (Base.isInvalid())
-    return ExprError();
-  
-  // We don't need to transform the getters/setters; they will never change.
-  
-  // If nothing changed, just retain the existing expression.
-  if (!getDerived().AlwaysRebuild() &&
-      Base.get() == E->getBase())
-    return SemaRef.Owned(E->Retain());
-  
-  return getDerived().RebuildObjCImplicitSetterGetterRefExpr(
-                                                          E->getGetterMethod(),
-                                                             E->getType(),
-                                                          E->getSetterMethod(),
-                                                             E->getLocation(),
-                                                             Base.get());
-                                                             
-}
+  if (E->isExplicitProperty())
+    return getDerived().RebuildObjCPropertyRefExpr(Base.get(),
+                                                   E->getExplicitProperty(),
+                                                   E->getLocation());
 
-template<typename Derived>
-ExprResult
-TreeTransform<Derived>::TransformObjCSuperExpr(ObjCSuperExpr *E) {
-  // Can never occur in a dependent context.
-  return SemaRef.Owned(E->Retain());
+  return getDerived().RebuildObjCPropertyRefExpr(Base.get(),
+                                                 E->getType(),
+                                                 E->getImplicitPropertyGetter(),
+                                                 E->getImplicitPropertySetter(),
+                                                 E->getLocation());
 }
 
 template<typename Derived>
@@ -6182,7 +7198,7 @@ TreeTransform<Derived>::TransformObjCIsaExpr(ObjCIsaExpr *E) {
   // If nothing changed, just retain the existing expression.
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   
   return getDerived().RebuildObjCIsaExpr(Base.get(), E->getIsaMemberLoc(),
                                          E->isArrow());
@@ -6193,18 +7209,14 @@ ExprResult
 TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
   bool ArgumentChanged = false;
   ASTOwningVector<Expr*> SubExprs(SemaRef);
-  for (unsigned I = 0, N = E->getNumSubExprs(); I != N; ++I) {
-    ExprResult SubExpr = getDerived().TransformExpr(E->getExpr(I));
-    if (SubExpr.isInvalid())
-      return ExprError();
-
-    ArgumentChanged = ArgumentChanged || SubExpr.get() != E->getExpr(I);
-    SubExprs.push_back(SubExpr.get());
-  }
+  SubExprs.reserve(E->getNumSubExprs());
+  if (getDerived().TransformExprs(E->getSubExprs(), E->getNumSubExprs(), false, 
+                                  SubExprs, &ArgumentChanged))
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       !ArgumentChanged)
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
 
   return getDerived().RebuildShuffleVectorExpr(E->getBuiltinLoc(),
                                                move_arg(SubExprs),
@@ -6214,52 +7226,92 @@ TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
-  SourceLocation CaretLoc(E->getExprLoc());
+  BlockDecl *oldBlock = E->getBlockDecl();
   
-  SemaRef.ActOnBlockStart(CaretLoc, /*Scope=*/0);
-  BlockScopeInfo *CurBlock = SemaRef.getCurBlock();
-  CurBlock->TheDecl->setIsVariadic(E->getBlockDecl()->isVariadic());
-  llvm::SmallVector<ParmVarDecl*, 4> Params;
-  llvm::SmallVector<QualType, 4> ParamTypes;
+  SemaRef.ActOnBlockStart(E->getCaretLocation(), /*Scope=*/0);
+  BlockScopeInfo *blockScope = SemaRef.getCurBlock();
+
+  blockScope->TheDecl->setIsVariadic(oldBlock->isVariadic());
+  llvm::SmallVector<ParmVarDecl*, 4> params;
+  llvm::SmallVector<QualType, 4> paramTypes;
   
   // Parameter substitution.
-  const BlockDecl *BD = E->getBlockDecl();
-  for (BlockDecl::param_const_iterator P = BD->param_begin(),
-       EN = BD->param_end(); P != EN; ++P) {
-    ParmVarDecl *OldParm = (*P);
-    ParmVarDecl *NewParm = getDerived().TransformFunctionTypeParam(OldParm);
-    QualType NewType = NewParm->getType();
-    Params.push_back(NewParm);
-    ParamTypes.push_back(NewParm->getType());
+  if (getDerived().TransformFunctionTypeParams(E->getCaretLocation(),
+                                               oldBlock->param_begin(),
+                                               oldBlock->param_size(),
+                                               0, paramTypes, &params))
+    return true;
+
+  const FunctionType *exprFunctionType = E->getFunctionType();
+  QualType exprResultType = exprFunctionType->getResultType();
+  if (!exprResultType.isNull()) {
+    if (!exprResultType->isDependentType())
+      blockScope->ReturnType = exprResultType;
+    else if (exprResultType != getSema().Context.DependentTy)
+      blockScope->ReturnType = getDerived().TransformType(exprResultType);
   }
   
-  const FunctionType *BExprFunctionType = E->getFunctionType();
-  QualType BExprResultType = BExprFunctionType->getResultType();
-  if (!BExprResultType.isNull()) {
-    if (!BExprResultType->isDependentType())
-      CurBlock->ReturnType = BExprResultType;
-    else if (BExprResultType != SemaRef.Context.DependentTy)
-      CurBlock->ReturnType = getDerived().TransformType(BExprResultType);
-  }
-    
-  // Transform the body
-  StmtResult Body = getDerived().TransformStmt(E->getBody());
-  if (Body.isInvalid())
+  // If the return type has not been determined yet, leave it as a dependent
+  // type; it'll get set when we process the body.
+  if (blockScope->ReturnType.isNull())
+    blockScope->ReturnType = getSema().Context.DependentTy;
+
+  // Don't allow returning a objc interface by value.
+  if (blockScope->ReturnType->isObjCObjectType()) {
+    getSema().Diag(E->getCaretLocation(), 
+                   diag::err_object_cannot_be_passed_returned_by_value) 
+      << 0 << blockScope->ReturnType;
     return ExprError();
+  }
+
+  QualType functionType = getDerived().RebuildFunctionProtoType(
+                                                        blockScope->ReturnType,
+                                                        paramTypes.data(),
+                                                        paramTypes.size(),
+                                                        oldBlock->isVariadic(),
+                                                        0, RQ_None,
+                                               exprFunctionType->getExtInfo());
+  blockScope->FunctionType = functionType;
+
   // Set the parameters on the block decl.
-  if (!Params.empty())
-    CurBlock->TheDecl->setParams(Params.data(), Params.size());
-    
-  QualType FunctionType = getDerived().RebuildFunctionProtoType(
-                                                        CurBlock->ReturnType,
-                                                        ParamTypes.data(),
-                                                        ParamTypes.size(),
-                                                        BD->isVariadic(),
-                                                        0,
-                                               BExprFunctionType->getExtInfo());
+  if (!params.empty())
+    blockScope->TheDecl->setParams(params.data(), params.size());
+  
+  // If the return type wasn't explicitly set, it will have been marked as a 
+  // dependent type (DependentTy); clear out the return type setting so 
+  // we will deduce the return type when type-checking the block's body.
+  if (blockScope->ReturnType == getSema().Context.DependentTy)
+    blockScope->ReturnType = QualType();
   
-  CurBlock->FunctionType = FunctionType;
-  return SemaRef.ActOnBlockStmtExpr(CaretLoc, Body.get(), /*Scope=*/0);
+  // Transform the body
+  StmtResult body = getDerived().TransformStmt(E->getBody());
+  if (body.isInvalid())
+    return ExprError();
+
+#ifndef NDEBUG
+  // In builds with assertions, make sure that we captured everything we
+  // captured before.
+
+  if (oldBlock->capturesCXXThis()) assert(blockScope->CapturesCXXThis);
+
+  for (BlockDecl::capture_iterator i = oldBlock->capture_begin(),
+         e = oldBlock->capture_end(); i != e; ++i) {
+    VarDecl *oldCapture = i->getVariable();
+
+    // Ignore parameter packs.
+    if (isa<ParmVarDecl>(oldCapture) &&
+        cast<ParmVarDecl>(oldCapture)->isParameterPack())
+      continue;
+
+    VarDecl *newCapture =
+      cast<VarDecl>(getDerived().TransformDecl(E->getCaretLocation(),
+                                               oldCapture));
+    assert(blockScope->CaptureMap.count(newCapture));
+  }
+#endif
+
+  return SemaRef.ActOnBlockStmtExpr(E->getCaretLocation(), body.get(),
+                                    /*Scope=*/0);
 }
 
 template<typename Derived>
@@ -6279,7 +7331,7 @@ TreeTransform<Derived>::TransformBlockDeclRefExpr(BlockDeclRefExpr *E) {
     // FIXME: this is a bit instantiation-specific.
     SemaRef.MarkDeclarationReferenced(E->getLocation(), ND);
     
-    return SemaRef.Owned(E->Retain());
+    return SemaRef.Owned(E);
   }
   
   DeclarationNameInfo NameInfo(E->getDecl()->getDeclName(), E->getLocation());
@@ -6403,10 +7455,10 @@ TreeTransform<Derived>::RebuildDependentSizedArrayType(QualType ElementType,
 
 template<typename Derived>
 QualType TreeTransform<Derived>::RebuildVectorType(QualType ElementType,
-                                     unsigned NumElements,
-                                     VectorType::AltiVecSpecific AltiVecSpec) {
+                                               unsigned NumElements,
+                                               VectorType::VectorKind VecKind) {
   // FIXME: semantic checking!
-  return SemaRef.Context.getVectorType(ElementType, NumElements, AltiVecSpec);
+  return SemaRef.Context.getVectorType(ElementType, NumElements, VecKind);
 }
 
 template<typename Derived>
@@ -6435,9 +7487,10 @@ QualType TreeTransform<Derived>::RebuildFunctionProtoType(QualType T,
                                                         unsigned NumParamTypes,
                                                           bool Variadic,
                                                           unsigned Quals,
+                                                  RefQualifierKind RefQualifier,
                                             const FunctionType::ExtInfo &Info) {
   return SemaRef.BuildFunctionType(T, ParamTypes, NumParamTypes, Variadic,
-                                   Quals,
+                                   Quals, RefQualifier,
                                    getDerived().getBaseLocation(),
                                    getDerived().getBaseEntity(),
                                    Info);
@@ -6474,8 +7527,9 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(Decl *D) {
 }
 
 template<typename Derived>
-QualType TreeTransform<Derived>::RebuildTypeOfExprType(Expr *E) {
-  return SemaRef.BuildTypeofExprType(E);
+QualType TreeTransform<Derived>::RebuildTypeOfExprType(Expr *E,
+                                                       SourceLocation Loc) {
+  return SemaRef.BuildTypeofExprType(E, Loc);
 }
 
 template<typename Derived>
@@ -6484,8 +7538,9 @@ QualType TreeTransform<Derived>::RebuildTypeOfType(QualType Underlying) {
 }
 
 template<typename Derived>
-QualType TreeTransform<Derived>::RebuildDecltypeType(Expr *E) {
-  return SemaRef.BuildDecltypeType(E);
+QualType TreeTransform<Derived>::RebuildDecltypeType(Expr *E,
+                                                     SourceLocation Loc) {
+  return SemaRef.BuildDecltypeType(E, Loc);
 }
 
 template<typename Derived>
@@ -6552,10 +7607,12 @@ TreeTransform<Derived>::RebuildTemplateName(NestedNameSpecifier *Qualifier,
 template<typename Derived>
 TemplateName
 TreeTransform<Derived>::RebuildTemplateName(NestedNameSpecifier *Qualifier,
+                                            SourceRange QualifierRange,
                                             const IdentifierInfo &II,
-                                            QualType ObjectType) {
+                                            QualType ObjectType,
+                                            NamedDecl *FirstQualifierInScope) {
   CXXScopeSpec SS;
-  SS.setRange(SourceRange(getDerived().getBaseLocation()));
+  SS.setRange(QualifierRange);
   SS.setScopeRep(Qualifier);
   UnqualifiedId Name;
   Name.setIdentifier(&II, /*FIXME:*/getDerived().getBaseLocation());
@@ -6567,7 +7624,7 @@ TreeTransform<Derived>::RebuildTemplateName(NestedNameSpecifier *Qualifier,
                                        ParsedType::make(ObjectType),
                                        /*EnteringContext=*/false,
                                        Template);
-  return Template.template getAsVal<TemplateName>();
+  return Template.get();
 }
 
 template<typename Derived>
diff --git a/lib/Sema/TypeLocBuilder.h b/lib/Sema/TypeLocBuilder.h
new file mode 100644
index 000000000000..3d20a5273d6d
--- /dev/null
+++ b/lib/Sema/TypeLocBuilder.h
@@ -0,0 +1,180 @@
+//===--- TypeLocBuilder.h - Type Source Info collector ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This files defines TypeLocBuilder, a class for building TypeLocs
+//  bottom-up.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_TYPELOCBUILDER_H
+#define LLVM_CLANG_SEMA_TYPELOCBUILDER_H
+
+#include "clang/AST/TypeLoc.h"
+#include "llvm/ADT/SmallVector.h"
+#include "clang/AST/ASTContext.h"
+
+namespace clang {
+
+class TypeLocBuilder {
+  enum { InlineCapacity = 8 * sizeof(SourceLocation) };
+
+  /// The underlying location-data buffer.  Data grows from the end
+  /// of the buffer backwards.
+  char *Buffer;
+
+  /// The capacity of the current buffer.
+  size_t Capacity;
+
+  /// The index of the first occupied byte in the buffer.
+  size_t Index;
+
+#ifndef NDEBUG
+  /// The last type pushed on this builder.
+  QualType LastTy;
+#endif
+    
+  /// The inline buffer.
+  char InlineBuffer[InlineCapacity];
+
+ public:
+  TypeLocBuilder()
+    : Buffer(InlineBuffer), Capacity(InlineCapacity), Index(InlineCapacity)
+  {}
+
+  ~TypeLocBuilder() {
+    if (Buffer != InlineBuffer)
+      delete[] Buffer;
+  }
+
+  /// Ensures that this buffer has at least as much capacity as described.
+  void reserve(size_t Requested) {
+    if (Requested > Capacity)
+      // For now, match the request exactly.
+      grow(Requested);
+  }
+
+  /// Pushes a copy of the given TypeLoc onto this builder.  The builder
+  /// must be empty for this to work.
+  void pushFullCopy(TypeLoc L) {
+    size_t Size = L.getFullDataSize();
+    TypeLoc Copy = pushFullUninitializedImpl(L.getType(), Size);
+    memcpy(Copy.getOpaqueData(), L.getOpaqueData(), Size);
+  }
+
+  /// Pushes uninitialized space for the given type.  The builder must
+  /// be empty.
+  TypeLoc pushFullUninitialized(QualType T) {
+    return pushFullUninitializedImpl(T, TypeLoc::getFullDataSizeForType(T));
+  }
+
+  /// Pushes space for a typespec TypeLoc.  Invalidates any TypeLocs
+  /// previously retrieved from this builder.
+  TypeSpecTypeLoc pushTypeSpec(QualType T) {
+    size_t LocalSize = TypeSpecTypeLoc::LocalDataSize;
+    return cast<TypeSpecTypeLoc>(pushImpl(T, LocalSize));
+  }
+
+  /// Resets this builder to the newly-initialized state.
+  void clear() {
+#ifndef NDEBUG
+    LastTy = QualType();
+#endif
+    Index = Capacity;
+  }  
+
+  /// Pushes space for a new TypeLoc of the given type.  Invalidates
+  /// any TypeLocs previously retrieved from this builder.
+  template <class TyLocType> TyLocType push(QualType T) {
+    size_t LocalSize = cast<TyLocType>(TypeLoc(T, 0)).getLocalDataSize();
+    return cast<TyLocType>(pushImpl(T, LocalSize));
+  }
+
+  /// Creates a TypeSourceInfo for the given type.
+  TypeSourceInfo *getTypeSourceInfo(ASTContext& Context, QualType T) {
+#ifndef NDEBUG
+    assert(T == LastTy && "type doesn't match last type pushed!");
+#endif
+
+    size_t FullDataSize = Capacity - Index;
+    TypeSourceInfo *DI = Context.CreateTypeSourceInfo(T, FullDataSize);
+    memcpy(DI->getTypeLoc().getOpaqueData(), &Buffer[Index], FullDataSize);
+    return DI;
+  }
+
+private:
+  TypeLoc pushImpl(QualType T, size_t LocalSize) {
+#ifndef NDEBUG
+    QualType TLast = TypeLoc(T, 0).getNextTypeLoc().getType();
+    assert(TLast == LastTy &&
+           "mismatch between last type and new type's inner type");
+    LastTy = T;
+#endif
+
+    // If we need to grow, grow by a factor of 2.
+    if (LocalSize > Index) {
+      size_t RequiredCapacity = Capacity + (LocalSize - Index);
+      size_t NewCapacity = Capacity * 2;
+      while (RequiredCapacity > NewCapacity)
+        NewCapacity *= 2;
+      grow(NewCapacity);
+    }
+
+    Index -= LocalSize;
+
+    return getTypeLoc(T);
+  }
+
+  /// Grow to the given capacity.
+  void grow(size_t NewCapacity) {
+    assert(NewCapacity > Capacity);
+
+    // Allocate the new buffer and copy the old data into it.
+    char *NewBuffer = new char[NewCapacity];
+    unsigned NewIndex = Index + NewCapacity - Capacity;
+    memcpy(&NewBuffer[NewIndex],
+           &Buffer[Index],
+           Capacity - Index);
+
+    if (Buffer != InlineBuffer)
+      delete[] Buffer;
+
+    Buffer = NewBuffer;
+    Capacity = NewCapacity;
+    Index = NewIndex;
+  }
+
+  TypeLoc pushFullUninitializedImpl(QualType T, size_t Size) {
+#ifndef NDEBUG
+    assert(LastTy.isNull() && "pushing full on non-empty TypeLocBuilder");
+    LastTy = T;
+#endif
+    assert(Index == Capacity && "pushing full on non-empty TypeLocBuilder");
+
+    reserve(Size);
+    Index -= Size;
+
+    return getTypeLoc(T);
+  }
+
+
+  // This is private because, when we kill off TypeSourceInfo in favor
+  // of TypeLoc, we'll want an interface that creates a TypeLoc given
+  // an ASTContext, and we don't want people to think they can just
+  // use this as an equivalent.
+  TypeLoc getTypeLoc(QualType T) {
+#ifndef NDEBUG
+    assert(LastTy == T && "type doesn't match last type pushed!");
+#endif
+    return TypeLoc(T, &Buffer[Index]);
+  }
+};
+
+}
+
+#endif
diff --git a/lib/Serialization/ASTCommon.cpp b/lib/Serialization/ASTCommon.cpp
index 77c1aff44b39..5e94f59ad493 100644
--- a/lib/Serialization/ASTCommon.cpp
+++ b/lib/Serialization/ASTCommon.cpp
@@ -12,11 +12,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "ASTCommon.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 
+// Give ASTDeserializationListener's VTable a home.
+ASTDeserializationListener::~ASTDeserializationListener() { }
+
 serialization::TypeIdx
 serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   unsigned ID = 0;
@@ -32,7 +36,8 @@ serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   case BuiltinType::UInt128:    ID = PREDEF_TYPE_UINT128_ID;    break;
   case BuiltinType::Char_S:     ID = PREDEF_TYPE_CHAR_S_ID;     break;
   case BuiltinType::SChar:      ID = PREDEF_TYPE_SCHAR_ID;      break;
-  case BuiltinType::WChar:      ID = PREDEF_TYPE_WCHAR_ID;      break;
+  case BuiltinType::WChar_S:
+  case BuiltinType::WChar_U:    ID = PREDEF_TYPE_WCHAR_ID;      break;
   case BuiltinType::Short:      ID = PREDEF_TYPE_SHORT_ID;      break;
   case BuiltinType::Int:        ID = PREDEF_TYPE_INT_ID;        break;
   case BuiltinType::Long:       ID = PREDEF_TYPE_LONG_ID;       break;
@@ -49,9 +54,6 @@ serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   case BuiltinType::ObjCId:     ID = PREDEF_TYPE_OBJC_ID;       break;
   case BuiltinType::ObjCClass:  ID = PREDEF_TYPE_OBJC_CLASS;    break;
   case BuiltinType::ObjCSel:    ID = PREDEF_TYPE_OBJC_SEL;      break;
-  case BuiltinType::UndeducedAuto:
-    assert(0 && "Should not see undeduced auto here");
-    break;
   }
 
   return TypeIdx(ID);
diff --git a/lib/Serialization/ASTCommon.h b/lib/Serialization/ASTCommon.h
index a0e2ecd8aa1b..d4166998d1cf 100644
--- a/lib/Serialization/ASTCommon.h
+++ b/lib/Serialization/ASTCommon.h
@@ -20,6 +20,12 @@ namespace clang {
 
 namespace serialization {
 
+enum DeclUpdateKind {
+  UPD_CXX_SET_DEFINITIONDATA,
+  UPD_CXX_ADDED_IMPLICIT_MEMBER,
+  UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION
+};
+
 TypeIdx TypeIdxFromBuiltin(const BuiltinType *BT);
 
 template <typename IdxForTypeTy>
@@ -28,7 +34,7 @@ TypeID MakeTypeID(QualType T, IdxForTypeTy IdxForType) {
     return PREDEF_TYPE_NULL_ID;
 
   unsigned FastQuals = T.getLocalFastQualifiers();
-  T.removeFastQualifiers();
+  T.removeLocalFastQualifiers();
 
   if (T.hasLocalNonFastQualifiers())
     return IdxForType(T).asTypeID(FastQuals);
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index f07215cb8f51..ce87b11c2695 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -33,17 +33,21 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceManagerInternals.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Version.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Bitcode/BitstreamReader.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/system_error.h"
 #include <algorithm>
 #include <iterator>
 #include <cstdio>
 #include <sys/stat.h>
+
 using namespace clang;
 using namespace clang::serialization;
 
@@ -74,6 +78,7 @@ PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
   PARSE_LANGOPT_BENIGN(HexFloats);
   PARSE_LANGOPT_IMPORTANT(C99, diag::warn_pch_c99);
   PARSE_LANGOPT_IMPORTANT(Microsoft, diag::warn_pch_microsoft_extensions);
+  PARSE_LANGOPT_BENIGN(MSCVersion);
   PARSE_LANGOPT_IMPORTANT(CPlusPlus, diag::warn_pch_cplusplus);
   PARSE_LANGOPT_IMPORTANT(CPlusPlus0x, diag::warn_pch_cplusplus0x);
   PARSE_LANGOPT_BENIGN(CXXOperatorName);
@@ -81,7 +86,10 @@ PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
   PARSE_LANGOPT_IMPORTANT(ObjC2, diag::warn_pch_objective_c2);
   PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI, diag::warn_pch_nonfragile_abi);
   PARSE_LANGOPT_IMPORTANT(ObjCNonFragileABI2, diag::warn_pch_nonfragile_abi2);
-  PARSE_LANGOPT_IMPORTANT(NoConstantCFStrings, 
+  PARSE_LANGOPT_IMPORTANT(AppleKext, diag::warn_pch_apple_kext);
+  PARSE_LANGOPT_IMPORTANT(ObjCDefaultSynthProperties,
+                          diag::warn_pch_objc_auto_properties);
+  PARSE_LANGOPT_IMPORTANT(NoConstantCFStrings,
                           diag::warn_pch_no_constant_cfstrings);
   PARSE_LANGOPT_BENIGN(PascalStrings);
   PARSE_LANGOPT_BENIGN(WritableStrings);
@@ -90,6 +98,8 @@ PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
   PARSE_LANGOPT_IMPORTANT(AltiVec, diag::warn_pch_altivec);
   PARSE_LANGOPT_IMPORTANT(Exceptions, diag::warn_pch_exceptions);
   PARSE_LANGOPT_IMPORTANT(SjLjExceptions, diag::warn_pch_sjlj_exceptions);
+  PARSE_LANGOPT_IMPORTANT(ObjCExceptions, diag::warn_pch_objc_exceptions);
+  PARSE_LANGOPT_IMPORTANT(MSBitfields, diag::warn_pch_ms_bitfields);
   PARSE_LANGOPT_IMPORTANT(NeXTRuntime, diag::warn_pch_objc_runtime);
   PARSE_LANGOPT_IMPORTANT(Freestanding, diag::warn_pch_freestanding);
   PARSE_LANGOPT_IMPORTANT(NoBuiltin, diag::warn_pch_builtins);
@@ -118,6 +128,7 @@ PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
   PARSE_LANGOPT_IMPORTANT(AccessControl, diag::warn_pch_access_control);
   PARSE_LANGOPT_IMPORTANT(CharIsSigned, diag::warn_pch_char_signed);
   PARSE_LANGOPT_IMPORTANT(ShortWChar, diag::warn_pch_short_wchar);
+  PARSE_LANGOPT_IMPORTANT(ShortEnums, diag::warn_pch_short_enums);
   if ((PPLangOpts.getGCMode() != 0) != (LangOpts.getGCMode() != 0)) {
     Reader.Diag(diag::warn_pch_gc_mode)
       << LangOpts.getGCMode() << PPLangOpts.getGCMode();
@@ -128,9 +139,11 @@ PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
                           diag::warn_pch_stack_protector);
   PARSE_LANGOPT_BENIGN(InstantiationDepth);
   PARSE_LANGOPT_IMPORTANT(OpenCL, diag::warn_pch_opencl);
+  PARSE_LANGOPT_IMPORTANT(CUDA, diag::warn_pch_cuda);
   PARSE_LANGOPT_BENIGN(CatchUndefined);
   PARSE_LANGOPT_IMPORTANT(ElideConstructors, diag::warn_pch_elide_constructors);
   PARSE_LANGOPT_BENIGN(SpellChecking);
+  PARSE_LANGOPT_BENIGN(DefaultFPContract);
 #undef PARSE_LANGOPT_IMPORTANT
 #undef PARSE_LANGOPT_BENIGN
 
@@ -146,12 +159,14 @@ bool PCHValidator::ReadTargetTriple(llvm::StringRef Triple) {
   return true;
 }
 
-struct EmptyStringRef {
-  bool operator ()(llvm::StringRef r) const { return r.empty(); }
-};
-struct EmptyBlock {
-  bool operator ()(const PCHPredefinesBlock &r) const { return r.Data.empty(); }
-};
+namespace {
+  struct EmptyStringRef {
+    bool operator ()(llvm::StringRef r) const { return r.empty(); }
+  };
+  struct EmptyBlock {
+    bool operator ()(const PCHPredefinesBlock &r) const {return r.Data.empty();}
+  };
+}
 
 static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
                                 PCHPredefinesBlocks R) {
@@ -263,7 +278,36 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
 
   llvm::SmallVector<llvm::StringRef, 8> CmdLineLines;
   Left.split(CmdLineLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
-  Right.split(CmdLineLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+
+  // Pick out implicit #includes after the PCH and don't consider them for
+  // validation; we will insert them into SuggestedPredefines so that the
+  // preprocessor includes them.
+  std::string IncludesAfterPCH;
+  llvm::SmallVector<llvm::StringRef, 8> AfterPCHLines;
+  Right.split(AfterPCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+  for (unsigned i = 0, e = AfterPCHLines.size(); i != e; ++i) {
+    if (AfterPCHLines[i].startswith("#include ")) {
+      IncludesAfterPCH += AfterPCHLines[i];
+      IncludesAfterPCH += '\n';
+    } else {
+      CmdLineLines.push_back(AfterPCHLines[i]);
+    }
+  }
+
+  // Make sure we add the includes last into SuggestedPredefines before we
+  // exit this function.
+  struct AddIncludesRAII {
+    std::string &SuggestedPredefines;
+    std::string &IncludesAfterPCH;
+
+    AddIncludesRAII(std::string &SuggestedPredefines,
+                    std::string &IncludesAfterPCH)
+      : SuggestedPredefines(SuggestedPredefines),
+        IncludesAfterPCH(IncludesAfterPCH) { }
+    ~AddIncludesRAII() {
+      SuggestedPredefines += IncludesAfterPCH;
+    }
+  } AddIncludes(SuggestedPredefines, IncludesAfterPCH);
 
   // Sort both sets of predefined buffer lines, since we allow some extra
   // definitions and they may appear at any point in the output.
@@ -281,6 +325,11 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
   bool ConflictingDefines = false;
   for (unsigned I = 0, N = MissingPredefines.size(); I != N; ++I) {
     llvm::StringRef Missing = MissingPredefines[I];
+    if (Missing.startswith("#include ")) {
+      // An -include was specified when generating the PCH; it is included in
+      // the PCH, just ignore it.
+      continue;
+    }
     if (!Missing.startswith("#define ")) {
       Reader.Diag(diag::warn_pch_compiler_options_mismatch);
       return true;
@@ -418,8 +467,6 @@ void PCHValidator::ReadCounter(unsigned Value) {
 void
 ASTReader::setDeserializationListener(ASTDeserializationListener *Listener) {
   DeserializationListener = Listener;
-  if (DeserializationListener)
-    DeserializationListener->SetReader(this);
 }
 
 
@@ -533,10 +580,10 @@ public:
 typedef OnDiskChainedHashTable<ASTSelectorLookupTrait>
   ASTSelectorLookupTable;
 
-namespace {
+namespace clang {
 class ASTIdentifierLookupTrait {
   ASTReader &Reader;
-  llvm::BitstreamCursor &Stream;
+  ASTReader::PerFileData &F;
 
   // If we know the IdentifierInfo in advance, it is here and we will
   // not build a new one. Used when deserializing information about an
@@ -550,9 +597,9 @@ public:
 
   typedef external_key_type internal_key_type;
 
-  ASTIdentifierLookupTrait(ASTReader &Reader, llvm::BitstreamCursor &Stream,
+  ASTIdentifierLookupTrait(ASTReader &Reader, ASTReader::PerFileData &F,
                            IdentifierInfo *II = 0)
-    : Reader(Reader), Stream(Stream), KnownII(II) { }
+    : Reader(Reader), F(F), KnownII(II) { }
 
   static bool EqualKey(const internal_key_type& a,
                        const internal_key_type& b) {
@@ -568,6 +615,10 @@ public:
   static const internal_key_type&
   GetInternalKey(const external_key_type& x) { return x; }
 
+  // This hopefully will just get inlined and removed by the optimizer.
+  static const external_key_type&
+  GetExternalKey(const internal_key_type& x) { return x; }
+
   static std::pair<unsigned, unsigned>
   ReadKeyDataLength(const unsigned char*& d) {
     using namespace clang::io;
@@ -644,7 +695,7 @@ public:
     // definition.
     if (hasMacroDefinition) {
       uint32_t Offset = ReadUnalignedLE32(d);
-      Reader.ReadMacroRecord(Stream, Offset);
+      Reader.SetIdentifierIsMacro(II, F, Offset);
       DataLen -= 4;
     }
 
@@ -752,7 +803,7 @@ public:
     case DeclarationName::CXXUsingDirective:
       break;
     }
-    
+
     return Key;
   }
 
@@ -814,7 +865,7 @@ public:
     case DeclarationName::ObjCZeroArgSelector:
     case DeclarationName::ObjCOneArgSelector:
     case DeclarationName::ObjCMultiArgSelector:
-      Key.Data = 
+      Key.Data =
          (uint64_t)Reader.DecodeSelector(ReadUnalignedLE32(d)).getAsOpaquePtr();
       break;
     case DeclarationName::CXXConstructorName:
@@ -831,7 +882,7 @@ public:
     case DeclarationName::CXXUsingDirective:
       break;
     }
-    
+
     return Key;
   }
 
@@ -868,8 +919,8 @@ bool ASTReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
       return true;
     }
 
-    Info.LexicalDecls = reinterpret_cast<const DeclID*>(Blob);
-    Info.NumLexicalDecls = BlobLen / sizeof(DeclID);
+    Info.LexicalDecls = reinterpret_cast<const KindDeclIDPair*>(Blob);
+    Info.NumLexicalDecls = BlobLen / sizeof(KindDeclIDPair);
   } else {
     Info.LexicalDecls = 0;
     Info.NumLexicalDecls = 0;
@@ -918,8 +969,9 @@ bool ASTReader::CheckPredefinesBuffers() {
 //===----------------------------------------------------------------------===//
 
 /// \brief Read the line table in the source manager block.
-/// \returns true if ther was an error.
-bool ASTReader::ParseLineTable(llvm::SmallVectorImpl<uint64_t> &Record) {
+/// \returns true if there was an error.
+bool ASTReader::ParseLineTable(PerFileData &F,
+                               llvm::SmallVectorImpl<uint64_t> &Record) {
   unsigned Idx = 0;
   LineTableInfo &LineTable = SourceMgr.getLineTable();
 
@@ -965,7 +1017,6 @@ namespace {
 
 class ASTStatData {
 public:
-  const bool hasStat;
   const ino_t ino;
   const dev_t dev;
   const mode_t mode;
@@ -973,10 +1024,7 @@ public:
   const off_t size;
 
   ASTStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
-  : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
-
-  ASTStatData()
-    : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
+    : ino(i), dev(d), mode(mo), mtime(m), size(s) {}
 };
 
 class ASTStatLookupTrait {
@@ -1011,9 +1059,6 @@ class ASTStatLookupTrait {
                             unsigned /*DataLen*/) {
     using namespace clang::io;
 
-    if (*d++ == 1)
-      return data_type();
-
     ino_t ino = (ino_t) ReadUnalignedLE32(d);
     dev_t dev = (dev_t) ReadUnalignedLE32(d);
     mode_t mode = (mode_t) ReadUnalignedLE16(d);
@@ -1027,44 +1072,40 @@ class ASTStatLookupTrait {
 ///
 /// This cache is very similar to the stat cache used by pretokenized
 /// headers.
-class ASTStatCache : public StatSysCallCache {
+class ASTStatCache : public FileSystemStatCache {
   typedef OnDiskChainedHashTable<ASTStatLookupTrait> CacheTy;
   CacheTy *Cache;
 
   unsigned &NumStatHits, &NumStatMisses;
 public:
-  ASTStatCache(const unsigned char *Buckets,
-               const unsigned char *Base,
-               unsigned &NumStatHits,
-               unsigned &NumStatMisses)
+  ASTStatCache(const unsigned char *Buckets, const unsigned char *Base,
+               unsigned &NumStatHits, unsigned &NumStatMisses)
     : Cache(0), NumStatHits(NumStatHits), NumStatMisses(NumStatMisses) {
     Cache = CacheTy::Create(Buckets, Base);
   }
 
   ~ASTStatCache() { delete Cache; }
 
-  int stat(const char *path, struct stat *buf) {
+  LookupResult getStat(const char *Path, struct stat &StatBuf,
+                       int *FileDescriptor) {
     // Do the lookup for the file's data in the AST file.
-    CacheTy::iterator I = Cache->find(path);
+    CacheTy::iterator I = Cache->find(Path);
 
     // If we don't get a hit in the AST file just forward to 'stat'.
     if (I == Cache->end()) {
       ++NumStatMisses;
-      return StatSysCallCache::stat(path, buf);
+      return statChained(Path, StatBuf, FileDescriptor);
     }
 
     ++NumStatHits;
     ASTStatData Data = *I;
 
-    if (!Data.hasStat)
-      return 1;
-
-    buf->st_ino = Data.ino;
-    buf->st_dev = Data.dev;
-    buf->st_mtime = Data.mtime;
-    buf->st_mode = Data.mode;
-    buf->st_size = Data.size;
-    return 0;
+    StatBuf.st_ino = Data.ino;
+    StatBuf.st_dev = Data.dev;
+    StatBuf.st_mtime = Data.mtime;
+    StatBuf.st_mode = Data.mode;
+    StatBuf.st_size = Data.size;
+    return CacheExists;
   }
 };
 } // end anonymous namespace
@@ -1129,7 +1170,7 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) {
       break;
 
     case SM_LINE_TABLE:
-      if (ParseLineTable(Record))
+      if (ParseLineTable(F, Record))
         return Failure;
       break;
 
@@ -1142,9 +1183,42 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) {
   }
 }
 
+/// \brief If a header file is not found at the path that we expect it to be
+/// and the PCH file was moved from its original location, try to resolve the
+/// file by assuming that header+PCH were moved together and the header is in
+/// the same place relative to the PCH.
+static std::string
+resolveFileRelativeToOriginalDir(const std::string &Filename,
+                                 const std::string &OriginalDir,
+                                 const std::string &CurrDir) {
+  assert(OriginalDir != CurrDir &&
+         "No point trying to resolve the file if the PCH dir didn't change");
+  using namespace llvm::sys;
+  llvm::SmallString<128> filePath(Filename);
+  fs::make_absolute(filePath);
+  assert(path::is_absolute(OriginalDir));
+  llvm::SmallString<128> currPCHPath(CurrDir);
+
+  path::const_iterator fileDirI = path::begin(path::parent_path(filePath)),
+                       fileDirE = path::end(path::parent_path(filePath));
+  path::const_iterator origDirI = path::begin(OriginalDir),
+                       origDirE = path::end(OriginalDir);
+  // Skip the common path components from filePath and OriginalDir.
+  while (fileDirI != fileDirE && origDirI != origDirE &&
+         *fileDirI == *origDirI) {
+    ++fileDirI;
+    ++origDirI;
+  }
+  for (; origDirI != origDirE; ++origDirI)
+    path::append(currPCHPath, "..");
+  path::append(currPCHPath, fileDirI, fileDirE);
+  path::append(currPCHPath, path::filename(Filename));
+  return currPCHPath.str();
+}
+
 /// \brief Get a cursor that's correctly positioned for reading the source
 /// location entry with the given ID.
-llvm::BitstreamCursor &ASTReader::SLocCursorForID(unsigned ID) {
+ASTReader::PerFileData *ASTReader::SLocCursorForID(unsigned ID) {
   assert(ID != 0 && ID <= TotalNumSLocEntries &&
          "SLocCursorForID should only be called for real IDs.");
 
@@ -1159,7 +1233,7 @@ llvm::BitstreamCursor &ASTReader::SLocCursorForID(unsigned ID) {
   assert(F && F->LocalNumSLocEntries > ID && "Chain corrupted");
 
   F->SLocEntryCursor.JumpToBit(F->SLocOffsets[ID]);
-  return F->SLocEntryCursor;
+  return F;
 }
 
 /// \brief Read in the source location entry with the given ID.
@@ -1172,7 +1246,8 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
     return Failure;
   }
 
-  llvm::BitstreamCursor &SLocEntryCursor = SLocCursorForID(ID);
+  PerFileData *F = SLocCursorForID(ID);
+  llvm::BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
 
   ++NumSLocEntriesRead;
   unsigned Code = SLocEntryCursor.ReadCode();
@@ -1195,6 +1270,17 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
     std::string Filename(BlobStart, BlobStart + BlobLen);
     MaybeAddSystemRootToFilename(Filename);
     const FileEntry *File = FileMgr.getFile(Filename);
+    if (File == 0 && !OriginalDir.empty() && !CurrentDir.empty() &&
+        OriginalDir != CurrentDir) {
+      std::string resolved = resolveFileRelativeToOriginalDir(Filename,
+                                                              OriginalDir,
+                                                              CurrentDir);
+      if (!resolved.empty())
+        File = FileMgr.getFile(resolved);
+    }
+    if (File == 0)
+      File = FileMgr.getVirtualFile(Filename, (off_t)Record[4],
+                                    (time_t)Record[5]);
     if (File == 0) {
       std::string ErrorStr = "could not find file '";
       ErrorStr += Filename;
@@ -1203,7 +1289,7 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
       return Failure;
     }
 
-    if (Record.size() < 10) {
+    if (Record.size() < 6) {
       Error("source location entry is incorrect");
       return Failure;
     }
@@ -1222,22 +1308,13 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
       return Failure;
     }
 
-    FileID FID = SourceMgr.createFileID(File,
-                                SourceLocation::getFromRawEncoding(Record[1]),
-                                       (SrcMgr::CharacteristicKind)Record[2],
+    FileID FID = SourceMgr.createFileID(File, ReadSourceLocation(*F, Record[1]),
+                                        (SrcMgr::CharacteristicKind)Record[2],
                                         ID, Record[0]);
     if (Record[3])
       const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile())
         .setHasLineDirectives();
-
-    // Reconstruct header-search information for this file.
-    HeaderFileInfo HFI;
-    HFI.isImport = Record[6];
-    HFI.DirInfo = Record[7];
-    HFI.NumIncludes = Record[8];
-    HFI.ControllingMacroID = Record[9];
-    if (Listener)
-      Listener->ReadHeaderFileInfo(HFI, File->getUID());
+    
     break;
   }
 
@@ -1271,11 +1348,10 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
   }
 
   case SM_SLOC_INSTANTIATION_ENTRY: {
-    SourceLocation SpellingLoc
-      = SourceLocation::getFromRawEncoding(Record[1]);
+    SourceLocation SpellingLoc = ReadSourceLocation(*F, Record[1]);
     SourceMgr.createInstantiationLoc(SpellingLoc,
-                              SourceLocation::getFromRawEncoding(Record[2]),
-                              SourceLocation::getFromRawEncoding(Record[3]),
+                                     ReadSourceLocation(*F, Record[2]),
+                                     ReadSourceLocation(*F, Record[3]),
                                      Record[4],
                                      ID,
                                      Record[0]);
@@ -1297,17 +1373,21 @@ bool ASTReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
   }
 
   while (true) {
+    uint64_t Offset = Cursor.GetCurrentBitNo();
     unsigned Code = Cursor.ReadCode();
 
     // We expect all abbrevs to be at the start of the block.
-    if (Code != llvm::bitc::DEFINE_ABBREV)
+    if (Code != llvm::bitc::DEFINE_ABBREV) {
+      Cursor.JumpToBit(Offset);
       return false;
+    }
     Cursor.ReadAbbrevRecord();
   }
 }
 
-void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
+PreprocessedEntity *ASTReader::ReadMacroRecord(PerFileData &F, uint64_t Offset) {
   assert(PP && "Forgot to set Preprocessor ?");
+  llvm::BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
   // after reading this macro.
@@ -1322,14 +1402,14 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
     unsigned Code = Stream.ReadCode();
     switch (Code) {
     case llvm::bitc::END_BLOCK:
-      return;
+      return 0;
 
     case llvm::bitc::ENTER_SUBBLOCK:
       // No known subblocks, always skip them.
       Stream.ReadSubBlockID();
       if (Stream.SkipBlock()) {
         Error("malformed block record in AST file");
-        return;
+        return 0;
       }
       continue;
 
@@ -1340,9 +1420,12 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
     }
 
     // Read a record.
+    const char *BlobStart = 0;
+    unsigned BlobLen = 0;
     Record.clear();
     PreprocessorRecordTypes RecType =
-      (PreprocessorRecordTypes)Stream.ReadRecord(Code, Record);
+      (PreprocessorRecordTypes)Stream.ReadRecord(Code, Record, BlobStart,
+                                                 BlobLen);
     switch (RecType) {
     case PP_MACRO_OBJECT_LIKE:
     case PP_MACRO_FUNCTION_LIKE: {
@@ -1350,14 +1433,14 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
       // of the definition of the macro we were looking for. We're
       // done.
       if (Macro)
-        return;
+        return 0;
 
       IdentifierInfo *II = DecodeIdentifierInfo(Record[0]);
       if (II == 0) {
         Error("macro must have a name in AST file");
-        return;
+        return 0;
       }
-      SourceLocation Loc = SourceLocation::getFromRawEncoding(Record[1]);
+      SourceLocation Loc = ReadSourceLocation(F, Record[1]);
       bool isUsed = Record[2];
 
       MacroInfo *MI = PP->AllocateMacroInfo(Loc);
@@ -1389,13 +1472,13 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
       // Remember that we saw this macro last so that we add the tokens that
       // form its body to it.
       Macro = MI;
-      
+
       if (NextIndex + 1 == Record.size() && PP->getPreprocessingRecord()) {
         // We have a macro definition. Load it now.
         PP->getPreprocessingRecord()->RegisterMacroDefinition(Macro,
                                         getMacroDefinition(Record[NextIndex]));
       }
-      
+
       ++NumMacrosRead;
       break;
     }
@@ -1407,7 +1490,7 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
 
       Token Tok;
       Tok.startToken();
-      Tok.setLocation(SourceLocation::getFromRawEncoding(Record[0]));
+      Tok.setLocation(ReadSourceLocation(F, Record[0]));
       Tok.setLength(Record[1]);
       if (IdentifierInfo *II = DecodeIdentifierInfo(Record[2]))
         Tok.setIdentifierInfo(II);
@@ -1416,92 +1499,242 @@ void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
       Macro->AddTokenToBody(Tok);
       break;
     }
-        
-    case PP_MACRO_INSTANTIATION: {
-      // If we already have a macro, that means that we've hit the end
-      // of the definition of the macro we were looking for. We're
-      // done.
-      if (Macro)
-        return;
-      
-      if (!PP->getPreprocessingRecord()) {
-        Error("missing preprocessing record in AST file");
-        return;
-      }
-        
-      PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-      if (PPRec.getPreprocessedEntity(Record[0]))
-        return;
-
-      MacroInstantiation *MI
-        = new (PPRec) MacroInstantiation(DecodeIdentifierInfo(Record[3]),
-                               SourceRange(
-                                 SourceLocation::getFromRawEncoding(Record[1]),
-                                 SourceLocation::getFromRawEncoding(Record[2])),
-                                         getMacroDefinition(Record[4]));
-      PPRec.SetPreallocatedEntity(Record[0], MI);
-      return;
-    }
+  }
+  }
+  
+  return 0;
+}
 
-    case PP_MACRO_DEFINITION: {
-      // If we already have a macro, that means that we've hit the end
-      // of the definition of the macro we were looking for. We're
-      // done.
-      if (Macro)
-        return;
+PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
+  assert(PP && "Forgot to set Preprocessor ?");
+  unsigned Code = F.PreprocessorDetailCursor.ReadCode();
+  switch (Code) {
+  case llvm::bitc::END_BLOCK:
+    return 0;
+    
+  case llvm::bitc::ENTER_SUBBLOCK:
+    Error("unexpected subblock record in preprocessor detail block");
+    return 0;
       
-      if (!PP->getPreprocessingRecord()) {
-        Error("missing preprocessing record in AST file");
-        return;
-      }
+  case llvm::bitc::DEFINE_ABBREV:
+    Error("unexpected abbrevation record in preprocessor detail block");
+    return 0;
       
-      PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
-      if (PPRec.getPreprocessedEntity(Record[0]))
-        return;
-        
-      if (Record[1] >= MacroDefinitionsLoaded.size()) {
-        Error("out-of-bounds macro definition record");
-        return;
-      }
+  default:
+    break;
+  }
 
+  if (!PP->getPreprocessingRecord()) {
+    Error("no preprocessing record");
+    return 0;
+  }
+  
+  // Read the record.
+  PreprocessingRecord &PPRec = *PP->getPreprocessingRecord();
+  const char *BlobStart = 0;
+  unsigned BlobLen = 0;
+  RecordData Record;
+  PreprocessorDetailRecordTypes RecType =
+    (PreprocessorDetailRecordTypes)F.PreprocessorDetailCursor.ReadRecord(
+                                             Code, Record, BlobStart, BlobLen);
+  switch (RecType) {
+  case PPD_MACRO_INSTANTIATION: {
+    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+      return PE;
+    
+    MacroInstantiation *MI
+      = new (PPRec) MacroInstantiation(DecodeIdentifierInfo(Record[3]),
+                                 SourceRange(ReadSourceLocation(F, Record[1]),
+                                             ReadSourceLocation(F, Record[2])),
+                                       getMacroDefinition(Record[4]));
+    PPRec.SetPreallocatedEntity(Record[0], MI);
+    return MI;
+  }
+      
+  case PPD_MACRO_DEFINITION: {
+    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+      return PE;
+    
+    if (Record[1] > MacroDefinitionsLoaded.size()) {
+      Error("out-of-bounds macro definition record");
+      return 0;
+    }
+    
+    // Decode the identifier info and then check again; if the macro is
+    // still defined and associated with the identifier,
+    IdentifierInfo *II = DecodeIdentifierInfo(Record[4]);
+    if (!MacroDefinitionsLoaded[Record[1] - 1]) {
       MacroDefinition *MD
-        = new (PPRec) MacroDefinition(DecodeIdentifierInfo(Record[4]),
-                                SourceLocation::getFromRawEncoding(Record[5]),
-                              SourceRange(
-                                SourceLocation::getFromRawEncoding(Record[2]),
-                                SourceLocation::getFromRawEncoding(Record[3])));
+        = new (PPRec) MacroDefinition(II,
+                                      ReadSourceLocation(F, Record[5]),
+                                      SourceRange(
+                                            ReadSourceLocation(F, Record[2]),
+                                            ReadSourceLocation(F, Record[3])));
+      
       PPRec.SetPreallocatedEntity(Record[0], MD);
-      MacroDefinitionsLoaded[Record[1]] = MD;
-      return;
+      MacroDefinitionsLoaded[Record[1] - 1] = MD;
+      
+      if (DeserializationListener)
+        DeserializationListener->MacroDefinitionRead(Record[1], MD);
     }
+    
+    return MacroDefinitionsLoaded[Record[1] - 1];
+  }
+      
+  case PPD_INCLUSION_DIRECTIVE: {
+    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+      return PE;
+    
+    const char *FullFileNameStart = BlobStart + Record[3];
+    const FileEntry *File
+      = PP->getFileManager().getFile(llvm::StringRef(FullFileNameStart,
+                                                     BlobLen - Record[3]));
+    
+    // FIXME: Stable encoding
+    InclusionDirective::InclusionKind Kind
+      = static_cast<InclusionDirective::InclusionKind>(Record[5]);
+    InclusionDirective *ID
+      = new (PPRec) InclusionDirective(PPRec, Kind,
+                                       llvm::StringRef(BlobStart, Record[3]),
+                                       Record[4],
+                                       File,
+                                 SourceRange(ReadSourceLocation(F, Record[1]),
+                                             ReadSourceLocation(F, Record[2])));
+    PPRec.SetPreallocatedEntity(Record[0], ID);
+    return ID;
   }
   }
+  
+  Error("invalid offset in preprocessor detail block");
+  return 0;
+}
+
+namespace {
+  /// \brief Trait class used to search the on-disk hash table containing all of
+  /// the header search information.
+  ///
+  /// The on-disk hash table contains a mapping from each header path to 
+  /// information about that header (how many times it has been included, its
+  /// controlling macro, etc.). Note that we actually hash based on the 
+  /// filename, and support "deep" comparisons of file names based on current
+  /// inode numbers, so that the search can cope with non-normalized path names
+  /// and symlinks.
+  class HeaderFileInfoTrait {
+    const char *SearchPath;
+    struct stat SearchPathStatBuf;
+    llvm::Optional<int> SearchPathStatResult;
+    
+    int StatSimpleCache(const char *Path, struct stat *StatBuf) {
+      if (Path == SearchPath) {
+        if (!SearchPathStatResult)
+          SearchPathStatResult = stat(Path, &SearchPathStatBuf);
+        
+        *StatBuf = SearchPathStatBuf;
+        return *SearchPathStatResult;
+      }
+      
+      return stat(Path, StatBuf);
+    }
+    
+  public:
+    typedef const char *external_key_type;
+    typedef const char *internal_key_type;
+    
+    typedef HeaderFileInfo data_type;
+    
+    HeaderFileInfoTrait(const char *SearchPath = 0) : SearchPath(SearchPath) { }
+    
+    static unsigned ComputeHash(const char *path) {
+      return llvm::HashString(llvm::sys::path::filename(path));
+    }
+    
+    static internal_key_type GetInternalKey(const char *path) { return path; }
+    
+    bool EqualKey(internal_key_type a, internal_key_type b) {
+      if (strcmp(a, b) == 0)
+        return true;
+      
+      if (llvm::sys::path::filename(a) != llvm::sys::path::filename(b))
+        return false;
+      
+      // The file names match, but the path names don't. stat() the files to
+      // see if they are the same.      
+      struct stat StatBufA, StatBufB;
+      if (StatSimpleCache(a, &StatBufA) || StatSimpleCache(b, &StatBufB))
+        return false;
+      
+      return StatBufA.st_ino == StatBufB.st_ino;
+    }
+    
+    static std::pair<unsigned, unsigned>
+    ReadKeyDataLength(const unsigned char*& d) {
+      unsigned KeyLen = (unsigned) clang::io::ReadUnalignedLE16(d);
+      unsigned DataLen = (unsigned) *d++;
+      return std::make_pair(KeyLen + 1, DataLen);
+    }
+    
+    static internal_key_type ReadKey(const unsigned char *d, unsigned) {
+      return (const char *)d;
+    }
+    
+    static data_type ReadData(const internal_key_type, const unsigned char *d,
+                              unsigned DataLen) {
+      const unsigned char *End = d + DataLen;
+      using namespace clang::io;
+      HeaderFileInfo HFI;
+      unsigned Flags = *d++;
+      HFI.isImport = (Flags >> 3) & 0x01;
+      HFI.DirInfo = (Flags >> 1) & 0x03;
+      HFI.Resolved = Flags & 0x01;
+      HFI.NumIncludes = ReadUnalignedLE16(d);
+      HFI.ControllingMacroID = ReadUnalignedLE32(d);
+      assert(End == d && "Wrong data length in HeaderFileInfo deserialization");
+      (void)End;
+      
+      // This HeaderFileInfo was externally loaded.
+      HFI.External = true;
+      return HFI;
+    }
+  };
+}
+
+/// \brief The on-disk hash table used for the global method pool.
+typedef OnDiskChainedHashTable<HeaderFileInfoTrait>
+  HeaderFileInfoLookupTable;
+
+void ASTReader::SetIdentifierIsMacro(IdentifierInfo *II, PerFileData &F,
+                                     uint64_t Offset) {
+  // Note that this identifier has a macro definition.
+  II->setHasMacroDefinition(true);
+  
+  // Adjust the offset based on our position in the chain.
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (Chain[I] == &F)
+      break;
+    
+    Offset += Chain[I]->SizeInBits;
+  }
+    
+  UnreadMacroRecordOffsets[II] = Offset;
 }
 
 void ASTReader::ReadDefinedMacros() {
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
-    llvm::BitstreamCursor &MacroCursor = Chain[N - I - 1]->MacroCursor;
+    PerFileData &F = *Chain[N - I - 1];
+    llvm::BitstreamCursor &MacroCursor = F.MacroCursor;
 
     // If there was no preprocessor block, skip this file.
     if (!MacroCursor.getBitStreamReader())
       continue;
 
     llvm::BitstreamCursor Cursor = MacroCursor;
-    if (Cursor.EnterSubBlock(PREPROCESSOR_BLOCK_ID)) {
-      Error("malformed preprocessor block record in AST file");
-      return;
-    }
+    Cursor.JumpToBit(F.MacroStartOffset);
 
     RecordData Record;
     while (true) {
       unsigned Code = Cursor.ReadCode();
-      if (Code == llvm::bitc::END_BLOCK) {
-        if (Cursor.ReadBlockEnd()) {
-          Error("error at end of preprocessor block in AST file");
-          return;
-        }
+      if (Code == llvm::bitc::END_BLOCK)
         break;
-      }
 
       if (Code == llvm::bitc::ENTER_SUBBLOCK) {
         // No known subblocks, always skip them.
@@ -1534,35 +1767,64 @@ void ASTReader::ReadDefinedMacros() {
       case PP_TOKEN:
         // Ignore tokens.
         break;
-        
-      case PP_MACRO_INSTANTIATION:
-      case PP_MACRO_DEFINITION:
-        // Read the macro record.
-        ReadMacroRecord(Chain[N - I - 1]->Stream, Cursor.GetCurrentBitNo());
-        break;
       }
     }
   }
+  
+  // Drain the unread macro-record offsets map.
+  while (!UnreadMacroRecordOffsets.empty())
+    LoadMacroDefinition(UnreadMacroRecordOffsets.begin());
+}
+
+void ASTReader::LoadMacroDefinition(
+                     llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos) {
+  assert(Pos != UnreadMacroRecordOffsets.end() && "Unknown macro definition");
+  PerFileData *F = 0;
+  uint64_t Offset = Pos->second;
+  UnreadMacroRecordOffsets.erase(Pos);
+  
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (Offset < Chain[I]->SizeInBits) {
+      F = Chain[I];
+      break;
+    }
+    
+    Offset -= Chain[I]->SizeInBits;
+  }    
+  if (!F) {
+    Error("Malformed macro record offset");
+    return;
+  }
+  
+  ReadMacroRecord(*F, Offset);
 }
 
-MacroDefinition *ASTReader::getMacroDefinition(IdentID ID) {
-  if (ID == 0 || ID >= MacroDefinitionsLoaded.size())
+void ASTReader::LoadMacroDefinition(IdentifierInfo *II) {
+  llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos
+    = UnreadMacroRecordOffsets.find(II);
+  LoadMacroDefinition(Pos);
+}
+
+MacroDefinition *ASTReader::getMacroDefinition(MacroID ID) {
+  if (ID == 0 || ID > MacroDefinitionsLoaded.size())
     return 0;
 
-  if (!MacroDefinitionsLoaded[ID]) {
-    unsigned Index = ID;
+  if (!MacroDefinitionsLoaded[ID - 1]) {
+    unsigned Index = ID - 1;
     for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
       PerFileData &F = *Chain[N - I - 1];
       if (Index < F.LocalNumMacroDefinitions) {
-        ReadMacroRecord(F.Stream, F.MacroDefinitionOffsets[Index]);
+        SavedStreamPosition SavedPosition(F.PreprocessorDetailCursor);  
+        F.PreprocessorDetailCursor.JumpToBit(F.MacroDefinitionOffsets[Index]);
+        LoadPreprocessedEntity(F);
         break;
       }
       Index -= F.LocalNumMacroDefinitions;
     }
-    assert(MacroDefinitionsLoaded[ID] && "Broken chain");
+    assert(MacroDefinitionsLoaded[ID - 1] && "Broken chain");
   }
 
-  return MacroDefinitionsLoaded[ID];
+  return MacroDefinitionsLoaded[ID - 1];
 }
 
 /// \brief If we are loading a relocatable PCH file, and the filename is
@@ -1573,7 +1835,7 @@ void ASTReader::MaybeAddSystemRootToFilename(std::string &Filename) {
   if (!RelocatablePCH)
     return;
 
-  if (Filename.empty() || llvm::sys::Path(Filename).isAbsolute())
+  if (Filename.empty() || llvm::sys::path::is_absolute(Filename))
     return;
 
   if (isysroot == 0) {
@@ -1628,17 +1890,38 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         }
         break;
 
+      case DECL_UPDATES_BLOCK_ID:
+        if (Stream.SkipBlock()) {
+          Error("malformed block record in AST file");
+          return Failure;
+        }
+        break;
+
       case PREPROCESSOR_BLOCK_ID:
         F.MacroCursor = Stream;
         if (PP)
           PP->setExternalSource(this);
 
-        if (Stream.SkipBlock()) {
+        if (Stream.SkipBlock() ||
+            ReadBlockAbbrevs(F.MacroCursor, PREPROCESSOR_BLOCK_ID)) {
           Error("malformed block record in AST file");
           return Failure;
         }
+        F.MacroStartOffset = F.MacroCursor.GetCurrentBitNo();
         break;
 
+      case PREPROCESSOR_DETAIL_BLOCK_ID:
+        F.PreprocessorDetailCursor = Stream;
+        if (Stream.SkipBlock() ||
+            ReadBlockAbbrevs(F.PreprocessorDetailCursor, 
+                             PREPROCESSOR_DETAIL_BLOCK_ID)) {
+          Error("malformed preprocessor detail record in AST file");
+          return Failure;
+        }
+        F.PreprocessorDetailStartOffset
+          = F.PreprocessorDetailCursor.GetCurrentBitNo();
+        break;
+        
       case SOURCE_MANAGER_BLOCK_ID:
         switch (ReadSourceManagerBlock(F)) {
         case Success:
@@ -1667,7 +1950,7 @@ ASTReader::ReadASTBlock(PerFileData &F) {
     const char *BlobStart = 0;
     unsigned BlobLen = 0;
     switch ((ASTRecordTypes)Stream.ReadRecord(Code, Record,
-                                                   &BlobStart, &BlobLen)) {
+                                              &BlobStart, &BlobLen)) {
     default:  // Default behavior: ignore.
       break;
 
@@ -1698,8 +1981,8 @@ ASTReader::ReadASTBlock(PerFileData &F) {
         return IgnorePCH;
       }
 
-      // Load the chained file.
-      switch(ReadASTCore(llvm::StringRef(BlobStart, BlobLen))) {
+      // Load the chained file, which is always a PCH file.
+      switch(ReadASTCore(llvm::StringRef(BlobStart, BlobLen), PCH)) {
       case Failure: return Failure;
         // If we have to ignore the dependency, we'll have to ignore this too.
       case IgnorePCH: return IgnorePCH;
@@ -1729,10 +2012,11 @@ ASTReader::ReadASTBlock(PerFileData &F) {
     case TU_UPDATE_LEXICAL: {
       DeclContextInfo Info = {
         /* No visible information */ 0,
-        reinterpret_cast<const DeclID *>(BlobStart),
-        BlobLen / sizeof(DeclID)
+        reinterpret_cast<const KindDeclIDPair *>(BlobStart),
+        BlobLen / sizeof(KindDeclIDPair)
       };
-      DeclContextOffsets[Context->getTranslationUnitDecl()].push_back(Info);
+      DeclContextOffsets[Context ? Context->getTranslationUnitDecl() : 0]
+        .push_back(Info);
       break;
     }
 
@@ -1742,7 +2026,7 @@ ASTReader::ReadASTBlock(PerFileData &F) {
                         (const unsigned char *)BlobStart + Record[1],
                         (const unsigned char *)BlobStart,
                         ASTDeclContextNameLookupTrait(*this));
-      if (ID == 1) { // Is it the TU?
+      if (ID == 1 && Context) { // Is it the TU?
         DeclContextInfo Info = {
           Table, /* No lexical inforamtion */ 0, 0
         };
@@ -1776,7 +2060,7 @@ ASTReader::ReadASTBlock(PerFileData &F) {
           = ASTIdentifierLookupTable::Create(
                        (const unsigned char *)F.IdentifierTableData + Record[0],
                        (const unsigned char *)F.IdentifierTableData,
-                       ASTIdentifierLookupTrait(*this, F.Stream));
+                       ASTIdentifierLookupTrait(*this, F));
         if (PP)
           PP->getIdentifierTable().setExternalIdentifierLookup(this);
       }
@@ -1863,11 +2147,9 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       TotalNumMethodPoolEntries += Record[1];
       break;
 
-    case REFERENCED_SELECTOR_POOL: {
-      ReferencedSelectorsData.insert(ReferencedSelectorsData.end(),
-          Record.begin(), Record.end());
+    case REFERENCED_SELECTOR_POOL:
+      F.ReferencedSelectorsData.swap(Record);
       break;
-    }
 
     case PP_COUNTER_VALUE:
       if (!Record.empty() && Listener)
@@ -1877,28 +2159,26 @@ ASTReader::ReadASTBlock(PerFileData &F) {
     case SOURCE_LOCATION_OFFSETS:
       F.SLocOffsets = (const uint32_t *)BlobStart;
       F.LocalNumSLocEntries = Record[0];
-      // We cannot delay this until the entire chain is loaded, because then
-      // source location preloads would also have to be delayed.
-      // FIXME: Is there a reason not to do that?
-      TotalNumSLocEntries += F.LocalNumSLocEntries;
-      SourceMgr.PreallocateSLocEntries(this, TotalNumSLocEntries, Record[1]);
+      F.LocalSLocSize = Record[1];
       break;
 
     case SOURCE_LOCATION_PRELOADS:
-      for (unsigned I = 0, N = Record.size(); I != N; ++I) {
-        ASTReadResult Result = ReadSLocEntryRecord(Record[I]);
-        if (Result != Success)
-          return Result;
-      }
+      if (PreloadSLocEntries.empty())
+        PreloadSLocEntries.swap(Record);
+      else
+        PreloadSLocEntries.insert(PreloadSLocEntries.end(),
+            Record.begin(), Record.end());
       break;
 
     case STAT_CACHE: {
-      ASTStatCache *MyStatCache =
-        new ASTStatCache((const unsigned char *)BlobStart + Record[0],
-                         (const unsigned char *)BlobStart,
-                         NumStatHits, NumStatMisses);
-      FileMgr.addStatCache(MyStatCache);
-      F.StatCache = MyStatCache;
+      if (!DisableStatCache) {
+        ASTStatCache *MyStatCache =
+          new ASTStatCache((const unsigned char *)BlobStart + Record[0],
+                           (const unsigned char *)BlobStart,
+                           NumStatHits, NumStatMisses);
+        FileMgr.addStatCache(MyStatCache);
+        F.StatCache = MyStatCache;
+      }
       break;
     }
 
@@ -1926,12 +2206,7 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
 
     case PENDING_IMPLICIT_INSTANTIATIONS:
-      // Optimization for the first block.
-      if (PendingInstantiations.empty())
-        PendingInstantiations.swap(Record);
-      else
-        PendingInstantiations.insert(PendingInstantiations.end(),
-                                     Record.begin(), Record.end());
+      F.PendingInstantiations.swap(Record);
       break;
 
     case SEMA_DECL_REFS:
@@ -1947,6 +2222,12 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       MaybeAddSystemRootToFilename(OriginalFileName);
       break;
 
+    case ORIGINAL_PCH_DIR:
+      // The primary AST will be the last to get here, so it will be the one
+      // that's used.
+      OriginalDir.assign(BlobStart, BlobLen);
+      break;
+
     case VERSION_CONTROL_BRANCH_REVISION: {
       const std::string &CurBranch = getClangFullRepositoryVersion();
       llvm::StringRef ASTBranch(BlobStart, BlobLen);
@@ -1963,6 +2244,17 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       F.LocalNumMacroDefinitions = Record[1];
       break;
 
+    case DECL_UPDATE_OFFSETS: {
+      if (Record.size() % 2 != 0) {
+        Error("invalid DECL_UPDATE_OFFSETS block in AST file");
+        return Failure;
+      }
+      for (unsigned I = 0, N = Record.size(); I != N; I += 2)
+        DeclUpdateOffsets[static_cast<DeclID>(Record[I])]
+            .push_back(std::make_pair(&F, Record[I+1]));
+      break;
+    }
+
     case DECL_REPLACEMENTS: {
       if (Record.size() % 2 != 0) {
         Error("invalid DECL_REPLACEMENTS block in AST file");
@@ -1973,13 +2265,57 @@ ASTReader::ReadASTBlock(PerFileData &F) {
             std::make_pair(&F, Record[I+1]);
       break;
     }
-
-    case ADDITIONAL_TEMPLATE_SPECIALIZATIONS: {
-      AdditionalTemplateSpecializations &ATS =
-          AdditionalTemplateSpecializationsPending[Record[0]];
-      ATS.insert(ATS.end(), Record.begin()+1, Record.end());
+        
+    case CXX_BASE_SPECIFIER_OFFSETS: {
+      if (F.LocalNumCXXBaseSpecifiers != 0) {
+        Error("duplicate CXX_BASE_SPECIFIER_OFFSETS record in AST file");
+        return Failure;
+      }
+      
+      F.LocalNumCXXBaseSpecifiers = Record[0];
+      F.CXXBaseSpecifiersOffsets = (const uint32_t *)BlobStart;
       break;
     }
+
+    case DIAG_PRAGMA_MAPPINGS:
+      if (Record.size() % 2 != 0) {
+        Error("invalid DIAG_USER_MAPPINGS block in AST file");
+        return Failure;
+      }
+      if (PragmaDiagMappings.empty())
+        PragmaDiagMappings.swap(Record);
+      else
+        PragmaDiagMappings.insert(PragmaDiagMappings.end(),
+                                Record.begin(), Record.end());
+      break;
+        
+    case CUDA_SPECIAL_DECL_REFS:
+      // Later tables overwrite earlier ones.
+      CUDASpecialDeclRefs.swap(Record);
+      break;
+
+    case HEADER_SEARCH_TABLE:
+      F.HeaderFileInfoTableData = BlobStart;
+      F.LocalNumHeaderFileInfos = Record[1];
+      if (Record[0]) {
+        F.HeaderFileInfoTable
+          = HeaderFileInfoLookupTable::Create(
+                   (const unsigned char *)F.HeaderFileInfoTableData + Record[0],
+                   (const unsigned char *)F.HeaderFileInfoTableData);
+        if (PP)
+          PP->getHeaderSearchInfo().SetExternalSource(this);
+      }
+      break;
+
+    case FP_PRAGMA_OPTIONS:
+      // Later tables overwrite earlier ones.
+      FPPragmaOptions.swap(Record);
+      break;
+
+    case OPENCL_EXTENSIONS:
+      // Later tables overwrite earlier ones.
+      OpenCLExtensions.swap(Record);
+      break;
     }
     First = false;
   }
@@ -1987,8 +2323,9 @@ ASTReader::ReadASTBlock(PerFileData &F) {
   return Failure;
 }
 
-ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
-  switch(ReadASTCore(FileName)) {
+ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
+                                            ASTFileType Type) {
+  switch(ReadASTCore(FileName, Type)) {
   case Failure: return Failure;
   case IgnorePCH: return IgnorePCH;
   case Success: break;
@@ -1996,11 +2333,13 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
 
   // Here comes stuff that we only do once the entire chain is loaded.
 
-  // Allocate space for loaded identifiers, decls and types.
+  // Allocate space for loaded slocentries, identifiers, decls and types.
   unsigned TotalNumIdentifiers = 0, TotalNumTypes = 0, TotalNumDecls = 0,
            TotalNumPreallocatedPreprocessingEntities = 0, TotalNumMacroDefs = 0,
            TotalNumSelectors = 0;
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    TotalNumSLocEntries += Chain[I]->LocalNumSLocEntries;
+    NextSLocOffset += Chain[I]->LocalSLocSize;
     TotalNumIdentifiers += Chain[I]->LocalNumIdentifiers;
     TotalNumTypes += Chain[I]->LocalNumTypes;
     TotalNumDecls += Chain[I]->LocalNumDecls;
@@ -2009,6 +2348,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
     TotalNumMacroDefs += Chain[I]->LocalNumMacroDefinitions;
     TotalNumSelectors += Chain[I]->LocalNumSelectors;
   }
+  SourceMgr.PreallocateSLocEntries(this, TotalNumSLocEntries, NextSLocOffset);
   IdentifiersLoaded.resize(TotalNumIdentifiers);
   TypesLoaded.resize(TotalNumTypes);
   DeclsLoaded.resize(TotalNumDecls);
@@ -2024,6 +2364,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
     }
   }
   SelectorsLoaded.resize(TotalNumSelectors);
+  // Preload SLocEntries.
+  for (unsigned I = 0, N = PreloadSLocEntries.size(); I != N; ++I) {
+    ASTReadResult Result = ReadSLocEntryRecord(PreloadSLocEntries[I]);
+    if (Result != Success)
+      return Result;
+  }
 
   // Check the predefines buffers.
   if (!DisableValidation && CheckPredefinesBuffers())
@@ -2058,7 +2404,7 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
       for (unsigned I = 0, N = Identifiers.size(); I != N; ++I) {
         IdentifierInfo *II = Identifiers[I];
         // Look in the on-disk hash tables for an entry for this identifier
-        ASTIdentifierLookupTrait Info(*this, Chain[J]->Stream, II);
+        ASTIdentifierLookupTrait Info(*this, *Chain[J], II);
         std::pair<const char*,unsigned> Key(II->getNameStart(),II->getLength());
         ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key, &Info);
         if (Pos == IdTable->end())
@@ -2074,21 +2420,54 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
   if (Context)
     InitializeContext(*Context);
 
+  if (DeserializationListener)
+    DeserializationListener->ReaderInitialized(this);
+
+  // If this AST file is a precompiled preamble, then set the main file ID of 
+  // the source manager to the file source file from which the preamble was
+  // built. This is the only valid way to use a precompiled preamble.
+  if (Type == Preamble) {
+    SourceLocation Loc
+      = SourceMgr.getLocation(FileMgr.getFile(getOriginalSourceFile()), 1, 1);
+    if (Loc.isValid()) {
+      std::pair<FileID, unsigned> Decomposed = SourceMgr.getDecomposedLoc(Loc);
+      SourceMgr.SetPreambleFileID(Decomposed.first);
+    }
+  }
+  
   return Success;
 }
 
-ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName) {
-  Chain.push_back(new PerFileData());
+ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
+                                                ASTFileType Type) {
+  PerFileData *Prev = Chain.empty() ? 0 : Chain.back();
+  Chain.push_back(new PerFileData(Type));
   PerFileData &F = *Chain.back();
+  if (Prev)
+    Prev->NextInSource = &F;
+  else
+    FirstInSource = &F;
+  F.Loaders.push_back(Prev);
 
   // Set the AST file name.
   F.FileName = FileName;
 
+  if (FileName != "-") {
+    CurrentDir = llvm::sys::path::parent_path(FileName);
+    if (CurrentDir.empty()) CurrentDir = ".";
+  }
+
   // Open the AST file.
   //
   // FIXME: This shouldn't be here, we should just take a raw_ostream.
   std::string ErrStr;
-  F.Buffer.reset(llvm::MemoryBuffer::getFileOrSTDIN(FileName, &ErrStr));
+  llvm::error_code ec;
+  if (FileName == "-") {
+    ec = llvm::MemoryBuffer::getSTDIN(F.Buffer);
+    if (ec)
+      ErrStr = ec.message();
+  } else
+    F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr));
   if (!F.Buffer) {
     Error(ErrStr.c_str());
     return IgnorePCH;
@@ -2185,6 +2564,18 @@ void ASTReader::InitializeContext(ASTContext &Ctx) {
   PP->getIdentifierTable().setExternalIdentifierLookup(this);
   PP->getHeaderSearchInfo().SetExternalLookup(this);
   PP->setExternalSource(this);
+  PP->getHeaderSearchInfo().SetExternalSource(this);
+  
+  // If we have an update block for the TU waiting, we have to add it before
+  // deserializing the decl.
+  DeclContextOffsetsMap::iterator DCU = DeclContextOffsets.find(0);
+  if (DCU != DeclContextOffsets.end()) {
+    // Insertion could invalidate map, so grab vector.
+    DeclContextInfos T;
+    T.swap(DCU->second);
+    DeclContextOffsets.erase(DCU);
+    DeclContextOffsets[Ctx.getTranslationUnitDecl()].swap(T);
+  }
 
   // Load the translation unit declaration
   GetTranslationUnitDecl();
@@ -2273,17 +2664,27 @@ void ASTReader::InitializeContext(ASTContext &Ctx) {
 
   if (SpecialTypes[SPECIAL_TYPE_INT128_INSTALLED])
     Context->setInt128Installed();
+
+  ReadPragmaDiagnosticMappings(Context->getDiagnostics());
+
+  // If there were any CUDA special declarations, deserialize them.
+  if (!CUDASpecialDeclRefs.empty()) {
+    assert(CUDASpecialDeclRefs.size() == 1 && "More decl refs than expected!");
+    Context->setcudaConfigureCallDecl(
+                           cast<FunctionDecl>(GetDecl(CUDASpecialDeclRefs[0])));
+  }
 }
 
 /// \brief Retrieve the name of the original source file name
 /// directly from the AST file, without actually loading the AST
 /// file.
 std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
+                                             FileManager &FileMgr,
                                              Diagnostic &Diags) {
   // Open the AST file.
   std::string ErrStr;
   llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
-  Buffer.reset(llvm::MemoryBuffer::getFile(ASTFileName.c_str(), &ErrStr));
+  Buffer.reset(FileMgr.getBufferForFile(ASTFileName, &ErrStr));
   if (!Buffer) {
     Diags.Report(diag::err_fe_unable_to_read_pch_file) << ErrStr;
     return std::string();
@@ -2390,6 +2791,8 @@ bool ASTReader::ParseLanguageOptions(
     PARSE_LANGOPT(ObjC2);
     PARSE_LANGOPT(ObjCNonFragileABI);
     PARSE_LANGOPT(ObjCNonFragileABI2);
+    PARSE_LANGOPT(AppleKext);
+    PARSE_LANGOPT(ObjCDefaultSynthProperties);
     PARSE_LANGOPT(NoConstantCFStrings);
     PARSE_LANGOPT(PascalStrings);
     PARSE_LANGOPT(WritableStrings);
@@ -2397,6 +2800,8 @@ bool ASTReader::ParseLanguageOptions(
     PARSE_LANGOPT(AltiVec);
     PARSE_LANGOPT(Exceptions);
     PARSE_LANGOPT(SjLjExceptions);
+    PARSE_LANGOPT(ObjCExceptions);
+    PARSE_LANGOPT(MSBitfields);
     PARSE_LANGOPT(NeXTRuntime);
     PARSE_LANGOPT(Freestanding);
     PARSE_LANGOPT(NoBuiltin);
@@ -2417,13 +2822,16 @@ bool ASTReader::ParseLanguageOptions(
     PARSE_LANGOPT(AccessControl);
     PARSE_LANGOPT(CharIsSigned);
     PARSE_LANGOPT(ShortWChar);
+    PARSE_LANGOPT(ShortEnums);
     LangOpts.setGCMode((LangOptions::GCMode)Record[Idx++]);
-    LangOpts.setVisibilityMode((LangOptions::VisibilityMode)Record[Idx++]);
+    LangOpts.setVisibilityMode((Visibility)Record[Idx++]);
     LangOpts.setStackProtectorMode((LangOptions::StackProtectorMode)
                                    Record[Idx++]);
     PARSE_LANGOPT(InstantiationDepth);
     PARSE_LANGOPT(OpenCL);
+    PARSE_LANGOPT(CUDA);
     PARSE_LANGOPT(CatchUndefined);
+    PARSE_LANGOPT(DefaultFPContract);
     // FIXME: Missing ElideConstructors?!
   #undef PARSE_LANGOPT
 
@@ -2434,7 +2842,83 @@ bool ASTReader::ParseLanguageOptions(
 }
 
 void ASTReader::ReadPreprocessedEntities() {
-  ReadDefinedMacros();
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    PerFileData &F = *Chain[I];
+    if (!F.PreprocessorDetailCursor.getBitStreamReader())
+      continue;
+
+    SavedStreamPosition SavedPosition(F.PreprocessorDetailCursor);  
+    F.PreprocessorDetailCursor.JumpToBit(F.PreprocessorDetailStartOffset);
+    while (LoadPreprocessedEntity(F)) { }
+  }
+}
+
+PreprocessedEntity *ASTReader::ReadPreprocessedEntityAtOffset(uint64_t Offset) {
+  PerFileData *F = 0;  
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (Offset < Chain[I]->SizeInBits) {
+      F = Chain[I];
+      break;
+    }
+    
+    Offset -= Chain[I]->SizeInBits;
+  }
+  
+  if (!F) {
+    Error("Malformed preprocessed entity offset");
+    return 0;
+  }
+
+  // Keep track of where we are in the stream, then jump back there
+  // after reading this entity.
+  SavedStreamPosition SavedPosition(F->PreprocessorDetailCursor);  
+  F->PreprocessorDetailCursor.JumpToBit(Offset);
+  return LoadPreprocessedEntity(*F);
+}
+
+HeaderFileInfo ASTReader::GetHeaderFileInfo(const FileEntry *FE) {
+  HeaderFileInfoTrait Trait(FE->getName());
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    PerFileData &F = *Chain[I];
+    HeaderFileInfoLookupTable *Table
+      = static_cast<HeaderFileInfoLookupTable *>(F.HeaderFileInfoTable);
+    if (!Table)
+      continue;
+    
+    // Look in the on-disk hash table for an entry for this file name.
+    HeaderFileInfoLookupTable::iterator Pos = Table->find(FE->getName(), 
+                                                          &Trait);
+    if (Pos == Table->end())
+      continue;
+
+    HeaderFileInfo HFI = *Pos;
+    if (Listener)
+      Listener->ReadHeaderFileInfo(HFI, FE->getUID());
+
+    return HFI;
+  }
+  
+  return HeaderFileInfo();
+}
+
+void ASTReader::ReadPragmaDiagnosticMappings(Diagnostic &Diag) {
+  unsigned Idx = 0;
+  while (Idx < PragmaDiagMappings.size()) {
+    SourceLocation
+      Loc = SourceLocation::getFromRawEncoding(PragmaDiagMappings[Idx++]);
+    while (1) {
+      assert(Idx < PragmaDiagMappings.size() &&
+             "Invalid data, didn't find '-1' marking end of diag/map pairs");
+      if (Idx >= PragmaDiagMappings.size())
+        break; // Something is messed up but at least avoid infinite loop in
+               // release build.
+      unsigned DiagID = PragmaDiagMappings[Idx++];
+      if (DiagID == (unsigned)-1)
+        break; // no more diag/map pairs for this location.
+      diag::Mapping Map = (diag::Mapping)PragmaDiagMappings[Idx++];
+      Diag.setDiagnosticMapping(DiagID, Map, Loc);
+    }
+  }
 }
 
 /// \brief Get the correct cursor and offset for loading a type.
@@ -2447,7 +2931,7 @@ ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
     Index -= F->LocalNumTypes;
   }
   assert(F && F->LocalNumTypes > Index && "Broken chain");
-  return RecordLocation(&F->DeclsCursor, F->TypeOffsets[Index]);
+  return RecordLocation(F, F->TypeOffsets[Index]);
 }
 
 /// \brief Read and return the type with the given index..
@@ -2458,7 +2942,7 @@ ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
 /// IDs.
 QualType ASTReader::ReadTypeRecord(unsigned Index) {
   RecordLocation Loc = TypeCursorForIndex(Index);
-  llvm::BitstreamCursor &DeclsCursor = *Loc.first;
+  llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor;
 
   // Keep track of where we are in the stream, then jump back there
   // after reading this type.
@@ -2469,7 +2953,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
   // Note that we are loading a type record.
   Deserializing AType(this);
 
-  DeclsCursor.JumpToBit(Loc.second);
+  DeclsCursor.JumpToBit(Loc.Offset);
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
   switch ((TypeCode)DeclsCursor.ReadRecord(Code, Record)) {
@@ -2535,6 +3019,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     }
     QualType PointeeType = GetType(Record[0]);
     QualType ClassType = GetType(Record[1]);
+    if (PointeeType.isNull() || ClassType.isNull())
+      return QualType();
+    
     return Context->getMemberPointerType(PointeeType, ClassType.getTypePtr());
   }
 
@@ -2559,9 +3046,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     QualType ElementType = GetType(Record[0]);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
-    SourceLocation LBLoc = SourceLocation::getFromRawEncoding(Record[3]);
-    SourceLocation RBLoc = SourceLocation::getFromRawEncoding(Record[4]);
-    return Context->getVariableArrayType(ElementType, ReadExpr(DeclsCursor),
+    SourceLocation LBLoc = ReadSourceLocation(*Loc.F, Record[3]);
+    SourceLocation RBLoc = ReadSourceLocation(*Loc.F, Record[4]);
+    return Context->getVariableArrayType(ElementType, ReadExpr(*Loc.F),
                                          ASM, IndexTypeQuals,
                                          SourceRange(LBLoc, RBLoc));
   }
@@ -2574,9 +3061,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
 
     QualType ElementType = GetType(Record[0]);
     unsigned NumElements = Record[1];
-    unsigned AltiVecSpec = Record[2];
+    unsigned VecKind = Record[2];
     return Context->getVectorType(ElementType, NumElements,
-                                  (VectorType::AltiVecSpecific)AltiVecSpec);
+                                  (VectorType::VectorKind)VecKind);
   }
 
   case TYPE_EXT_VECTOR: {
@@ -2602,28 +3089,30 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
 
   case TYPE_FUNCTION_PROTO: {
     QualType ResultType = GetType(Record[0]);
-    bool NoReturn = Record[1];
-    unsigned RegParm = Record[2];
-    CallingConv CallConv = (CallingConv)Record[3];
+
+    FunctionProtoType::ExtProtoInfo EPI;
+    EPI.ExtInfo = FunctionType::ExtInfo(/*noreturn*/ Record[1],
+                                        /*regparm*/ Record[2],
+                                        static_cast<CallingConv>(Record[3]));
+
     unsigned Idx = 4;
     unsigned NumParams = Record[Idx++];
     llvm::SmallVector<QualType, 16> ParamTypes;
     for (unsigned I = 0; I != NumParams; ++I)
       ParamTypes.push_back(GetType(Record[Idx++]));
-    bool isVariadic = Record[Idx++];
-    unsigned Quals = Record[Idx++];
-    bool hasExceptionSpec = Record[Idx++];
-    bool hasAnyExceptionSpec = Record[Idx++];
-    unsigned NumExceptions = Record[Idx++];
+
+    EPI.Variadic = Record[Idx++];
+    EPI.TypeQuals = Record[Idx++];
+    EPI.RefQualifier = static_cast<RefQualifierKind>(Record[Idx++]);
+    EPI.HasExceptionSpec = Record[Idx++];
+    EPI.HasAnyExceptionSpec = Record[Idx++];
+    EPI.NumExceptions = Record[Idx++];
     llvm::SmallVector<QualType, 2> Exceptions;
-    for (unsigned I = 0; I != NumExceptions; ++I)
+    for (unsigned I = 0; I != EPI.NumExceptions; ++I)
       Exceptions.push_back(GetType(Record[Idx++]));
+    EPI.Exceptions = Exceptions.data();
     return Context->getFunctionType(ResultType, ParamTypes.data(), NumParams,
-                                    isVariadic, Quals, hasExceptionSpec,
-                                    hasAnyExceptionSpec, NumExceptions,
-                                    Exceptions.data(),
-                                    FunctionType::ExtInfo(NoReturn, RegParm,
-                                                          CallConv));
+                                    EPI);
   }
 
   case TYPE_UNRESOLVED_USING:
@@ -2637,11 +3126,13 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     }
     TypedefDecl *Decl = cast<TypedefDecl>(GetDecl(Record[0]));
     QualType Canonical = GetType(Record[1]);
+    if (!Canonical.isNull())
+      Canonical = Context->getCanonicalType(Canonical);
     return Context->getTypedefType(Decl, Canonical);
   }
 
   case TYPE_TYPEOF_EXPR:
-    return Context->getTypeOfExprType(ReadExpr(DeclsCursor));
+    return Context->getTypeOfExprType(ReadExpr(*Loc.F));
 
   case TYPE_TYPEOF: {
     if (Record.size() != 1) {
@@ -2653,7 +3144,10 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
   }
 
   case TYPE_DECLTYPE:
-    return Context->getDecltypeType(ReadExpr(DeclsCursor));
+    return Context->getDecltypeType(ReadExpr(*Loc.F));
+
+  case TYPE_AUTO:
+    return Context->getAutoType(GetType(Record[0]));
 
   case TYPE_RECORD: {
     if (Record.size() != 2) {
@@ -2662,7 +3156,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     }
     bool IsDependent = Record[0];
     QualType T = Context->getRecordType(cast<RecordDecl>(GetDecl(Record[1])));
-    T->Dependent = IsDependent;
+    const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
 
@@ -2673,10 +3167,44 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     }
     bool IsDependent = Record[0];
     QualType T = Context->getEnumType(cast<EnumDecl>(GetDecl(Record[1])));
-    T->Dependent = IsDependent;
+    const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
 
+  case TYPE_ATTRIBUTED: {
+    if (Record.size() != 3) {
+      Error("incorrect encoding of attributed type");
+      return QualType();
+    }
+    QualType modifiedType = GetType(Record[0]);
+    QualType equivalentType = GetType(Record[1]);
+    AttributedType::Kind kind = static_cast<AttributedType::Kind>(Record[2]);
+    return Context->getAttributedType(kind, modifiedType, equivalentType);
+  }
+
+  case TYPE_PAREN: {
+    if (Record.size() != 1) {
+      Error("incorrect encoding of paren type");
+      return QualType();
+    }
+    QualType InnerType = GetType(Record[0]);
+    return Context->getParenType(InnerType);
+  }
+
+  case TYPE_PACK_EXPANSION: {
+    if (Record.size() != 2) {
+      Error("incorrect encoding of pack expansion type");
+      return QualType();
+    }
+    QualType Pattern = GetType(Record[0]);
+    if (Pattern.isNull())
+      return QualType();
+    llvm::Optional<unsigned> NumExpansions;
+    if (Record[1])
+      NumExpansions = Record[1] - 1;
+    return Context->getPackExpansionType(Pattern, NumExpansions);
+  }
+
   case TYPE_ELABORATED: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
@@ -2698,7 +3226,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     llvm::SmallVector<ObjCProtocolDecl*, 4> Protos;
     for (unsigned I = 0; I != NumProtos; ++I)
       Protos.push_back(cast<ObjCProtocolDecl>(GetDecl(Record[Idx++])));
-    return Context->getObjCObjectType(Base, Protos.data(), NumProtos);    
+    return Context->getObjCObjectType(Base, Protos.data(), NumProtos);
   }
 
   case TYPE_OBJC_OBJECT_POINTER: {
@@ -2716,6 +3244,15 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
                                             Replacement);
   }
 
+  case TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK: {
+    unsigned Idx = 0;
+    QualType Parm = GetType(Record[Idx++]);
+    TemplateArgument ArgPack = ReadTemplateArgument(*Loc.F, Record, Idx);
+    return Context->getSubstTemplateTypeParmPackType(
+                                               cast<TemplateTypeParmType>(Parm),
+                                                     ArgPack);
+  }
+
   case TYPE_INJECTED_CLASS_NAME: {
     CXXRecordDecl *D = cast<CXXRecordDecl>(GetDecl(Record[0]));
     QualType TST = GetType(Record[1]); // probably derivable
@@ -2724,7 +3261,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     return
       QualType(new (*Context, TypeAlignment) InjectedClassNameType(D, TST), 0);
   }
-  
+
   case TYPE_TEMPLATE_TYPE_PARM: {
     unsigned Idx = 0;
     unsigned Depth = Record[Idx++];
@@ -2733,16 +3270,18 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     IdentifierInfo *Name = GetIdentifierInfo(Record, Idx);
     return Context->getTemplateTypeParmType(Depth, Index, Pack, Name);
   }
-  
+
   case TYPE_DEPENDENT_NAME: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
     const IdentifierInfo *Name = this->GetIdentifierInfo(Record, Idx);
     QualType Canon = GetType(Record[Idx++]);
+    if (!Canon.isNull())
+      Canon = Context->getCanonicalType(Canon);
     return Context->getDependentNameType(Keyword, NNS, Name, Canon);
   }
-  
+
   case TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
@@ -2752,11 +3291,11 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     llvm::SmallVector<TemplateArgument, 8> Args;
     Args.reserve(NumArgs);
     while (NumArgs--)
-      Args.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
+      Args.push_back(ReadTemplateArgument(*Loc.F, Record, Idx));
     return Context->getDependentTemplateSpecializationType(Keyword, NNS, Name,
                                                       Args.size(), Args.data());
   }
-  
+
   case TYPE_DEPENDENT_SIZED_ARRAY: {
     unsigned Idx = 0;
 
@@ -2767,8 +3306,8 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     unsigned IndexTypeQuals = Record[Idx++];
 
     // DependentSizedArrayType
-    Expr *NumElts = ReadExpr(DeclsCursor);
-    SourceRange Brackets = ReadSourceRange(Record, Idx);
+    Expr *NumElts = ReadExpr(*Loc.F);
+    SourceRange Brackets = ReadSourceRange(*Loc.F, Record, Idx);
 
     return Context->getDependentSizedArrayType(ElementType, NumElts, ASM,
                                                IndexTypeQuals, Brackets);
@@ -2777,9 +3316,9 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
   case TYPE_TEMPLATE_SPECIALIZATION: {
     unsigned Idx = 0;
     bool IsDependent = Record[Idx++];
-    TemplateName Name = ReadTemplateName(Record, Idx);
+    TemplateName Name = ReadTemplateName(*Loc.F, Record, Idx);
     llvm::SmallVector<TemplateArgument, 8> Args;
-    ReadTemplateArgumentList(Args, DeclsCursor, Record, Idx);
+    ReadTemplateArgumentList(Args, *Loc.F, Record, Idx);
     QualType Canon = GetType(Record[Idx++]);
     QualType T;
     if (Canon.isNull())
@@ -2788,7 +3327,7 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
     else
       T = Context->getTemplateSpecializationType(Name, Args.data(),
                                                  Args.size(), Canon);
-    T->Dependent = IsDependent;
+    const_cast<Type*>(T.getTypePtr())->setDependent(IsDependent);
     return T;
   }
   }
@@ -2796,18 +3335,23 @@ QualType ASTReader::ReadTypeRecord(unsigned Index) {
   return QualType();
 }
 
-namespace {
-
-class TypeLocReader : public TypeLocVisitor<TypeLocReader> {
+class clang::TypeLocReader : public TypeLocVisitor<TypeLocReader> {
   ASTReader &Reader;
+  ASTReader::PerFileData &F;
   llvm::BitstreamCursor &DeclsCursor;
   const ASTReader::RecordData &Record;
   unsigned &Idx;
 
+  SourceLocation ReadSourceLocation(const ASTReader::RecordData &R,
+                                    unsigned &I) {
+    return Reader.ReadSourceLocation(F, R, I);
+  }
+
 public:
-  TypeLocReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
+  TypeLocReader(ASTReader &Reader, ASTReader::PerFileData &F,
                 const ASTReader::RecordData &Record, unsigned &Idx)
-    : Reader(Reader), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
+    : Reader(Reader), F(F), DeclsCursor(F.DeclsCursor), Record(Record), Idx(Idx)
+  { }
 
   // We want compile-time assurance that we've enumerated all of
   // these, so unfortunately we have to declare them first, then
@@ -2821,13 +3365,11 @@ public:
   void VisitArrayTypeLoc(ArrayTypeLoc);
 };
 
-}
-
 void TypeLocReader::VisitQualifiedTypeLoc(QualifiedTypeLoc TL) {
   // nothing to do
 }
 void TypeLocReader::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
-  TL.setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setBuiltinLoc(ReadSourceLocation(Record, Idx));
   if (TL.needsExtraLocalData()) {
     TL.setWrittenTypeSpec(static_cast<DeclSpec::TST>(Record[Idx++]));
     TL.setWrittenSignSpec(static_cast<DeclSpec::TSS>(Record[Idx++]));
@@ -2836,28 +3378,28 @@ void TypeLocReader::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
   }
 }
 void TypeLocReader::VisitComplexTypeLoc(ComplexTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitPointerTypeLoc(PointerTypeLoc TL) {
-  TL.setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setStarLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitBlockPointerTypeLoc(BlockPointerTypeLoc TL) {
-  TL.setCaretLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setCaretLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitLValueReferenceTypeLoc(LValueReferenceTypeLoc TL) {
-  TL.setAmpLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setAmpLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitRValueReferenceTypeLoc(RValueReferenceTypeLoc TL) {
-  TL.setAmpAmpLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setAmpAmpLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitMemberPointerTypeLoc(MemberPointerTypeLoc TL) {
-  TL.setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setStarLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitArrayTypeLoc(ArrayTypeLoc TL) {
-  TL.setLBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setLBracketLoc(ReadSourceLocation(Record, Idx));
+  TL.setRBracketLoc(ReadSourceLocation(Record, Idx));
   if (Record[Idx++])
-    TL.setSizeExpr(Reader.ReadExpr(DeclsCursor));
+    TL.setSizeExpr(Reader.ReadExpr(F));
   else
     TL.setSizeExpr(0);
 }
@@ -2876,17 +3418,18 @@ void TypeLocReader::VisitDependentSizedArrayTypeLoc(
 }
 void TypeLocReader::VisitDependentSizedExtVectorTypeLoc(
                                         DependentSizedExtVectorTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitVectorTypeLoc(VectorTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
-  TL.setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setTrailingReturn(Record[Idx++]);
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
     TL.setArg(i, cast_or_null<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
   }
@@ -2898,87 +3441,119 @@ void TypeLocReader::VisitFunctionNoProtoTypeLoc(FunctionNoProtoTypeLoc TL) {
   VisitFunctionTypeLoc(TL);
 }
 void TypeLocReader::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitTypeOfExprTypeLoc(TypeOfExprTypeLoc TL) {
-  TL.setTypeofLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setTypeofLoc(ReadSourceLocation(Record, Idx));
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitTypeOfTypeLoc(TypeOfTypeLoc TL) {
-  TL.setTypeofLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setUnderlyingTInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+  TL.setTypeofLoc(ReadSourceLocation(Record, Idx));
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setUnderlyingTInfo(Reader.GetTypeSourceInfo(F, Record, Idx));
 }
 void TypeLocReader::VisitDecltypeTypeLoc(DecltypeTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
+}
+void TypeLocReader::VisitAutoTypeLoc(AutoTypeLoc TL) {
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitRecordTypeLoc(RecordTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitEnumTypeLoc(EnumTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
+}
+void TypeLocReader::VisitAttributedTypeLoc(AttributedTypeLoc TL) {
+  TL.setAttrNameLoc(ReadSourceLocation(Record, Idx));
+  if (TL.hasAttrOperand()) {
+    SourceRange range;
+    range.setBegin(ReadSourceLocation(Record, Idx));
+    range.setEnd(ReadSourceLocation(Record, Idx));
+    TL.setAttrOperandParensRange(range);
+  }
+  if (TL.hasAttrExprOperand()) {
+    if (Record[Idx++])
+      TL.setAttrExprOperand(Reader.ReadExpr(F));
+    else
+      TL.setAttrExprOperand(0);
+  } else if (TL.hasAttrEnumOperand())
+    TL.setAttrEnumOperandLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitSubstTemplateTypeParmTypeLoc(
                                             SubstTemplateTypeParmTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
+}
+void TypeLocReader::VisitSubstTemplateTypeParmPackTypeLoc(
+                                          SubstTemplateTypeParmPackTypeLoc TL) {
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitTemplateSpecializationTypeLoc(
                                            TemplateSpecializationTypeLoc TL) {
-  TL.setTemplateNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setLAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setTemplateNameLoc(ReadSourceLocation(Record, Idx));
+  TL.setLAngleLoc(ReadSourceLocation(Record, Idx));
+  TL.setRAngleLoc(ReadSourceLocation(Record, Idx));
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     TL.setArgLocInfo(i,
-        Reader.GetTemplateArgumentLocInfo(TL.getTypePtr()->getArg(i).getKind(),
-                                          DeclsCursor, Record, Idx));
+        Reader.GetTemplateArgumentLocInfo(F,
+                                          TL.getTypePtr()->getArg(i).getKind(),
+                                          Record, Idx));
+}
+void TypeLocReader::VisitParenTypeLoc(ParenTypeLoc TL) {
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-  TL.setKeywordLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setQualifierRange(Reader.ReadSourceRange(Record, Idx));
+  TL.setKeywordLoc(ReadSourceLocation(Record, Idx));
+  TL.setQualifierRange(Reader.ReadSourceRange(F, Record, Idx));
 }
 void TypeLocReader::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
-  TL.setKeywordLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setKeywordLoc(ReadSourceLocation(Record, Idx));
+  TL.setQualifierRange(Reader.ReadSourceRange(F, Record, Idx));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitDependentTemplateSpecializationTypeLoc(
        DependentTemplateSpecializationTypeLoc TL) {
-  TL.setKeywordLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setLAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setKeywordLoc(ReadSourceLocation(Record, Idx));
+  TL.setQualifierRange(Reader.ReadSourceRange(F, Record, Idx));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
+  TL.setLAngleLoc(ReadSourceLocation(Record, Idx));
+  TL.setRAngleLoc(ReadSourceLocation(Record, Idx));
   for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I)
     TL.setArgLocInfo(I,
-        Reader.GetTemplateArgumentLocInfo(TL.getTypePtr()->getArg(I).getKind(),
-                                          DeclsCursor, Record, Idx));
+        Reader.GetTemplateArgumentLocInfo(F,
+                                          TL.getTypePtr()->getArg(I).getKind(),
+                                          Record, Idx));
+}
+void TypeLocReader::VisitPackExpansionTypeLoc(PackExpansionTypeLoc TL) {
+  TL.setEllipsisLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL) {
-  TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setNameLoc(ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitObjCObjectTypeLoc(ObjCObjectTypeLoc TL) {
   TL.setHasBaseTypeAsWritten(Record[Idx++]);
-  TL.setLAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setRAngleLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setLAngleLoc(ReadSourceLocation(Record, Idx));
+  TL.setRAngleLoc(ReadSourceLocation(Record, Idx));
   for (unsigned i = 0, e = TL.getNumProtocols(); i != e; ++i)
-    TL.setProtocolLoc(i, SourceLocation::getFromRawEncoding(Record[Idx++]));
+    TL.setProtocolLoc(i, ReadSourceLocation(Record, Idx));
 }
 void TypeLocReader::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
-  TL.setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TL.setStarLoc(ReadSourceLocation(Record, Idx));
 }
 
-TypeSourceInfo *ASTReader::GetTypeSourceInfo(llvm::BitstreamCursor &DeclsCursor,
+TypeSourceInfo *ASTReader::GetTypeSourceInfo(PerFileData &F,
                                              const RecordData &Record,
                                              unsigned &Idx) {
   QualType InfoTy = GetType(Record[Idx++]);
@@ -2986,7 +3561,7 @@ TypeSourceInfo *ASTReader::GetTypeSourceInfo(llvm::BitstreamCursor &DeclsCursor,
     return 0;
 
   TypeSourceInfo *TInfo = getContext()->CreateTypeSourceInfo(InfoTy);
-  TypeLocReader TLR(*this, DeclsCursor, Record, Idx);
+  TypeLocReader TLR(*this, F, Record, Idx);
   for (TypeLoc TL = TInfo->getTypeLoc(); !TL.isNull(); TL = TL.getNextTypeLoc())
     TLR.Visit(TL);
   return TInfo;
@@ -3043,6 +3618,9 @@ QualType ASTReader::GetType(TypeID ID) {
   assert(Index < TypesLoaded.size() && "Type index out-of-range");
   if (TypesLoaded[Index].isNull()) {
     TypesLoaded[Index] = ReadTypeRecord(Index);
+    if (TypesLoaded[Index].isNull())
+      return QualType();
+
     TypesLoaded[Index]->setFromAST();
     TypeIdxs[TypesLoaded[Index]] = TypeIdx::fromTypeID(ID);
     if (DeserializationListener)
@@ -3074,20 +3652,36 @@ TypeIdx ASTReader::GetTypeIdx(QualType T) const {
   return I->second;
 }
 
+unsigned ASTReader::getTotalNumCXXBaseSpecifiers() const {
+  unsigned Result = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
+    Result += Chain[I]->LocalNumCXXBaseSpecifiers;
+  
+  return Result;
+}
+
 TemplateArgumentLocInfo
-ASTReader::GetTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
-                                      llvm::BitstreamCursor &DeclsCursor,
+ASTReader::GetTemplateArgumentLocInfo(PerFileData &F,
+                                      TemplateArgument::ArgKind Kind,
                                       const RecordData &Record,
                                       unsigned &Index) {
   switch (Kind) {
   case TemplateArgument::Expression:
-    return ReadExpr(DeclsCursor);
+    return ReadExpr(F);
   case TemplateArgument::Type:
-    return GetTypeSourceInfo(DeclsCursor, Record, Index);
+    return GetTypeSourceInfo(F, Record, Index);
   case TemplateArgument::Template: {
-    SourceRange QualifierRange = ReadSourceRange(Record, Index);
-    SourceLocation TemplateNameLoc = ReadSourceLocation(Record, Index);
-    return TemplateArgumentLocInfo(QualifierRange, TemplateNameLoc);
+    SourceRange QualifierRange = ReadSourceRange(F, Record, Index);
+    SourceLocation TemplateNameLoc = ReadSourceLocation(F, Record, Index);
+    return TemplateArgumentLocInfo(QualifierRange, TemplateNameLoc,
+                                   SourceLocation());
+  }
+  case TemplateArgument::TemplateExpansion: {
+    SourceRange QualifierRange = ReadSourceRange(F, Record, Index);
+    SourceLocation TemplateNameLoc = ReadSourceLocation(F, Record, Index);
+    SourceLocation EllipsisLoc = ReadSourceLocation(F, Record, Index);
+    return TemplateArgumentLocInfo(QualifierRange, TemplateNameLoc, 
+                                   EllipsisLoc);
   }
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
@@ -3100,16 +3694,15 @@ ASTReader::GetTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
 }
 
 TemplateArgumentLoc
-ASTReader::ReadTemplateArgumentLoc(llvm::BitstreamCursor &DeclsCursor,
+ASTReader::ReadTemplateArgumentLoc(PerFileData &F,
                                    const RecordData &Record, unsigned &Index) {
-  TemplateArgument Arg = ReadTemplateArgument(DeclsCursor, Record, Index);
+  TemplateArgument Arg = ReadTemplateArgument(F, Record, Index);
 
   if (Arg.getKind() == TemplateArgument::Expression) {
     if (Record[Index++]) // bool InfoHasSameExpr.
       return TemplateArgumentLoc(Arg, TemplateArgumentLocInfo(Arg.getAsExpr()));
   }
-  return TemplateArgumentLoc(Arg, GetTemplateArgumentLocInfo(Arg.getKind(),
-                                                             DeclsCursor,
+  return TemplateArgumentLoc(Arg, GetTemplateArgumentLocInfo(F, Arg.getKind(),
                                                              Record, Index));
 }
 
@@ -3117,6 +3710,63 @@ Decl *ASTReader::GetExternalDecl(uint32_t ID) {
   return GetDecl(ID);
 }
 
+uint64_t 
+ASTReader::GetCXXBaseSpecifiersOffset(serialization::CXXBaseSpecifiersID ID) {
+  if (ID == 0)
+    return 0;
+  
+  --ID;
+  uint64_t Offset = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (ID < Chain[I]->LocalNumCXXBaseSpecifiers)
+      return Offset + Chain[I]->CXXBaseSpecifiersOffsets[ID];
+    
+    ID -= Chain[I]->LocalNumCXXBaseSpecifiers;
+    Offset += Chain[I]->SizeInBits;
+  }
+  
+  assert(false && "CXXBaseSpecifiers not found");
+  return 0;
+}
+
+CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
+  // Figure out which AST file contains this offset.
+  PerFileData *F = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    if (Offset < Chain[I]->SizeInBits) {
+      F = Chain[I];
+      break;
+    }
+    
+    Offset -= Chain[I]->SizeInBits;
+  }
+  
+  if (!F) {
+    Error("Malformed AST file: C++ base specifiers at impossible offset");
+    return 0;
+  }
+  
+  llvm::BitstreamCursor &Cursor = F->DeclsCursor;
+  SavedStreamPosition SavedPosition(Cursor);
+  Cursor.JumpToBit(Offset);
+  ReadingKindTracker ReadingKind(Read_Decl, *this);
+  RecordData Record;
+  unsigned Code = Cursor.ReadCode();
+  unsigned RecCode = Cursor.ReadRecord(Code, Record);
+  if (RecCode != DECL_CXX_BASE_SPECIFIERS) {
+    Error("Malformed AST file: missing C++ base specifiers");
+    return 0;
+  }
+
+  unsigned Idx = 0;
+  unsigned NumBases = Record[Idx++];
+  void *Mem = Context->Allocate(sizeof(CXXBaseSpecifier) * NumBases);
+  CXXBaseSpecifier *Bases = new (Mem) CXXBaseSpecifier [NumBases];
+  for (unsigned I = 0; I != NumBases; ++I)
+    Bases[I] = ReadCXXBaseSpecifier(*F, Record, Idx);
+  return Bases;
+}
+
 TranslationUnitDecl *ASTReader::GetTranslationUnitDecl() {
   if (!DeclsLoaded[0]) {
     ReadDeclRecord(0, 1);
@@ -3152,6 +3802,9 @@ Decl *ASTReader::GetDecl(DeclID ID) {
 /// source each time it is called, and is meant to be used via a
 /// LazyOffsetPtr (which is used by Decls for the body of functions, etc).
 Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
+  // Switch case IDs are per Decl.
+  ClearSwitchCaseIDs();
+
   // Offset here is a global offset across the entire chain.
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
     PerFileData &F = *Chain[N - I - 1];
@@ -3159,7 +3812,7 @@ Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
       // Since we know that this statement is part of a decl, make sure to use
       // the decl cursor to read it.
       F.DeclsCursor.JumpToBit(Offset);
-      return ReadStmtFromStream(F.DeclsCursor);
+      return ReadStmtFromStream(F);
     }
     Offset -= F.SizeInBits;
   }
@@ -3167,13 +3820,16 @@ Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
 }
 
 bool ASTReader::FindExternalLexicalDecls(const DeclContext *DC,
+                                         bool (*isKindWeWant)(Decl::Kind),
                                          llvm::SmallVectorImpl<Decl*> &Decls) {
   assert(DC->hasExternalLexicalStorage() &&
          "DeclContext has no lexical decls in storage");
 
   // There might be lexical decls in multiple parts of the chain, for the TU
   // at least.
-  DeclContextInfos &Infos = DeclContextOffsets[DC];
+  // DeclContextOffsets might reallocate as we load additional decls below,
+  // so make a copy of the vector.
+  DeclContextInfos Infos = DeclContextOffsets[DC];
   for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
        I != E; ++I) {
     // IDs can be 0 if this context doesn't contain declarations.
@@ -3181,10 +3837,15 @@ bool ASTReader::FindExternalLexicalDecls(const DeclContext *DC,
       continue;
 
     // Load all of the declaration IDs
-    for (const DeclID *ID = I->LexicalDecls,
-                           *IDE = ID + I->NumLexicalDecls;
-         ID != IDE; ++ID)
-      Decls.push_back(GetDecl(*ID));
+    for (const KindDeclIDPair *ID = I->LexicalDecls,
+                              *IDE = ID + I->NumLexicalDecls; ID != IDE; ++ID) {
+      if (isKindWeWant && !isKindWeWant((Decl::Kind)ID->first))
+        continue;
+
+      Decl *D = GetDecl(ID->second);
+      assert(D && "Null decl in lexical decls");
+      Decls.push_back(D);
+    }
   }
 
   ++NumLexicalDeclContextsRead;
@@ -3355,11 +4016,11 @@ void ASTReader::InitializeSema(Sema &S) {
 
   // Makes sure any declarations that were deserialized "too early"
   // still get added to the identifier's declaration chains.
-  if (SemaObj->TUScope) {
-    for (unsigned I = 0, N = PreloadedDecls.size(); I != N; ++I) {
+  for (unsigned I = 0, N = PreloadedDecls.size(); I != N; ++I) {
+    if (SemaObj->TUScope)
       SemaObj->TUScope->AddDecl(PreloadedDecls[I]);
-      SemaObj->IdResolver.AddDecl(PreloadedDecls[I]);
-    }
+
+    SemaObj->IdResolver.AddDecl(PreloadedDecls[I]);
   }
   PreloadedDecls.clear();
 
@@ -3377,21 +4038,6 @@ void ASTReader::InitializeSema(Sema &S) {
     SemaObj->UnusedFileScopedDecls.push_back(D);
   }
 
-  // If there were any weak undeclared identifiers, deserialize them and add to
-  // Sema's list of weak undeclared identifiers.
-  if (!WeakUndeclaredIdentifiers.empty()) {
-    unsigned Idx = 0;
-    for (unsigned I = 0, N = WeakUndeclaredIdentifiers[Idx++]; I != N; ++I) {
-      IdentifierInfo *WeakId = GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
-      IdentifierInfo *AliasId=GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
-      SourceLocation Loc = ReadSourceLocation(WeakUndeclaredIdentifiers, Idx);
-      bool Used = WeakUndeclaredIdentifiers[Idx++];
-      Sema::WeakInfo WI(AliasId, Loc);
-      WI.setUsed(Used);
-      SemaObj->WeakUndeclaredIdentifiers.insert(std::make_pair(WeakId, WI));
-    }
-  }
-
   // If there were any locally-scoped external declarations,
   // deserialize them and add them to Sema's table of locally-scoped
   // external declarations.
@@ -3409,34 +4055,12 @@ void ASTReader::InitializeSema(Sema &S) {
   // FIXME: Do VTable uses and dynamic classes deserialize too much ?
   // Can we cut them down before writing them ?
 
-  // If there were any VTable uses, deserialize the information and add it
-  // to Sema's vector and map of VTable uses.
-  if (!VTableUses.empty()) {
-    unsigned Idx = 0;
-    for (unsigned I = 0, N = VTableUses[Idx++]; I != N; ++I) {
-      CXXRecordDecl *Class = cast<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
-      SourceLocation Loc = ReadSourceLocation(VTableUses, Idx);
-      bool DefinitionRequired = VTableUses[Idx++];
-      SemaObj->VTableUses.push_back(std::make_pair(Class, Loc));
-      SemaObj->VTablesUsed[Class] = DefinitionRequired;
-    }
-  }
-
   // If there were any dynamic classes declarations, deserialize them
   // and add them to Sema's vector of such declarations.
   for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I)
     SemaObj->DynamicClasses.push_back(
                                cast<CXXRecordDecl>(GetDecl(DynamicClasses[I])));
 
-  // If there were any pending implicit instantiations, deserialize them
-  // and add them to Sema's queue of such instantiations.
-  assert(PendingInstantiations.size() % 2 == 0 && "Expected pairs of entries");
-  for (unsigned Idx = 0, N = PendingInstantiations.size(); Idx < N;) {
-    ValueDecl *D=cast<ValueDecl>(GetDecl(PendingInstantiations[Idx++]));
-    SourceLocation Loc = ReadSourceLocation(PendingInstantiations, Idx);
-    SemaObj->PendingInstantiations.push_back(std::make_pair(D, Loc));
-  }
-
   // Load the offsets of the declarations that Sema references.
   // They will be lazily deserialized when needed.
   if (!SemaDeclRefs.empty()) {
@@ -3445,18 +4069,76 @@ void ASTReader::InitializeSema(Sema &S) {
     SemaObj->StdBadAlloc = SemaDeclRefs[1];
   }
 
-  // If there are @selector references added them to its pool. This is for
-  // implementation of -Wselector.
-  if (!ReferencedSelectorsData.empty()) {
-    unsigned int DataSize = ReferencedSelectorsData.size()-1;
-    unsigned I = 0;
-    while (I < DataSize) {
-      Selector Sel = DecodeSelector(ReferencedSelectorsData[I++]);
-      SourceLocation SelLoc = 
-        SourceLocation::getFromRawEncoding(ReferencedSelectorsData[I++]);
-      SemaObj->ReferencedSelectors.insert(std::make_pair(Sel, SelLoc));
+  for (PerFileData *F = FirstInSource; F; F = F->NextInSource) {
+
+    // If there are @selector references added them to its pool. This is for
+    // implementation of -Wselector.
+    if (!F->ReferencedSelectorsData.empty()) {
+      unsigned int DataSize = F->ReferencedSelectorsData.size()-1;
+      unsigned I = 0;
+      while (I < DataSize) {
+        Selector Sel = DecodeSelector(F->ReferencedSelectorsData[I++]);
+        SourceLocation SelLoc = ReadSourceLocation(
+                                    *F, F->ReferencedSelectorsData, I);
+        SemaObj->ReferencedSelectors.insert(std::make_pair(Sel, SelLoc));
+      }
+    }
+
+    // If there were any pending implicit instantiations, deserialize them
+    // and add them to Sema's queue of such instantiations.
+    assert(F->PendingInstantiations.size() % 2 == 0 &&
+           "Expected pairs of entries");
+    for (unsigned Idx = 0, N = F->PendingInstantiations.size(); Idx < N;) {
+      ValueDecl *D=cast<ValueDecl>(GetDecl(F->PendingInstantiations[Idx++]));
+      SourceLocation Loc = ReadSourceLocation(*F, F->PendingInstantiations,Idx);
+      SemaObj->PendingInstantiations.push_back(std::make_pair(D, Loc));
+    }
+  }
+
+  // The two special data sets below always come from the most recent PCH,
+  // which is at the front of the chain.
+  PerFileData &F = *Chain.front();
+
+  // If there were any weak undeclared identifiers, deserialize them and add to
+  // Sema's list of weak undeclared identifiers.
+  if (!WeakUndeclaredIdentifiers.empty()) {
+    unsigned Idx = 0;
+    for (unsigned I = 0, N = WeakUndeclaredIdentifiers[Idx++]; I != N; ++I) {
+      IdentifierInfo *WeakId = GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
+      IdentifierInfo *AliasId= GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
+      SourceLocation Loc = ReadSourceLocation(F, WeakUndeclaredIdentifiers,Idx);
+      bool Used = WeakUndeclaredIdentifiers[Idx++];
+      Sema::WeakInfo WI(AliasId, Loc);
+      WI.setUsed(Used);
+      SemaObj->WeakUndeclaredIdentifiers.insert(std::make_pair(WeakId, WI));
+    }
+  }
+
+  // If there were any VTable uses, deserialize the information and add it
+  // to Sema's vector and map of VTable uses.
+  if (!VTableUses.empty()) {
+    unsigned Idx = 0;
+    for (unsigned I = 0, N = VTableUses[Idx++]; I != N; ++I) {
+      CXXRecordDecl *Class = cast<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
+      SourceLocation Loc = ReadSourceLocation(F, VTableUses, Idx);
+      bool DefinitionRequired = VTableUses[Idx++];
+      SemaObj->VTableUses.push_back(std::make_pair(Class, Loc));
+      SemaObj->VTablesUsed[Class] = DefinitionRequired;
     }
   }
+
+  if (!FPPragmaOptions.empty()) {
+    assert(FPPragmaOptions.size() == 1 && "Wrong number of FP_PRAGMA_OPTIONS");
+    SemaObj->FPFeatures.fp_contract = FPPragmaOptions[0];
+  }
+
+  if (!OpenCLExtensions.empty()) {
+    unsigned I = 0;
+#define OPENCLEXT(nm)  SemaObj->OpenCLFeatures.nm = OpenCLExtensions[I++];
+#include "clang/Basic/OpenCLExtensions.def"
+
+    assert(OpenCLExtensions.size() == I && "Wrong number of OPENCL_EXTENSIONS");
+  }
 }
 
 IdentifierInfo* ASTReader::get(const char *NameStart, const char *NameEnd) {
@@ -3480,6 +4162,64 @@ IdentifierInfo* ASTReader::get(const char *NameStart, const char *NameEnd) {
   return 0;
 }
 
+namespace clang {
+  /// \brief An identifier-lookup iterator that enumerates all of the
+  /// identifiers stored within a set of AST files.
+  class ASTIdentifierIterator : public IdentifierIterator {
+    /// \brief The AST reader whose identifiers are being enumerated.
+    const ASTReader &Reader;
+
+    /// \brief The current index into the chain of AST files stored in
+    /// the AST reader.
+    unsigned Index;
+
+    /// \brief The current position within the identifier lookup table
+    /// of the current AST file.
+    ASTIdentifierLookupTable::key_iterator Current;
+
+    /// \brief The end position within the identifier lookup table of
+    /// the current AST file.
+    ASTIdentifierLookupTable::key_iterator End;
+
+  public:
+    explicit ASTIdentifierIterator(const ASTReader &Reader);
+
+    virtual llvm::StringRef Next();
+  };
+}
+
+ASTIdentifierIterator::ASTIdentifierIterator(const ASTReader &Reader)
+  : Reader(Reader), Index(Reader.Chain.size() - 1) {
+  ASTIdentifierLookupTable *IdTable
+    = (ASTIdentifierLookupTable *)Reader.Chain[Index]->IdentifierLookupTable;
+  Current = IdTable->key_begin();
+  End = IdTable->key_end();
+}
+
+llvm::StringRef ASTIdentifierIterator::Next() {
+  while (Current == End) {
+    // If we have exhausted all of our AST files, we're done.
+    if (Index == 0)
+      return llvm::StringRef();
+
+    --Index;
+    ASTIdentifierLookupTable *IdTable
+      = (ASTIdentifierLookupTable *)Reader.Chain[Index]->IdentifierLookupTable;
+    Current = IdTable->key_begin();
+    End = IdTable->key_end();
+  }
+
+  // We have any identifiers remaining in the current AST file; return
+  // the next one.
+  std::pair<const char*, unsigned> Key = *Current;
+  ++Current;
+  return llvm::StringRef(Key.first, Key.second);
+}
+
+IdentifierIterator *ASTReader::getIdentifiers() const {
+  return new ASTIdentifierIterator(*this);
+}
+
 std::pair<ObjCMethodList, ObjCMethodList>
 ASTReader::ReadMethodPool(Selector Sel) {
   // Find this selector in a hash table. We want to find the most recent entry.
@@ -3545,8 +4285,7 @@ ASTReader::SetGloballyVisibleDecls(IdentifierInfo *II,
     PendingIdentifierInfos.push_back(PendingIdentifierInfo());
     PendingIdentifierInfo &PII = PendingIdentifierInfos.back();
     PII.II = II;
-    for (unsigned I = 0, N = DeclIDs.size(); I != N; ++I)
-      PII.DeclIDs.push_back(DeclIDs[I]);
+    PII.DeclIDs.append(DeclIDs.begin(), DeclIDs.end());
     return;
   }
 
@@ -3558,8 +4297,8 @@ ASTReader::SetGloballyVisibleDecls(IdentifierInfo *II,
         // and add it to the declaration chain for this identifier, so
         // that (unqualified) name lookup will find it.
         SemaObj->TUScope->AddDecl(D);
-        SemaObj->IdResolver.AddDeclToIdentifierChain(II, D);
       }
+      SemaObj->IdResolver.AddDeclToIdentifierChain(II, D);
     } else {
       // Queue this declaration so that it will be added to the
       // translation unit scope and identifier's declaration chain
@@ -3644,7 +4383,7 @@ Selector ASTReader::DecodeSelector(unsigned ID) {
   return SelectorsLoaded[ID - 1];
 }
 
-Selector ASTReader::GetExternalSelector(uint32_t ID) { 
+Selector ASTReader::GetExternalSelector(uint32_t ID) {
   return DecodeSelector(ID);
 }
 
@@ -3693,9 +4432,65 @@ ASTReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
   return DeclarationName();
 }
 
+void ASTReader::ReadDeclarationNameLoc(PerFileData &F,
+                                       DeclarationNameLoc &DNLoc,
+                                       DeclarationName Name,
+                                      const RecordData &Record, unsigned &Idx) {
+  switch (Name.getNameKind()) {
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    DNLoc.NamedType.TInfo = GetTypeSourceInfo(F, Record, Idx);
+    break;
+
+  case DeclarationName::CXXOperatorName:
+    DNLoc.CXXOperatorName.BeginOpNameLoc
+        = ReadSourceLocation(F, Record, Idx).getRawEncoding();
+    DNLoc.CXXOperatorName.EndOpNameLoc
+        = ReadSourceLocation(F, Record, Idx).getRawEncoding();
+    break;
+
+  case DeclarationName::CXXLiteralOperatorName:
+    DNLoc.CXXLiteralOperatorName.OpNameLoc
+        = ReadSourceLocation(F, Record, Idx).getRawEncoding();
+    break;
+
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXUsingDirective:
+    break;
+  }
+}
+
+void ASTReader::ReadDeclarationNameInfo(PerFileData &F,
+                                        DeclarationNameInfo &NameInfo,
+                                      const RecordData &Record, unsigned &Idx) {
+  NameInfo.setName(ReadDeclarationName(Record, Idx));
+  NameInfo.setLoc(ReadSourceLocation(F, Record, Idx));
+  DeclarationNameLoc DNLoc;
+  ReadDeclarationNameLoc(F, DNLoc, NameInfo.getName(), Record, Idx);
+  NameInfo.setInfo(DNLoc);
+}
+
+void ASTReader::ReadQualifierInfo(PerFileData &F, QualifierInfo &Info,
+                                  const RecordData &Record, unsigned &Idx) {
+  Info.NNS = ReadNestedNameSpecifier(Record, Idx);
+  Info.NNSRange = ReadSourceRange(F, Record, Idx);
+  unsigned NumTPLists = Record[Idx++];
+  Info.NumTemplParamLists = NumTPLists;
+  if (NumTPLists) {
+    Info.TemplParamLists = new (*Context) TemplateParameterList*[NumTPLists];
+    for (unsigned i=0; i != NumTPLists; ++i)
+      Info.TemplParamLists[i] = ReadTemplateParameterList(F, Record, Idx);
+  }
+}
+
 TemplateName
-ASTReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
-  TemplateName::NameKind Kind = (TemplateName::NameKind)Record[Idx++]; 
+ASTReader::ReadTemplateName(PerFileData &F, const RecordData &Record, 
+                            unsigned &Idx) {
+  TemplateName::NameKind Kind = (TemplateName::NameKind)Record[Idx++];
   switch (Kind) {
   case TemplateName::Template:
     return TemplateName(cast_or_null<TemplateDecl>(GetDecl(Record[Idx++])));
@@ -3708,14 +4503,14 @@ ASTReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
 
     return Context->getOverloadedTemplateName(Decls.begin(), Decls.end());
   }
-    
+
   case TemplateName::QualifiedTemplate: {
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
     bool hasTemplKeyword = Record[Idx++];
     TemplateDecl *Template = cast<TemplateDecl>(GetDecl(Record[Idx++]));
     return Context->getQualifiedTemplateName(NNS, hasTemplKeyword, Template);
   }
-    
+
   case TemplateName::DependentTemplate: {
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
     if (Record[Idx++])  // isIdentifier
@@ -3724,16 +4519,30 @@ ASTReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
     return Context->getDependentTemplateName(NNS,
                                          (OverloadedOperatorKind)Record[Idx++]);
   }
+      
+  case TemplateName::SubstTemplateTemplateParmPack: {
+    TemplateTemplateParmDecl *Param 
+      = cast_or_null<TemplateTemplateParmDecl>(GetDecl(Record[Idx++]));
+    if (!Param)
+      return TemplateName();
+    
+    TemplateArgument ArgPack = ReadTemplateArgument(F, Record, Idx);
+    if (ArgPack.getKind() != TemplateArgument::Pack)
+      return TemplateName();
+    
+    return Context->getSubstTemplateTemplateParmPack(Param, ArgPack);
+  }
   }
-  
+
   assert(0 && "Unhandled template name kind!");
   return TemplateName();
 }
 
 TemplateArgument
-ASTReader::ReadTemplateArgument(llvm::BitstreamCursor &DeclsCursor,
+ASTReader::ReadTemplateArgument(PerFileData &F,
                                 const RecordData &Record, unsigned &Idx) {
-  switch ((TemplateArgument::ArgKind)Record[Idx++]) {
+  TemplateArgument::ArgKind Kind = (TemplateArgument::ArgKind)Record[Idx++];
+  switch (Kind) {
   case TemplateArgument::Null:
     return TemplateArgument();
   case TemplateArgument::Type:
@@ -3745,39 +4554,44 @@ ASTReader::ReadTemplateArgument(llvm::BitstreamCursor &DeclsCursor,
     QualType T = GetType(Record[Idx++]);
     return TemplateArgument(Value, T);
   }
-  case TemplateArgument::Template:
-    return TemplateArgument(ReadTemplateName(Record, Idx));
+  case TemplateArgument::Template: 
+    return TemplateArgument(ReadTemplateName(F, Record, Idx));
+  case TemplateArgument::TemplateExpansion: {
+    TemplateName Name = ReadTemplateName(F, Record, Idx);
+    llvm::Optional<unsigned> NumTemplateExpansions;
+    if (unsigned NumExpansions = Record[Idx++])
+      NumTemplateExpansions = NumExpansions - 1;
+    return TemplateArgument(Name, NumTemplateExpansions);
+  }
   case TemplateArgument::Expression:
-    return TemplateArgument(ReadExpr(DeclsCursor));
+    return TemplateArgument(ReadExpr(F));
   case TemplateArgument::Pack: {
     unsigned NumArgs = Record[Idx++];
-    llvm::SmallVector<TemplateArgument, 8> Args;
-    Args.reserve(NumArgs);
-    while (NumArgs--)
-      Args.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
-    TemplateArgument TemplArg;
-    TemplArg.setArgumentPack(Args.data(), Args.size(), /*CopyArgs=*/true);
-    return TemplArg;
+    TemplateArgument *Args = new (*Context) TemplateArgument[NumArgs];
+    for (unsigned I = 0; I != NumArgs; ++I)
+      Args[I] = ReadTemplateArgument(F, Record, Idx);
+    return TemplateArgument(Args, NumArgs);
   }
   }
-  
+
   assert(0 && "Unhandled template argument kind!");
   return TemplateArgument();
 }
 
 TemplateParameterList *
-ASTReader::ReadTemplateParameterList(const RecordData &Record, unsigned &Idx) {
-  SourceLocation TemplateLoc = ReadSourceLocation(Record, Idx);
-  SourceLocation LAngleLoc = ReadSourceLocation(Record, Idx);
-  SourceLocation RAngleLoc = ReadSourceLocation(Record, Idx);
+ASTReader::ReadTemplateParameterList(PerFileData &F,
+                                     const RecordData &Record, unsigned &Idx) {
+  SourceLocation TemplateLoc = ReadSourceLocation(F, Record, Idx);
+  SourceLocation LAngleLoc = ReadSourceLocation(F, Record, Idx);
+  SourceLocation RAngleLoc = ReadSourceLocation(F, Record, Idx);
 
   unsigned NumParams = Record[Idx++];
   llvm::SmallVector<NamedDecl *, 16> Params;
   Params.reserve(NumParams);
   while (NumParams--)
     Params.push_back(cast<NamedDecl>(GetDecl(Record[Idx++])));
-    
-  TemplateParameterList* TemplateParams = 
+
+  TemplateParameterList* TemplateParams =
     TemplateParameterList::Create(*Context, TemplateLoc, LAngleLoc,
                                   Params.data(), Params.size(), RAngleLoc);
   return TemplateParams;
@@ -3786,12 +4600,12 @@ ASTReader::ReadTemplateParameterList(const RecordData &Record, unsigned &Idx) {
 void
 ASTReader::
 ReadTemplateArgumentList(llvm::SmallVector<TemplateArgument, 8> &TemplArgs,
-                         llvm::BitstreamCursor &DeclsCursor,
-                         const RecordData &Record, unsigned &Idx) {
+                         PerFileData &F, const RecordData &Record,
+                         unsigned &Idx) {
   unsigned NumTemplateArgs = Record[Idx++];
   TemplArgs.reserve(NumTemplateArgs);
   while (NumTemplateArgs--)
-    TemplArgs.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
+    TemplArgs.push_back(ReadTemplateArgument(F, Record, Idx));
 }
 
 /// \brief Read a UnresolvedSet structure.
@@ -3806,45 +4620,52 @@ void ASTReader::ReadUnresolvedSet(UnresolvedSetImpl &Set,
 }
 
 CXXBaseSpecifier
-ASTReader::ReadCXXBaseSpecifier(llvm::BitstreamCursor &DeclsCursor,
+ASTReader::ReadCXXBaseSpecifier(PerFileData &F,
                                 const RecordData &Record, unsigned &Idx) {
   bool isVirtual = static_cast<bool>(Record[Idx++]);
   bool isBaseOfClass = static_cast<bool>(Record[Idx++]);
   AccessSpecifier AS = static_cast<AccessSpecifier>(Record[Idx++]);
-  TypeSourceInfo *TInfo = GetTypeSourceInfo(DeclsCursor, Record, Idx);
-  SourceRange Range = ReadSourceRange(Record, Idx);
-  return CXXBaseSpecifier(Range, isVirtual, isBaseOfClass, AS, TInfo);
+  bool inheritConstructors = static_cast<bool>(Record[Idx++]);
+  TypeSourceInfo *TInfo = GetTypeSourceInfo(F, Record, Idx);
+  SourceRange Range = ReadSourceRange(F, Record, Idx);
+  SourceLocation EllipsisLoc = ReadSourceLocation(F, Record, Idx);
+  CXXBaseSpecifier Result(Range, isVirtual, isBaseOfClass, AS, TInfo, 
+                          EllipsisLoc);
+  Result.setInheritConstructors(inheritConstructors);
+  return Result;
 }
 
-std::pair<CXXBaseOrMemberInitializer **, unsigned>
-ASTReader::ReadCXXBaseOrMemberInitializers(llvm::BitstreamCursor &Cursor,
-                                           const RecordData &Record,
-                                           unsigned &Idx) {
-  CXXBaseOrMemberInitializer **BaseOrMemberInitializers = 0;
+std::pair<CXXCtorInitializer **, unsigned>
+ASTReader::ReadCXXCtorInitializers(PerFileData &F, const RecordData &Record,
+                                   unsigned &Idx) {
+  CXXCtorInitializer **CtorInitializers = 0;
   unsigned NumInitializers = Record[Idx++];
   if (NumInitializers) {
     ASTContext &C = *getContext();
 
-    BaseOrMemberInitializers
-        = new (C) CXXBaseOrMemberInitializer*[NumInitializers];
+    CtorInitializers
+        = new (C) CXXCtorInitializer*[NumInitializers];
     for (unsigned i=0; i != NumInitializers; ++i) {
       TypeSourceInfo *BaseClassInfo = 0;
       bool IsBaseVirtual = false;
       FieldDecl *Member = 0;
-  
+      IndirectFieldDecl *IndirectMember = 0;
+
       bool IsBaseInitializer = Record[Idx++];
       if (IsBaseInitializer) {
-        BaseClassInfo = GetTypeSourceInfo(Cursor, Record, Idx);
+        BaseClassInfo = GetTypeSourceInfo(F, Record, Idx);
         IsBaseVirtual = Record[Idx++];
       } else {
-        Member = cast<FieldDecl>(GetDecl(Record[Idx++]));
+        bool IsIndirectMemberInitializer = Record[Idx++];
+        if (IsIndirectMemberInitializer)
+          IndirectMember = cast<IndirectFieldDecl>(GetDecl(Record[Idx++]));
+        else
+          Member = cast<FieldDecl>(GetDecl(Record[Idx++]));
       }
-      SourceLocation MemberLoc = ReadSourceLocation(Record, Idx);
-      Expr *Init = ReadExpr(Cursor);
-      FieldDecl *AnonUnionMember
-          = cast_or_null<FieldDecl>(GetDecl(Record[Idx++]));
-      SourceLocation LParenLoc = ReadSourceLocation(Record, Idx);
-      SourceLocation RParenLoc = ReadSourceLocation(Record, Idx);
+      SourceLocation MemberOrEllipsisLoc = ReadSourceLocation(F, Record, Idx);
+      Expr *Init = ReadExpr(F);
+      SourceLocation LParenLoc = ReadSourceLocation(F, Record, Idx);
+      SourceLocation RParenLoc = ReadSourceLocation(F, Record, Idx);
       bool IsWritten = Record[Idx++];
       unsigned SourceOrderOrNumArrayIndices;
       llvm::SmallVector<VarDecl *, 8> Indices;
@@ -3856,28 +4677,33 @@ ASTReader::ReadCXXBaseOrMemberInitializers(llvm::BitstreamCursor &Cursor,
         for (unsigned i=0; i != SourceOrderOrNumArrayIndices; ++i)
           Indices.push_back(cast<VarDecl>(GetDecl(Record[Idx++])));
       }
-      
-      CXXBaseOrMemberInitializer *BOMInit;
+
+      CXXCtorInitializer *BOMInit;
       if (IsBaseInitializer) {
-        BOMInit = new (C) CXXBaseOrMemberInitializer(C, BaseClassInfo,
-                                                     IsBaseVirtual, LParenLoc,
-                                                     Init, RParenLoc);
+        BOMInit = new (C) CXXCtorInitializer(C, BaseClassInfo, IsBaseVirtual,
+                                             LParenLoc, Init, RParenLoc,
+                                             MemberOrEllipsisLoc);
       } else if (IsWritten) {
-        BOMInit = new (C) CXXBaseOrMemberInitializer(C, Member, MemberLoc,
-                                                     LParenLoc, Init, RParenLoc);
+        if (Member)
+          BOMInit = new (C) CXXCtorInitializer(C, Member, MemberOrEllipsisLoc,
+                                               LParenLoc, Init, RParenLoc);
+        else 
+          BOMInit = new (C) CXXCtorInitializer(C, IndirectMember,
+                                               MemberOrEllipsisLoc, LParenLoc,
+                                               Init, RParenLoc);
       } else {
-        BOMInit = CXXBaseOrMemberInitializer::Create(C, Member, MemberLoc,
-                                                     LParenLoc, Init, RParenLoc,
-                                                     Indices.data(),
-                                                     Indices.size());
+        BOMInit = CXXCtorInitializer::Create(C, Member, MemberOrEllipsisLoc,
+                                             LParenLoc, Init, RParenLoc,
+                                             Indices.data(), Indices.size());
       }
 
-      BOMInit->setAnonUnionMember(AnonUnionMember);
-      BaseOrMemberInitializers[i] = BOMInit;
+      if (IsWritten)
+        BOMInit->setSourceOrder(SourceOrderOrNumArrayIndices);
+      CtorInitializers[i] = BOMInit;
     }
   }
 
-  return std::make_pair(BaseOrMemberInitializers, NumInitializers);
+  return std::make_pair(CtorInitializers, NumInitializers);
 }
 
 NestedNameSpecifier *
@@ -3902,7 +4728,10 @@ ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
 
     case NestedNameSpecifier::TypeSpec:
     case NestedNameSpecifier::TypeSpecWithTemplate: {
-      Type *T = GetType(Record[Idx++]).getTypePtr();
+      const Type *T = GetType(Record[Idx++]).getTypePtrOrNull();
+      if (!T)
+        return 0;
+      
       bool Template = Record[Idx++];
       NNS = NestedNameSpecifier::Create(*Context, Prev, Template, T);
       break;
@@ -3920,9 +4749,10 @@ ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
 }
 
 SourceRange
-ASTReader::ReadSourceRange(const RecordData &Record, unsigned &Idx) {
-  SourceLocation beg = SourceLocation::getFromRawEncoding(Record[Idx++]);
-  SourceLocation end = SourceLocation::getFromRawEncoding(Record[Idx++]);
+ASTReader::ReadSourceRange(PerFileData &F, const RecordData &Record,
+                           unsigned &Idx) {
+  SourceLocation beg = ReadSourceLocation(F, Record, Idx);
+  SourceLocation end = ReadSourceLocation(F, Record, Idx);
   return SourceRange(beg, end);
 }
 
@@ -3965,7 +4795,7 @@ DiagnosticBuilder ASTReader::Diag(unsigned DiagID) {
 }
 
 DiagnosticBuilder ASTReader::Diag(SourceLocation Loc, unsigned DiagID) {
-  return Diags.Report(FullSourceLoc(Loc, SourceMgr), DiagID);
+  return Diags.Report(Loc, DiagID);
 }
 
 /// \brief Retrieve the identifier table associated with the
@@ -3988,70 +4818,8 @@ SwitchCase *ASTReader::getSwitchCaseWithID(unsigned ID) {
   return SwitchCaseStmts[ID];
 }
 
-/// \brief Record that the given label statement has been
-/// deserialized and has the given ID.
-void ASTReader::RecordLabelStmt(LabelStmt *S, unsigned ID) {
-  assert(LabelStmts.find(ID) == LabelStmts.end() &&
-         "Deserialized label twice");
-  LabelStmts[ID] = S;
-
-  // If we've already seen any goto statements that point to this
-  // label, resolve them now.
-  typedef std::multimap<unsigned, GotoStmt *>::iterator GotoIter;
-  std::pair<GotoIter, GotoIter> Gotos = UnresolvedGotoStmts.equal_range(ID);
-  for (GotoIter Goto = Gotos.first; Goto != Gotos.second; ++Goto)
-    Goto->second->setLabel(S);
-  UnresolvedGotoStmts.erase(Gotos.first, Gotos.second);
-
-  // If we've already seen any address-label statements that point to
-  // this label, resolve them now.
-  typedef std::multimap<unsigned, AddrLabelExpr *>::iterator AddrLabelIter;
-  std::pair<AddrLabelIter, AddrLabelIter> AddrLabels
-    = UnresolvedAddrLabelExprs.equal_range(ID);
-  for (AddrLabelIter AddrLabel = AddrLabels.first;
-       AddrLabel != AddrLabels.second; ++AddrLabel)
-    AddrLabel->second->setLabel(S);
-  UnresolvedAddrLabelExprs.erase(AddrLabels.first, AddrLabels.second);
-}
-
-/// \brief Set the label of the given statement to the label
-/// identified by ID.
-///
-/// Depending on the order in which the label and other statements
-/// referencing that label occur, this operation may complete
-/// immediately (updating the statement) or it may queue the
-/// statement to be back-patched later.
-void ASTReader::SetLabelOf(GotoStmt *S, unsigned ID) {
-  std::map<unsigned, LabelStmt *>::iterator Label = LabelStmts.find(ID);
-  if (Label != LabelStmts.end()) {
-    // We've already seen this label, so set the label of the goto and
-    // we're done.
-    S->setLabel(Label->second);
-  } else {
-    // We haven't seen this label yet, so add this goto to the set of
-    // unresolved goto statements.
-    UnresolvedGotoStmts.insert(std::make_pair(ID, S));
-  }
-}
-
-/// \brief Set the label of the given expression to the label
-/// identified by ID.
-///
-/// Depending on the order in which the label and other statements
-/// referencing that label occur, this operation may complete
-/// immediately (updating the statement) or it may queue the
-/// statement to be back-patched later.
-void ASTReader::SetLabelOf(AddrLabelExpr *S, unsigned ID) {
-  std::map<unsigned, LabelStmt *>::iterator Label = LabelStmts.find(ID);
-  if (Label != LabelStmts.end()) {
-    // We've already seen this label, so set the label of the
-    // label-address expression and we're done.
-    S->setLabel(Label->second);
-  } else {
-    // We haven't seen this label yet, so add this label-address
-    // expression to the set of unresolved label-address expressions.
-    UnresolvedAddrLabelExprs.insert(std::make_pair(ID, S));
-  }
+void ASTReader::ClearSwitchCaseIDs() {
+  SwitchCaseStmts.clear();
 }
 
 void ASTReader::FinishedDeserializing() {
@@ -4066,43 +4834,57 @@ void ASTReader::FinishedDeserializing() {
       PendingIdentifierInfos.pop_front();
     }
 
+    // Ready to load previous declarations of Decls that were delayed.
+    while (!PendingPreviousDecls.empty()) {
+      loadAndAttachPreviousDecl(PendingPreviousDecls.front().first,
+                                PendingPreviousDecls.front().second);
+      PendingPreviousDecls.pop_front();
+    }
+
     // We are not in recursive loading, so it's safe to pass the "interesting"
     // decls to the consumer.
     if (Consumer)
       PassInterestingDeclsToConsumer();
+
+    assert(PendingForwardRefs.size() == 0 &&
+           "Some forward refs did not get linked to the definition!");
   }
   --NumCurrentElementsDeserializing;
 }
 
 ASTReader::ASTReader(Preprocessor &PP, ASTContext *Context,
-                     const char *isysroot, bool DisableValidation)
+                     const char *isysroot, bool DisableValidation,
+                     bool DisableStatCache)
   : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
     SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
     Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context),
     Consumer(0), isysroot(isysroot), DisableValidation(DisableValidation),
-    NumStatHits(0), NumStatMisses(0), NumSLocEntriesRead(0),
-    TotalNumSLocEntries(0), NumStatementsRead(0), TotalNumStatements(0),
-    NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0),
-    NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0),
-    TotalNumMethodPoolEntries(0), NumLexicalDeclContextsRead(0),
-    TotalLexicalDeclContexts(0), NumVisibleDeclContextsRead(0),
-    TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) {
+    DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), 
+    NumSLocEntriesRead(0), TotalNumSLocEntries(0), NextSLocOffset(0), 
+    NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0), 
+    TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0), 
+    NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0), 
+    NumLexicalDeclContextsRead(0), TotalLexicalDeclContexts(0), 
+    NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0), 
+    NumCurrentElementsDeserializing(0) 
+{
   RelocatablePCH = false;
 }
 
 ASTReader::ASTReader(SourceManager &SourceMgr, FileManager &FileMgr,
                      Diagnostic &Diags, const char *isysroot,
-                     bool DisableValidation)
+                     bool DisableValidation, bool DisableStatCache)
   : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr),
     Diags(Diags), SemaObj(0), PP(0), Context(0), Consumer(0),
-    isysroot(isysroot), DisableValidation(DisableValidation), NumStatHits(0),
-    NumStatMisses(0), NumSLocEntriesRead(0), TotalNumSLocEntries(0),
-    NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0),
-    TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0),
-    NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0),
-    NumLexicalDeclContextsRead(0), TotalLexicalDeclContexts(0),
-    NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0),
-    NumCurrentElementsDeserializing(0) {
+    isysroot(isysroot), DisableValidation(DisableValidation), 
+    DisableStatCache(DisableStatCache), NumStatHits(0), NumStatMisses(0), 
+    NumSLocEntriesRead(0), TotalNumSLocEntries(0),
+    NextSLocOffset(0), NumStatementsRead(0), TotalNumStatements(0),
+    NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0),
+    NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0),
+    TotalNumMethodPoolEntries(0), NumLexicalDeclContextsRead(0),
+    TotalLexicalDeclContexts(0), NumVisibleDeclContextsRead(0),
+    TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) {
   RelocatablePCH = false;
 }
 
@@ -4131,17 +4913,23 @@ ASTReader::~ASTReader() {
   }
 }
 
-ASTReader::PerFileData::PerFileData()
-  : StatCache(0), LocalNumSLocEntries(0), LocalNumTypes(0), TypeOffsets(0),
-    LocalNumDecls(0), DeclOffsets(0), LocalNumIdentifiers(0),
-    IdentifierOffsets(0), IdentifierTableData(0), IdentifierLookupTable(0),
-    LocalNumMacroDefinitions(0), MacroDefinitionOffsets(0),
-    NumPreallocatedPreprocessingEntities(0), SelectorLookupTable(0),
-    SelectorLookupTableData(0), SelectorOffsets(0), LocalNumSelectors(0)
+ASTReader::PerFileData::PerFileData(ASTFileType Ty)
+  : Type(Ty), SizeInBits(0), LocalNumSLocEntries(0), SLocOffsets(0), LocalSLocSize(0),
+    LocalNumIdentifiers(0), IdentifierOffsets(0), IdentifierTableData(0),
+    IdentifierLookupTable(0), LocalNumMacroDefinitions(0),
+    MacroDefinitionOffsets(0), 
+    LocalNumHeaderFileInfos(0), HeaderFileInfoTableData(0),
+    HeaderFileInfoTable(0),
+    LocalNumSelectors(0), SelectorOffsets(0),
+    SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0),
+    DeclOffsets(0), LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0),
+    LocalNumTypes(0), TypeOffsets(0), StatCache(0),
+    NumPreallocatedPreprocessingEntities(0), NextInSource(0)
 {}
 
 ASTReader::PerFileData::~PerFileData() {
   delete static_cast<ASTIdentifierLookupTable *>(IdentifierLookupTable);
+  delete static_cast<HeaderFileInfoLookupTable *>(HeaderFileInfoTable);
   delete static_cast<ASTSelectorLookupTable *>(SelectorLookupTable);
 }
 
diff --git a/lib/Serialization/ASTReaderDecl.cpp b/lib/Serialization/ASTReaderDecl.cpp
index 7adbe1220ff7..dec15dd694f5 100644
--- a/lib/Serialization/ASTReaderDecl.cpp
+++ b/lib/Serialization/ASTReaderDecl.cpp
@@ -12,6 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ASTCommon.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
@@ -30,26 +31,60 @@ using namespace clang::serialization;
 namespace clang {
   class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
     ASTReader &Reader;
+    ASTReader::PerFileData &F;
     llvm::BitstreamCursor &Cursor;
     const DeclID ThisDeclID;
-    const ASTReader::RecordData &Record;
+    typedef ASTReader::RecordData RecordData;
+    const RecordData &Record;
     unsigned &Idx;
     TypeID TypeIDForTypeDecl;
 
     uint64_t GetCurrentCursorOffset();
+    SourceLocation ReadSourceLocation(const RecordData &R, unsigned &I) {
+      return Reader.ReadSourceLocation(F, R, I);
+    }
+    SourceRange ReadSourceRange(const RecordData &R, unsigned &I) {
+      return Reader.ReadSourceRange(F, R, I);
+    }
+    TypeSourceInfo *GetTypeSourceInfo(const RecordData &R, unsigned &I) {
+      return Reader.GetTypeSourceInfo(F, R, I);
+    }
+    void ReadQualifierInfo(QualifierInfo &Info,
+                           const RecordData &R, unsigned &I) {
+      Reader.ReadQualifierInfo(F, Info, R, I);
+    }
+    void ReadDeclarationNameLoc(DeclarationNameLoc &DNLoc, DeclarationName Name,
+                                const RecordData &R, unsigned &I) {
+      Reader.ReadDeclarationNameLoc(F, DNLoc, Name, R, I);
+    }
+    void ReadDeclarationNameInfo(DeclarationNameInfo &NameInfo,
+                                const RecordData &R, unsigned &I) {
+      Reader.ReadDeclarationNameInfo(F, NameInfo, R, I);
+    }
+
+    void ReadCXXDefinitionData(struct CXXRecordDecl::DefinitionData &Data,
+                               const RecordData &R, unsigned &I);
 
+    void InitializeCXXDefinitionData(CXXRecordDecl *D,
+                                     CXXRecordDecl *DefinitionDecl,
+                                     const RecordData &Record, unsigned &Idx);
   public:
-    ASTDeclReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
-                  DeclID thisDeclID, const ASTReader::RecordData &Record,
-                  unsigned &Idx)
-      : Reader(Reader), Cursor(Cursor), ThisDeclID(thisDeclID), Record(Record),
-        Idx(Idx), TypeIDForTypeDecl(0) { }
+    ASTDeclReader(ASTReader &Reader, ASTReader::PerFileData &F,
+                  llvm::BitstreamCursor &Cursor, DeclID thisDeclID,
+                  const RecordData &Record, unsigned &Idx)
+      : Reader(Reader), F(F), Cursor(Cursor), ThisDeclID(thisDeclID),
+        Record(Record), Idx(Idx), TypeIDForTypeDecl(0) { }
+
+    static void attachPreviousDecl(Decl *D, Decl *previous);
 
     void Visit(Decl *D);
 
+    void UpdateDecl(Decl *D, const RecordData &Record);
+
     void VisitDecl(Decl *D);
     void VisitTranslationUnitDecl(TranslationUnitDecl *TU);
     void VisitNamedDecl(NamedDecl *ND);
+    void VisitLabelDecl(LabelDecl *LD);
     void VisitNamespaceDecl(NamespaceDecl *D);
     void VisitUsingDirectiveDecl(UsingDirectiveDecl *D);
     void VisitNamespaceAliasDecl(NamespaceAliasDecl *D);
@@ -75,6 +110,7 @@ namespace clang {
     void VisitCXXDestructorDecl(CXXDestructorDecl *D);
     void VisitCXXConversionDecl(CXXConversionDecl *D);
     void VisitFieldDecl(FieldDecl *FD);
+    void VisitIndirectFieldDecl(IndirectFieldDecl *FD);
     void VisitVarDecl(VarDecl *VD);
     void VisitImplicitParamDecl(ImplicitParamDecl *PD);
     void VisitParmVarDecl(ParmVarDecl *PD);
@@ -134,7 +170,7 @@ void ASTDeclReader::Visit(Decl *D) {
 
   if (TypeDecl *TD = dyn_cast<TypeDecl>(D)) {
     // if we have a fully initialized TypeDecl, we can safely read its type now.
-    TD->setTypeForDecl(Reader.GetType(TypeIDForTypeDecl).getTypePtr());
+    TD->setTypeForDecl(Reader.GetType(TypeIDForTypeDecl).getTypePtrOrNull());
   } else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // FunctionDecl's body was written last after all other Stmts/Exprs.
     if (Record[Idx++])
@@ -146,17 +182,17 @@ void ASTDeclReader::VisitDecl(Decl *D) {
   D->setDeclContext(cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
   D->setLexicalDeclContext(
                      cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
-  D->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  D->setLocation(ReadSourceLocation(Record, Idx));
   D->setInvalidDecl(Record[Idx++]);
-  if (Record[Idx++]) {
+  if (Record[Idx++]) { // hasAttrs
     AttrVec Attrs;
-    Reader.ReadAttributes(Cursor, Attrs);
+    Reader.ReadAttributes(F, Attrs, Record, Idx);
     D->setAttrs(Attrs);
   }
   D->setImplicit(Record[Idx++]);
   D->setUsed(Record[Idx++]);
   D->setAccess((AccessSpecifier)Record[Idx++]);
-  D->setPCHLevel(Record[Idx++] + 1);
+  D->setPCHLevel(Record[Idx++] + (F.Type <= ASTReader::PCH));
 }
 
 void ASTDeclReader::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
@@ -178,29 +214,39 @@ void ASTDeclReader::VisitTypeDecl(TypeDecl *TD) {
 
 void ASTDeclReader::VisitTypedefDecl(TypedefDecl *TD) {
   VisitTypeDecl(TD);
-  TD->setTypeSourceInfo(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
+  TD->setTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
 }
 
 void ASTDeclReader::VisitTagDecl(TagDecl *TD) {
   VisitTypeDecl(TD);
-  TD->IdentifierNamespace = Record[Idx++];
   VisitRedeclarable(TD);
+  TD->IdentifierNamespace = Record[Idx++];
   TD->setTagKind((TagDecl::TagKind)Record[Idx++]);
   TD->setDefinition(Record[Idx++]);
   TD->setEmbeddedInDeclarator(Record[Idx++]);
-  TD->setRBraceLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TD->setTagKeywordLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  // FIXME: maybe read optional qualifier and its range.
-  TD->setTypedefForAnonDecl(
-                    cast_or_null<TypedefDecl>(Reader.GetDecl(Record[Idx++])));
+  TD->setRBraceLoc(ReadSourceLocation(Record, Idx));
+  TD->setTagKeywordLoc(ReadSourceLocation(Record, Idx));
+  if (Record[Idx++]) { // hasExtInfo
+    TagDecl::ExtInfo *Info = new (*Reader.getContext()) TagDecl::ExtInfo();
+    ReadQualifierInfo(*Info, Record, Idx);
+    TD->TypedefDeclOrQualifier = Info;
+  } else
+    TD->setTypedefForAnonDecl(
+                      cast_or_null<TypedefDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
 void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
   VisitTagDecl(ED);
-  ED->setIntegerType(Reader.GetType(Record[Idx++]));
+  if (TypeSourceInfo *TI = Reader.GetTypeSourceInfo(F, Record, Idx))
+    ED->setIntegerTypeSourceInfo(TI);
+  else
+    ED->setIntegerType(Reader.GetType(Record[Idx++]));
   ED->setPromotionType(Reader.GetType(Record[Idx++]));
   ED->setNumPositiveBits(Record[Idx++]);
   ED->setNumNegativeBits(Record[Idx++]);
+  ED->IsScoped = Record[Idx++];
+  ED->IsScopedUsingClassTag = Record[Idx++];
+  ED->IsFixed = Record[Idx++];
   ED->setInstantiationOfMemberEnum(
                          cast_or_null<EnumDecl>(Reader.GetDecl(Record[Idx++])));
 }
@@ -220,22 +266,27 @@ void ASTDeclReader::VisitValueDecl(ValueDecl *VD) {
 void ASTDeclReader::VisitEnumConstantDecl(EnumConstantDecl *ECD) {
   VisitValueDecl(ECD);
   if (Record[Idx++])
-    ECD->setInitExpr(Reader.ReadExpr(Cursor));
+    ECD->setInitExpr(Reader.ReadExpr(F));
   ECD->setInitVal(Reader.ReadAPSInt(Record, Idx));
 }
 
 void ASTDeclReader::VisitDeclaratorDecl(DeclaratorDecl *DD) {
   VisitValueDecl(DD);
-  TypeSourceInfo *TInfo = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
-  if (TInfo)
-    DD->setTypeSourceInfo(TInfo);
-  // FIXME: read optional qualifier and its range.
+  if (Record[Idx++]) { // hasExtInfo
+    DeclaratorDecl::ExtInfo *Info
+        = new (*Reader.getContext()) DeclaratorDecl::ExtInfo();
+    ReadQualifierInfo(*Info, Record, Idx);
+    Info->TInfo = GetTypeSourceInfo(Record, Idx);
+    DD->DeclInfo = Info;
+  } else
+    DD->DeclInfo = GetTypeSourceInfo(Record, Idx);
 }
 
 void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   VisitDeclaratorDecl(FD);
-  // FIXME: read DeclarationNameLoc.
+  VisitRedeclarable(FD);
 
+  ReadDeclarationNameLoc(FD->DNLoc, FD->getDeclName(), Record, Idx);
   FD->IdentifierNamespace = Record[Idx++];
   switch ((FunctionDecl::TemplatedKind)Record[Idx++]) {
   default: assert(false && "Unhandled TemplatedKind!");
@@ -249,8 +300,8 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   case FunctionDecl::TK_MemberSpecialization: {
     FunctionDecl *InstFD = cast<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
-    SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
-    FD->setInstantiationOfMemberFunction(InstFD, TSK);
+    SourceLocation POI = ReadSourceLocation(Record, Idx);
+    FD->setInstantiationOfMemberFunction(*Reader.getContext(), InstFD, TSK);
     FD->getMemberSpecializationInfo()->setPointOfInstantiation(POI);
     break;
   }
@@ -261,7 +312,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     
     // Template arguments.
     llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-    Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
+    Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
     
     // Template args as written.
     llvm::SmallVector<TemplateArgumentLoc, 8> TemplArgLocs;
@@ -271,20 +322,45 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
       TemplArgLocs.reserve(NumTemplateArgLocs);
       for (unsigned i=0; i != NumTemplateArgLocs; ++i)
         TemplArgLocs.push_back(
-            Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx));
+            Reader.ReadTemplateArgumentLoc(F, Record, Idx));
   
-      LAngleLoc = Reader.ReadSourceLocation(Record, Idx);
-      RAngleLoc = Reader.ReadSourceLocation(Record, Idx);
+      LAngleLoc = ReadSourceLocation(Record, Idx);
+      RAngleLoc = ReadSourceLocation(Record, Idx);
     }
     
-    SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
-
-    if (FD->isCanonicalDecl()) // if canonical add to template's set.
-      FD->setFunctionTemplateSpecialization(Template, TemplArgs.size(),
-                                            TemplArgs.data(), TSK,
-                                            TemplArgLocs.size(),
-                                            TemplArgLocs.data(),
-                                            LAngleLoc, RAngleLoc, POI);
+    SourceLocation POI = ReadSourceLocation(Record, Idx);
+
+    ASTContext &C = *Reader.getContext();
+    TemplateArgumentList *TemplArgList
+      = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), TemplArgs.size());
+    TemplateArgumentListInfo *TemplArgsInfo
+      = new (C) TemplateArgumentListInfo(LAngleLoc, RAngleLoc);
+    for (unsigned i=0, e = TemplArgLocs.size(); i != e; ++i)
+      TemplArgsInfo->addArgument(TemplArgLocs[i]);
+    FunctionTemplateSpecializationInfo *FTInfo
+        = FunctionTemplateSpecializationInfo::Create(C, FD, Template, TSK,
+                                                     TemplArgList,
+                                                     TemplArgsInfo, POI);
+    FD->TemplateOrSpecialization = FTInfo;
+
+    if (FD->isCanonicalDecl()) { // if canonical add to template's set.
+      // The template that contains the specializations set. It's not safe to
+      // use getCanonicalDecl on Template since it may still be initializing.
+      FunctionTemplateDecl *CanonTemplate
+        = cast<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+      // Get the InsertPos by FindNodeOrInsertPos() instead of calling
+      // InsertNode(FTInfo) directly to avoid the getASTContext() call in
+      // FunctionTemplateSpecializationInfo's Profile().
+      // We avoid getASTContext because a decl in the parent hierarchy may
+      // be initializing.
+      llvm::FoldingSetNodeID ID;
+      FunctionTemplateSpecializationInfo::Profile(ID, TemplArgs.data(),
+                                                  TemplArgs.size(), C);
+      void *InsertPos = 0;
+      CanonTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
+      assert(InsertPos && "Another specialization already inserted!");
+      CanonTemplate->getSpecializations().InsertNode(FTInfo, InsertPos);
+    }
     break;
   }
   case FunctionDecl::TK_DependentFunctionTemplateSpecialization: {
@@ -298,9 +374,9 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     TemplateArgumentListInfo TemplArgs;
     unsigned NumArgs = Record[Idx++];
     while (NumArgs--)
-      TemplArgs.addArgument(Reader.ReadTemplateArgumentLoc(Cursor,Record, Idx));
-    TemplArgs.setLAngleLoc(Reader.ReadSourceLocation(Record, Idx));
-    TemplArgs.setRAngleLoc(Reader.ReadSourceLocation(Record, Idx));
+      TemplArgs.addArgument(Reader.ReadTemplateArgumentLoc(F, Record, Idx));
+    TemplArgs.setLAngleLoc(ReadSourceLocation(Record, Idx));
+    TemplArgs.setRAngleLoc(ReadSourceLocation(Record, Idx));
     
     FD->setDependentTemplateSpecialization(*Reader.getContext(),
                                            TemplDecls, TemplArgs);
@@ -311,19 +387,18 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   // FunctionDecl's body is handled last at ASTDeclReader::Visit,
   // after everything else is read.
 
-  VisitRedeclarable(FD);
-  FD->setStorageClass((StorageClass)Record[Idx++]);
-  FD->setStorageClassAsWritten((StorageClass)Record[Idx++]);
-  FD->setInlineSpecified(Record[Idx++]);
-  FD->setVirtualAsWritten(Record[Idx++]);
-  FD->setPure(Record[Idx++]);
-  FD->setHasInheritedPrototype(Record[Idx++]);
-  FD->setHasWrittenPrototype(Record[Idx++]);
-  FD->setDeleted(Record[Idx++]);
-  FD->setTrivial(Record[Idx++]);
-  FD->setCopyAssignment(Record[Idx++]);
-  FD->setHasImplicitReturnZero(Record[Idx++]);
-  FD->setLocEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  FD->SClass = (StorageClass)Record[Idx++];
+  FD->SClassAsWritten = (StorageClass)Record[Idx++];
+  FD->IsInline = Record[Idx++];
+  FD->IsInlineSpecified = Record[Idx++];
+  FD->IsVirtualAsWritten = Record[Idx++];
+  FD->IsPure = Record[Idx++];
+  FD->HasInheritedPrototype = Record[Idx++];
+  FD->HasWrittenPrototype = Record[Idx++];
+  FD->IsDeleted = Record[Idx++];
+  FD->IsTrivial = Record[Idx++];
+  FD->HasImplicitReturnZero = Record[Idx++];
+  FD->EndRangeLoc = ReadSourceLocation(Record, Idx);
 
   // Read in the parameters.
   unsigned NumParams = Record[Idx++];
@@ -331,7 +406,7 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   Params.reserve(NumParams);
   for (unsigned I = 0; I != NumParams; ++I)
     Params.push_back(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
-  FD->setParams(Params.data(), NumParams);
+  FD->setParams(*Reader.getContext(), Params.data(), NumParams);
 }
 
 void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
@@ -339,7 +414,7 @@ void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
   if (Record[Idx++]) {
     // In practice, this won't be executed (since method definitions
     // don't occur in header files).
-    MD->setBody(Reader.ReadStmt(Cursor));
+    MD->setBody(Reader.ReadStmt(F));
     MD->setSelfDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
     MD->setCmdDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
   }
@@ -351,8 +426,8 @@ void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
   MD->setObjCDeclQualifier((Decl::ObjCDeclQualifier)Record[Idx++]);
   MD->setNumSelectorArgs(unsigned(Record[Idx++]));
   MD->setResultType(Reader.GetType(Record[Idx++]));
-  MD->setResultTypeSourceInfo(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
-  MD->setEndLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  MD->setResultTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
+  MD->setEndLoc(ReadSourceLocation(Record, Idx));
   unsigned NumParams = Record[Idx++];
   llvm::SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
@@ -364,14 +439,14 @@ void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
 
 void ASTDeclReader::VisitObjCContainerDecl(ObjCContainerDecl *CD) {
   VisitNamedDecl(CD);
-  SourceLocation A = SourceLocation::getFromRawEncoding(Record[Idx++]);
-  SourceLocation B = SourceLocation::getFromRawEncoding(Record[Idx++]);
+  SourceLocation A = ReadSourceLocation(Record, Idx);
+  SourceLocation B = ReadSourceLocation(Record, Idx);
   CD->setAtEndRange(SourceRange(A, B));
 }
 
 void ASTDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   VisitObjCContainerDecl(ID);
-  ID->setTypeForDecl(Reader.GetType(Record[Idx++]).getTypePtr());
+  ID->setTypeForDecl(Reader.GetType(Record[Idx++]).getTypePtrOrNull());
   ID->setSuperClass(cast_or_null<ObjCInterfaceDecl>
                        (Reader.GetDecl(Record[Idx++])));
   
@@ -384,7 +459,7 @@ void ASTDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   llvm::SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtocols);
   for (unsigned I = 0; I != NumProtocols; ++I)
-    ProtoLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
+    ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   ID->setProtocolList(Protocols.data(), NumProtocols, ProtoLocs.data(),
                       *Reader.getContext());
   
@@ -409,9 +484,9 @@ void ASTDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   ID->setIvarList(0);
   ID->setForwardDecl(Record[Idx++]);
   ID->setImplicitInterfaceDecl(Record[Idx++]);
-  ID->setClassLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  ID->setSuperClassLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  ID->setLocEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  ID->setClassLoc(ReadSourceLocation(Record, Idx));
+  ID->setSuperClassLoc(ReadSourceLocation(Record, Idx));
+  ID->setLocEnd(ReadSourceLocation(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCIvarDecl(ObjCIvarDecl *IVD) {
@@ -426,7 +501,7 @@ void ASTDeclReader::VisitObjCIvarDecl(ObjCIvarDecl *IVD) {
 void ASTDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
   VisitObjCContainerDecl(PD);
   PD->setForwardDecl(Record[Idx++]);
-  PD->setLocEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  PD->setLocEnd(ReadSourceLocation(Record, Idx));
   unsigned NumProtoRefs = Record[Idx++];
   llvm::SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
   ProtoRefs.reserve(NumProtoRefs);
@@ -435,7 +510,7 @@ void ASTDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
   llvm::SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
+    ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   PD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
                       *Reader.getContext());
 }
@@ -454,7 +529,7 @@ void ASTDeclReader::VisitObjCClassDecl(ObjCClassDecl *CD) {
   llvm::SmallVector<SourceLocation, 16> SLocs;
   SLocs.reserve(NumClassRefs);
   for (unsigned I = 0; I != NumClassRefs; ++I)
-    SLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
+    SLocs.push_back(ReadSourceLocation(Record, Idx));
   CD->setClassList(*Reader.getContext(), ClassRefs.data(), SLocs.data(),
                    NumClassRefs);
 }
@@ -469,7 +544,7 @@ void ASTDeclReader::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *FPD) {
   llvm::SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
+    ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   FPD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
                        *Reader.getContext());
 }
@@ -485,13 +560,13 @@ void ASTDeclReader::VisitObjCCategoryDecl(ObjCCategoryDecl *CD) {
   llvm::SmallVector<SourceLocation, 16> ProtoLocs;
   ProtoLocs.reserve(NumProtoRefs);
   for (unsigned I = 0; I != NumProtoRefs; ++I)
-    ProtoLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
+    ProtoLocs.push_back(ReadSourceLocation(Record, Idx));
   CD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
                       *Reader.getContext());
   CD->setNextClassCategory(cast_or_null<ObjCCategoryDecl>(Reader.GetDecl(Record[Idx++])));
   CD->setHasSynthBitfield(Record[Idx++]);
-  CD->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  CD->setCategoryNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  CD->setAtLoc(ReadSourceLocation(Record, Idx));
+  CD->setCategoryNameLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTDeclReader::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *CAD) {
@@ -501,8 +576,8 @@ void ASTDeclReader::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *CAD) {
 
 void ASTDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   VisitNamedDecl(D);
-  D->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  D->setType(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
+  D->setAtLoc(ReadSourceLocation(Record, Idx));
+  D->setType(GetTypeSourceInfo(Record, Idx));
   // FIXME: stable encoding
   D->setPropertyAttributes(
                       (ObjCPropertyDecl::PropertyAttributeKind)Record[Idx++]);
@@ -537,27 +612,28 @@ void ASTDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   D->setSuperClass(
               cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
   llvm::tie(D->IvarInitializers, D->NumIvarInitializers)
-      = Reader.ReadCXXBaseOrMemberInitializers(Cursor, Record, Idx);
+      = Reader.ReadCXXCtorInitializers(F, Record, Idx);
   D->setHasSynthBitfield(Record[Idx++]);
 }
 
 
 void ASTDeclReader::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
   VisitDecl(D);
-  D->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  D->setAtLoc(ReadSourceLocation(Record, Idx));
   D->setPropertyDecl(
                cast_or_null<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setPropertyIvarDecl(
-                   cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setGetterCXXConstructor(Reader.ReadExpr(Cursor));
-  D->setSetterCXXAssignment(Reader.ReadExpr(Cursor));
+  D->PropertyIvarDecl = 
+                   cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++]));
+  D->IvarLoc = ReadSourceLocation(Record, Idx);
+  D->setGetterCXXConstructor(Reader.ReadExpr(F));
+  D->setSetterCXXAssignment(Reader.ReadExpr(F));
 }
 
 void ASTDeclReader::VisitFieldDecl(FieldDecl *FD) {
   VisitDeclaratorDecl(FD);
   FD->setMutable(Record[Idx++]);
   if (Record[Idx++])
-    FD->setBitWidth(Reader.ReadExpr(Cursor));
+    FD->setBitWidth(Reader.ReadExpr(F));
   if (!FD->getDeclName()) {
     FieldDecl *Tmpl = cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++]));
     if (Tmpl)
@@ -565,22 +641,33 @@ void ASTDeclReader::VisitFieldDecl(FieldDecl *FD) {
   }
 }
 
+void ASTDeclReader::VisitIndirectFieldDecl(IndirectFieldDecl *FD) {
+  VisitValueDecl(FD);
+
+  FD->ChainingSize = Record[Idx++];
+  assert(FD->ChainingSize >= 2 && "Anonymous chaining must be >= 2");
+  FD->Chaining = new (*Reader.getContext())NamedDecl*[FD->ChainingSize];
+
+  for (unsigned I = 0; I != FD->ChainingSize; ++I)
+    FD->Chaining[I] = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+}
+
 void ASTDeclReader::VisitVarDecl(VarDecl *VD) {
   VisitDeclaratorDecl(VD);
-  VD->setStorageClass((StorageClass)Record[Idx++]);
+  VisitRedeclarable(VD);
+  VD->SClass = (StorageClass)Record[Idx++];
   VD->setStorageClassAsWritten((StorageClass)Record[Idx++]);
   VD->setThreadSpecified(Record[Idx++]);
   VD->setCXXDirectInitializer(Record[Idx++]);
   VD->setExceptionVariable(Record[Idx++]);
   VD->setNRVOVariable(Record[Idx++]);
-  VisitRedeclarable(VD);
   if (Record[Idx++])
-    VD->setInit(Reader.ReadExpr(Cursor));
+    VD->setInit(Reader.ReadExpr(F));
 
   if (Record[Idx++]) { // HasMemberSpecializationInfo.
     VarDecl *Tmpl = cast<VarDecl>(Reader.GetDecl(Record[Idx++]));
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
-    SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
+    SourceLocation POI = ReadSourceLocation(Record, Idx);
     Reader.getContext()->setInstantiatedFromStaticDataMember(VD, Tmpl, TSK,POI);
   }
 }
@@ -594,24 +681,40 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) {
   PD->setObjCDeclQualifier((Decl::ObjCDeclQualifier)Record[Idx++]);
   PD->setHasInheritedDefaultArg(Record[Idx++]);
   if (Record[Idx++]) // hasUninstantiatedDefaultArg.
-    PD->setUninstantiatedDefaultArg(Reader.ReadExpr(Cursor));
+    PD->setUninstantiatedDefaultArg(Reader.ReadExpr(F));
 }
 
 void ASTDeclReader::VisitFileScopeAsmDecl(FileScopeAsmDecl *AD) {
   VisitDecl(AD);
-  AD->setAsmString(cast<StringLiteral>(Reader.ReadExpr(Cursor)));
+  AD->setAsmString(cast<StringLiteral>(Reader.ReadExpr(F)));
 }
 
 void ASTDeclReader::VisitBlockDecl(BlockDecl *BD) {
   VisitDecl(BD);
-  BD->setBody(cast_or_null<CompoundStmt>(Reader.ReadStmt(Cursor)));
-  BD->setSignatureAsWritten(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
+  BD->setBody(cast_or_null<CompoundStmt>(Reader.ReadStmt(F)));
+  BD->setSignatureAsWritten(GetTypeSourceInfo(Record, Idx));
   unsigned NumParams = Record[Idx++];
   llvm::SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
   for (unsigned I = 0; I != NumParams; ++I)
     Params.push_back(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
   BD->setParams(Params.data(), NumParams);
+
+  bool capturesCXXThis = Record[Idx++];
+  unsigned numCaptures = Record[Idx++];
+  llvm::SmallVector<BlockDecl::Capture, 16> captures;
+  captures.reserve(numCaptures);
+  for (unsigned i = 0; i != numCaptures; ++i) {
+    VarDecl *decl = cast<VarDecl>(Reader.GetDecl(Record[Idx++]));
+    unsigned flags = Record[Idx++];
+    bool byRef = (flags & 1);
+    bool nested = (flags & 2);
+    Expr *copyExpr = ((flags & 4) ? Reader.ReadExpr(F) : 0);
+
+    captures.push_back(BlockDecl::Capture(decl, byRef, nested, copyExpr));
+  }
+  BD->setCaptures(*Reader.getContext(), captures.begin(),
+                  captures.end(), capturesCXXThis);
 }
 
 void ASTDeclReader::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
@@ -620,12 +723,17 @@ void ASTDeclReader::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
   D->setHasBraces(Record[Idx++]);
 }
 
+void ASTDeclReader::VisitLabelDecl(LabelDecl *D) {
+  VisitNamedDecl(D);
+}
+
+
 void ASTDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
   VisitNamedDecl(D);
-  D->setLBracLoc(Reader.ReadSourceLocation(Record, Idx));
-  D->setRBracLoc(Reader.ReadSourceLocation(Record, Idx));
-  D->setNextNamespace(
-                    cast_or_null<NamespaceDecl>(Reader.GetDecl(Record[Idx++])));
+  D->IsInline = Record[Idx++];
+  D->LBracLoc = ReadSourceLocation(Record, Idx);
+  D->RBracLoc = ReadSourceLocation(Record, Idx);
+  D->NextNamespace = Record[Idx++];
 
   bool IsOriginal = Record[Idx++];
   D->OrigOrAnonNamespace.setInt(IsOriginal);
@@ -635,29 +743,20 @@ void ASTDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
 
 void ASTDeclReader::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   VisitNamedDecl(D);
-  D->NamespaceLoc = Reader.ReadSourceLocation(Record, Idx);
-  D->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
+  D->NamespaceLoc = ReadSourceLocation(Record, Idx);
+  D->setQualifierRange(ReadSourceRange(Record, Idx));
   D->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  D->IdentLoc = Reader.ReadSourceLocation(Record, Idx);
+  D->IdentLoc = ReadSourceLocation(Record, Idx);
   D->Namespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
 }
 
 void ASTDeclReader::VisitUsingDecl(UsingDecl *D) {
   VisitNamedDecl(D);
-  D->setUsingLocation(Reader.ReadSourceLocation(Record, Idx));
-  D->setNestedNameRange(Reader.ReadSourceRange(Record, Idx));
+  D->setUsingLocation(ReadSourceLocation(Record, Idx));
+  D->setNestedNameRange(ReadSourceRange(Record, Idx));
   D->setTargetNestedNameDecl(Reader.ReadNestedNameSpecifier(Record, Idx));
-  // FIXME: read the DNLoc component.
-
-  // FIXME: It would probably be more efficient to read these into a vector
-  // and then re-cosntruct the shadow decl set over that vector since it
-  // would avoid existence checks.
-  unsigned NumShadows = Record[Idx++];
-  for(unsigned I = 0; I != NumShadows; ++I) {
-    // Avoid invariant checking of UsingDecl::addShadowDecl, the decl may still
-    // be initializing.
-    D->Shadows.insert(cast<UsingShadowDecl>(Reader.GetDecl(Record[Idx++])));
-  }
+  ReadDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record, Idx);
+  D->FirstUsingShadow = cast_or_null<UsingShadowDecl>(Reader.GetDecl(Record[Idx++]));
   D->setTypeName(Record[Idx++]);
   NamedDecl *Pattern = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   if (Pattern)
@@ -667,7 +766,7 @@ void ASTDeclReader::VisitUsingDecl(UsingDecl *D) {
 void ASTDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
   VisitNamedDecl(D);
   D->setTargetDecl(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setUsingDecl(cast<UsingDecl>(Reader.GetDecl(Record[Idx++])));
+  D->UsingOrNextShadow = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   UsingShadowDecl *Pattern
       = cast_or_null<UsingShadowDecl>(Reader.GetDecl(Record[Idx++]));
   if (Pattern)
@@ -676,9 +775,9 @@ void ASTDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
 
 void ASTDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   VisitNamedDecl(D);
-  D->UsingLoc = Reader.ReadSourceLocation(Record, Idx);
-  D->NamespaceLoc = Reader.ReadSourceLocation(Record, Idx);
-  D->QualifierRange = Reader.ReadSourceRange(Record, Idx);
+  D->UsingLoc = ReadSourceLocation(Record, Idx);
+  D->NamespaceLoc = ReadSourceLocation(Record, Idx);
+  D->QualifierRange = ReadSourceRange(Record, Idx);
   D->Qualifier = Reader.ReadNestedNameSpecifier(Record, Idx);
   D->NominatedNamespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   D->CommonAncestor = cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++]));
@@ -686,88 +785,99 @@ void ASTDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
 
 void ASTDeclReader::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
   VisitValueDecl(D);
-  D->setTargetNestedNameRange(Reader.ReadSourceRange(Record, Idx));
-  D->setUsingLoc(Reader.ReadSourceLocation(Record, Idx));
+  D->setTargetNestedNameRange(ReadSourceRange(Record, Idx));
+  D->setUsingLoc(ReadSourceLocation(Record, Idx));
   D->setTargetNestedNameSpecifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  // FIXME: read the DNLoc component.
+  ReadDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record, Idx);
 }
 
 void ASTDeclReader::VisitUnresolvedUsingTypenameDecl(
                                                UnresolvedUsingTypenameDecl *D) {
   VisitTypeDecl(D);
-  D->TargetNestedNameRange = Reader.ReadSourceRange(Record, Idx);
-  D->UsingLocation = Reader.ReadSourceLocation(Record, Idx);
-  D->TypenameLocation = Reader.ReadSourceLocation(Record, Idx);
+  D->TargetNestedNameRange = ReadSourceRange(Record, Idx);
+  D->UsingLocation = ReadSourceLocation(Record, Idx);
+  D->TypenameLocation = ReadSourceLocation(Record, Idx);
   D->TargetNestedNameSpecifier = Reader.ReadNestedNameSpecifier(Record, Idx);
 }
 
-void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
+void ASTDeclReader::ReadCXXDefinitionData(
+                                   struct CXXRecordDecl::DefinitionData &Data,
+                                   const RecordData &Record, unsigned &Idx) {
+  Data.UserDeclaredConstructor = Record[Idx++];
+  Data.UserDeclaredCopyConstructor = Record[Idx++];
+  Data.UserDeclaredCopyAssignment = Record[Idx++];
+  Data.UserDeclaredDestructor = Record[Idx++];
+  Data.Aggregate = Record[Idx++];
+  Data.PlainOldData = Record[Idx++];
+  Data.Empty = Record[Idx++];
+  Data.Polymorphic = Record[Idx++];
+  Data.Abstract = Record[Idx++];
+  Data.HasTrivialConstructor = Record[Idx++];
+  Data.HasTrivialCopyConstructor = Record[Idx++];
+  Data.HasTrivialCopyAssignment = Record[Idx++];
+  Data.HasTrivialDestructor = Record[Idx++];
+  Data.ComputedVisibleConversions = Record[Idx++];
+  Data.DeclaredDefaultConstructor = Record[Idx++];
+  Data.DeclaredCopyConstructor = Record[Idx++];
+  Data.DeclaredCopyAssignment = Record[Idx++];
+  Data.DeclaredDestructor = Record[Idx++];
+
+  Data.NumBases = Record[Idx++];
+  if (Data.NumBases)
+    Data.Bases = Reader.GetCXXBaseSpecifiersOffset(Record[Idx++]);
+  Data.NumVBases = Record[Idx++];
+  if (Data.NumVBases)
+    Data.VBases = Reader.GetCXXBaseSpecifiersOffset(Record[Idx++]);
+  
+  Reader.ReadUnresolvedSet(Data.Conversions, Record, Idx);
+  Reader.ReadUnresolvedSet(Data.VisibleConversions, Record, Idx);
+  assert(Data.Definition && "Data.Definition should be already set!");
+  Data.FirstFriend
+      = cast_or_null<FriendDecl>(Reader.GetDecl(Record[Idx++]));
+}
+
+void ASTDeclReader::InitializeCXXDefinitionData(CXXRecordDecl *D,
+                                                CXXRecordDecl *DefinitionDecl,
+                                                const RecordData &Record,
+                                                unsigned &Idx) {
   ASTContext &C = *Reader.getContext();
 
-  // We need to allocate the DefinitionData struct ahead of VisitRecordDecl
-  // so that the other CXXRecordDecls can get a pointer even when the owner
-  // is still initializing.
-  bool OwnsDefinitionData = false;
-  enum DataOwnership { Data_NoDefData, Data_Owner, Data_NotOwner };
-  switch ((DataOwnership)Record[Idx++]) {
-  default:
-    assert(0 && "Out of sync with ASTDeclWriter or messed up reading");
-  case Data_NoDefData:
-    break;
-  case Data_Owner:
-    OwnsDefinitionData = true;
+  if (D == DefinitionDecl) {
     D->DefinitionData = new (C) struct CXXRecordDecl::DefinitionData(D);
-    break;
-  case Data_NotOwner:
-    D->DefinitionData
-        = cast<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]))->DefinitionData;
-    break;
+    ReadCXXDefinitionData(*D->DefinitionData, Record, Idx);
+    // We read the definition info. Check if there are pending forward
+    // references that need to point to this DefinitionData pointer.
+    ASTReader::PendingForwardRefsMap::iterator
+        FindI = Reader.PendingForwardRefs.find(D);
+    if (FindI != Reader.PendingForwardRefs.end()) {
+      ASTReader::ForwardRefs &Refs = FindI->second;
+      for (ASTReader::ForwardRefs::iterator
+             I = Refs.begin(), E = Refs.end(); I != E; ++I)
+        (*I)->DefinitionData = D->DefinitionData;
+#ifndef NDEBUG
+      // We later check whether PendingForwardRefs is empty to make sure all
+      // pending references were linked.
+      Reader.PendingForwardRefs.erase(D);
+#endif
+    }
+  } else if (DefinitionDecl) {
+    if (DefinitionDecl->DefinitionData) {
+      D->DefinitionData = DefinitionDecl->DefinitionData;
+    } else {
+      // The definition is still initializing.
+      Reader.PendingForwardRefs[DefinitionDecl].push_back(D);
+    }
   }
+}
 
+void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   VisitRecordDecl(D);
 
-  if (OwnsDefinitionData) {
-    assert(D->DefinitionData);
-    struct CXXRecordDecl::DefinitionData &Data = *D->DefinitionData;
-
-    Data.UserDeclaredConstructor = Record[Idx++];
-    Data.UserDeclaredCopyConstructor = Record[Idx++];
-    Data.UserDeclaredCopyAssignment = Record[Idx++];
-    Data.UserDeclaredDestructor = Record[Idx++];
-    Data.Aggregate = Record[Idx++];
-    Data.PlainOldData = Record[Idx++];
-    Data.Empty = Record[Idx++];
-    Data.Polymorphic = Record[Idx++];
-    Data.Abstract = Record[Idx++];
-    Data.HasTrivialConstructor = Record[Idx++];
-    Data.HasTrivialCopyConstructor = Record[Idx++];
-    Data.HasTrivialCopyAssignment = Record[Idx++];
-    Data.HasTrivialDestructor = Record[Idx++];
-    Data.ComputedVisibleConversions = Record[Idx++];
-    Data.DeclaredDefaultConstructor = Record[Idx++];
-    Data.DeclaredCopyConstructor = Record[Idx++];
-    Data.DeclaredCopyAssignment = Record[Idx++];
-    Data.DeclaredDestructor = Record[Idx++];
-
-    // setBases() is unsuitable since it may try to iterate the bases of an
-    // uninitialized base.
-    Data.NumBases = Record[Idx++];
-    Data.Bases = new(C) CXXBaseSpecifier [Data.NumBases];
-    for (unsigned i = 0; i != Data.NumBases; ++i)
-      Data.Bases[i] = Reader.ReadCXXBaseSpecifier(Cursor, Record, Idx);
-
-    // FIXME: Make VBases lazily computed when needed to avoid storing them.
-    Data.NumVBases = Record[Idx++];
-    Data.VBases = new(C) CXXBaseSpecifier [Data.NumVBases];
-    for (unsigned i = 0; i != Data.NumVBases; ++i)
-      Data.VBases[i] = Reader.ReadCXXBaseSpecifier(Cursor, Record, Idx);
-
-    Reader.ReadUnresolvedSet(Data.Conversions, Record, Idx);
-    Reader.ReadUnresolvedSet(Data.VisibleConversions, Record, Idx);
-    assert(Data.Definition && "Data.Definition should be already set!");
-    Data.FirstFriend
-        = cast_or_null<FriendDecl>(Reader.GetDecl(Record[Idx++]));
-  }
+  CXXRecordDecl *DefinitionDecl
+      = cast_or_null<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+  InitializeCXXDefinitionData(D, DefinitionDecl, Record, Idx);
+
+  ASTContext &C = *Reader.getContext();
 
   enum CXXRecKind {
     CXXRecNotTemplate = 0, CXXRecTemplate, CXXRecMemberSpecialization
@@ -778,18 +888,28 @@ void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   case CXXRecNotTemplate:
     break;
   case CXXRecTemplate:
-    D->setDescribedClassTemplate(
-                        cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++])));
+    D->TemplateOrInstantiation
+        = cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
     break;
   case CXXRecMemberSpecialization: {
     CXXRecordDecl *RD = cast<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
     TemplateSpecializationKind TSK = (TemplateSpecializationKind)Record[Idx++];
-    SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
-    D->setInstantiationOfMemberClass(RD, TSK);
-    D->getMemberSpecializationInfo()->setPointOfInstantiation(POI);
+    SourceLocation POI = ReadSourceLocation(Record, Idx);
+    MemberSpecializationInfo *MSI = new (C) MemberSpecializationInfo(RD, TSK);
+    MSI->setPointOfInstantiation(POI);
+    D->TemplateOrInstantiation = MSI;
     break;
   }
   }
+
+  // Load the key function to avoid deserializing every method so we can
+  // compute it.
+  if (D->IsDefinition) {
+    CXXMethodDecl *Key
+        = cast_or_null<CXXMethodDecl>(Reader.GetDecl(Record[Idx++]));
+    if (Key)
+      C.KeyFunctions[D] = Key;
+  }
 }
 
 void ASTDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
@@ -808,8 +928,8 @@ void ASTDeclReader::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
   
   D->IsExplicitSpecified = Record[Idx++];
   D->ImplicitlyDefined = Record[Idx++];
-  llvm::tie(D->BaseOrMemberInitializers, D->NumBaseOrMemberInitializers)
-      = Reader.ReadCXXBaseOrMemberInitializers(Cursor, Record, Idx);
+  llvm::tie(D->CtorInitializers, D->NumCtorInitializers)
+      = Reader.ReadCXXCtorInitializers(F, Record, Idx);
 }
 
 void ASTDeclReader::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
@@ -826,17 +946,18 @@ void ASTDeclReader::VisitCXXConversionDecl(CXXConversionDecl *D) {
 
 void ASTDeclReader::VisitAccessSpecDecl(AccessSpecDecl *D) {
   VisitDecl(D);
-  D->setColonLoc(Reader.ReadSourceLocation(Record, Idx));
+  D->setColonLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   VisitDecl(D);
   if (Record[Idx++])
-    D->Friend = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
+    D->Friend = GetTypeSourceInfo(Record, Idx);
   else
     D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
-  D->NextFriend = cast_or_null<FriendDecl>(Reader.GetDecl(Record[Idx++]));
-  D->FriendLoc = Reader.ReadSourceLocation(Record, Idx);
+  D->NextFriend = Record[Idx++];
+  D->UnsupportedFriend = (Record[Idx++] != 0);
+  D->FriendLoc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
@@ -845,12 +966,12 @@ void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
   D->NumParams = NumParams;
   D->Params = new TemplateParameterList*[NumParams];
   for (unsigned i = 0; i != NumParams; ++i)
-    D->Params[i] = Reader.ReadTemplateParameterList(Record, Idx);
+    D->Params[i] = Reader.ReadTemplateParameterList(F, Record, Idx);
   if (Record[Idx++]) // HasFriendDecl
     D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   else
-    D->Friend = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
-  D->FriendLoc = Reader.ReadSourceLocation(Record, Idx);
+    D->Friend = GetTypeSourceInfo(Record, Idx);
+  D->FriendLoc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTDeclReader::VisitTemplateDecl(TemplateDecl *D) {
@@ -859,21 +980,32 @@ void ASTDeclReader::VisitTemplateDecl(TemplateDecl *D) {
   NamedDecl *TemplatedDecl
     = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   TemplateParameterList* TemplateParams
-      = Reader.ReadTemplateParameterList(Record, Idx); 
+      = Reader.ReadTemplateParameterList(F, Record, Idx); 
   D->init(TemplatedDecl, TemplateParams);
 }
 
 void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
-  VisitTemplateDecl(D);
-
-  D->IdentifierNamespace = Record[Idx++];
-  RedeclarableTemplateDecl *PrevDecl =
-      cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]));
-  assert((PrevDecl == 0 || PrevDecl->getKind() == D->getKind()) &&
-         "PrevDecl kind mismatch");
-  if (PrevDecl)
-    D->CommonOrPrev = PrevDecl;
-  if (PrevDecl == 0) {
+  // Initialize CommonOrPrev before VisitTemplateDecl so that getCommonPtr()
+  // can be used while this is still initializing.
+
+  assert(D->CommonOrPrev.isNull() && "getCommonPtr was called earlier on this");
+  DeclID PreviousDeclID = Record[Idx++];
+  DeclID FirstDeclID =  PreviousDeclID ? Record[Idx++] : 0;
+  // We delay loading of the redeclaration chain to avoid deeply nested calls.
+  // We temporarily set the first (canonical) declaration as the previous one
+  // which is the one that matters and mark the real previous DeclID to be
+  // loaded & attached later on.
+  RedeclarableTemplateDecl *FirstDecl =
+      cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(FirstDeclID));
+  assert((FirstDecl == 0 || FirstDecl->getKind() == D->getKind()) &&
+         "FirstDecl kind mismatch");
+  if (FirstDecl) {
+    D->CommonOrPrev = FirstDecl;
+    // Mark the real previous DeclID to be loaded & attached later on.
+    if (PreviousDeclID != FirstDeclID)
+      Reader.PendingPreviousDecls.push_back(std::make_pair(D, PreviousDeclID));
+  } else {
+    D->CommonOrPrev = D->newCommon(*Reader.getContext());
     if (RedeclarableTemplateDecl *RTD
           = cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]))) {
       assert(RTD->getKind() == D->getKind() &&
@@ -897,9 +1029,9 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
       Decl *NewLatest = Reader.GetDecl(I->second);
       assert((LatestDecl->getLocation().isInvalid() ||
               NewLatest->getLocation().isInvalid()  ||
-              Reader.SourceMgr.isBeforeInTranslationUnit(
-                                                   LatestDecl->getLocation(),
-                                                   NewLatest->getLocation())) &&
+              !Reader.SourceMgr.isBeforeInTranslationUnit(
+                                                  NewLatest->getLocation(),
+                                                  LatestDecl->getLocation())) &&
              "The new latest is supposed to come after the previous latest");
       LatestDecl = cast<RedeclarableTemplateDecl>(NewLatest);
     }
@@ -907,24 +1039,42 @@ void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
     assert(LatestDecl->getKind() == D->getKind() && "Latest kind mismatch");
     D->getCommonPtr()->Latest = LatestDecl;
   }
+
+  VisitTemplateDecl(D);
+  D->IdentifierNamespace = Record[Idx++];
 }
 
 void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   VisitRedeclarableTemplateDecl(D);
 
   if (D->getPreviousDeclaration() == 0) {
-    // This ClassTemplateDecl owns a CommonPtr; read it.
-
-    // FoldingSets are filled in VisitClassTemplateSpecializationDecl.
-    unsigned size = Record[Idx++];
-    while (size--)
-      cast<ClassTemplateSpecializationDecl>(Reader.GetDecl(Record[Idx++]));
-
-    size = Record[Idx++];
-    while (size--)
-      cast<ClassTemplatePartialSpecializationDecl>(
-                                                 Reader.GetDecl(Record[Idx++]));
-
+    // This ClassTemplateDecl owns a CommonPtr; read it to keep track of all of
+    // the specializations.
+    llvm::SmallVector<serialization::DeclID, 2> SpecIDs;
+    SpecIDs.push_back(0);
+    
+    // Specializations.
+    unsigned Size = Record[Idx++];
+    SpecIDs[0] += Size;
+    SpecIDs.append(Record.begin() + Idx, Record.begin() + Idx + Size);
+    Idx += Size;
+
+    // Partial specializations.
+    Size = Record[Idx++];
+    SpecIDs[0] += Size;
+    SpecIDs.append(Record.begin() + Idx, Record.begin() + Idx + Size);
+    Idx += Size;
+
+    if (SpecIDs[0]) {
+      typedef serialization::DeclID DeclID;
+      
+      ClassTemplateDecl::Common *CommonPtr = D->getCommonPtr();
+      CommonPtr->LazySpecializations
+        = new (*Reader.getContext()) DeclID [SpecIDs.size()];
+      memcpy(CommonPtr->LazySpecializations, SpecIDs.data(), 
+             SpecIDs.size() * sizeof(DeclID));
+    }
+    
     // InjectedClassNameType is computed.
   }
 }
@@ -932,41 +1082,52 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 void ASTDeclReader::VisitClassTemplateSpecializationDecl(
                                            ClassTemplateSpecializationDecl *D) {
   VisitCXXRecordDecl(D);
-
+  
+  ASTContext &C = *Reader.getContext();
   if (Decl *InstD = Reader.GetDecl(Record[Idx++])) {
     if (ClassTemplateDecl *CTD = dyn_cast<ClassTemplateDecl>(InstD)) {
-      D->setInstantiationOf(CTD);
+      D->SpecializedTemplate = CTD;
     } else {
       llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-      Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
-      D->setInstantiationOf(cast<ClassTemplatePartialSpecializationDecl>(InstD),
-                            TemplArgs.data(), TemplArgs.size());
+      Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
+      TemplateArgumentList *ArgList
+        = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), 
+                                           TemplArgs.size());
+      ClassTemplateSpecializationDecl::SpecializedPartialSpecialization *PS
+          = new (C) ClassTemplateSpecializationDecl::
+                                             SpecializedPartialSpecialization();
+      PS->PartialSpecialization
+          = cast<ClassTemplatePartialSpecializationDecl>(InstD);
+      PS->TemplateArgs = ArgList;
+      D->SpecializedTemplate = PS;
     }
   }
 
   // Explicit info.
-  if (TypeSourceInfo *TyInfo = Reader.GetTypeSourceInfo(Cursor, Record, Idx)) {
-    D->setTypeAsWritten(TyInfo);
-    D->setExternLoc(Reader.ReadSourceLocation(Record, Idx));
-    D->setTemplateKeywordLoc(Reader.ReadSourceLocation(Record, Idx));
+  if (TypeSourceInfo *TyInfo = GetTypeSourceInfo(Record, Idx)) {
+    ClassTemplateSpecializationDecl::ExplicitSpecializationInfo *ExplicitInfo
+        = new (C) ClassTemplateSpecializationDecl::ExplicitSpecializationInfo;
+    ExplicitInfo->TypeAsWritten = TyInfo;
+    ExplicitInfo->ExternLoc = ReadSourceLocation(Record, Idx);
+    ExplicitInfo->TemplateKeywordLoc = ReadSourceLocation(Record, Idx);
+    D->ExplicitInfo = ExplicitInfo;
   }
 
   llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-  Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
-  D->initTemplateArgs(TemplArgs.data(), TemplArgs.size());
-  SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
-  if (POI.isValid())
-    D->setPointOfInstantiation(POI);
-  D->setSpecializationKind((TemplateSpecializationKind)Record[Idx++]);
-
+  Reader.ReadTemplateArgumentList(TemplArgs, F, Record, Idx);
+  D->TemplateArgs = TemplateArgumentList::CreateCopy(C, TemplArgs.data(), 
+                                                     TemplArgs.size());
+  D->PointOfInstantiation = ReadSourceLocation(Record, Idx);
+  D->SpecializationKind = (TemplateSpecializationKind)Record[Idx++];
+  
   if (D->isCanonicalDecl()) { // It's kept in the folding set.
     ClassTemplateDecl *CanonPattern
                        = cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
     if (ClassTemplatePartialSpecializationDecl *Partial
-            = dyn_cast<ClassTemplatePartialSpecializationDecl>(D)) {
-      CanonPattern->getPartialSpecializations().InsertNode(Partial);
+                       = dyn_cast<ClassTemplatePartialSpecializationDecl>(D)) {
+      CanonPattern->getCommonPtr()->PartialSpecializations.InsertNode(Partial);
     } else {
-      CanonPattern->getSpecializations().InsertNode(D);
+      CanonPattern->getCommonPtr()->Specializations.InsertNode(D);
     }
   }
 }
@@ -975,23 +1136,25 @@ void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   VisitClassTemplateSpecializationDecl(D);
 
-  D->initTemplateParameters(Reader.ReadTemplateParameterList(Record, Idx));
-  
-  TemplateArgumentListInfo ArgInfos;
+  ASTContext &C = *Reader.getContext();
+  D->TemplateParams = Reader.ReadTemplateParameterList(F, Record, Idx);
+
   unsigned NumArgs = Record[Idx++];
-  while (NumArgs--)
-    ArgInfos.addArgument(Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx));
-  D->initTemplateArgsAsWritten(ArgInfos);
-  
-  D->setSequenceNumber(Record[Idx++]);
+  if (NumArgs) {
+    D->NumArgsAsWritten = NumArgs;
+    D->ArgsAsWritten = new (C) TemplateArgumentLoc[NumArgs];
+    for (unsigned i=0; i != NumArgs; ++i)
+      D->ArgsAsWritten[i] = Reader.ReadTemplateArgumentLoc(F, Record, Idx);
+  }
+
+  D->SequenceNumber = Record[Idx++];
 
   // These are read/set from/to the first declaration.
   if (D->getPreviousDeclaration() == 0) {
-    D->setInstantiatedFromMember(
+    D->InstantiatedFromMember.setPointer(
         cast_or_null<ClassTemplatePartialSpecializationDecl>(
                                                 Reader.GetDecl(Record[Idx++])));
-    if (Record[Idx++])
-      D->setMemberSpecialization();
+    D->InstantiatedFromMember.setInt(Record[Idx++]);
   }
 }
 
@@ -1003,7 +1166,7 @@ void ASTDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
 
     // Read the function specialization declarations.
     // FunctionTemplateDecl's FunctionTemplateSpecializationInfos are filled
-    // through the specialized FunctionDecl's setFunctionTemplateSpecialization.
+    // when reading the specialized FunctionDecl.
     unsigned NumSpecs = Record[Idx++];
     while (NumSpecs--)
       Reader.GetDecl(Record[Idx++]);
@@ -1017,21 +1180,30 @@ void ASTDeclReader::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
   D->setParameterPack(Record[Idx++]);
 
   bool Inherited = Record[Idx++];
-  TypeSourceInfo *DefArg = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
+  TypeSourceInfo *DefArg = GetTypeSourceInfo(Record, Idx);
   D->setDefaultArgument(DefArg, Inherited);
 }
 
 void ASTDeclReader::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
-  VisitVarDecl(D);
+  VisitDeclaratorDecl(D);
   // TemplateParmPosition.
   D->setDepth(Record[Idx++]);
   D->setPosition(Record[Idx++]);
-  // Rest of NonTypeTemplateParmDecl.
-  if (Record[Idx++]) {
-    Expr *DefArg = Reader.ReadExpr(Cursor);
-    bool Inherited = Record[Idx++];
-    D->setDefaultArgument(DefArg, Inherited);
- }
+  if (D->isExpandedParameterPack()) {
+    void **Data = reinterpret_cast<void **>(D + 1);
+    for (unsigned I = 0, N = D->getNumExpansionTypes(); I != N; ++I) {
+      Data[2*I] = Reader.GetType(Record[Idx++]).getAsOpaquePtr();
+      Data[2*I + 1] = GetTypeSourceInfo(Record, Idx);
+    }
+  } else {
+    // Rest of NonTypeTemplateParmDecl.
+    D->ParameterPack = Record[Idx++];
+    if (Record[Idx++]) {
+      Expr *DefArg = Reader.ReadExpr(F);
+      bool Inherited = Record[Idx++];
+      D->setDefaultArgument(DefArg, Inherited);
+   }
+  }
 }
 
 void ASTDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
@@ -1040,15 +1212,16 @@ void ASTDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   D->setDepth(Record[Idx++]);
   D->setPosition(Record[Idx++]);
   // Rest of TemplateTemplateParmDecl.
-  TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx);
+  TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(F, Record, Idx);
   bool IsInherited = Record[Idx++];
   D->setDefaultArgument(Arg, IsInherited);
+  D->ParameterPack = Record[Idx++];
 }
 
 void ASTDeclReader::VisitStaticAssertDecl(StaticAssertDecl *D) {
   VisitDecl(D);
-  D->AssertExpr = Reader.ReadExpr(Cursor);
-  D->Message = cast<StringLiteral>(Reader.ReadExpr(Cursor));
+  D->AssertExpr = Reader.ReadExpr(F);
+  D->Message = cast<StringLiteral>(Reader.ReadExpr(F));
 }
 
 std::pair<uint64_t, uint64_t>
@@ -1068,10 +1241,20 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
                 " reading");
   case NoRedeclaration:
     break;
-  case PointsToPrevious:
+  case PointsToPrevious: {
+    DeclID PreviousDeclID = Record[Idx++];
+    DeclID FirstDeclID = Record[Idx++];
+    // We delay loading of the redeclaration chain to avoid deeply nested calls.
+    // We temporarily set the first (canonical) declaration as the previous one
+    // which is the one that matters and mark the real previous DeclID to be
+    // loaded & attached later on.
     D->RedeclLink = typename Redeclarable<T>::PreviousDeclLink(
-                                cast_or_null<T>(Reader.GetDecl(Record[Idx++])));
+                                cast_or_null<T>(Reader.GetDecl(FirstDeclID)));
+    if (PreviousDeclID != FirstDeclID)
+      Reader.PendingPreviousDecls.push_back(std::make_pair(static_cast<T*>(D),
+                                                           PreviousDeclID));
     break;
+  }
   case PointsToLatest:
     D->RedeclLink = typename Redeclarable<T>::LatestDeclLink(
                                 cast_or_null<T>(Reader.GetDecl(Record[Idx++])));
@@ -1095,12 +1278,6 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
       = Reader.FirstLatestDeclIDs.find(ThisDeclID);
   if (I != Reader.FirstLatestDeclIDs.end()) {
     Decl *NewLatest = Reader.GetDecl(I->second);
-    assert((D->getMostRecentDeclaration()->getLocation().isInvalid() ||
-            NewLatest->getLocation().isInvalid() ||
-            Reader.SourceMgr.isBeforeInTranslationUnit(
-                                   D->getMostRecentDeclaration()->getLocation(),
-                                   NewLatest->getLocation())) &&
-           "The new latest is supposed to come after the previous latest");
     D->RedeclLink
         = typename Redeclarable<T>::LatestDeclLink(cast_or_null<T>(NewLatest));
   }
@@ -1111,28 +1288,16 @@ void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
 //===----------------------------------------------------------------------===//
 
 /// \brief Reads attributes from the current stream position.
-void ASTReader::ReadAttributes(llvm::BitstreamCursor &DeclsCursor,
-                               AttrVec &Attrs) {
-  unsigned Code = DeclsCursor.ReadCode();
-  assert(Code == llvm::bitc::UNABBREV_RECORD &&
-         "Expected unabbreviated record"); (void)Code;
-
-  RecordData Record;
-  unsigned Idx = 0;
-  unsigned RecCode = DeclsCursor.ReadRecord(Code, Record);
-  assert(RecCode == DECL_ATTR && "Expected attribute record");
-  (void)RecCode;
-
-  while (Idx < Record.size()) {
+void ASTReader::ReadAttributes(PerFileData &F, AttrVec &Attrs,
+                               const RecordData &Record, unsigned &Idx) {
+  for (unsigned i = 0, e = Record[Idx++]; i != e; ++i) {
     Attr *New = 0;
     attr::Kind Kind = (attr::Kind)Record[Idx++];
-    SourceLocation Loc = SourceLocation::getFromRawEncoding(Record[Idx++]);
-    bool isInherited = Record[Idx++];
+    SourceLocation Loc = ReadSourceLocation(F, Record, Idx);
 
 #include "clang/Serialization/AttrPCHRead.inc"
 
     assert(New && "Unable to decode attribute?");
-    New->setInherited(isInherited);
     Attrs.push_back(New);
   }
 }
@@ -1176,7 +1341,7 @@ ASTReader::DeclCursorForIndex(unsigned Index, DeclID ID) {
   // See if there's an override.
   DeclReplacementMap::iterator It = ReplacedDecls.find(ID);
   if (It != ReplacedDecls.end())
-    return RecordLocation(&It->second.first->DeclsCursor, It->second.second);
+    return RecordLocation(It->second.first, It->second.second);
 
   PerFileData *F = 0;
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
@@ -1186,13 +1351,32 @@ ASTReader::DeclCursorForIndex(unsigned Index, DeclID ID) {
     Index -= F->LocalNumDecls;
   }
   assert(F && F->LocalNumDecls > Index && "Broken chain");
-  return RecordLocation(&F->DeclsCursor, F->DeclOffsets[Index]);
+  return RecordLocation(F, F->DeclOffsets[Index]);
+}
+
+void ASTDeclReader::attachPreviousDecl(Decl *D, Decl *previous) {
+  assert(D && previous);
+  if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
+    TD->RedeclLink.setPointer(cast<TagDecl>(previous));
+  } else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    FD->RedeclLink.setPointer(cast<FunctionDecl>(previous));
+  } else if (VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    VD->RedeclLink.setPointer(cast<VarDecl>(previous));
+  } else {
+    RedeclarableTemplateDecl *TD = cast<RedeclarableTemplateDecl>(D);
+    TD->CommonOrPrev = cast<RedeclarableTemplateDecl>(previous);
+  }
+}
+
+void ASTReader::loadAndAttachPreviousDecl(Decl *D, serialization::DeclID ID) {
+  Decl *previous = GetDecl(ID);
+  ASTDeclReader::attachPreviousDecl(D, previous);
 }
 
 /// \brief Read the declaration at the given offset from the AST file.
 Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   RecordLocation Loc = DeclCursorForIndex(Index, ID);
-  llvm::BitstreamCursor &DeclsCursor = *Loc.first;
+  llvm::BitstreamCursor &DeclsCursor = Loc.F->DeclsCursor;
   // Keep track of where we are in the stream, then jump back there
   // after reading this declaration.
   SavedStreamPosition SavedPosition(DeclsCursor);
@@ -1202,15 +1386,14 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   // Note that we are loading a declaration record.
   Deserializing ADecl(this);
 
-  DeclsCursor.JumpToBit(Loc.second);
+  DeclsCursor.JumpToBit(Loc.Offset);
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
   unsigned Idx = 0;
-  ASTDeclReader Reader(*this, DeclsCursor, ID, Record, Idx);
+  ASTDeclReader Reader(*this, *Loc.F, DeclsCursor, ID, Record, Idx);
 
   Decl *D = 0;
   switch ((DeclCode)DeclsCursor.ReadRecord(Code, Record)) {
-  case DECL_ATTR:
   case DECL_CONTEXT_LEXICAL:
   case DECL_CONTEXT_VISIBLE:
     assert(false && "Record cannot be de-serialized with ReadDeclRecord");
@@ -1241,6 +1424,9 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
                                 (LinkageSpecDecl::LanguageIDs)0,
                                 false);
     break;
+  case DECL_LABEL:
+    D = LabelDecl::Create(*Context, 0, SourceLocation(), 0);
+    break;
   case DECL_NAMESPACE:
     D = NamespaceDecl::Create(*Context, 0, SourceLocation(), 0);
     break;
@@ -1289,8 +1475,7 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
     D = CXXConversionDecl::Create(*Context, Decl::EmptyShell());
     break;
   case DECL_ACCESS_SPEC:
-    D = AccessSpecDecl::Create(*Context, AS_none, 0, SourceLocation(),
-                               SourceLocation());
+    D = AccessSpecDecl::Create(*Context, Decl::EmptyShell());
     break;
   case DECL_FRIEND:
     D = FriendDecl::Create(*Context, Decl::EmptyShell());
@@ -1318,10 +1503,16 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
     break;
   case DECL_NON_TYPE_TEMPLATE_PARM:
     D = NonTypeTemplateParmDecl::Create(*Context, 0, SourceLocation(), 0,0,0,
-                                        QualType(),0);
+                                        QualType(), false, 0);
+    break;
+  case DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK:
+    D = NonTypeTemplateParmDecl::Create(*Context, 0, SourceLocation(), 0, 0,
+                                        0, QualType(), 0, 0, Record[Idx++],
+                                        0);
     break;
   case DECL_TEMPLATE_TEMPLATE_PARM:
-    D = TemplateTemplateParmDecl::Create(*Context, 0, SourceLocation(),0,0,0,0);
+    D = TemplateTemplateParmDecl::Create(*Context, 0, SourceLocation(), 0, 0,
+                                         false, 0, 0);
     break;
   case DECL_STATIC_ASSERT:
     D = StaticAssertDecl::Create(*Context, 0, SourceLocation(), 0, 0);
@@ -1371,12 +1562,17 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   case DECL_OBJC_PROPERTY_IMPL:
     D = ObjCPropertyImplDecl::Create(*Context, 0, SourceLocation(),
                                      SourceLocation(), 0,
-                                     ObjCPropertyImplDecl::Dynamic, 0);
+                                     ObjCPropertyImplDecl::Dynamic, 0,
+                                     SourceLocation());
     break;
   case DECL_FIELD:
     D = FieldDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0, 0,
                           false);
     break;
+  case DECL_INDIRECTFIELD:
+    D = IndirectFieldDecl::Create(*Context, 0, SourceLocation(), 0, QualType(),
+                                  0, 0);
+    break;
   case DECL_VAR:
     D = VarDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0,
                         SC_None, SC_None);
@@ -1396,6 +1592,9 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
   case DECL_BLOCK:
     D = BlockDecl::Create(*Context, 0, SourceLocation());
     break;
+  case DECL_CXX_BASE_SPECIFIERS:
+    Error("attempt to read a C++ base-specifier record as a declaration");
+    return 0;
   }
 
   assert(D && "Unknown declaration reading AST file");
@@ -1435,21 +1634,29 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
       }
     }
   }
+  assert(Idx == Record.size());
 
-  // If this is a template, read additional specializations that may be in a
-  // different part of the chain.
-  if (isa<RedeclarableTemplateDecl>(D)) {
-    AdditionalTemplateSpecializationsMap::iterator F =
-        AdditionalTemplateSpecializationsPending.find(ID);
-    if (F != AdditionalTemplateSpecializationsPending.end()) {
-      for (AdditionalTemplateSpecializations::iterator I = F->second.begin(),
-                                                       E = F->second.end();
-           I != E; ++I)
-        GetDecl(*I);
-      AdditionalTemplateSpecializationsPending.erase(F);
+  // The declaration may have been modified by files later in the chain.
+  // If this is the case, read the record containing the updates from each file
+  // and pass it to ASTDeclReader to make the modifications.
+  DeclUpdateOffsetsMap::iterator UpdI = DeclUpdateOffsets.find(ID);
+  if (UpdI != DeclUpdateOffsets.end()) {
+    FileOffsetsTy &UpdateOffsets = UpdI->second;
+    for (FileOffsetsTy::iterator
+           I = UpdateOffsets.begin(), E = UpdateOffsets.end(); I != E; ++I) {
+      PerFileData *F = I->first;
+      uint64_t Offset = I->second;
+      llvm::BitstreamCursor &Cursor = F->DeclsCursor;
+      SavedStreamPosition SavedPosition(Cursor);
+      Cursor.JumpToBit(Offset);
+      RecordData Record;
+      unsigned Code = Cursor.ReadCode();
+      unsigned RecCode = Cursor.ReadRecord(Code, Record);
+      (void)RecCode;
+      assert(RecCode == DECL_UPDATES && "Expected DECL_UPDATES record!");
+      Reader.UpdateDecl(D, Record);
     }
   }
-  assert(Idx == Record.size());
 
   // If we have deserialized a declaration that has a definition the
   // AST consumer might need to know about, queue it.
@@ -1460,3 +1667,27 @@ Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
 
   return D;
 }
+
+void ASTDeclReader::UpdateDecl(Decl *D, const RecordData &Record) {
+  unsigned Idx = 0;
+  while (Idx < Record.size()) {
+    switch ((DeclUpdateKind)Record[Idx++]) {
+    case UPD_CXX_SET_DEFINITIONDATA: {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(D);
+      CXXRecordDecl *
+          DefinitionDecl = cast<CXXRecordDecl>(Reader.GetDecl(Record[Idx++]));
+      assert(!RD->DefinitionData && "DefinitionData is already set!");
+      InitializeCXXDefinitionData(RD, DefinitionDecl, Record, Idx);
+      break;
+    }
+
+    case UPD_CXX_ADDED_IMPLICIT_MEMBER:
+      cast<CXXRecordDecl>(D)->addedMember(Reader.GetDecl(Record[Idx++]));
+      break;
+
+    case UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION:
+      // It will be added to the template's specializations set when loaded.
+      Reader.GetDecl(Record[Idx++]);
+    }
+  }
+}
diff --git a/lib/Serialization/ASTReaderStmt.cpp b/lib/Serialization/ASTReaderStmt.cpp
index ee5d40a3698e..4e91c989260d 100644
--- a/lib/Serialization/ASTReaderStmt.cpp
+++ b/lib/Serialization/ASTReaderStmt.cpp
@@ -14,6 +14,7 @@
 
 #include "clang/Serialization/ASTReader.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/StmtVisitor.h"
 using namespace clang;
 using namespace clang::serialization;
@@ -22,14 +23,36 @@ namespace clang {
 
   class ASTStmtReader : public StmtVisitor<ASTStmtReader> {
     ASTReader &Reader;
+    ASTReader::PerFileData &F;
     llvm::BitstreamCursor &DeclsCursor;
     const ASTReader::RecordData &Record;
     unsigned &Idx;
 
+    SourceLocation ReadSourceLocation(const ASTReader::RecordData &R,
+                                      unsigned &I) {
+      return Reader.ReadSourceLocation(F, R, I);
+    }
+    SourceRange ReadSourceRange(const ASTReader::RecordData &R, unsigned &I) {
+      return Reader.ReadSourceRange(F, R, I);
+    }
+    TypeSourceInfo *GetTypeSourceInfo(const ASTReader::RecordData &R,
+                                      unsigned &I) {
+      return Reader.GetTypeSourceInfo(F, R, I);
+    }
+    void ReadDeclarationNameLoc(DeclarationNameLoc &DNLoc, DeclarationName Name,
+                                const ASTReader::RecordData &R, unsigned &I) {
+      Reader.ReadDeclarationNameLoc(F, DNLoc, Name, R, I);
+    }
+    void ReadDeclarationNameInfo(DeclarationNameInfo &NameInfo,
+                                const ASTReader::RecordData &R, unsigned &I) {
+      Reader.ReadDeclarationNameInfo(F, NameInfo, R, I);
+    }
+
   public:
-    ASTStmtReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
+    ASTStmtReader(ASTReader &Reader, ASTReader::PerFileData &F,
+                  llvm::BitstreamCursor &Cursor,
                   const ASTReader::RecordData &Record, unsigned &Idx)
-      : Reader(Reader), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
+      : Reader(Reader), F(F), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
 
     /// \brief The number of record fields required for the Stmt class
     /// itself.
@@ -37,7 +60,7 @@ namespace clang {
 
     /// \brief The number of record fields required for the Expr class
     /// itself.
-    static const unsigned NumExprFields = NumStmtFields + 3;
+    static const unsigned NumExprFields = NumStmtFields + 6;
     
     /// \brief Read and initialize a ExplicitTemplateArgumentList structure.
     void ReadExplicitTemplateArgumentList(ExplicitTemplateArgumentList &ArgList,
@@ -82,6 +105,7 @@ namespace clang {
     void VisitBinaryOperator(BinaryOperator *E);
     void VisitCompoundAssignOperator(CompoundAssignOperator *E);
     void VisitConditionalOperator(ConditionalOperator *E);
+    void VisitBinaryConditionalOperator(BinaryConditionalOperator *E);
     void VisitImplicitCastExpr(ImplicitCastExpr *E);
     void VisitExplicitCastExpr(ExplicitCastExpr *E);
     void VisitCStyleCastExpr(CStyleCastExpr *E);
@@ -93,7 +117,6 @@ namespace clang {
     void VisitVAArgExpr(VAArgExpr *E);
     void VisitAddrLabelExpr(AddrLabelExpr *E);
     void VisitStmtExpr(StmtExpr *E);
-    void VisitTypesCompatibleExpr(TypesCompatibleExpr *E);
     void VisitChooseExpr(ChooseExpr *E);
     void VisitGNUNullExpr(GNUNullExpr *E);
     void VisitShuffleVectorExpr(ShuffleVectorExpr *E);
@@ -105,10 +128,7 @@ namespace clang {
     void VisitObjCProtocolExpr(ObjCProtocolExpr *E);
     void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E);
     void VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E);
-    void VisitObjCImplicitSetterGetterRefExpr(
-                            ObjCImplicitSetterGetterRefExpr *E);
     void VisitObjCMessageExpr(ObjCMessageExpr *E);
-    void VisitObjCSuperExpr(ObjCSuperExpr *E);
     void VisitObjCIsaExpr(ObjCIsaExpr *E);
 
     void VisitObjCForCollectionStmt(ObjCForCollectionStmt *);
@@ -134,6 +154,7 @@ namespace clang {
     void VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E);
     void VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E);
     void VisitCXXTypeidExpr(CXXTypeidExpr *E);
+    void VisitCXXUuidofExpr(CXXUuidofExpr *E);
     void VisitCXXThisExpr(CXXThisExpr *E);
     void VisitCXXThrowExpr(CXXThrowExpr *E);
     void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E);
@@ -144,7 +165,7 @@ namespace clang {
     void VisitCXXDeleteExpr(CXXDeleteExpr *E);
     void VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E);
     
-    void VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E);
+    void VisitExprWithCleanups(ExprWithCleanups *E);
     
     void VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E);
     void VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E);
@@ -155,6 +176,16 @@ namespace clang {
     void VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E);
 
     void VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E);
+    void VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *E);
+    void VisitCXXNoexceptExpr(CXXNoexceptExpr *E);
+    void VisitPackExpansionExpr(PackExpansionExpr *E);
+    void VisitSizeOfPackExpr(SizeOfPackExpr *E);
+    void VisitSubstNonTypeTemplateParmPackExpr(
+                                           SubstNonTypeTemplateParmPackExpr *E);
+    void VisitOpaqueValueExpr(OpaqueValueExpr *E);
+    
+    // CUDA Expressions
+    void VisitCUDAKernelCallExpr(CUDAKernelCallExpr *E);
   };
 }
 
@@ -162,11 +193,11 @@ void ASTStmtReader::
 ReadExplicitTemplateArgumentList(ExplicitTemplateArgumentList &ArgList,
                                  unsigned NumTemplateArgs) {
   TemplateArgumentListInfo ArgInfo;
-  ArgInfo.setLAngleLoc(Reader.ReadSourceLocation(Record, Idx));
-  ArgInfo.setRAngleLoc(Reader.ReadSourceLocation(Record, Idx));
+  ArgInfo.setLAngleLoc(ReadSourceLocation(Record, Idx));
+  ArgInfo.setRAngleLoc(ReadSourceLocation(Record, Idx));
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
     ArgInfo.addArgument(
-        Reader.ReadTemplateArgumentLoc(DeclsCursor, Record, Idx));
+        Reader.ReadTemplateArgumentLoc(F, Record, Idx));
   ArgList.initializeFrom(ArgInfo);
 }
 
@@ -176,7 +207,8 @@ void ASTStmtReader::VisitStmt(Stmt *S) {
 
 void ASTStmtReader::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
-  S->setSemiLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setSemiLoc(ReadSourceLocation(Record, Idx));
+  S->LeadingEmptyMacro = Record[Idx++];
 }
 
 void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) {
@@ -186,8 +218,8 @@ void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) {
   while (NumStmts--)
     Stmts.push_back(Reader.ReadSubStmt());
   S->setStmts(*Reader.getContext(), Stmts.data(), Stmts.size());
-  S->setLBracLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRBracLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setLBracLoc(ReadSourceLocation(Record, Idx));
+  S->setRBracLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitSwitchCase(SwitchCase *S) {
@@ -200,24 +232,25 @@ void ASTStmtReader::VisitCaseStmt(CaseStmt *S) {
   S->setLHS(Reader.ReadSubExpr());
   S->setRHS(Reader.ReadSubExpr());
   S->setSubStmt(Reader.ReadSubStmt());
-  S->setCaseLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setEllipsisLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setCaseLoc(ReadSourceLocation(Record, Idx));
+  S->setEllipsisLoc(ReadSourceLocation(Record, Idx));
+  S->setColonLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitDefaultStmt(DefaultStmt *S) {
   VisitSwitchCase(S);
   S->setSubStmt(Reader.ReadSubStmt());
-  S->setDefaultLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setDefaultLoc(ReadSourceLocation(Record, Idx));
+  S->setColonLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
-  S->setID(Reader.GetIdentifierInfo(Record, Idx));
+  LabelDecl *LD = cast<LabelDecl>(Reader.GetDecl(Record[Idx++]));
+  LD->setStmt(S);
+  S->setDecl(LD);
   S->setSubStmt(Reader.ReadSubStmt());
-  S->setIdentLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  Reader.RecordLabelStmt(S, Record[Idx++]);
+  S->setIdentLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitIfStmt(IfStmt *S) {
@@ -227,8 +260,8 @@ void ASTStmtReader::VisitIfStmt(IfStmt *S) {
   S->setCond(Reader.ReadSubExpr());
   S->setThen(Reader.ReadSubStmt());
   S->setElse(Reader.ReadSubStmt());
-  S->setIfLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setElseLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setIfLoc(ReadSourceLocation(Record, Idx));
+  S->setElseLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
@@ -237,7 +270,10 @@ void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
-  S->setSwitchLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setSwitchLoc(ReadSourceLocation(Record, Idx));
+  if (Record[Idx++])
+    S->setAllEnumCasesCovered();
+
   SwitchCase *PrevSC = 0;
   for (unsigned N = Record.size(); Idx != N; ++Idx) {
     SwitchCase *SC = Reader.getSwitchCaseWithID(Record[Idx]);
@@ -246,9 +282,6 @@ void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
     else
       S->setSwitchCaseList(SC);
 
-    // Retain this SwitchCase, since SwitchStmt::addSwitchCase() would
-    // normally retain it (but we aren't calling addSwitchCase).
-    SC->Retain();
     PrevSC = SC;
   }
 }
@@ -259,16 +292,16 @@ void ASTStmtReader::VisitWhileStmt(WhileStmt *S) {
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
-  S->setWhileLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setWhileLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitDoStmt(DoStmt *S) {
   VisitStmt(S);
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
-  S->setDoLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setWhileLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setDoLoc(ReadSourceLocation(Record, Idx));
+  S->setWhileLoc(ReadSourceLocation(Record, Idx));
+  S->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitForStmt(ForStmt *S) {
@@ -279,46 +312,46 @@ void ASTStmtReader::VisitForStmt(ForStmt *S) {
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
   S->setInc(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
-  S->setForLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setForLoc(ReadSourceLocation(Record, Idx));
+  S->setLParenLoc(ReadSourceLocation(Record, Idx));
+  S->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
-  Reader.SetLabelOf(S, Record[Idx++]);
-  S->setGotoLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setLabelLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setLabel(cast<LabelDecl>(Reader.GetDecl(Record[Idx++])));
+  S->setGotoLoc(ReadSourceLocation(Record, Idx));
+  S->setLabelLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
   VisitStmt(S);
-  S->setGotoLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setGotoLoc(ReadSourceLocation(Record, Idx));
+  S->setStarLoc(ReadSourceLocation(Record, Idx));
   S->setTarget(Reader.ReadSubExpr());
 }
 
 void ASTStmtReader::VisitContinueStmt(ContinueStmt *S) {
   VisitStmt(S);
-  S->setContinueLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setContinueLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitBreakStmt(BreakStmt *S) {
   VisitStmt(S);
-  S->setBreakLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setBreakLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
   S->setRetValue(Reader.ReadSubExpr());
-  S->setReturnLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setReturnLoc(ReadSourceLocation(Record, Idx));
   S->setNRVOCandidate(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
 void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
   VisitStmt(S);
-  S->setStartLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setEndLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setStartLoc(ReadSourceLocation(Record, Idx));
+  S->setEndLoc(ReadSourceLocation(Record, Idx));
 
   if (Idx + 1 == Record.size()) {
     // Single declaration
@@ -339,8 +372,8 @@ void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
   unsigned NumOutputs = Record[Idx++];
   unsigned NumInputs = Record[Idx++];
   unsigned NumClobbers = Record[Idx++];
-  S->setAsmLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setAsmLoc(ReadSourceLocation(Record, Idx));
+  S->setRParenLoc(ReadSourceLocation(Record, Idx));
   S->setVolatile(Record[Idx++]);
   S->setSimple(Record[Idx++]);
   S->setMSAsm(Record[Idx++]);
@@ -373,12 +406,15 @@ void ASTStmtReader::VisitExpr(Expr *E) {
   E->setType(Reader.GetType(Record[Idx++]));
   E->setTypeDependent(Record[Idx++]);
   E->setValueDependent(Record[Idx++]);
+  E->ExprBits.ContainsUnexpandedParameterPack = Record[Idx++];
+  E->setValueKind(static_cast<ExprValueKind>(Record[Idx++]));
+  E->setObjectKind(static_cast<ExprObjectKind>(Record[Idx++]));
   assert(Idx == NumExprFields && "Incorrect expression field count");
 }
 
 void ASTStmtReader::VisitPredefinedExpr(PredefinedExpr *E) {
   VisitExpr(E);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setIdentType((PredefinedExpr::IdentType)Record[Idx++]);
 }
 
@@ -386,28 +422,31 @@ void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
   VisitExpr(E);
 
   bool HasQualifier = Record[Idx++];
-  unsigned NumTemplateArgs = Record[Idx++];
+  bool HasExplicitTemplateArgs = Record[Idx++];
   
   E->DecoratedD.setInt((HasQualifier? DeclRefExpr::HasQualifierFlag : 0) |
-      (NumTemplateArgs ? DeclRefExpr::HasExplicitTemplateArgumentListFlag : 0));
+      (HasExplicitTemplateArgs 
+         ? DeclRefExpr::HasExplicitTemplateArgumentListFlag : 0));
   
   if (HasQualifier) {
     E->getNameQualifier()->NNS = Reader.ReadNestedNameSpecifier(Record, Idx);
-    E->getNameQualifier()->Range = Reader.ReadSourceRange(Record, Idx);
+    E->getNameQualifier()->Range = ReadSourceRange(Record, Idx);
   }
 
-  if (NumTemplateArgs)
+  if (HasExplicitTemplateArgs) {
+    unsigned NumTemplateArgs = Record[Idx++];
     ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
                                      NumTemplateArgs);
+  }
 
   E->setDecl(cast<ValueDecl>(Reader.GetDecl(Record[Idx++])));
-  // FIXME: read DeclarationNameLoc.
-  E->setLocation(Reader.ReadSourceLocation(Record, Idx));
+  E->setLocation(ReadSourceLocation(Record, Idx));
+  ReadDeclarationNameLoc(E->DNLoc, E->getDecl()->getDeclName(), Record, Idx);
 }
 
 void ASTStmtReader::VisitIntegerLiteral(IntegerLiteral *E) {
   VisitExpr(E);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setValue(*Reader.getContext(), Reader.ReadAPInt(Record, Idx));
 }
 
@@ -415,7 +454,7 @@ void ASTStmtReader::VisitFloatingLiteral(FloatingLiteral *E) {
   VisitExpr(E);
   E->setValue(*Reader.getContext(), Reader.ReadAPFloat(Record, Idx));
   E->setExact(Record[Idx++]);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitImaginaryLiteral(ImaginaryLiteral *E) {
@@ -438,20 +477,20 @@ void ASTStmtReader::VisitStringLiteral(StringLiteral *E) {
 
   // Read source locations
   for (unsigned I = 0, N = E->getNumConcatenated(); I != N; ++I)
-    E->setStrTokenLoc(I, SourceLocation::getFromRawEncoding(Record[Idx++]));
+    E->setStrTokenLoc(I, ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCharacterLiteral(CharacterLiteral *E) {
   VisitExpr(E);
   E->setValue(Record[Idx++]);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setWide(Record[Idx++]);
 }
 
 void ASTStmtReader::VisitParenExpr(ParenExpr *E) {
   VisitExpr(E);
-  E->setLParen(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParen(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLParen(ReadSourceLocation(Record, Idx));
+  E->setRParen(ReadSourceLocation(Record, Idx));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
@@ -462,15 +501,15 @@ void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) {
   for (unsigned i = 0; i != NumExprs; ++i)
     E->Exprs[i] = Reader.ReadSubStmt();
   E->NumExprs = NumExprs;
-  E->LParenLoc = Reader.ReadSourceLocation(Record, Idx);
-  E->RParenLoc = Reader.ReadSourceLocation(Record, Idx);
+  E->LParenLoc = ReadSourceLocation(Record, Idx);
+  E->RParenLoc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitUnaryOperator(UnaryOperator *E) {
   VisitExpr(E);
   E->setSubExpr(Reader.ReadSubExpr());
   E->setOpcode((UnaryOperator::Opcode)Record[Idx++]);
-  E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
@@ -480,13 +519,13 @@ void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
   ++Idx;
   assert(E->getNumExpressions() == Record[Idx]);
   ++Idx;
-  E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
+  E->setTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
   for (unsigned I = 0, N = E->getNumComponents(); I != N; ++I) {
     Node::Kind Kind = static_cast<Node::Kind>(Record[Idx++]);
-    SourceLocation Start = SourceLocation::getFromRawEncoding(Record[Idx++]);
-    SourceLocation End = SourceLocation::getFromRawEncoding(Record[Idx++]);
+    SourceLocation Start = ReadSourceLocation(Record, Idx);
+    SourceLocation End = ReadSourceLocation(Record, Idx);
     switch (Kind) {
     case Node::Array:
       E->setComponent(I, Node(Start, Record[Idx++], End));
@@ -505,7 +544,7 @@ void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
         
     case Node::Base: {
       CXXBaseSpecifier *Base = new (*Reader.getContext()) CXXBaseSpecifier();
-      *Base = Reader.ReadCXXBaseSpecifier(DeclsCursor, Record, Idx);
+      *Base = Reader.ReadCXXBaseSpecifier(F, Record, Idx);
       E->setComponent(I, Node(Base));
       break;
     }
@@ -523,23 +562,23 @@ void ASTStmtReader::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
     E->setArgument(Reader.ReadSubExpr());
     ++Idx;
   } else {
-    E->setArgument(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+    E->setArgument(GetTypeSourceInfo(Record, Idx));
   }
-  E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
   VisitExpr(E);
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
-  E->setRBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setRBracketLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   E->setNumArgs(*Reader.getContext(), Record[Idx++]);
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
   E->setCallee(Reader.ReadSubExpr());
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
@@ -554,7 +593,7 @@ void ASTStmtReader::VisitMemberExpr(MemberExpr *E) {
 void ASTStmtReader::VisitObjCIsaExpr(ObjCIsaExpr *E) {
   VisitExpr(E);
   E->setBase(Reader.ReadSubExpr());
-  E->setIsaMemberLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setIsaMemberLoc(ReadSourceLocation(Record, Idx));
   E->setArrow(Record[Idx++]);
 }
 
@@ -567,7 +606,7 @@ void ASTStmtReader::VisitCastExpr(CastExpr *E) {
   CastExpr::path_iterator BaseI = E->path_begin();
   while (NumBaseSpecs--) {
     CXXBaseSpecifier *BaseSpec = new (*Reader.getContext()) CXXBaseSpecifier;
-    *BaseSpec = Reader.ReadCXXBaseSpecifier(DeclsCursor, Record, Idx);
+    *BaseSpec = Reader.ReadCXXBaseSpecifier(F, Record, Idx);
     *BaseI++ = BaseSpec;
   }
 }
@@ -577,7 +616,7 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
   E->setOpcode((BinaryOperator::Opcode)Record[Idx++]);
-  E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
@@ -588,34 +627,46 @@ void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
 
 void ASTStmtReader::VisitConditionalOperator(ConditionalOperator *E) {
   VisitExpr(E);
-  E->setCond(Reader.ReadSubExpr());
-  E->setLHS(Reader.ReadSubExpr());
-  E->setRHS(Reader.ReadSubExpr());
-  E->setSAVE(Reader.ReadSubExpr());
-  E->setQuestionLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->SubExprs[ConditionalOperator::COND] = Reader.ReadSubExpr();
+  E->SubExprs[ConditionalOperator::LHS] = Reader.ReadSubExpr();
+  E->SubExprs[ConditionalOperator::RHS] = Reader.ReadSubExpr();
+  E->QuestionLoc = ReadSourceLocation(Record, Idx);
+  E->ColonLoc = ReadSourceLocation(Record, Idx);
+}
+
+void
+ASTStmtReader::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) {
+  VisitExpr(E);
+  E->OpaqueValue = cast<OpaqueValueExpr>(Reader.ReadSubExpr());
+  E->SubExprs[BinaryConditionalOperator::COMMON] = Reader.ReadSubExpr();
+  E->SubExprs[BinaryConditionalOperator::COND] = Reader.ReadSubExpr();
+  E->SubExprs[BinaryConditionalOperator::LHS] = Reader.ReadSubExpr();
+  E->SubExprs[BinaryConditionalOperator::RHS] = Reader.ReadSubExpr();
+  E->QuestionLoc = ReadSourceLocation(Record, Idx);
+  E->ColonLoc = ReadSourceLocation(Record, Idx);
+
+  E->getOpaqueValue()->setSourceExpr(E->getCommon());
 }
 
 void ASTStmtReader::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   VisitCastExpr(E);
-  E->setValueKind(static_cast<ExprValueKind>(Record[Idx++]));
 }
 
 void ASTStmtReader::VisitExplicitCastExpr(ExplicitCastExpr *E) {
   VisitCastExpr(E);
-  E->setTypeInfoAsWritten(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+  E->setTypeInfoAsWritten(GetTypeSourceInfo(Record, Idx));
 }
 
 void ASTStmtReader::VisitCStyleCastExpr(CStyleCastExpr *E) {
   VisitExplicitCastExpr(E);
-  E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLParenLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
   VisitExpr(E);
-  E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+  E->setLParenLoc(ReadSourceLocation(Record, Idx));
+  E->setTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
   E->setInitializer(Reader.ReadSubExpr());
   E->setFileScope(Record[Idx++]);
 }
@@ -624,7 +675,7 @@ void ASTStmtReader::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
   VisitExpr(E);
   E->setBase(Reader.ReadSubExpr());
   E->setAccessor(Reader.GetIdentifierInfo(Record, Idx));
-  E->setAccessorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setAccessorLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitInitListExpr(InitListExpr *E) {
@@ -634,8 +685,8 @@ void ASTStmtReader::VisitInitListExpr(InitListExpr *E) {
   for (unsigned I = 0; I != NumInits; ++I)
     E->updateInit(*Reader.getContext(), I, Reader.ReadSubExpr());
   E->setSyntacticForm(cast_or_null<InitListExpr>(Reader.ReadSubStmt()));
-  E->setLBraceLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRBraceLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLBraceLoc(ReadSourceLocation(Record, Idx));
+  E->setRBraceLoc(ReadSourceLocation(Record, Idx));
   E->setInitializedFieldInUnion(
                       cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++])));
   E->sawArrayRangeDesignator(Record[Idx++]);
@@ -649,7 +700,7 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
   assert(NumSubExprs == E->getNumSubExprs() && "Wrong number of subexprs");
   for (unsigned I = 0; I != NumSubExprs; ++I)
     E->setSubExpr(I, Reader.ReadSubExpr());
-  E->setEqualOrColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setEqualOrColonLoc(ReadSourceLocation(Record, Idx));
   E->setGNUSyntax(Record[Idx++]);
 
   llvm::SmallVector<Designator, 4> Designators;
@@ -658,9 +709,9 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     case DESIG_FIELD_DECL: {
       FieldDecl *Field = cast<FieldDecl>(Reader.GetDecl(Record[Idx++]));
       SourceLocation DotLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       SourceLocation FieldLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       Designators.push_back(Designator(Field->getIdentifier(), DotLoc,
                                        FieldLoc));
       Designators.back().setField(Field);
@@ -670,9 +721,9 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     case DESIG_FIELD_NAME: {
       const IdentifierInfo *Name = Reader.GetIdentifierInfo(Record, Idx);
       SourceLocation DotLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       SourceLocation FieldLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       Designators.push_back(Designator(Name, DotLoc, FieldLoc));
       break;
     }
@@ -680,9 +731,9 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     case DESIG_ARRAY: {
       unsigned Index = Record[Idx++];
       SourceLocation LBracketLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       SourceLocation RBracketLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       Designators.push_back(Designator(Index, LBracketLoc, RBracketLoc));
       break;
     }
@@ -690,11 +741,11 @@ void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
     case DESIG_ARRAY_RANGE: {
       unsigned Index = Record[Idx++];
       SourceLocation LBracketLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       SourceLocation EllipsisLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       SourceLocation RBracketLoc
-        = SourceLocation::getFromRawEncoding(Record[Idx++]);
+        = ReadSourceLocation(Record, Idx);
       Designators.push_back(Designator(Index, LBracketLoc, EllipsisLoc,
                                        RBracketLoc));
       break;
@@ -712,45 +763,37 @@ void ASTStmtReader::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
 void ASTStmtReader::VisitVAArgExpr(VAArgExpr *E) {
   VisitExpr(E);
   E->setSubExpr(Reader.ReadSubExpr());
-  E->setWrittenTypeInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
-  E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setWrittenTypeInfo(GetTypeSourceInfo(Record, Idx));
+  E->setBuiltinLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitAddrLabelExpr(AddrLabelExpr *E) {
   VisitExpr(E);
-  E->setAmpAmpLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setLabelLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  Reader.SetLabelOf(E, Record[Idx++]);
+  E->setAmpAmpLoc(ReadSourceLocation(Record, Idx));
+  E->setLabelLoc(ReadSourceLocation(Record, Idx));
+  E->setLabel(cast<LabelDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
 void ASTStmtReader::VisitStmtExpr(StmtExpr *E) {
   VisitExpr(E);
-  E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLParenLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
   E->setSubStmt(cast_or_null<CompoundStmt>(Reader.ReadSubStmt()));
 }
 
-void ASTStmtReader::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
-  VisitExpr(E);
-  E->setArgTInfo1(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
-  E->setArgTInfo2(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
-  E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-}
-
 void ASTStmtReader::VisitChooseExpr(ChooseExpr *E) {
   VisitExpr(E);
   E->setCond(Reader.ReadSubExpr());
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
-  E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setBuiltinLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitGNUNullExpr(GNUNullExpr *E) {
   VisitExpr(E);
-  E->setTokenLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setTokenLocation(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
@@ -760,23 +803,21 @@ void ASTStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   while (NumExprs--)
     Exprs.push_back(Reader.ReadSubExpr());
   E->setExprs(*Reader.getContext(), Exprs.data(), Exprs.size());
-  E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setBuiltinLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitBlockExpr(BlockExpr *E) {
   VisitExpr(E);
   E->setBlockDecl(cast_or_null<BlockDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setHasBlockDeclRefExprs(Record[Idx++]);
 }
 
 void ASTStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
   VisitExpr(E);
-  E->setDecl(cast<ValueDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setDecl(cast<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setByRef(Record[Idx++]);
   E->setConstQualAdded(Record[Idx++]);
-  E->setCopyConstructorExpr(Reader.ReadSubExpr());
 }
 
 //===----------------------------------------------------------------------===//
@@ -785,34 +826,34 @@ void ASTStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
 void ASTStmtReader::VisitObjCStringLiteral(ObjCStringLiteral *E) {
   VisitExpr(E);
   E->setString(cast<StringLiteral>(Reader.ReadSubStmt()));
-  E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setAtLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
   VisitExpr(E);
-  E->setEncodedTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor,Record,Idx));
-  E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setEncodedTypeSourceInfo(GetTypeSourceInfo(Record, Idx));
+  E->setAtLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
   VisitExpr(E);
   E->setSelector(Reader.GetSelector(Record, Idx));
-  E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setAtLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
   VisitExpr(E);
   E->setProtocol(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setAtLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   VisitExpr(E);
   E->setDecl(cast<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setBase(Reader.ReadSubExpr());
   E->setIsArrow(Record[Idx++]);
   E->setIsFreeIvar(Record[Idx++]);
@@ -820,23 +861,31 @@ void ASTStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
 
 void ASTStmtReader::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   VisitExpr(E);
-  E->setProperty(cast<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setBase(Reader.ReadSubExpr());
-}
-
-void ASTStmtReader::VisitObjCImplicitSetterGetterRefExpr(
-                                      ObjCImplicitSetterGetterRefExpr *E) {
-  VisitExpr(E);
-  E->setGetterMethod(
-                 cast_or_null<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setSetterMethod(
-                 cast_or_null<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setInterfaceDecl(
-              cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setBase(Reader.ReadSubExpr());
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setClassLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  bool Implicit = Record[Idx++] != 0;
+  if (Implicit) {
+    ObjCMethodDecl *Getter =
+      cast<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++]));
+    ObjCMethodDecl *Setter =
+      cast<ObjCMethodDecl>(Reader.GetDecl(Record[Idx++]));
+    E->setImplicitProperty(Getter, Setter);
+  } else {
+    E->setExplicitProperty(
+                      cast<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
+  }
+  E->setLocation(ReadSourceLocation(Record, Idx));
+  E->setReceiverLocation(ReadSourceLocation(Record, Idx));
+  switch (Record[Idx++]) {
+  case 0:
+    E->setBase(Reader.ReadSubExpr());
+    break;
+  case 1:
+    E->setSuperReceiver(Reader.GetType(Record[Idx++]));
+    break;
+  case 2:
+    E->setClassReceiver(
+                     cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
+    break;
+  }
 }
 
 void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
@@ -851,13 +900,13 @@ void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
     break;
 
   case ObjCMessageExpr::Class:
-    E->setClassReceiver(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+    E->setClassReceiver(GetTypeSourceInfo(Record, Idx));
     break;
 
   case ObjCMessageExpr::SuperClass:
   case ObjCMessageExpr::SuperInstance: {
     QualType T = Reader.GetType(Record[Idx++]);
-    SourceLocation SuperLoc = SourceLocation::getFromRawEncoding(Record[Idx++]);
+    SourceLocation SuperLoc = ReadSourceLocation(Record, Idx);
     E->setSuper(SuperLoc, T, Kind == ObjCMessageExpr::SuperInstance);
     break;
   }
@@ -870,39 +919,35 @@ void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   else
     E->setSelector(Reader.GetSelector(Record, Idx));
 
-  E->setLeftLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRightLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->LBracLoc = ReadSourceLocation(Record, Idx);
+  E->RBracLoc = ReadSourceLocation(Record, Idx);
+  E->SelectorLoc = ReadSourceLocation(Record, Idx);
 
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
 }
 
-void ASTStmtReader::VisitObjCSuperExpr(ObjCSuperExpr *E) {
-  VisitExpr(E);
-  E->setLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-}
-
 void ASTStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   VisitStmt(S);
   S->setElement(Reader.ReadSubStmt());
   S->setCollection(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
-  S->setForLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setForLoc(ReadSourceLocation(Record, Idx));
+  S->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   VisitStmt(S);
   S->setCatchBody(Reader.ReadSubStmt());
   S->setCatchParamDecl(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
-  S->setAtCatchLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setAtCatchLoc(ReadSourceLocation(Record, Idx));
+  S->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
   VisitStmt(S);
   S->setFinallyBody(Reader.ReadSubStmt());
-  S->setAtFinallyLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setAtFinallyLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
@@ -916,20 +961,20 @@ void ASTStmtReader::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
 
   if (HasFinally)
     S->setFinallyStmt(Reader.ReadSubStmt());
-  S->setAtTryLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setAtTryLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   VisitStmt(S);
   S->setSynchExpr(Reader.ReadSubStmt());
   S->setSynchBody(Reader.ReadSubStmt());
-  S->setAtSynchronizedLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setAtSynchronizedLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
   VisitStmt(S);
   S->setThrowExpr(Reader.ReadSubStmt());
-  S->setThrowLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  S->setThrowLoc(ReadSourceLocation(Record, Idx));
 }
 
 //===----------------------------------------------------------------------===//
@@ -938,7 +983,7 @@ void ASTStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
 
 void ASTStmtReader::VisitCXXCatchStmt(CXXCatchStmt *S) {
   VisitStmt(S);
-  S->CatchLoc = Reader.ReadSourceLocation(Record, Idx);
+  S->CatchLoc = ReadSourceLocation(Record, Idx);
   S->ExceptionDecl = cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++]));
   S->HandlerBlock = Reader.ReadSubStmt();
 }
@@ -947,7 +992,7 @@ void ASTStmtReader::VisitCXXTryStmt(CXXTryStmt *S) {
   VisitStmt(S);
   assert(Record[Idx] == S->getNumHandlers() && "NumStmtFields is wrong ?");
   ++Idx;
-  S->TryLoc = Reader.ReadSourceLocation(Record, Idx);
+  S->TryLoc = ReadSourceLocation(Record, Idx);
   S->getStmts()[0] = Reader.ReadSubStmt();
   for (unsigned i = 0, e = S->getNumHandlers(); i != e; ++i)
     S->getStmts()[i + 1] = Reader.ReadSubStmt();
@@ -966,21 +1011,23 @@ void ASTStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
   E->setConstructor(cast<CXXConstructorDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setElidable(Record[Idx++]);  
   E->setRequiresZeroInitialization(Record[Idx++]);
   E->setConstructionKind((CXXConstructExpr::ConstructionKind)Record[Idx++]);
+  E->ParenRange = ReadSourceRange(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
   VisitCXXConstructExpr(E);
-  E->TyBeginLoc = Reader.ReadSourceLocation(Record, Idx);
-  E->RParenLoc = Reader.ReadSourceLocation(Record, Idx);
+  E->Type = GetTypeSourceInfo(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
   VisitExplicitCastExpr(E);
-  E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  SourceRange R = ReadSourceRange(Record, Idx);
+  E->Loc = R.getBegin();
+  E->RParenLoc = R.getEnd();
 }
 
 void ASTStmtReader::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
@@ -1001,43 +1048,55 @@ void ASTStmtReader::VisitCXXConstCastExpr(CXXConstCastExpr *E) {
 
 void ASTStmtReader::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E) {
   VisitExplicitCastExpr(E);
-  E->setTypeBeginLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setTypeBeginLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
   VisitExpr(E);
   E->setValue(Record[Idx++]);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E) {
   VisitExpr(E);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
   VisitExpr(E);
-  E->setSourceRange(Reader.ReadSourceRange(Record, Idx));
+  E->setSourceRange(ReadSourceRange(Record, Idx));
   if (E->isTypeOperand()) { // typeid(int)
     E->setTypeOperandSourceInfo(
-        Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+        GetTypeSourceInfo(Record, Idx));
     return;
   }
   
   // typeid(42+2)
   E->setExprOperand(Reader.ReadSubExpr());
 }
+void ASTStmtReader::VisitCXXUuidofExpr(CXXUuidofExpr *E) {
+  VisitExpr(E);
+  E->setSourceRange(ReadSourceRange(Record, Idx));
+  if (E->isTypeOperand()) { // __uuidof(ComType)
+    E->setTypeOperandSourceInfo(
+        GetTypeSourceInfo(Record, Idx));
+    return;
+  }
+  
+  // __uuidof(expr)
+  E->setExprOperand(Reader.ReadSubExpr());
+}
 
 void ASTStmtReader::VisitCXXThisExpr(CXXThisExpr *E) {
   VisitExpr(E);
-  E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setLocation(ReadSourceLocation(Record, Idx));
   E->setImplicit(Record[Idx++]);
 }
 
 void ASTStmtReader::VisitCXXThrowExpr(CXXThrowExpr *E) {
   VisitExpr(E);
-  E->setThrowLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->setThrowLoc(ReadSourceLocation(Record, Idx));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
@@ -1047,7 +1106,7 @@ void ASTStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   assert(Record[Idx] == E->Param.getInt() && "We messed up at creation ?");
   ++Idx; // HasOtherExprStored and SubExpr was handled during creation.
   E->Param.setPointer(cast<ParmVarDecl>(Reader.GetDecl(Record[Idx++])));
-  E->Loc = Reader.ReadSourceLocation(Record, Idx);
+  E->Loc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
@@ -1058,14 +1117,15 @@ void ASTStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
 
 void ASTStmtReader::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
   VisitExpr(E);
-  E->setTypeBeginLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->TypeInfo = GetTypeSourceInfo(Record, Idx);
+  E->RParenLoc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
   VisitExpr(E);
-  E->setGlobalNew(Record[Idx++]);
-  E->setHasInitializer(Record[Idx++]);
+  E->GlobalNew = Record[Idx++];
+  E->Initializer = Record[Idx++];
+  E->UsualArrayDeleteWantsSize = Record[Idx++];
   bool isArray = Record[Idx++];
   unsigned NumPlacementArgs = Record[Idx++];
   unsigned NumCtorArgs = Record[Idx++];
@@ -1074,13 +1134,16 @@ void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
                     cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++])));
   E->setConstructor(
                cast_or_null<CXXConstructorDecl>(Reader.GetDecl(Record[Idx++])));
+  E->AllocatedTypeInfo = GetTypeSourceInfo(Record, Idx);
   SourceRange TypeIdParens;
-  TypeIdParens.setBegin(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TypeIdParens.setEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  TypeIdParens.setBegin(ReadSourceLocation(Record, Idx));
+  TypeIdParens.setEnd(ReadSourceLocation(Record, Idx));
   E->TypeIdParens = TypeIdParens;
-  E->setStartLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setEndLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  
+  E->StartLoc = ReadSourceLocation(Record, Idx);
+  E->EndLoc = ReadSourceLocation(Record, Idx);
+  E->ConstructorLParen = ReadSourceLocation(Record, Idx);
+  E->ConstructorRParen = ReadSourceLocation(Record, Idx);
+
   E->AllocateArgsArray(*Reader.getContext(), isArray, NumPlacementArgs,
                        NumCtorArgs);
 
@@ -1092,12 +1155,13 @@ void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
 
 void ASTStmtReader::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   VisitExpr(E);
-  E->setGlobalDelete(Record[Idx++]);
-  E->setArrayForm(Record[Idx++]);
-  E->setOperatorDelete(
-                     cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++])));
-  E->setArgument(Reader.ReadSubExpr());
-  E->setStartLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
+  E->GlobalDelete = Record[Idx++];
+  E->ArrayForm = Record[Idx++];
+  E->ArrayFormAsWritten = Record[Idx++];
+  E->UsualArrayDeleteWantsSize = Record[Idx++];
+  E->OperatorDelete = cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
+  E->Argument = Reader.ReadSubExpr();
+  E->Loc = ReadSourceLocation(Record, Idx);
 }
 
 void ASTStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
@@ -1105,21 +1169,21 @@ void ASTStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
 
   E->setBase(Reader.ReadSubExpr());
   E->setArrow(Record[Idx++]);
-  E->setOperatorLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  E->setScopeTypeInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
-  E->setColonColonLoc(Reader.ReadSourceLocation(Record, Idx));
-  E->setTildeLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->setQualifierRange(ReadSourceRange(Record, Idx));
+  E->setScopeTypeInfo(GetTypeSourceInfo(Record, Idx));
+  E->setColonColonLoc(ReadSourceLocation(Record, Idx));
+  E->setTildeLoc(ReadSourceLocation(Record, Idx));
   
   IdentifierInfo *II = Reader.GetIdentifierInfo(Record, Idx);
   if (II)
-    E->setDestroyedType(II, Reader.ReadSourceLocation(Record, Idx));
+    E->setDestroyedType(II, ReadSourceLocation(Record, Idx));
   else
-    E->setDestroyedType(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+    E->setDestroyedType(GetTypeSourceInfo(Record, Idx));
 }
 
-void ASTStmtReader::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
+void ASTStmtReader::VisitExprWithCleanups(ExprWithCleanups *E) {
   VisitExpr(E);
   unsigned NumTemps = Record[Idx++];
   if (NumTemps) {
@@ -1134,41 +1198,31 @@ void
 ASTStmtReader::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   VisitExpr(E);
   
-  unsigned NumTemplateArgs = Record[Idx++];
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
-         "Read wrong record during creation ?");
-  if (E->hasExplicitTemplateArgs())
+  if (Record[Idx++])
     ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
-                                     NumTemplateArgs);
+                                     Record[Idx++]);
 
   E->setBase(Reader.ReadSubExpr());
   E->setBaseType(Reader.GetType(Record[Idx++]));
   E->setArrow(Record[Idx++]);
-  E->setOperatorLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
+  E->setQualifierRange(ReadSourceRange(Record, Idx));
   E->setFirstQualifierFoundInScope(
                         cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++])));
-  // FIXME: read whole DeclarationNameInfo.
-  E->setMember(Reader.ReadDeclarationName(Record, Idx));
-  E->setMemberLoc(Reader.ReadSourceLocation(Record, Idx));
+  ReadDeclarationNameInfo(E->MemberNameInfo, Record, Idx);
 }
 
 void
 ASTStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   VisitExpr(E);
   
-  unsigned NumTemplateArgs = Record[Idx++];
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
-         "Read wrong record during creation ?");
-  if (E->hasExplicitTemplateArgs())
-    ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
-                                     NumTemplateArgs);
-
-  // FIXME: read whole DeclarationNameInfo.
-  E->setDeclName(Reader.ReadDeclarationName(Record, Idx));
-  E->setLocation(Reader.ReadSourceLocation(Record, Idx));
-  E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
+  if (Record[Idx++])
+    ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(), 
+                                     Record[Idx++]);
+  
+  ReadDeclarationNameInfo(E->NameInfo, Record, Idx);
+  E->setQualifierRange(ReadSourceRange(Record, Idx));
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
 }
 
@@ -1179,21 +1233,18 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   ++Idx; // NumArgs;
   for (unsigned I = 0, N = E->arg_size(); I != N; ++I)
     E->setArg(I, Reader.ReadSubExpr());
-  E->setTypeBeginLoc(Reader.ReadSourceLocation(Record, Idx));
-  E->setTypeAsWritten(Reader.GetType(Record[Idx++]));
-  E->setLParenLoc(Reader.ReadSourceLocation(Record, Idx));
-  E->setRParenLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->Type = GetTypeSourceInfo(Record, Idx);
+  E->setLParenLoc(ReadSourceLocation(Record, Idx));
+  E->setRParenLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) {
   VisitExpr(E);
   
-  unsigned NumTemplateArgs = Record[Idx++];
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
-         "Read wrong record during creation ?");
-  if (E->hasExplicitTemplateArgs())
+  // Read the explicit template argument list, if available.
+  if (Record[Idx++])
     ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
-                                     NumTemplateArgs);
+                                     Record[Idx++]);
 
   unsigned NumDecls = Record[Idx++];
   UnresolvedSet<8> Decls;
@@ -1204,11 +1255,9 @@ void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) {
   }
   E->initializeResults(*Reader.getContext(), Decls.begin(), Decls.end());
 
-  // FIXME: read whole DeclarationNameInfo.
-  E->setName(Reader.ReadDeclarationName(Record, Idx));
+  ReadDeclarationNameInfo(E->NameInfo, Record, Idx);
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  E->setNameLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->setQualifierRange(ReadSourceRange(Record, Idx));
 }
 
 void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
@@ -1217,7 +1266,7 @@ void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
   E->setHasUnresolvedUsing(Record[Idx++]);
   E->setBase(Reader.ReadSubExpr());
   E->setBaseType(Reader.GetType(Record[Idx++]));
-  E->setOperatorLoc(Reader.ReadSourceLocation(Record, Idx));
+  E->setOperatorLoc(ReadSourceLocation(Record, Idx));
 }
 
 void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
@@ -1230,17 +1279,81 @@ void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
 void ASTStmtReader::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   VisitExpr(E);
   E->UTT = (UnaryTypeTrait)Record[Idx++];
-  SourceRange Range = Reader.ReadSourceRange(Record, Idx);
+  E->Value = (bool)Record[Idx++];
+  SourceRange Range = ReadSourceRange(Record, Idx);
   E->Loc = Range.getBegin();
   E->RParen = Range.getEnd();
-  E->QueriedType = Reader.GetType(Record[Idx++]);
+  E->QueriedType = GetTypeSourceInfo(Record, Idx);
+}
+
+void ASTStmtReader::VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *E) {
+  VisitExpr(E);
+  E->BTT = (BinaryTypeTrait)Record[Idx++];
+  E->Value = (bool)Record[Idx++];
+  SourceRange Range = ReadSourceRange(Record, Idx);
+  E->Loc = Range.getBegin();
+  E->RParen = Range.getEnd();
+  E->LhsType = GetTypeSourceInfo(Record, Idx);
+  E->RhsType = GetTypeSourceInfo(Record, Idx);
+}
+
+void ASTStmtReader::VisitCXXNoexceptExpr(CXXNoexceptExpr *E) {
+  VisitExpr(E);
+  E->Value = (bool)Record[Idx++];
+  E->Range = ReadSourceRange(Record, Idx);
+  E->Operand = Reader.ReadSubExpr();
+}
+
+void ASTStmtReader::VisitPackExpansionExpr(PackExpansionExpr *E) {
+  VisitExpr(E);
+  E->EllipsisLoc = ReadSourceLocation(Record, Idx);
+  E->NumExpansions = Record[Idx++];
+  E->Pattern = Reader.ReadSubExpr();  
+}
+
+void ASTStmtReader::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
+  VisitExpr(E);
+  E->OperatorLoc = ReadSourceLocation(Record, Idx);
+  E->PackLoc = ReadSourceLocation(Record, Idx);
+  E->RParenLoc = ReadSourceLocation(Record, Idx);
+  E->Length = Record[Idx++];
+  E->Pack = cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+}
+
+void ASTStmtReader::VisitSubstNonTypeTemplateParmPackExpr(
+                                          SubstNonTypeTemplateParmPackExpr *E) {
+  VisitExpr(E);
+  E->Param
+    = cast_or_null<NonTypeTemplateParmDecl>(Reader.GetDecl(Record[Idx++]));
+  TemplateArgument ArgPack = Reader.ReadTemplateArgument(F, Record, Idx);
+  if (ArgPack.getKind() != TemplateArgument::Pack)
+    return;
+  
+  E->Arguments = ArgPack.pack_begin();
+  E->NumArguments = ArgPack.pack_size();
+  E->NameLoc = ReadSourceLocation(Record, Idx);
+}
+
+void ASTStmtReader::VisitOpaqueValueExpr(OpaqueValueExpr *E) {
+  VisitExpr(E);
+  Idx++; // skip ID
+  E->Loc = ReadSourceLocation(Record, Idx);
 }
 
-Stmt *ASTReader::ReadStmt(llvm::BitstreamCursor &Cursor) {
+//===----------------------------------------------------------------------===//
+// CUDA Expressions and Statements
+//===----------------------------------------------------------------------===//
+
+void ASTStmtReader::VisitCUDAKernelCallExpr(CUDAKernelCallExpr *E) {
+  VisitCallExpr(E);
+  E->setConfig(cast<CallExpr>(Reader.ReadSubExpr()));
+}
+
+Stmt *ASTReader::ReadStmt(PerFileData &F) {
   switch (ReadingKind) {
   case Read_Decl:
   case Read_Type:
-    return ReadStmtFromStream(Cursor);
+    return ReadStmtFromStream(F);
   case Read_Stmt:
     return ReadSubStmt();
   }
@@ -1249,8 +1362,8 @@ Stmt *ASTReader::ReadStmt(llvm::BitstreamCursor &Cursor) {
   return 0;
 }
 
-Expr *ASTReader::ReadExpr(llvm::BitstreamCursor &Cursor) {
-  return cast_or_null<Expr>(ReadStmt(Cursor));
+Expr *ASTReader::ReadExpr(PerFileData &F) {
+  return cast_or_null<Expr>(ReadStmt(F));
 }
 
 Expr *ASTReader::ReadSubExpr() {
@@ -1264,17 +1377,18 @@ Expr *ASTReader::ReadSubExpr() {
 // the stack, with expressions having operands removing those operands from the
 // stack. Evaluation terminates when we see a STMT_STOP record, and
 // the single remaining expression on the stack is our result.
-Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
+Stmt *ASTReader::ReadStmtFromStream(PerFileData &F) {
 
   ReadingKindTracker ReadingKind(Read_Stmt, *this);
-  
+  llvm::BitstreamCursor &Cursor = F.DeclsCursor;
+
 #ifndef NDEBUG
   unsigned PrevNumStmts = StmtStack.size();
 #endif
 
   RecordData Record;
   unsigned Idx;
-  ASTStmtReader Reader(*this, Cursor, Record, Idx);
+  ASTStmtReader Reader(*this, F, Cursor, Record, Idx);
   Stmt::EmptyShell Empty;
 
   while (true) {
@@ -1390,7 +1504,10 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
     case EXPR_DECL_REF:
       S = DeclRefExpr::CreateEmpty(*Context,
                          /*HasQualifier=*/Record[ASTStmtReader::NumExprFields],
-                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1]);
+          /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1]
+                                   ? Record[ASTStmtReader::NumExprFields + 2] 
+                                   : 0);
       break;
 
     case EXPR_INTEGER_LITERAL:
@@ -1454,16 +1571,17 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       SourceRange QualifierRange;
       if (Record[Idx++]) { // HasQualifier.
         NNS = ReadNestedNameSpecifier(Record, Idx);
-        QualifierRange = ReadSourceRange(Record, Idx);
+        QualifierRange = ReadSourceRange(F, Record, Idx);
       }
 
       TemplateArgumentListInfo ArgInfo;
-      unsigned NumTemplateArgs = Record[Idx++];
-      if (NumTemplateArgs) {
-        ArgInfo.setLAngleLoc(ReadSourceLocation(Record, Idx));
-        ArgInfo.setRAngleLoc(ReadSourceLocation(Record, Idx));
+      bool HasExplicitTemplateArgs = Record[Idx++];
+      if (HasExplicitTemplateArgs) {
+        unsigned NumTemplateArgs = Record[Idx++];
+        ArgInfo.setLAngleLoc(ReadSourceLocation(F, Record, Idx));
+        ArgInfo.setRAngleLoc(ReadSourceLocation(F, Record, Idx));
         for (unsigned i = 0; i != NumTemplateArgs; ++i)
-          ArgInfo.addArgument(ReadTemplateArgumentLoc(Cursor, Record, Idx));
+          ArgInfo.addArgument(ReadTemplateArgumentLoc(F, Record, Idx));
       }
       
       NamedDecl *FoundD = cast_or_null<NamedDecl>(GetDecl(Record[Idx++]));
@@ -1471,16 +1589,19 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       DeclAccessPair FoundDecl = DeclAccessPair::make(FoundD, AS);
 
       QualType T = GetType(Record[Idx++]);
+      ExprValueKind VK = static_cast<ExprValueKind>(Record[Idx++]);
+      ExprObjectKind OK = static_cast<ExprObjectKind>(Record[Idx++]);
       Expr *Base = ReadSubExpr();
       ValueDecl *MemberD = cast<ValueDecl>(GetDecl(Record[Idx++]));
-      // FIXME: read DeclarationNameLoc.
-      SourceLocation MemberLoc = ReadSourceLocation(Record, Idx);
+      SourceLocation MemberLoc = ReadSourceLocation(F, Record, Idx);
       DeclarationNameInfo MemberNameInfo(MemberD->getDeclName(), MemberLoc);
       bool IsArrow = Record[Idx++];
 
       S = MemberExpr::Create(*Context, Base, IsArrow, NNS, QualifierRange,
                              MemberD, FoundDecl, MemberNameInfo,
-                             NumTemplateArgs ? &ArgInfo : 0, T);
+                             HasExplicitTemplateArgs ? &ArgInfo : 0, T, VK, OK);
+      ReadDeclarationNameLoc(F, cast<MemberExpr>(S)->MemberDNLoc,
+                             MemberD->getDeclName(), Record, Idx);
       break;
     }
 
@@ -1496,6 +1617,10 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       S = new (Context) ConditionalOperator(Empty);
       break;
 
+    case EXPR_BINARY_CONDITIONAL_OPERATOR:
+      S = new (Context) BinaryConditionalOperator(Empty);
+      break;
+
     case EXPR_IMPLICIT_CAST:
       S = ImplicitCastExpr::CreateEmpty(*Context,
                        /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
@@ -1540,10 +1665,6 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       S = new (Context) StmtExpr(Empty);
       break;
 
-    case EXPR_TYPES_COMPATIBLE:
-      S = new (Context) TypesCompatibleExpr(Empty);
-      break;
-
     case EXPR_CHOOSE:
       S = new (Context) ChooseExpr(Empty);
       break;
@@ -1583,15 +1704,12 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       S = new (Context) ObjCPropertyRefExpr(Empty);
       break;
     case EXPR_OBJC_KVC_REF_EXPR:
-      S = new (Context) ObjCImplicitSetterGetterRefExpr(Empty);
+      llvm_unreachable("mismatching AST file");
       break;
     case EXPR_OBJC_MESSAGE_EXPR:
       S = ObjCMessageExpr::CreateEmpty(*Context,
                                      Record[ASTStmtReader::NumExprFields]);
       break;
-    case EXPR_OBJC_SUPER_EXPR:
-      S = new (Context) ObjCSuperExpr(Empty);
-      break;
     case EXPR_OBJC_ISA:
       S = new (Context) ObjCIsaExpr(Empty);
       break;
@@ -1678,6 +1796,12 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
     case EXPR_CXX_TYPEID_TYPE:
       S = new (Context) CXXTypeidExpr(Empty, false);
       break;
+    case EXPR_CXX_UUIDOF_EXPR:
+      S = new (Context) CXXUuidofExpr(Empty, true);
+      break;
+    case EXPR_CXX_UUIDOF_TYPE:
+      S = new (Context) CXXUuidofExpr(Empty, false);
+      break;
     case EXPR_CXX_THIS:
       S = new (Context) CXXThisExpr(Empty);
       break;
@@ -1710,18 +1834,24 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       S = new (Context) CXXPseudoDestructorExpr(Empty);
       break;
         
-    case EXPR_CXX_EXPR_WITH_TEMPORARIES:
-      S = new (Context) CXXExprWithTemporaries(Empty);
+    case EXPR_EXPR_WITH_CLEANUPS:
+      S = new (Context) ExprWithCleanups(Empty);
       break;
       
     case EXPR_CXX_DEPENDENT_SCOPE_MEMBER:
       S = CXXDependentScopeMemberExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
+          /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
+                                   ? Record[ASTStmtReader::NumExprFields + 1] 
+                                   : 0);
       break;
       
     case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF:
       S = DependentScopeDeclRefExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
+          /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
+                                   ? Record[ASTStmtReader::NumExprFields + 1] 
+                                   : 0);
       break;
       
     case EXPR_CXX_UNRESOLVED_CONSTRUCT:
@@ -1731,17 +1861,62 @@ Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       
     case EXPR_CXX_UNRESOLVED_MEMBER:
       S = UnresolvedMemberExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
+          /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
+                                   ? Record[ASTStmtReader::NumExprFields + 1] 
+                                   : 0);
       break;
       
     case EXPR_CXX_UNRESOLVED_LOOKUP:
       S = UnresolvedLookupExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
+          /*HasExplicitTemplateArgs=*/Record[ASTStmtReader::NumExprFields],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]
+                                   ? Record[ASTStmtReader::NumExprFields + 1] 
+                                   : 0);
       break;
       
     case EXPR_CXX_UNARY_TYPE_TRAIT:
       S = new (Context) UnaryTypeTraitExpr(Empty);
       break;
+
+    case EXPR_BINARY_TYPE_TRAIT:
+      S = new (Context) BinaryTypeTraitExpr(Empty);
+      break;
+
+    case EXPR_CXX_NOEXCEPT:
+      S = new (Context) CXXNoexceptExpr(Empty);
+      break;
+
+    case EXPR_PACK_EXPANSION:
+      S = new (Context) PackExpansionExpr(Empty);
+      break;
+        
+    case EXPR_SIZEOF_PACK:
+      S = new (Context) SizeOfPackExpr(Empty);
+      break;
+        
+    case EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK:
+      S = new (Context) SubstNonTypeTemplateParmPackExpr(Empty);
+      break;
+        
+    case EXPR_OPAQUE_VALUE: {
+      unsigned key = Record[ASTStmtReader::NumExprFields];
+      OpaqueValueExpr *&expr = OpaqueValueExprs[key];
+
+      // If we already have an entry for this opaque value expression,
+      // don't bother reading it again.
+      if (expr) {
+        StmtStack.push_back(expr);
+        continue;
+      }
+
+      S = expr = new (Context) OpaqueValueExpr(Empty);
+      break;
+    }
+
+    case EXPR_CUDA_KERNEL_CALL:
+      S = new (Context) CUDAKernelCallExpr(*Context, Empty);
+      break;
     }
     
     // We hit a STMT_STOP, so we're done with this expression.
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index 3b6b218e744c..8fcb535a9c89 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Serialization/ASTWriter.h"
+#include "clang/Serialization/ASTSerializationListener.h"
 #include "ASTCommon.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/IdentifierResolver.h"
@@ -19,6 +20,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclContextInternals.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/AST/DeclFriend.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/Type.h"
@@ -29,6 +31,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "clang/Basic/OnDiskHashTable.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceManagerInternals.h"
@@ -38,9 +41,11 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 #include <cstdio>
+#include <string.h>
 using namespace clang;
 using namespace clang::serialization;
 
@@ -60,13 +65,13 @@ const T *data(const std::vector<T, Allocator> &v) {
 namespace {
   class ASTTypeWriter {
     ASTWriter &Writer;
-    ASTWriter::RecordData &Record;
+    ASTWriter::RecordDataImpl &Record;
 
   public:
     /// \brief Type code that corresponds to the record generated.
     TypeCode Code;
 
-    ASTTypeWriter(ASTWriter &Writer, ASTWriter::RecordData &Record)
+    ASTTypeWriter(ASTWriter &Writer, ASTWriter::RecordDataImpl &Record)
       : Writer(Writer), Record(Record), Code(TYPE_EXT_QUAL) { }
 
     void VisitArrayType(const ArrayType *T);
@@ -142,7 +147,7 @@ void ASTTypeWriter::VisitVariableArrayType(const VariableArrayType *T) {
 void ASTTypeWriter::VisitVectorType(const VectorType *T) {
   Writer.AddTypeRef(T->getElementType(), Record);
   Record.push_back(T->getNumElements());
-  Record.push_back(T->getAltiVecSpecific());
+  Record.push_back(T->getVectorKind());
   Code = TYPE_VECTOR;
 }
 
@@ -172,6 +177,7 @@ void ASTTypeWriter::VisitFunctionProtoType(const FunctionProtoType *T) {
     Writer.AddTypeRef(T->getArgType(I), Record);
   Record.push_back(T->isVariadic());
   Record.push_back(T->getTypeQuals());
+  Record.push_back(static_cast<unsigned>(T->getRefQualifier()));
   Record.push_back(T->hasExceptionSpec());
   Record.push_back(T->hasAnyExceptionSpec());
   Record.push_back(T->getNumExceptions());
@@ -207,6 +213,11 @@ void ASTTypeWriter::VisitDecltypeType(const DecltypeType *T) {
   Code = TYPE_DECLTYPE;
 }
 
+void ASTTypeWriter::VisitAutoType(const AutoType *T) {
+  Writer.AddTypeRef(T->getDeducedType(), Record);
+  Code = TYPE_AUTO;
+}
+
 void ASTTypeWriter::VisitTagType(const TagType *T) {
   Record.push_back(T->isDependentType());
   Writer.AddDeclRef(T->getDecl(), Record);
@@ -224,6 +235,13 @@ void ASTTypeWriter::VisitEnumType(const EnumType *T) {
   Code = TYPE_ENUM;
 }
 
+void ASTTypeWriter::VisitAttributedType(const AttributedType *T) {
+  Writer.AddTypeRef(T->getModifiedType(), Record);
+  Writer.AddTypeRef(T->getEquivalentType(), Record);
+  Record.push_back(T->getAttrKind());
+  Code = TYPE_ATTRIBUTED;
+}
+
 void
 ASTTypeWriter::VisitSubstTemplateTypeParmType(
                                         const SubstTemplateTypeParmType *T) {
@@ -233,6 +251,14 @@ ASTTypeWriter::VisitSubstTemplateTypeParmType(
 }
 
 void
+ASTTypeWriter::VisitSubstTemplateTypeParmPackType(
+                                      const SubstTemplateTypeParmPackType *T) {
+  Writer.AddTypeRef(QualType(T->getReplacedParameter(), 0), Record);
+  Writer.AddTemplateArgument(T->getArgumentPack(), Record);
+  Code = TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK;
+}
+
+void
 ASTTypeWriter::VisitTemplateSpecializationType(
                                        const TemplateSpecializationType *T) {
   Record.push_back(T->isDependentType());
@@ -295,6 +321,20 @@ ASTTypeWriter::VisitDependentTemplateSpecializationType(
   Code = TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION;
 }
 
+void ASTTypeWriter::VisitPackExpansionType(const PackExpansionType *T) {
+  Writer.AddTypeRef(T->getPattern(), Record);
+  if (llvm::Optional<unsigned> NumExpansions = T->getNumExpansions())
+    Record.push_back(*NumExpansions + 1);
+  else
+    Record.push_back(0);
+  Code = TYPE_PACK_EXPANSION;
+}
+
+void ASTTypeWriter::VisitParenType(const ParenType *T) {
+  Writer.AddTypeRef(T->getInnerType(), Record);
+  Code = TYPE_PAREN;
+}
+
 void ASTTypeWriter::VisitElaboratedType(const ElaboratedType *T) {
   Record.push_back(T->getKeyword());
   Writer.AddNestedNameSpecifier(T->getQualifier(), Record);
@@ -332,10 +372,10 @@ namespace {
 
 class TypeLocWriter : public TypeLocVisitor<TypeLocWriter> {
   ASTWriter &Writer;
-  ASTWriter::RecordData &Record;
+  ASTWriter::RecordDataImpl &Record;
 
 public:
-  TypeLocWriter(ASTWriter &Writer, ASTWriter::RecordData &Record)
+  TypeLocWriter(ASTWriter &Writer, ASTWriter::RecordDataImpl &Record)
     : Writer(Writer), Record(Record) { }
 
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
@@ -412,6 +452,7 @@ void TypeLocWriter::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
 void TypeLocWriter::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   Writer.AddSourceLocation(TL.getLParenLoc(), Record);
   Writer.AddSourceLocation(TL.getRParenLoc(), Record);
+  Record.push_back(TL.getTrailingReturn());
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     Writer.AddDeclRef(TL.getArg(i), Record);
 }
@@ -441,12 +482,30 @@ void TypeLocWriter::VisitTypeOfTypeLoc(TypeOfTypeLoc TL) {
 void TypeLocWriter::VisitDecltypeTypeLoc(DecltypeTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
+void TypeLocWriter::VisitAutoTypeLoc(AutoTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getNameLoc(), Record);
+}
 void TypeLocWriter::VisitRecordTypeLoc(RecordTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
 void TypeLocWriter::VisitEnumTypeLoc(EnumTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
+void TypeLocWriter::VisitAttributedTypeLoc(AttributedTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getAttrNameLoc(), Record);
+  if (TL.hasAttrOperand()) {
+    SourceRange range = TL.getAttrOperandParensRange();
+    Writer.AddSourceLocation(range.getBegin(), Record);
+    Writer.AddSourceLocation(range.getEnd(), Record);
+  }
+  if (TL.hasAttrExprOperand()) {
+    Expr *operand = TL.getAttrExprOperand();
+    Record.push_back(operand ? 1 : 0);
+    if (operand) Writer.AddStmt(operand);
+  } else if (TL.hasAttrEnumOperand()) {
+    Writer.AddSourceLocation(TL.getAttrEnumOperandLoc(), Record);
+  }
+}
 void TypeLocWriter::VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
@@ -454,6 +513,10 @@ void TypeLocWriter::VisitSubstTemplateTypeParmTypeLoc(
                                             SubstTemplateTypeParmTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
+void TypeLocWriter::VisitSubstTemplateTypeParmPackTypeLoc(
+                                          SubstTemplateTypeParmPackTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getNameLoc(), Record);
+}
 void TypeLocWriter::VisitTemplateSpecializationTypeLoc(
                                            TemplateSpecializationTypeLoc TL) {
   Writer.AddSourceLocation(TL.getTemplateNameLoc(), Record);
@@ -463,6 +526,10 @@ void TypeLocWriter::VisitTemplateSpecializationTypeLoc(
     Writer.AddTemplateArgumentLocInfo(TL.getArgLoc(i).getArgument().getKind(),
                                       TL.getArgLoc(i).getLocInfo(), Record);
 }
+void TypeLocWriter::VisitParenTypeLoc(ParenTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getLParenLoc(), Record);
+  Writer.AddSourceLocation(TL.getRParenLoc(), Record);
+}
 void TypeLocWriter::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
   Writer.AddSourceLocation(TL.getKeywordLoc(), Record);
   Writer.AddSourceRange(TL.getQualifierRange(), Record);
@@ -486,6 +553,9 @@ void TypeLocWriter::VisitDependentTemplateSpecializationTypeLoc(
     Writer.AddTemplateArgumentLocInfo(TL.getArgLoc(I).getArgument().getKind(),
                                       TL.getArgLoc(I).getLocInfo(), Record);
 }
+void TypeLocWriter::VisitPackExpansionTypeLoc(PackExpansionTypeLoc TL) {
+  Writer.AddSourceLocation(TL.getEllipsisLoc(), Record);
+}
 void TypeLocWriter::VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL) {
   Writer.AddSourceLocation(TL.getNameLoc(), Record);
 }
@@ -506,7 +576,7 @@ void TypeLocWriter::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
 
 static void EmitBlockID(unsigned ID, const char *Name,
                         llvm::BitstreamWriter &Stream,
-                        ASTWriter::RecordData &Record) {
+                        ASTWriter::RecordDataImpl &Record) {
   Record.clear();
   Record.push_back(ID);
   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
@@ -521,7 +591,7 @@ static void EmitBlockID(unsigned ID, const char *Name,
 
 static void EmitRecordID(unsigned ID, const char *Name,
                          llvm::BitstreamWriter &Stream,
-                         ASTWriter::RecordData &Record) {
+                         ASTWriter::RecordDataImpl &Record) {
   Record.clear();
   Record.push_back(ID);
   while (*Name)
@@ -530,7 +600,7 @@ static void EmitRecordID(unsigned ID, const char *Name,
 }
 
 static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
-                          ASTWriter::RecordData &Record) {
+                          ASTWriter::RecordDataImpl &Record) {
 #define RECORD(X) EmitRecordID(X, #X, Stream, Record)
   RECORD(STMT_STOP);
   RECORD(STMT_NULL_PTR);
@@ -577,7 +647,6 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
   RECORD(EXPR_VA_ARG);
   RECORD(EXPR_ADDR_LABEL);
   RECORD(EXPR_STMT);
-  RECORD(EXPR_TYPES_COMPATIBLE);
   RECORD(EXPR_CHOOSE);
   RECORD(EXPR_GNU_NULL);
   RECORD(EXPR_SHUFFLE_VECTOR);
@@ -591,7 +660,6 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
   RECORD(EXPR_OBJC_PROPERTY_REF_EXPR);
   RECORD(EXPR_OBJC_KVC_REF_EXPR);
   RECORD(EXPR_OBJC_MESSAGE_EXPR);
-  RECORD(EXPR_OBJC_SUPER_EXPR);
   RECORD(STMT_OBJC_FOR_COLLECTION);
   RECORD(STMT_OBJC_CATCH);
   RECORD(STMT_OBJC_FINALLY);
@@ -607,6 +675,32 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
   RECORD(EXPR_CXX_FUNCTIONAL_CAST);
   RECORD(EXPR_CXX_BOOL_LITERAL);
   RECORD(EXPR_CXX_NULL_PTR_LITERAL);
+  RECORD(EXPR_CXX_TYPEID_EXPR);
+  RECORD(EXPR_CXX_TYPEID_TYPE);
+  RECORD(EXPR_CXX_UUIDOF_EXPR);
+  RECORD(EXPR_CXX_UUIDOF_TYPE);
+  RECORD(EXPR_CXX_THIS);
+  RECORD(EXPR_CXX_THROW);
+  RECORD(EXPR_CXX_DEFAULT_ARG);
+  RECORD(EXPR_CXX_BIND_TEMPORARY);
+  RECORD(EXPR_CXX_SCALAR_VALUE_INIT);
+  RECORD(EXPR_CXX_NEW);
+  RECORD(EXPR_CXX_DELETE);
+  RECORD(EXPR_CXX_PSEUDO_DESTRUCTOR);
+  RECORD(EXPR_EXPR_WITH_CLEANUPS);
+  RECORD(EXPR_CXX_DEPENDENT_SCOPE_MEMBER);
+  RECORD(EXPR_CXX_DEPENDENT_SCOPE_DECL_REF);
+  RECORD(EXPR_CXX_UNRESOLVED_CONSTRUCT);
+  RECORD(EXPR_CXX_UNRESOLVED_MEMBER);
+  RECORD(EXPR_CXX_UNRESOLVED_LOOKUP);
+  RECORD(EXPR_CXX_UNARY_TYPE_TRAIT);
+  RECORD(EXPR_CXX_NOEXCEPT);
+  RECORD(EXPR_OPAQUE_VALUE);
+  RECORD(EXPR_BINARY_TYPE_TRAIT);
+  RECORD(EXPR_PACK_EXPANSION);
+  RECORD(EXPR_SIZEOF_PACK);
+  RECORD(EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK);
+  RECORD(EXPR_CUDA_KERNEL_CALL);
 #undef RECORD
 }
 
@@ -643,6 +737,21 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(MACRO_DEFINITION_OFFSETS);
   RECORD(CHAINED_METADATA);
   RECORD(REFERENCED_SELECTOR_POOL);
+  RECORD(TU_UPDATE_LEXICAL);
+  RECORD(REDECLS_UPDATE_LATEST);
+  RECORD(SEMA_DECL_REFS);
+  RECORD(WEAK_UNDECLARED_IDENTIFIERS);
+  RECORD(PENDING_IMPLICIT_INSTANTIATIONS);
+  RECORD(DECL_REPLACEMENTS);
+  RECORD(UPDATE_VISIBLE);
+  RECORD(DECL_UPDATE_OFFSETS);
+  RECORD(DECL_UPDATES);
+  RECORD(CXX_BASE_SPECIFIER_OFFSETS);
+  RECORD(DIAG_PRAGMA_MAPPINGS);
+  RECORD(CUDA_SPECIAL_DECL_REFS);
+  RECORD(HEADER_SEARCH_TABLE);
+  RECORD(FP_PRAGMA_OPTIONS);
+  RECORD(OPENCL_EXTENSIONS);
   
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
@@ -657,8 +766,6 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(PP_MACRO_OBJECT_LIKE);
   RECORD(PP_MACRO_FUNCTION_LIKE);
   RECORD(PP_TOKEN);
-  RECORD(PP_MACRO_INSTANTIATION);
-  RECORD(PP_MACRO_DEFINITION);
   
   // Decls and Types block.
   BLOCK(DECLTYPES_BLOCK);
@@ -684,7 +791,21 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(TYPE_OBJC_INTERFACE);
   RECORD(TYPE_OBJC_OBJECT);
   RECORD(TYPE_OBJC_OBJECT_POINTER);
-  RECORD(DECL_ATTR);
+  RECORD(TYPE_DECLTYPE);
+  RECORD(TYPE_ELABORATED);
+  RECORD(TYPE_SUBST_TEMPLATE_TYPE_PARM);
+  RECORD(TYPE_UNRESOLVED_USING);
+  RECORD(TYPE_INJECTED_CLASS_NAME);
+  RECORD(TYPE_OBJC_OBJECT);
+  RECORD(TYPE_TEMPLATE_TYPE_PARM);
+  RECORD(TYPE_TEMPLATE_SPECIALIZATION);
+  RECORD(TYPE_DEPENDENT_NAME);
+  RECORD(TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION);
+  RECORD(TYPE_DEPENDENT_SIZED_ARRAY);
+  RECORD(TYPE_PAREN);
+  RECORD(TYPE_PACK_EXPANSION);
+  RECORD(TYPE_ATTRIBUTED);
+  RECORD(TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK);
   RECORD(DECL_TRANSLATION_UNIT);
   RECORD(DECL_TYPEDEF);
   RECORD(DECL_ENUM);
@@ -712,6 +833,39 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(DECL_BLOCK);
   RECORD(DECL_CONTEXT_LEXICAL);
   RECORD(DECL_CONTEXT_VISIBLE);
+  RECORD(DECL_NAMESPACE);
+  RECORD(DECL_NAMESPACE_ALIAS);
+  RECORD(DECL_USING);
+  RECORD(DECL_USING_SHADOW);
+  RECORD(DECL_USING_DIRECTIVE);
+  RECORD(DECL_UNRESOLVED_USING_VALUE);
+  RECORD(DECL_UNRESOLVED_USING_TYPENAME);
+  RECORD(DECL_LINKAGE_SPEC);
+  RECORD(DECL_CXX_RECORD);
+  RECORD(DECL_CXX_METHOD);
+  RECORD(DECL_CXX_CONSTRUCTOR);
+  RECORD(DECL_CXX_DESTRUCTOR);
+  RECORD(DECL_CXX_CONVERSION);
+  RECORD(DECL_ACCESS_SPEC);
+  RECORD(DECL_FRIEND);
+  RECORD(DECL_FRIEND_TEMPLATE);
+  RECORD(DECL_CLASS_TEMPLATE);
+  RECORD(DECL_CLASS_TEMPLATE_SPECIALIZATION);
+  RECORD(DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION);
+  RECORD(DECL_FUNCTION_TEMPLATE);
+  RECORD(DECL_TEMPLATE_TYPE_PARM);
+  RECORD(DECL_NON_TYPE_TEMPLATE_PARM);
+  RECORD(DECL_TEMPLATE_TEMPLATE_PARM);
+  RECORD(DECL_STATIC_ASSERT);
+  RECORD(DECL_CXX_BASE_SPECIFIERS);
+  RECORD(DECL_INDIRECTFIELD);
+  RECORD(DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK);
+  
+  BLOCK(PREPROCESSOR_DETAIL_BLOCK);
+  RECORD(PPD_MACRO_INSTANTIATION);
+  RECORD(PPD_MACRO_DEFINITION);
+  RECORD(PPD_INCLUSION_DIRECTIVE);
+  
   // Statements and Exprs can occur in the Decls and Types block.
   AddStmtsExprs(Stream, Record);
 #undef RECORD
@@ -756,7 +910,8 @@ adjustFilenameForRelocatablePCH(const char *Filename, const char *isysroot) {
 }
 
 /// \brief Write the AST metadata (e.g., i686-apple-darwin9).
-void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
+void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot,
+                              const std::string &OutputFile) {
   using namespace llvm;
 
   // Metadata
@@ -792,9 +947,9 @@ void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
     FileAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
     unsigned FileAbbrevCode = Stream.EmitAbbrev(FileAbbrev);
 
-    llvm::sys::Path MainFilePath(MainFile->getName());
+    llvm::SmallString<128> MainFilePath(MainFile->getName());
 
-    MainFilePath.makeAbsolute();
+    llvm::sys::fs::make_absolute(MainFilePath);
 
     const char *MainFileNameStr = MainFilePath.c_str();
     MainFileNameStr = adjustFilenameForRelocatablePCH(MainFileNameStr,
@@ -804,6 +959,23 @@ void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
     Stream.EmitRecordWithBlob(FileAbbrevCode, Record, MainFileNameStr);
   }
 
+  // Original PCH directory
+  if (!OutputFile.empty() && OutputFile != "-") {
+    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(ORIGINAL_PCH_DIR));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
+    unsigned AbbrevCode = Stream.EmitAbbrev(Abbrev);
+
+    llvm::SmallString<128> OutputPath(OutputFile);
+
+    llvm::sys::fs::make_absolute(OutputPath);
+    StringRef origDir = llvm::sys::path::parent_path(OutputPath);
+
+    RecordData Record;
+    Record.push_back(ORIGINAL_PCH_DIR);
+    Stream.EmitRecordWithBlob(AbbrevCode, Record, origDir);
+  }
+
   // Repository branch/version information.
   BitCodeAbbrev *RepoAbbrev = new BitCodeAbbrev();
   RepoAbbrev->Add(BitCodeAbbrevOp(VERSION_CONTROL_BRANCH_REVISION));
@@ -829,6 +1001,8 @@ void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   Record.push_back(LangOpts.HexFloats);  // C99 Hexadecimal float constants.
   Record.push_back(LangOpts.C99);  // C99 Support
   Record.push_back(LangOpts.Microsoft);  // Microsoft extensions.
+  // LangOpts.MSCVersion is ignored because all it does it set a macro, which is
+  // already saved elsewhere.
   Record.push_back(LangOpts.CPlusPlus);  // C++ Support
   Record.push_back(LangOpts.CPlusPlus0x);  // C++0x Support
   Record.push_back(LangOpts.CXXOperatorNames);  // Treat C++ operator names as keywords.
@@ -839,6 +1013,9 @@ void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
                                                  // modern abi enabled.
   Record.push_back(LangOpts.ObjCNonFragileABI2); // Objective-C enhanced
                                                  // modern abi enabled.
+  Record.push_back(LangOpts.AppleKext);          // Apple's kernel extensions ABI
+  Record.push_back(LangOpts.ObjCDefaultSynthProperties); // Objective-C auto-synthesized
+                                                      // properties enabled.
   Record.push_back(LangOpts.NoConstantCFStrings); // non cfstring generation enabled..
 
   Record.push_back(LangOpts.PascalStrings);  // Allow Pascal strings
@@ -847,7 +1024,9 @@ void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   Record.push_back(LangOpts.AltiVec);
   Record.push_back(LangOpts.Exceptions);  // Support exception handling.
   Record.push_back(LangOpts.SjLjExceptions);
+  Record.push_back(LangOpts.ObjCExceptions);
 
+  Record.push_back(LangOpts.MSBitfields); // MS-compatible structure layout
   Record.push_back(LangOpts.NeXTRuntime); // Use NeXT runtime.
   Record.push_back(LangOpts.Freestanding); // Freestanding implementation
   Record.push_back(LangOpts.NoBuiltin); // Do not use builtin functions (-fno-builtin)
@@ -879,12 +1058,17 @@ void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   Record.push_back(LangOpts.CharIsSigned); // Whether char is a signed or
                                            // unsigned type
   Record.push_back(LangOpts.ShortWChar);  // force wchar_t to be unsigned short
+  Record.push_back(LangOpts.ShortEnums);  // Should the enum type be equivalent
+                                          // to the smallest integer type with
+                                          // enough room.
   Record.push_back(LangOpts.getGCMode());
   Record.push_back(LangOpts.getVisibilityMode());
   Record.push_back(LangOpts.getStackProtectorMode());
   Record.push_back(LangOpts.InstantiationDepth);
   Record.push_back(LangOpts.OpenCL);
+  Record.push_back(LangOpts.CUDA);
   Record.push_back(LangOpts.CatchUndefined);
+  Record.push_back(LangOpts.DefaultFPContract);
   Record.push_back(LangOpts.ElideConstructors);
   Record.push_back(LangOpts.SpellChecking);
   Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
@@ -901,8 +1085,8 @@ public:
   typedef const char * key_type;
   typedef key_type key_type_ref;
 
-  typedef std::pair<int, struct stat> data_type;
-  typedef const data_type& data_type_ref;
+  typedef struct stat data_type;
+  typedef const data_type &data_type_ref;
 
   static unsigned ComputeHash(const char *path) {
     return llvm::HashString(path);
@@ -913,9 +1097,7 @@ public:
                       data_type_ref Data) {
     unsigned StrLen = strlen(path);
     clang::io::Emit16(Out, StrLen);
-    unsigned DataLen = 1; // result value
-    if (Data.first == 0)
-      DataLen += 4 + 4 + 2 + 8 + 8;
+    unsigned DataLen = 4 + 4 + 2 + 8 + 8;
     clang::io::Emit8(Out, DataLen);
     return std::make_pair(StrLen + 1, DataLen);
   }
@@ -924,21 +1106,16 @@ public:
     Out.write(path, KeyLen);
   }
 
-  void EmitData(llvm::raw_ostream& Out, key_type_ref,
+  void EmitData(llvm::raw_ostream &Out, key_type_ref,
                 data_type_ref Data, unsigned DataLen) {
     using namespace clang::io;
     uint64_t Start = Out.tell(); (void)Start;
 
-    // Result of stat()
-    Emit8(Out, Data.first? 1 : 0);
-
-    if (Data.first == 0) {
-      Emit32(Out, (uint32_t) Data.second.st_ino);
-      Emit32(Out, (uint32_t) Data.second.st_dev);
-      Emit16(Out, (uint16_t) Data.second.st_mode);
-      Emit64(Out, (uint64_t) Data.second.st_mtime);
-      Emit64(Out, (uint64_t) Data.second.st_size);
-    }
+    Emit32(Out, (uint32_t) Data.st_ino);
+    Emit32(Out, (uint32_t) Data.st_dev);
+    Emit16(Out, (uint16_t) Data.st_mode);
+    Emit64(Out, (uint64_t) Data.st_mtime);
+    Emit64(Out, (uint64_t) Data.st_size);
 
     assert(Out.tell() - Start == DataLen && "Wrong data length");
   }
@@ -1002,11 +1179,6 @@ static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &Stream) {
   // FileEntry fields.
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Size
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // Modification time
-  // HeaderFileInfo fields.
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImport
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // DirInfo
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // NumIncludes
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // ControllingMacro
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
   return Stream.EmitAbbrev(Abbrev);
 }
@@ -1049,6 +1221,135 @@ static unsigned CreateSLocInstantiationAbbrev(llvm::BitstreamWriter &Stream) {
   return Stream.EmitAbbrev(Abbrev);
 }
 
+namespace {
+  // Trait used for the on-disk hash table of header search information.
+  class HeaderFileInfoTrait {
+    ASTWriter &Writer;
+    HeaderSearch &HS;
+    
+  public:
+    HeaderFileInfoTrait(ASTWriter &Writer, HeaderSearch &HS) 
+      : Writer(Writer), HS(HS) { }
+    
+    typedef const char *key_type;
+    typedef key_type key_type_ref;
+    
+    typedef HeaderFileInfo data_type;
+    typedef const data_type &data_type_ref;
+    
+    static unsigned ComputeHash(const char *path) {
+      // The hash is based only on the filename portion of the key, so that the
+      // reader can match based on filenames when symlinking or excess path
+      // elements ("foo/../", "../") change the form of the name. However,
+      // complete path is still the key.
+      return llvm::HashString(llvm::sys::path::filename(path));
+    }
+    
+    std::pair<unsigned,unsigned>
+    EmitKeyDataLength(llvm::raw_ostream& Out, const char *path,
+                      data_type_ref Data) {
+      unsigned StrLen = strlen(path);
+      clang::io::Emit16(Out, StrLen);
+      unsigned DataLen = 1 + 2 + 4;
+      clang::io::Emit8(Out, DataLen);
+      return std::make_pair(StrLen + 1, DataLen);
+    }
+    
+    void EmitKey(llvm::raw_ostream& Out, const char *path, unsigned KeyLen) {
+      Out.write(path, KeyLen);
+    }
+    
+    void EmitData(llvm::raw_ostream &Out, key_type_ref,
+                  data_type_ref Data, unsigned DataLen) {
+      using namespace clang::io;
+      uint64_t Start = Out.tell(); (void)Start;
+      
+      unsigned char Flags = (Data.isImport << 3)
+                          | (Data.DirInfo << 1)
+                          | Data.Resolved;
+      Emit8(Out, (uint8_t)Flags);
+      Emit16(Out, (uint16_t) Data.NumIncludes);
+      
+      if (!Data.ControllingMacro)
+        Emit32(Out, (uint32_t)Data.ControllingMacroID);
+      else
+        Emit32(Out, (uint32_t)Writer.getIdentifierRef(Data.ControllingMacro));
+      assert(Out.tell() - Start == DataLen && "Wrong data length");
+    }
+  };
+} // end anonymous namespace
+
+/// \brief Write the header search block for the list of files that 
+///
+/// \param HS The header search structure to save.
+///
+/// \param Chain Whether we're creating a chained AST file.
+void ASTWriter::WriteHeaderSearch(HeaderSearch &HS, const char* isysroot) {
+  llvm::SmallVector<const FileEntry *, 16> FilesByUID;
+  HS.getFileMgr().GetUniqueIDMapping(FilesByUID);
+  
+  if (FilesByUID.size() > HS.header_file_size())
+    FilesByUID.resize(HS.header_file_size());
+  
+  HeaderFileInfoTrait GeneratorTrait(*this, HS);
+  OnDiskChainedHashTableGenerator<HeaderFileInfoTrait> Generator;  
+  llvm::SmallVector<const char *, 4> SavedStrings;
+  unsigned NumHeaderSearchEntries = 0;
+  for (unsigned UID = 0, LastUID = FilesByUID.size(); UID != LastUID; ++UID) {
+    const FileEntry *File = FilesByUID[UID];
+    if (!File)
+      continue;
+
+    const HeaderFileInfo &HFI = HS.header_file_begin()[UID];
+    if (HFI.External && Chain)
+      continue;
+
+    // Turn the file name into an absolute path, if it isn't already.
+    const char *Filename = File->getName();
+    Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
+      
+    // If we performed any translation on the file name at all, we need to
+    // save this string, since the generator will refer to it later.
+    if (Filename != File->getName()) {
+      Filename = strdup(Filename);
+      SavedStrings.push_back(Filename);
+    }
+    
+    Generator.insert(Filename, HFI, GeneratorTrait);
+    ++NumHeaderSearchEntries;
+  }
+  
+  // Create the on-disk hash table in a buffer.
+  llvm::SmallString<4096> TableData;
+  uint32_t BucketOffset;
+  {
+    llvm::raw_svector_ostream Out(TableData);
+    // Make sure that no bucket is at offset 0
+    clang::io::Emit32(Out, 0);
+    BucketOffset = Generator.Emit(Out, GeneratorTrait);
+  }
+
+  // Create a blob abbreviation
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_TABLE));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  unsigned TableAbbrev = Stream.EmitAbbrev(Abbrev);
+  
+  // Write the stat cache
+  RecordData Record;
+  Record.push_back(HEADER_SEARCH_TABLE);
+  Record.push_back(BucketOffset);
+  Record.push_back(NumHeaderSearchEntries);
+  Stream.EmitRecordWithBlob(TableAbbrev, Record, TableData.str());
+  
+  // Free all of the strings we had to duplicate.
+  for (unsigned I = 0, N = SavedStrings.size(); I != N; ++I)
+    free((void*)SavedStrings[I]);
+}
+
 /// \brief Writes the block containing the serialized form of the
 /// source manager.
 ///
@@ -1150,29 +1451,14 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         Record.push_back(Content->Entry->getSize());
         Record.push_back(Content->Entry->getModificationTime());
 
-        // Emit header-search information associated with this file.
-        HeaderFileInfo HFI;
-        HeaderSearch &HS = PP.getHeaderSearchInfo();
-        if (Content->Entry->getUID() < HS.header_file_size())
-          HFI = HS.header_file_begin()[Content->Entry->getUID()];
-        Record.push_back(HFI.isImport);
-        Record.push_back(HFI.DirInfo);
-        Record.push_back(HFI.NumIncludes);
-        AddIdentifierRef(HFI.ControllingMacro, Record);
-
         // Turn the file name into an absolute path, if it isn't already.
         const char *Filename = Content->Entry->getName();
-        llvm::sys::Path FilePath(Filename, strlen(Filename));
-        FilePath.makeAbsolute();
+        llvm::SmallString<128> FilePath(Filename);
+        llvm::sys::fs::make_absolute(FilePath);
         Filename = FilePath.c_str();
 
         Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
         Stream.EmitRecordWithBlob(SLocFileAbbrv, Record, Filename);
-
-        // FIXME: For now, preload all file source locations, so that
-        // we get the appropriate File entries in the reader. This is
-        // a temporary measure.
-        PreloadSLocs.push_back(BaseSLocID + SLocEntryOffsets.size());
       } else {
         // The source location entry is a buffer. The blob associated
         // with this entry contains the contents of the buffer.
@@ -1228,7 +1514,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   Record.clear();
   Record.push_back(SOURCE_LOCATION_OFFSETS);
   Record.push_back(SLocEntryOffsets.size());
-  Record.push_back(SourceMgr.getNextOffset());
+  unsigned BaseOffset = Chain ? Chain->getNextSLocOffset() : 0;
+  Record.push_back(SourceMgr.getNextOffset() - BaseOffset);
   Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record,
                             (const char *)data(SLocEntryOffsets),
                            SLocEntryOffsets.size()*sizeof(SLocEntryOffsets[0]));
@@ -1242,6 +1529,14 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
 // Preprocessor Serialization
 //===----------------------------------------------------------------------===//
 
+static int compareMacroDefinitions(const void *XPtr, const void *YPtr) {
+  const std::pair<const IdentifierInfo *, MacroInfo *> &X =
+    *(const std::pair<const IdentifierInfo *, MacroInfo *>*)XPtr;
+  const std::pair<const IdentifierInfo *, MacroInfo *> &Y =
+    *(const std::pair<const IdentifierInfo *, MacroInfo *>*)YPtr;
+  return X.first->getName().compare(Y.first->getName());
+}
+
 /// \brief Writes the block containing the serialized form of the
 /// preprocessor.
 ///
@@ -1256,30 +1551,63 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
   }
 
   // Enter the preprocessor block.
-  Stream.EnterSubblock(PREPROCESSOR_BLOCK_ID, 2);
+  Stream.EnterSubblock(PREPROCESSOR_BLOCK_ID, 3);
 
   // If the AST file contains __DATE__ or __TIME__ emit a warning about this.
   // FIXME: use diagnostics subsystem for localization etc.
   if (PP.SawDateOrTime())
     fprintf(stderr, "warning: precompiled header used __DATE__ or __TIME__.\n");
 
+
   // Loop over all the macro definitions that are live at the end of the file,
   // emitting each to the PP section.
   PreprocessingRecord *PPRec = PP.getPreprocessingRecord();
-  for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
-       I != E; ++I) {
-    // FIXME: This emits macros in hash table order, we should do it in a stable
-    // order so that output is reproducible.
-    MacroInfo *MI = I->second;
 
+  // Construct the list of macro definitions that need to be serialized.
+  llvm::SmallVector<std::pair<const IdentifierInfo *, MacroInfo *>, 2> 
+    MacrosToEmit;
+  llvm::SmallPtrSet<const IdentifierInfo*, 4> MacroDefinitionsSeen;
+  for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0), 
+                                    E = PP.macro_end(Chain == 0);
+       I != E; ++I) {
+    MacroDefinitionsSeen.insert(I->first);
+    MacrosToEmit.push_back(std::make_pair(I->first, I->second));
+  }
+  
+  // Sort the set of macro definitions that need to be serialized by the
+  // name of the macro, to provide a stable ordering.
+  llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(), 
+                       &compareMacroDefinitions);
+  
+  // Resolve any identifiers that defined macros at the time they were
+  // deserialized, adding them to the list of macros to emit (if appropriate).
+  for (unsigned I = 0, N = DeserializedMacroNames.size(); I != N; ++I) {
+    IdentifierInfo *Name
+      = const_cast<IdentifierInfo *>(DeserializedMacroNames[I]);
+    if (Name->hasMacroDefinition() && MacroDefinitionsSeen.insert(Name))
+      MacrosToEmit.push_back(std::make_pair(Name, PP.getMacroInfo(Name)));
+  }
+  
+  for (unsigned I = 0, N = MacrosToEmit.size(); I != N; ++I) {
+    const IdentifierInfo *Name = MacrosToEmit[I].first;
+    MacroInfo *MI = MacrosToEmit[I].second;
+    if (!MI)
+      continue;
+    
     // Don't emit builtin macros like __LINE__ to the AST file unless they have
     // been redefined by the header (in which case they are not isBuiltinMacro).
     // Also skip macros from a AST file if we're chaining.
-    if (MI->isBuiltinMacro() || (Chain && MI->isFromAST()))
+
+    // FIXME: There is a (probably minor) optimization we could do here, if
+    // the macro comes from the original PCH but the identifier comes from a
+    // chained PCH, by storing the offset into the original PCH rather than
+    // writing the macro definition a second time.
+    if (MI->isBuiltinMacro() ||
+        (Chain && Name->isFromAST() && MI->isFromAST()))
       continue;
 
-    AddIdentifierRef(I->first, Record);
-    MacroOffsets[I->first] = Stream.GetCurrentBitNo();
+    AddIdentifierRef(Name, Record);
+    MacroOffsets[Name] = Stream.GetCurrentBitNo();
     Record.push_back(MI->getDefinitionLoc().getRawEncoding());
     Record.push_back(MI->isUsed());
 
@@ -1296,12 +1624,12 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
            I != E; ++I)
         AddIdentifierRef(*I, Record);
     }
-    
+
     // If we have a detailed preprocessing record, record the macro definition
     // ID that corresponds to this macro.
     if (PPRec)
       Record.push_back(getMacroDefinitionID(PPRec->findMacroDefinition(MI)));
-    
+
     Stream.EmitRecord(Code, Record);
     Record.clear();
 
@@ -1318,7 +1646,6 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
       // FIXME: When reading literal tokens, reconstruct the literal pointer if
       // it is needed.
       AddIdentifierRef(Tok.getIdentifierInfo(), Record);
-
       // FIXME: Should translate token kind to a stable encoding.
       Record.push_back(Tok.getKind());
       // FIXME: Should translate token flags to a stable encoding.
@@ -1329,49 +1656,111 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
     }
     ++NumMacros;
   }
+  Stream.ExitBlock();
+
+  if (PPRec)
+    WritePreprocessorDetail(*PPRec);
+}
+
+void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
+  if (PPRec.begin(Chain) == PPRec.end(Chain))
+    return;
   
+  // Enter the preprocessor block.
+  Stream.EnterSubblock(PREPROCESSOR_DETAIL_BLOCK_ID, 3);
+
   // If the preprocessor has a preprocessing record, emit it.
   unsigned NumPreprocessingRecords = 0;
-  if (PPRec) {
-    for (PreprocessingRecord::iterator E = PPRec->begin(), EEnd = PPRec->end();
-         E != EEnd; ++E) {
-      Record.clear();
+  using namespace llvm;
+  
+  // Set up the abbreviation for 
+  unsigned InclusionAbbrev = 0;
+  {
+    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(PPD_INCLUSION_DIRECTIVE));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // index
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // start location
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // end location
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // filename length
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // in quotes
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // kind
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+    InclusionAbbrev = Stream.EmitAbbrev(Abbrev);
+  }
+  
+  unsigned IndexBase = Chain ? PPRec.getNumPreallocatedEntities() : 0;
+  RecordData Record;
+  for (PreprocessingRecord::iterator E = PPRec.begin(Chain),
+                                  EEnd = PPRec.end(Chain);
+       E != EEnd; ++E) {
+    Record.clear();
+
+    if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
+      // Record this macro definition's location.
+      MacroID ID = getMacroDefinitionID(MD);
       
-      if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(*E)) {
-        Record.push_back(NumPreprocessingRecords++);
-        AddSourceLocation(MI->getSourceRange().getBegin(), Record);
-        AddSourceLocation(MI->getSourceRange().getEnd(), Record);
-        AddIdentifierRef(MI->getName(), Record);
-        Record.push_back(getMacroDefinitionID(MI->getDefinition()));
-        Stream.EmitRecord(PP_MACRO_INSTANTIATION, Record);
+      // Don't write the macro definition if it is from another AST file.
+      if (ID < FirstMacroID)
         continue;
-      }
       
-      if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
-        // Record this macro definition's location.
-        IdentID ID = getMacroDefinitionID(MD);
-        if (ID != MacroDefinitionOffsets.size()) {
-          if (ID > MacroDefinitionOffsets.size())
-            MacroDefinitionOffsets.resize(ID + 1);
-          
-          MacroDefinitionOffsets[ID] = Stream.GetCurrentBitNo();            
-        } else
-          MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo());
+      // Notify the serialization listener that we're serializing this entity.
+      if (SerializationListener)
+        SerializationListener->SerializedPreprocessedEntity(*E, 
+                                                    Stream.GetCurrentBitNo());
+
+      unsigned Position = ID - FirstMacroID;
+      if (Position != MacroDefinitionOffsets.size()) {
+        if (Position > MacroDefinitionOffsets.size())
+          MacroDefinitionOffsets.resize(Position + 1);
         
-        Record.push_back(NumPreprocessingRecords++);
-        Record.push_back(ID);
-        AddSourceLocation(MD->getSourceRange().getBegin(), Record);
-        AddSourceLocation(MD->getSourceRange().getEnd(), Record);
-        AddIdentifierRef(MD->getName(), Record);
-        AddSourceLocation(MD->getLocation(), Record);
-        Stream.EmitRecord(PP_MACRO_DEFINITION, Record);
-        continue;
-      }
+        MacroDefinitionOffsets[Position] = Stream.GetCurrentBitNo();
+      } else
+        MacroDefinitionOffsets.push_back(Stream.GetCurrentBitNo());
+      
+      Record.push_back(IndexBase + NumPreprocessingRecords++);
+      Record.push_back(ID);
+      AddSourceLocation(MD->getSourceRange().getBegin(), Record);
+      AddSourceLocation(MD->getSourceRange().getEnd(), Record);
+      AddIdentifierRef(MD->getName(), Record);
+      AddSourceLocation(MD->getLocation(), Record);
+      Stream.EmitRecord(PPD_MACRO_DEFINITION, Record);
+      continue;
+    }
+
+    // Notify the serialization listener that we're serializing this entity.
+    if (SerializationListener)
+      SerializationListener->SerializedPreprocessedEntity(*E, 
+                                                    Stream.GetCurrentBitNo());
+
+    if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(*E)) {          
+      Record.push_back(IndexBase + NumPreprocessingRecords++);
+      AddSourceLocation(MI->getSourceRange().getBegin(), Record);
+      AddSourceLocation(MI->getSourceRange().getEnd(), Record);
+      AddIdentifierRef(MI->getName(), Record);
+      Record.push_back(getMacroDefinitionID(MI->getDefinition()));
+      Stream.EmitRecord(PPD_MACRO_INSTANTIATION, Record);
+      continue;
     }
+
+    if (InclusionDirective *ID = dyn_cast<InclusionDirective>(*E)) {
+      Record.push_back(PPD_INCLUSION_DIRECTIVE);
+      Record.push_back(IndexBase + NumPreprocessingRecords++);
+      AddSourceLocation(ID->getSourceRange().getBegin(), Record);
+      AddSourceLocation(ID->getSourceRange().getEnd(), Record);
+      Record.push_back(ID->getFileName().size());
+      Record.push_back(ID->wasInQuotes());
+      Record.push_back(static_cast<unsigned>(ID->getKind()));
+      llvm::SmallString<64> Buffer;
+      Buffer += ID->getFileName();
+      Buffer += ID->getFile()->getName();
+      Stream.EmitRecordWithBlob(InclusionAbbrev, Record, Buffer);
+      continue;
+    }
+    
+    llvm_unreachable("Unhandled PreprocessedEntity in ASTWriter");
   }
-  
   Stream.ExitBlock();
-  
+
   // Write the offsets table for the preprocessing record.
   if (NumPreprocessingRecords > 0) {
     // Write the offsets table for identifier IDs.
@@ -1382,7 +1771,7 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macro defs
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned MacroDefOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
-    
+
     Record.clear();
     Record.push_back(MACRO_DEFINITION_OFFSETS);
     Record.push_back(NumPreprocessingRecords);
@@ -1393,6 +1782,32 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
   }
 }
 
+void ASTWriter::WritePragmaDiagnosticMappings(const Diagnostic &Diag) {
+  RecordData Record;
+  for (Diagnostic::DiagStatePointsTy::const_iterator
+         I = Diag.DiagStatePoints.begin(), E = Diag.DiagStatePoints.end();
+         I != E; ++I) {
+    const Diagnostic::DiagStatePoint &point = *I; 
+    if (point.Loc.isInvalid())
+      continue;
+
+    Record.push_back(point.Loc.getRawEncoding());
+    for (Diagnostic::DiagState::iterator
+           I = point.State->begin(), E = point.State->end(); I != E; ++I) {
+      unsigned diag = I->first, map = I->second;
+      if (map & 0x10) { // mapping from a diagnostic pragma.
+        Record.push_back(diag);
+        Record.push_back(map & 0x7);
+      }
+    }
+    Record.push_back(-1); // mark the end of the diag/map pairs for this
+                          // location.
+  }
+
+  if (!Record.empty())
+    Stream.EmitRecord(DIAG_PRAGMA_MAPPINGS, Record);
+}
+
 //===----------------------------------------------------------------------===//
 // Type Serialization
 //===----------------------------------------------------------------------===//
@@ -1403,6 +1818,8 @@ void ASTWriter::WriteType(QualType T) {
   if (Idx.getIndex() == 0) // we haven't seen this type before.
     Idx = TypeIdx(NextTypeID++);
 
+  assert(Idx.getIndex() >= FirstTypeID && "Re-writing a type from a prior AST");
+
   // Record the offset for this type.
   unsigned Index = Idx.getIndex() - FirstTypeID;
   if (TypeOffsets.size() == Index)
@@ -1457,14 +1874,15 @@ uint64_t ASTWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
   uint64_t Offset = Stream.GetCurrentBitNo();
   RecordData Record;
   Record.push_back(DECL_CONTEXT_LEXICAL);
-  llvm::SmallVector<DeclID, 64> Decls;
+  llvm::SmallVector<KindDeclIDPair, 64> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
          D != DEnd; ++D)
-    Decls.push_back(GetDeclRef(*D));
+    Decls.push_back(std::make_pair((*D)->getKind(), GetDeclRef(*D)));
 
   ++NumLexicalDeclContexts;
   Stream.EmitRecordWithBlob(DeclContextLexicalAbbrev, Record,
-     reinterpret_cast<char*>(Decls.data()), Decls.size() * sizeof(DeclID));
+                            reinterpret_cast<char*>(Decls.data()),
+                            Decls.size() * sizeof(KindDeclIDPair));
   return Offset;
 }
 
@@ -1674,7 +2092,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
     // Create a blob abbreviation for the selector table offsets.
     Abbrev = new BitCodeAbbrev();
     Abbrev->Add(BitCodeAbbrevOp(SELECTOR_OFFSETS));
-    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // index
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // size
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned SelectorOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
@@ -2088,13 +2506,6 @@ uint64_t ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
 /// DeclContext in a dependent AST file. As such, they only exist for the TU
 /// (in C++) and for namespaces.
 void ASTWriter::WriteDeclContextVisibleUpdate(const DeclContext *DC) {
-  assert((DC->isTranslationUnit() || DC->isNamespace()) &&
-         "Only TU and namespaces should have visible decl updates.");
-
-  // Make the context build its lookup table, but don't make it load external
-  // decls.
-  DC->lookup(DeclarationName());
-
   StoredDeclsMap *Map = static_cast<StoredDeclsMap*>(DC->getLookupPtr());
   if (!Map || Map->empty())
     return;
@@ -2130,19 +2541,23 @@ void ASTWriter::WriteDeclContextVisibleUpdate(const DeclContext *DC) {
   Stream.EmitRecordWithBlob(UpdateVisibleAbbrev, Record, LookupTable.str());
 }
 
-/// \brief Write ADDITIONAL_TEMPLATE_SPECIALIZATIONS blocks for all templates
-/// that have new specializations in the current AST file.
-void ASTWriter::WriteAdditionalTemplateSpecializations() {
+/// \brief Write an FP_PRAGMA_OPTIONS block for the given FPOptions.
+void ASTWriter::WriteFPPragmaOptions(const FPOptions &Opts) {
   RecordData Record;
-  for (AdditionalTemplateSpecializationsMap::iterator
-           I = AdditionalTemplateSpecializations.begin(),
-           E = AdditionalTemplateSpecializations.end();
-       I != E; ++I) {
-    Record.clear();
-    Record.push_back(I->first);
-    Record.insert(Record.end(), I->second.begin(), I->second.end());
-    Stream.EmitRecord(ADDITIONAL_TEMPLATE_SPECIALIZATIONS, Record);
-  }
+  Record.push_back(Opts.fp_contract);
+  Stream.EmitRecord(FP_PRAGMA_OPTIONS, Record);
+}
+
+/// \brief Write an OPENCL_EXTENSIONS block for the given OpenCLOptions.
+void ASTWriter::WriteOpenCLExtensions(Sema &SemaRef) {
+  if (!SemaRef.Context.getLangOptions().OpenCL)
+    return;
+
+  const OpenCLOptions &Opts = SemaRef.getOpenCLOptions();
+  RecordData Record;
+#define OPENCLEXT(nm)  Record.push_back(Opts.nm);
+#include "clang/Basic/OpenCLExtensions.def"
+  Stream.EmitRecord(OPENCL_EXTENSIONS, Record);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2150,22 +2565,19 @@ void ASTWriter::WriteAdditionalTemplateSpecializations() {
 //===----------------------------------------------------------------------===//
 
 /// \brief Write a record containing the given attributes.
-void ASTWriter::WriteAttributeRecord(const AttrVec &Attrs) {
-  RecordData Record;
+void ASTWriter::WriteAttributes(const AttrVec &Attrs, RecordDataImpl &Record) {
+  Record.push_back(Attrs.size());
   for (AttrVec::const_iterator i = Attrs.begin(), e = Attrs.end(); i != e; ++i){
     const Attr * A = *i;
     Record.push_back(A->getKind()); // FIXME: stable encoding, target attrs
     AddSourceLocation(A->getLocation(), Record);
-    Record.push_back(A->isInherited());
 
 #include "clang/Serialization/AttrPCHWrite.inc"
 
   }
-
-  Stream.EmitRecord(DECL_ATTR, Record);
 }
 
-void ASTWriter::AddString(llvm::StringRef Str, RecordData &Record) {
+void ASTWriter::AddString(llvm::StringRef Str, RecordDataImpl &Record) {
   Record.push_back(Str.size());
   Record.insert(Record.end(), Str.begin(), Str.end());
 }
@@ -2193,15 +2605,20 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
 }
 
 ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
-  : Stream(Stream), Chain(0), FirstDeclID(1), NextDeclID(FirstDeclID),
+  : Stream(Stream), Chain(0), SerializationListener(0), 
+    FirstDeclID(1), NextDeclID(FirstDeclID),
     FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID),
     FirstIdentID(1), NextIdentID(FirstIdentID), FirstSelectorID(1),
-    NextSelectorID(FirstSelectorID), CollectedStmts(&StmtsToEmit),
+    NextSelectorID(FirstSelectorID), FirstMacroID(1), NextMacroID(FirstMacroID),
+    CollectedStmts(&StmtsToEmit),
     NumStatements(0), NumMacros(0), NumLexicalDeclContexts(0),
-    NumVisibleDeclContexts(0) {
+    NumVisibleDeclContexts(0), FirstCXXBaseSpecifiersID(1),
+    NextCXXBaseSpecifiersID(1)
+{
 }
 
 void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+                         const std::string &OutputFile,
                          const char *isysroot) {
   // Emit the file header.
   Stream.Emit((unsigned)'C', 8);
@@ -2214,11 +2631,12 @@ void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   if (Chain)
     WriteASTChain(SemaRef, StatCalls, isysroot);
   else
-    WriteASTCore(SemaRef, StatCalls, isysroot);
+    WriteASTCore(SemaRef, StatCalls, isysroot, OutputFile);
 }
 
 void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
-                             const char *isysroot) {
+                             const char *isysroot,
+                             const std::string &OutputFile) {
   using namespace llvm;
 
   ASTContext &Context = SemaRef.Context;
@@ -2318,10 +2736,15 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
     AddDeclRef(SemaRef.getStdBadAlloc(), SemaDeclRefs);
   }
 
+  RecordData CUDASpecialDeclRefs;
+  if (Context.getcudaConfigureCallDecl()) {
+    AddDeclRef(Context.getcudaConfigureCallDecl(), CUDASpecialDeclRefs);
+  }
+
   // Write the remaining AST contents.
   RecordData Record;
   Stream.EnterSubblock(AST_BLOCK_ID, 5);
-  WriteMetadata(Context, isysroot);
+  WriteMetadata(Context, isysroot, OutputFile);
   WriteLanguageOptions(Context.getLangOptions());
   if (StatCalls && !isysroot)
     WriteStatCache(*StatCalls);
@@ -2363,12 +2786,36 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Stream.ExitBlock();
 
   WritePreprocessor(PP);
+  WriteHeaderSearch(PP.getHeaderSearchInfo(), isysroot);
   WriteSelectors(SemaRef);
   WriteReferencedSelectorsPool(SemaRef);
   WriteIdentifierTable(PP);
+  WriteFPPragmaOptions(SemaRef.getFPOptions());
+  WriteOpenCLExtensions(SemaRef);
 
   WriteTypeDeclOffsets();
+  WritePragmaDiagnosticMappings(Context.getDiagnostics());
 
+  // Write the C++ base-specifier set offsets.
+  if (!CXXBaseSpecifiersOffsets.empty()) {
+    // Create a blob abbreviation for the C++ base specifiers offsets.
+    using namespace llvm;
+    
+    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+    Abbrev->Add(BitCodeAbbrevOp(CXX_BASE_SPECIFIER_OFFSETS));
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // size
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+    unsigned BaseSpecifierOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+    
+    // Write the selector offsets table.
+    Record.clear();
+    Record.push_back(CXX_BASE_SPECIFIER_OFFSETS);
+    Record.push_back(CXXBaseSpecifiersOffsets.size());
+    Stream.EmitRecordWithBlob(BaseSpecifierOffsetAbbrev, Record,
+                              (const char *)CXXBaseSpecifiersOffsets.data(),
+                            CXXBaseSpecifiersOffsets.size() * sizeof(uint32_t));
+  }
+  
   // Write the record containing external, unnamed definitions.
   if (!ExternalDefinitions.empty())
     Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
@@ -2411,6 +2858,10 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   if (!SemaDeclRefs.empty())
     Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);
 
+  // Write the record containing CUDA-specific declaration references.
+  if (!CUDASpecialDeclRefs.empty())
+    Stream.EmitRecord(CUDA_SPECIAL_DECL_REFS, CUDASpecialDeclRefs);
+
   // Some simple statistics
   Record.clear();
   Record.push_back(NumStatements);
@@ -2425,21 +2876,12 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                               const char *isysroot) {
   using namespace llvm;
 
-  FirstDeclID += Chain->getTotalNumDecls();
-  FirstTypeID += Chain->getTotalNumTypes();
-  FirstIdentID += Chain->getTotalNumIdentifiers();
-  FirstSelectorID += Chain->getTotalNumSelectors();
-  NextDeclID = FirstDeclID;
-  NextTypeID = FirstTypeID;
-  NextIdentID = FirstIdentID;
-  NextSelectorID = FirstSelectorID;
-
   ASTContext &Context = SemaRef.Context;
   Preprocessor &PP = SemaRef.PP;
 
   RecordData Record;
   Stream.EnterSubblock(AST_BLOCK_ID, 5);
-  WriteMetadata(Context, isysroot);
+  WriteMetadata(Context, isysroot, "");
   if (StatCalls && !isysroot)
     WriteStatCache(*StatCalls);
   // FIXME: Source manager block should only write new stuff, which could be
@@ -2451,12 +2893,12 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // We don't start with the translation unit, but with its decls that
   // don't come from the chained PCH.
   const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
-  llvm::SmallVector<DeclID, 64> NewGlobalDecls;
+  llvm::SmallVector<KindDeclIDPair, 64> NewGlobalDecls;
   for (DeclContext::decl_iterator I = TU->noload_decls_begin(),
                                   E = TU->noload_decls_end();
        I != E; ++I) {
     if ((*I)->getPCHLevel() == 0)
-      NewGlobalDecls.push_back(GetDeclRef(*I));
+      NewGlobalDecls.push_back(std::make_pair((*I)->getKind(), GetDeclRef(*I)));
     else if ((*I)->isChangedSinceDeserialization())
       (void)GetDeclRef(*I); // Make sure it's written, but don't record it.
   }
@@ -2469,17 +2911,15 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Record.push_back(TU_UPDATE_LEXICAL);
   Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record,
                           reinterpret_cast<const char*>(NewGlobalDecls.data()),
-                          NewGlobalDecls.size() * sizeof(DeclID));
-  // And in C++, a visible updates block for the TU.
-  if (Context.getLangOptions().CPlusPlus) {
-    Abv = new llvm::BitCodeAbbrev();
-    Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE));
-    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
-    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, 32));
-    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
-    UpdateVisibleAbbrev = Stream.EmitAbbrev(Abv);
-    WriteDeclContextVisibleUpdate(TU);
-  }
+                          NewGlobalDecls.size() * sizeof(KindDeclIDPair));
+  // And a visible updates block for the DeclContexts.
+  Abv = new llvm::BitCodeAbbrev();
+  Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, 32));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  UpdateVisibleAbbrev = Stream.EmitAbbrev(Abv);
+  WriteDeclContextVisibleUpdate(TU);
 
   // Build a record containing all of the new tentative definitions in this
   // file, in TentativeDefinitions order.
@@ -2574,6 +3014,9 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   Stream.EnterSubblock(DECLTYPES_BLOCK_ID, 3);
   WriteDeclsBlockAbbrevs();
+  for (DeclsToRewriteTy::iterator
+         I = DeclsToRewrite.begin(), E = DeclsToRewrite.end(); I != E; ++I)
+    DeclTypesToEmit.push(const_cast<Decl*>(*I));
   while (!DeclTypesToEmit.empty()) {
     DeclOrType DOT = DeclTypesToEmit.front();
     DeclTypesToEmit.pop();
@@ -2588,7 +3031,13 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   WriteSelectors(SemaRef);
   WriteReferencedSelectorsPool(SemaRef);
   WriteIdentifierTable(PP);
+  WriteFPPragmaOptions(SemaRef.getFPOptions());
+  WriteOpenCLExtensions(SemaRef);
+
   WriteTypeDeclOffsets();
+  // FIXME: For chained PCH only write the new mappings (we currently
+  // write all of them again).
+  WritePragmaDiagnosticMappings(Context.getDiagnostics());
 
   /// Build a record containing first declarations from a chained PCH and the
   /// most recent declarations in this AST that they point to.
@@ -2645,28 +3094,50 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   if (!SemaDeclRefs.empty())
     Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);
 
-  // Write the updates to C++ namespaces.
-  for (llvm::SmallPtrSet<const NamespaceDecl *, 16>::iterator
-           I = UpdatedNamespaces.begin(),
-           E = UpdatedNamespaces.end();
+  // Write the updates to DeclContexts.
+  for (llvm::SmallPtrSet<const DeclContext *, 16>::iterator
+           I = UpdatedDeclContexts.begin(),
+           E = UpdatedDeclContexts.end();
          I != E; ++I)
     WriteDeclContextVisibleUpdate(*I);
 
-  // Write the updates to C++ template specialization lists.
-  if (!AdditionalTemplateSpecializations.empty())
-    WriteAdditionalTemplateSpecializations();
+  WriteDeclUpdatesBlocks();
 
   Record.clear();
   Record.push_back(NumStatements);
   Record.push_back(NumMacros);
   Record.push_back(NumLexicalDeclContexts);
   Record.push_back(NumVisibleDeclContexts);
-  WriteDeclUpdateBlock();
+  WriteDeclReplacementsBlock();
   Stream.EmitRecord(STATISTICS, Record);
   Stream.ExitBlock();
 }
 
-void ASTWriter::WriteDeclUpdateBlock() {
+void ASTWriter::WriteDeclUpdatesBlocks() {
+  if (DeclUpdates.empty())
+    return;
+
+  RecordData OffsetsRecord;
+  Stream.EnterSubblock(DECL_UPDATES_BLOCK_ID, 3);
+  for (DeclUpdateMap::iterator
+         I = DeclUpdates.begin(), E = DeclUpdates.end(); I != E; ++I) {
+    const Decl *D = I->first;
+    UpdateRecord &URec = I->second;
+
+    if (DeclsToRewrite.count(D))
+      continue; // The decl will be written completely,no need to store updates.
+
+    uint64_t Offset = Stream.GetCurrentBitNo();
+    Stream.EmitRecord(DECL_UPDATES, URec);
+
+    OffsetsRecord.push_back(GetDeclRef(D));
+    OffsetsRecord.push_back(Offset);
+  }
+  Stream.ExitBlock();
+  Stream.EmitRecord(DECL_UPDATE_OFFSETS, OffsetsRecord);
+}
+
+void ASTWriter::WriteDeclReplacementsBlock() {
   if (ReplacedDecls.empty())
     return;
 
@@ -2679,33 +3150,31 @@ void ASTWriter::WriteDeclUpdateBlock() {
   Stream.EmitRecord(DECL_REPLACEMENTS, Record);
 }
 
-void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordData &Record) {
+void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record) {
   Record.push_back(Loc.getRawEncoding());
 }
 
-void ASTWriter::AddSourceRange(SourceRange Range, RecordData &Record) {
+void ASTWriter::AddSourceRange(SourceRange Range, RecordDataImpl &Record) {
   AddSourceLocation(Range.getBegin(), Record);
   AddSourceLocation(Range.getEnd(), Record);
 }
 
-void ASTWriter::AddAPInt(const llvm::APInt &Value, RecordData &Record) {
+void ASTWriter::AddAPInt(const llvm::APInt &Value, RecordDataImpl &Record) {
   Record.push_back(Value.getBitWidth());
-  unsigned N = Value.getNumWords();
-  const uint64_t* Words = Value.getRawData();
-  for (unsigned I = 0; I != N; ++I)
-    Record.push_back(Words[I]);
+  const uint64_t *Words = Value.getRawData();
+  Record.append(Words, Words + Value.getNumWords());
 }
 
-void ASTWriter::AddAPSInt(const llvm::APSInt &Value, RecordData &Record) {
+void ASTWriter::AddAPSInt(const llvm::APSInt &Value, RecordDataImpl &Record) {
   Record.push_back(Value.isUnsigned());
   AddAPInt(Value, Record);
 }
 
-void ASTWriter::AddAPFloat(const llvm::APFloat &Value, RecordData &Record) {
+void ASTWriter::AddAPFloat(const llvm::APFloat &Value, RecordDataImpl &Record) {
   AddAPInt(Value.bitcastToAPInt(), Record);
 }
 
-void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordData &Record) {
+void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Record) {
   Record.push_back(getIdentifierRef(II));
 }
 
@@ -2719,17 +3188,17 @@ IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   return ID;
 }
 
-IdentID ASTWriter::getMacroDefinitionID(MacroDefinition *MD) {
+MacroID ASTWriter::getMacroDefinitionID(MacroDefinition *MD) {
   if (MD == 0)
     return 0;
-  
-  IdentID &ID = MacroDefinitions[MD];
+
+  MacroID &ID = MacroDefinitions[MD];
   if (ID == 0)
-    ID = MacroDefinitions.size();
+    ID = NextMacroID++;
   return ID;
 }
 
-void ASTWriter::AddSelectorRef(const Selector SelRef, RecordData &Record) {
+void ASTWriter::AddSelectorRef(const Selector SelRef, RecordDataImpl &Record) {
   Record.push_back(getSelectorRef(SelRef));
 }
 
@@ -2750,13 +3219,23 @@ SelectorID ASTWriter::getSelectorRef(Selector Sel) {
   return SID;
 }
 
-void ASTWriter::AddCXXTemporary(const CXXTemporary *Temp, RecordData &Record) {
+void ASTWriter::AddCXXTemporary(const CXXTemporary *Temp, RecordDataImpl &Record) {
   AddDeclRef(Temp->getDestructor(), Record);
 }
 
+void ASTWriter::AddCXXBaseSpecifiersRef(CXXBaseSpecifier const *Bases,
+                                      CXXBaseSpecifier const *BasesEnd,
+                                        RecordDataImpl &Record) {
+  assert(Bases != BasesEnd && "Empty base-specifier sets are not recorded");
+  CXXBaseSpecifiersToWrite.push_back(
+                                QueuedCXXBaseSpecifiers(NextCXXBaseSpecifiersID,
+                                                        Bases, BasesEnd));
+  Record.push_back(NextCXXBaseSpecifiersID++);
+}
+
 void ASTWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
                                            const TemplateArgumentLocInfo &Arg,
-                                           RecordData &Record) {
+                                           RecordDataImpl &Record) {
   switch (Kind) {
   case TemplateArgument::Expression:
     AddStmt(Arg.getAsExpr());
@@ -2768,6 +3247,11 @@ void ASTWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
     AddSourceRange(Arg.getTemplateQualifierRange(), Record);
     AddSourceLocation(Arg.getTemplateNameLoc(), Record);
     break;
+  case TemplateArgument::TemplateExpansion:
+    AddSourceRange(Arg.getTemplateQualifierRange(), Record);
+    AddSourceLocation(Arg.getTemplateNameLoc(), Record);
+    AddSourceLocation(Arg.getTemplateEllipsisLoc(), Record);
+    break;
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
   case TemplateArgument::Declaration:
@@ -2777,7 +3261,7 @@ void ASTWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
 }
 
 void ASTWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg,
-                                       RecordData &Record) {
+                                       RecordDataImpl &Record) {
   AddTemplateArgument(Arg.getArgument(), Record);
 
   if (Arg.getArgument().getKind() == TemplateArgument::Expression) {
@@ -2791,7 +3275,7 @@ void ASTWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg,
                              Record);
 }
 
-void ASTWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordData &Record) {
+void ASTWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordDataImpl &Record) {
   if (TInfo == 0) {
     AddTypeRef(QualType(), Record);
     return;
@@ -2803,7 +3287,7 @@ void ASTWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordData &Record) {
     TLW.Visit(TL);
 }
 
-void ASTWriter::AddTypeRef(QualType T, RecordData &Record) {
+void ASTWriter::AddTypeRef(QualType T, RecordDataImpl &Record) {
   Record.push_back(GetOrCreateTypeID(T));
 }
 
@@ -2842,7 +3326,7 @@ TypeIdx ASTWriter::getTypeIdx(QualType T) const {
   return I->second;
 }
 
-void ASTWriter::AddDeclRef(const Decl *D, RecordData &Record) {
+void ASTWriter::AddDeclRef(const Decl *D, RecordDataImpl &Record) {
   Record.push_back(GetDeclRef(D));
 }
 
@@ -2850,7 +3334,7 @@ DeclID ASTWriter::GetDeclRef(const Decl *D) {
   if (D == 0) {
     return 0;
   }
-
+  assert(!(reinterpret_cast<uintptr_t>(D) & 0x01) && "Invalid decl pointer");
   DeclID &ID = DeclIDs[D];
   if (ID == 0) {
     // We haven't seen this declaration before. Give it a new ID and
@@ -2876,7 +3360,7 @@ DeclID ASTWriter::getDeclID(const Decl *D) {
   return DeclIDs[D];
 }
 
-void ASTWriter::AddDeclarationName(DeclarationName Name, RecordData &Record) {
+void ASTWriter::AddDeclarationName(DeclarationName Name, RecordDataImpl &Record) {
   // FIXME: Emit a stable enum for NameKind.  0 = Identifier etc.
   Record.push_back(Name.getNameKind());
   switch (Name.getNameKind()) {
@@ -2910,8 +3394,57 @@ void ASTWriter::AddDeclarationName(DeclarationName Name, RecordData &Record) {
   }
 }
 
+void ASTWriter::AddDeclarationNameLoc(const DeclarationNameLoc &DNLoc,
+                                     DeclarationName Name, RecordDataImpl &Record) {
+  switch (Name.getNameKind()) {
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    AddTypeSourceInfo(DNLoc.NamedType.TInfo, Record);
+    break;
+
+  case DeclarationName::CXXOperatorName:
+    AddSourceLocation(
+       SourceLocation::getFromRawEncoding(DNLoc.CXXOperatorName.BeginOpNameLoc),
+       Record);
+    AddSourceLocation(
+        SourceLocation::getFromRawEncoding(DNLoc.CXXOperatorName.EndOpNameLoc),
+        Record);
+    break;
+
+  case DeclarationName::CXXLiteralOperatorName:
+    AddSourceLocation(
+     SourceLocation::getFromRawEncoding(DNLoc.CXXLiteralOperatorName.OpNameLoc),
+     Record);
+    break;
+
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXUsingDirective:
+    break;
+  }
+}
+
+void ASTWriter::AddDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
+                                       RecordDataImpl &Record) {
+  AddDeclarationName(NameInfo.getName(), Record);
+  AddSourceLocation(NameInfo.getLoc(), Record);
+  AddDeclarationNameLoc(NameInfo.getInfo(), NameInfo.getName(), Record);
+}
+
+void ASTWriter::AddQualifierInfo(const QualifierInfo &Info,
+                                 RecordDataImpl &Record) {
+  AddNestedNameSpecifier(Info.NNS, Record);
+  AddSourceRange(Info.NNSRange, Record);
+  Record.push_back(Info.NumTemplParamLists);
+  for (unsigned i=0, e=Info.NumTemplParamLists; i != e; ++i)
+    AddTemplateParameterList(Info.TemplParamLists[i], Record);
+}
+
 void ASTWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
-                                       RecordData &Record) {
+                                       RecordDataImpl &Record) {
   // Nested name specifiers usually aren't too long. I think that 8 would
   // typically accomodate the vast majority.
   llvm::SmallVector<NestedNameSpecifier *, 8> NestedNames;
@@ -2949,8 +3482,8 @@ void ASTWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
   }
 }
 
-void ASTWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
-  TemplateName::NameKind Kind = Name.getKind(); 
+void ASTWriter::AddTemplateName(TemplateName Name, RecordDataImpl &Record) {
+  TemplateName::NameKind Kind = Name.getKind();
   Record.push_back(Kind);
   switch (Kind) {
   case TemplateName::Template:
@@ -2965,7 +3498,7 @@ void ASTWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
       AddDeclRef(*I, Record);
     break;
   }
-    
+
   case TemplateName::QualifiedTemplate: {
     QualifiedTemplateName *QualT = Name.getAsQualifiedTemplateName();
     AddNestedNameSpecifier(QualT->getQualifier(), Record);
@@ -2973,7 +3506,7 @@ void ASTWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
     AddDeclRef(QualT->getTemplateDecl(), Record);
     break;
   }
-    
+
   case TemplateName::DependentTemplate: {
     DependentTemplateName *DepT = Name.getAsDependentTemplateName();
     AddNestedNameSpecifier(DepT->getQualifier(), Record);
@@ -2984,11 +3517,19 @@ void ASTWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
       Record.push_back(DepT->getOperator());
     break;
   }
+      
+  case TemplateName::SubstTemplateTemplateParmPack: {
+    SubstTemplateTemplateParmPackStorage *SubstPack
+      = Name.getAsSubstTemplateTemplateParmPack();
+    AddDeclRef(SubstPack->getParameterPack(), Record);
+    AddTemplateArgument(SubstPack->getArgumentPack(), Record);
+    break;
+  }
   }
 }
 
-void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg, 
-                                    RecordData &Record) {
+void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg,
+                                    RecordDataImpl &Record) {
   Record.push_back(Arg.getKind());
   switch (Arg.getKind()) {
   case TemplateArgument::Null:
@@ -3004,7 +3545,14 @@ void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg,
     AddTypeRef(Arg.getIntegralType(), Record);
     break;
   case TemplateArgument::Template:
-    AddTemplateName(Arg.getAsTemplate(), Record);
+    AddTemplateName(Arg.getAsTemplateOrTemplatePattern(), Record);
+    break;
+  case TemplateArgument::TemplateExpansion:
+    AddTemplateName(Arg.getAsTemplateOrTemplatePattern(), Record);
+    if (llvm::Optional<unsigned> NumExpansions = Arg.getNumTemplateExpansions())
+      Record.push_back(*NumExpansions + 1);
+    else
+      Record.push_back(0);
     break;
   case TemplateArgument::Expression:
     AddStmt(Arg.getAsExpr());
@@ -3020,7 +3568,7 @@ void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg,
 
 void
 ASTWriter::AddTemplateParameterList(const TemplateParameterList *TemplateParams,
-                                    RecordData &Record) {
+                                    RecordDataImpl &Record) {
   assert(TemplateParams && "No TemplateParams!");
   AddSourceLocation(TemplateParams->getTemplateLoc(), Record);
   AddSourceLocation(TemplateParams->getLAngleLoc(), Record);
@@ -3035,16 +3583,16 @@ ASTWriter::AddTemplateParameterList(const TemplateParameterList *TemplateParams,
 /// \brief Emit a template argument list.
 void
 ASTWriter::AddTemplateArgumentList(const TemplateArgumentList *TemplateArgs,
-                                   RecordData &Record) {
+                                   RecordDataImpl &Record) {
   assert(TemplateArgs && "No TemplateArgs!");
-  Record.push_back(TemplateArgs->flat_size());
-  for (int i=0, e = TemplateArgs->flat_size(); i != e; ++i)
+  Record.push_back(TemplateArgs->size());
+  for (int i=0, e = TemplateArgs->size(); i != e; ++i)
     AddTemplateArgument(TemplateArgs->get(i), Record);
 }
 
 
 void
-ASTWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordData &Record) {
+ASTWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordDataImpl &Record) {
   Record.push_back(Set.size());
   for (UnresolvedSetImpl::const_iterator
          I = Set.begin(), E = Set.end(); I != E; ++I) {
@@ -3054,31 +3602,69 @@ ASTWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordData &Record) {
 }
 
 void ASTWriter::AddCXXBaseSpecifier(const CXXBaseSpecifier &Base,
-                                    RecordData &Record) {
+                                    RecordDataImpl &Record) {
   Record.push_back(Base.isVirtual());
   Record.push_back(Base.isBaseOfClass());
   Record.push_back(Base.getAccessSpecifierAsWritten());
+  Record.push_back(Base.getInheritConstructors());
   AddTypeSourceInfo(Base.getTypeSourceInfo(), Record);
   AddSourceRange(Base.getSourceRange(), Record);
+  AddSourceLocation(Base.isPackExpansion()? Base.getEllipsisLoc() 
+                                          : SourceLocation(),
+                    Record);
+}
+
+void ASTWriter::FlushCXXBaseSpecifiers() {
+  RecordData Record;
+  for (unsigned I = 0, N = CXXBaseSpecifiersToWrite.size(); I != N; ++I) {
+    Record.clear();
+    
+    // Record the offset of this base-specifier set.
+    unsigned Index = CXXBaseSpecifiersToWrite[I].ID - FirstCXXBaseSpecifiersID;
+    if (Index == CXXBaseSpecifiersOffsets.size())
+      CXXBaseSpecifiersOffsets.push_back(Stream.GetCurrentBitNo());
+    else {
+      if (Index > CXXBaseSpecifiersOffsets.size())
+        CXXBaseSpecifiersOffsets.resize(Index + 1);
+      CXXBaseSpecifiersOffsets[Index] = Stream.GetCurrentBitNo();
+    }
+
+    const CXXBaseSpecifier *B = CXXBaseSpecifiersToWrite[I].Bases,
+                        *BEnd = CXXBaseSpecifiersToWrite[I].BasesEnd;
+    Record.push_back(BEnd - B);
+    for (; B != BEnd; ++B)
+      AddCXXBaseSpecifier(*B, Record);
+    Stream.EmitRecord(serialization::DECL_CXX_BASE_SPECIFIERS, Record);
+    
+    // Flush any expressions that were written as part of the base specifiers.
+    FlushStmts();
+  }
+
+  CXXBaseSpecifiersToWrite.clear();
 }
 
-void ASTWriter::AddCXXBaseOrMemberInitializers(
-                        const CXXBaseOrMemberInitializer * const *BaseOrMembers,
-                        unsigned NumBaseOrMembers, RecordData &Record) {
-  Record.push_back(NumBaseOrMembers);
-  for (unsigned i=0; i != NumBaseOrMembers; ++i) {
-    const CXXBaseOrMemberInitializer *Init = BaseOrMembers[i];
+void ASTWriter::AddCXXCtorInitializers(
+                             const CXXCtorInitializer * const *CtorInitializers,
+                             unsigned NumCtorInitializers,
+                             RecordDataImpl &Record) {
+  Record.push_back(NumCtorInitializers);
+  for (unsigned i=0; i != NumCtorInitializers; ++i) {
+    const CXXCtorInitializer *Init = CtorInitializers[i];
 
     Record.push_back(Init->isBaseInitializer());
     if (Init->isBaseInitializer()) {
       AddTypeSourceInfo(Init->getBaseClassInfo(), Record);
       Record.push_back(Init->isBaseVirtual());
     } else {
-      AddDeclRef(Init->getMember(), Record);
+      Record.push_back(Init->isIndirectMemberInitializer());
+      if (Init->isIndirectMemberInitializer())
+        AddDeclRef(Init->getIndirectMember(), Record);
+      else
+        AddDeclRef(Init->getMember(), Record);
     }
+
     AddSourceLocation(Init->getMemberLocation(), Record);
     AddStmt(Init->getInit());
-    AddDeclRef(Init->getAnonUnionMember(), Record);
     AddSourceLocation(Init->getLParenLoc(), Record);
     AddSourceLocation(Init->getRParenLoc(), Record);
     Record.push_back(Init->isWritten());
@@ -3092,22 +3678,86 @@ void ASTWriter::AddCXXBaseOrMemberInitializers(
   }
 }
 
-void ASTWriter::SetReader(ASTReader *Reader) {
+void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Record) {
+  assert(D->DefinitionData);
+  struct CXXRecordDecl::DefinitionData &Data = *D->DefinitionData;
+  Record.push_back(Data.UserDeclaredConstructor);
+  Record.push_back(Data.UserDeclaredCopyConstructor);
+  Record.push_back(Data.UserDeclaredCopyAssignment);
+  Record.push_back(Data.UserDeclaredDestructor);
+  Record.push_back(Data.Aggregate);
+  Record.push_back(Data.PlainOldData);
+  Record.push_back(Data.Empty);
+  Record.push_back(Data.Polymorphic);
+  Record.push_back(Data.Abstract);
+  Record.push_back(Data.HasTrivialConstructor);
+  Record.push_back(Data.HasTrivialCopyConstructor);
+  Record.push_back(Data.HasTrivialCopyAssignment);
+  Record.push_back(Data.HasTrivialDestructor);
+  Record.push_back(Data.ComputedVisibleConversions);
+  Record.push_back(Data.DeclaredDefaultConstructor);
+  Record.push_back(Data.DeclaredCopyConstructor);
+  Record.push_back(Data.DeclaredCopyAssignment);
+  Record.push_back(Data.DeclaredDestructor);
+
+  Record.push_back(Data.NumBases);
+  if (Data.NumBases > 0)
+    AddCXXBaseSpecifiersRef(Data.getBases(), Data.getBases() + Data.NumBases, 
+                            Record);
+  
+  // FIXME: Make VBases lazily computed when needed to avoid storing them.
+  Record.push_back(Data.NumVBases);
+  if (Data.NumVBases > 0)
+    AddCXXBaseSpecifiersRef(Data.getVBases(), Data.getVBases() + Data.NumVBases, 
+                            Record);
+
+  AddUnresolvedSet(Data.Conversions, Record);
+  AddUnresolvedSet(Data.VisibleConversions, Record);
+  // Data.Definition is the owning decl, no need to write it. 
+  AddDeclRef(Data.FirstFriend, Record);
+}
+
+void ASTWriter::ReaderInitialized(ASTReader *Reader) {
   assert(Reader && "Cannot remove chain");
+  assert(!Chain && "Cannot replace chain");
   assert(FirstDeclID == NextDeclID &&
          FirstTypeID == NextTypeID &&
          FirstIdentID == NextIdentID &&
          FirstSelectorID == NextSelectorID &&
+         FirstMacroID == NextMacroID &&
+         FirstCXXBaseSpecifiersID == NextCXXBaseSpecifiersID &&
          "Setting chain after writing has started.");
   Chain = Reader;
+
+  FirstDeclID += Chain->getTotalNumDecls();
+  FirstTypeID += Chain->getTotalNumTypes();
+  FirstIdentID += Chain->getTotalNumIdentifiers();
+  FirstSelectorID += Chain->getTotalNumSelectors();
+  FirstMacroID += Chain->getTotalNumMacroDefinitions();
+  FirstCXXBaseSpecifiersID += Chain->getTotalNumCXXBaseSpecifiers();
+  NextDeclID = FirstDeclID;
+  NextTypeID = FirstTypeID;
+  NextIdentID = FirstIdentID;
+  NextSelectorID = FirstSelectorID;
+  NextMacroID = FirstMacroID;
+  NextCXXBaseSpecifiersID = FirstCXXBaseSpecifiersID;
 }
 
 void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
   IdentifierIDs[II] = ID;
+  if (II->hasMacroDefinition())
+    DeserializedMacroNames.push_back(II);
 }
 
 void ASTWriter::TypeRead(TypeIdx Idx, QualType T) {
-  TypeIdxs[T] = Idx;
+  // Always take the highest-numbered type index. This copes with an interesting
+  // case for chained AST writing where we schedule writing the type and then,
+  // later, deserialize the type from another AST. In this case, we want to
+  // keep the higher-numbered entry so that we can properly write it out to
+  // the AST file.
+  TypeIdx &StoredIdx = TypeIdxs[T];
+  if (Idx.getIndex() >= StoredIdx.getIndex())
+    StoredIdx = Idx;
 }
 
 void ASTWriter::DeclRead(DeclID ID, const Decl *D) {
@@ -3117,3 +3767,75 @@ void ASTWriter::DeclRead(DeclID ID, const Decl *D) {
 void ASTWriter::SelectorRead(SelectorID ID, Selector S) {
   SelectorIDs[S] = ID;
 }
+
+void ASTWriter::MacroDefinitionRead(serialization::MacroID ID,
+                                    MacroDefinition *MD) {
+  MacroDefinitions[MD] = ID;
+}
+
+void ASTWriter::CompletedTagDefinition(const TagDecl *D) {
+  assert(D->isDefinition());
+  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
+    // We are interested when a PCH decl is modified.
+    if (RD->getPCHLevel() > 0) {
+      // A forward reference was mutated into a definition. Rewrite it.
+      // FIXME: This happens during template instantiation, should we
+      // have created a new definition decl instead ?
+      RewriteDecl(RD);
+    }
+
+    for (CXXRecordDecl::redecl_iterator
+           I = RD->redecls_begin(), E = RD->redecls_end(); I != E; ++I) {
+      CXXRecordDecl *Redecl = cast<CXXRecordDecl>(*I);
+      if (Redecl == RD)
+        continue;
+
+      // We are interested when a PCH decl is modified.
+      if (Redecl->getPCHLevel() > 0) {
+        UpdateRecord &Record = DeclUpdates[Redecl];
+        Record.push_back(UPD_CXX_SET_DEFINITIONDATA);
+        assert(Redecl->DefinitionData);
+        assert(Redecl->DefinitionData->Definition == D);
+        AddDeclRef(D, Record); // the DefinitionDecl
+      }
+    }
+  }
+}
+void ASTWriter::AddedVisibleDecl(const DeclContext *DC, const Decl *D) {
+  // TU and namespaces are handled elsewhere.
+  if (isa<TranslationUnitDecl>(DC) || isa<NamespaceDecl>(DC))
+    return;
+
+  if (!(D->getPCHLevel() == 0 && cast<Decl>(DC)->getPCHLevel() > 0))
+    return; // Not a source decl added to a DeclContext from PCH.
+
+  AddUpdatedDeclContext(DC);
+}
+
+void ASTWriter::AddedCXXImplicitMember(const CXXRecordDecl *RD, const Decl *D) {
+  assert(D->isImplicit());
+  if (!(D->getPCHLevel() == 0 && RD->getPCHLevel() > 0))
+    return; // Not a source member added to a class from PCH.
+  if (!isa<CXXMethodDecl>(D))
+    return; // We are interested in lazily declared implicit methods.
+
+  // A decl coming from PCH was modified.
+  assert(RD->isDefinition());
+  UpdateRecord &Record = DeclUpdates[RD];
+  Record.push_back(UPD_CXX_ADDED_IMPLICIT_MEMBER);
+  AddDeclRef(D, Record);
+}
+
+void ASTWriter::AddedCXXTemplateSpecialization(const ClassTemplateDecl *TD,
+                                     const ClassTemplateSpecializationDecl *D) {
+  // The specializations set is kept in the canonical template.
+  TD = TD->getCanonicalDecl();
+  if (!(D->getPCHLevel() == 0 && TD->getPCHLevel() > 0))
+    return; // Not a source specialization added to a template from PCH.
+
+  UpdateRecord &Record = DeclUpdates[TD];
+  Record.push_back(UPD_CXX_ADDED_TEMPLATE_SPECIALIZATION);
+  AddDeclRef(D, Record);
+}
+
+ASTSerializationListener::~ASTSerializationListener() { }
diff --git a/lib/Serialization/ASTWriterDecl.cpp b/lib/Serialization/ASTWriterDecl.cpp
index ce39a1076b8e..ce07e1389b82 100644
--- a/lib/Serialization/ASTWriterDecl.cpp
+++ b/lib/Serialization/ASTWriterDecl.cpp
@@ -16,6 +16,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/DeclContextInternals.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -30,22 +31,23 @@ namespace clang {
 
     ASTWriter &Writer;
     ASTContext &Context;
-    ASTWriter::RecordData &Record;
+    typedef ASTWriter::RecordData RecordData;
+    RecordData &Record;
 
   public:
     serialization::DeclCode Code;
     unsigned AbbrevToUse;
 
-    ASTDeclWriter(ASTWriter &Writer, ASTContext &Context,
-                  ASTWriter::RecordData &Record)
+    ASTDeclWriter(ASTWriter &Writer, ASTContext &Context, RecordData &Record)
       : Writer(Writer), Context(Context), Record(Record) {
     }
-    
+
     void Visit(Decl *D);
 
     void VisitDecl(Decl *D);
     void VisitTranslationUnitDecl(TranslationUnitDecl *D);
     void VisitNamedDecl(NamedDecl *D);
+    void VisitLabelDecl(LabelDecl *LD);
     void VisitNamespaceDecl(NamespaceDecl *D);
     void VisitUsingDirectiveDecl(UsingDirectiveDecl *D);
     void VisitNamespaceAliasDecl(NamespaceAliasDecl *D);
@@ -71,6 +73,7 @@ namespace clang {
     void VisitCXXDestructorDecl(CXXDestructorDecl *D);
     void VisitCXXConversionDecl(CXXConversionDecl *D);
     void VisitFieldDecl(FieldDecl *D);
+    void VisitIndirectFieldDecl(IndirectFieldDecl *D);
     void VisitVarDecl(VarDecl *D);
     void VisitImplicitParamDecl(ImplicitParamDecl *D);
     void VisitParmVarDecl(ParmVarDecl *D);
@@ -133,6 +136,8 @@ void ASTDeclWriter::VisitDecl(Decl *D) {
   Writer.AddSourceLocation(D->getLocation(), Record);
   Record.push_back(D->isInvalidDecl());
   Record.push_back(D->hasAttrs());
+  if (D->hasAttrs())
+    Writer.WriteAttributes(D->getAttrs(), Record);
   Record.push_back(D->isImplicit());
   Record.push_back(D->isUsed(false));
   Record.push_back(D->getAccess());
@@ -163,23 +168,31 @@ void ASTDeclWriter::VisitTypedefDecl(TypedefDecl *D) {
 
 void ASTDeclWriter::VisitTagDecl(TagDecl *D) {
   VisitTypeDecl(D);
-  Record.push_back(D->getIdentifierNamespace());
   VisitRedeclarable(D);
+  Record.push_back(D->getIdentifierNamespace());
   Record.push_back((unsigned)D->getTagKind()); // FIXME: stable encoding
   Record.push_back(D->isDefinition());
   Record.push_back(D->isEmbeddedInDeclarator());
   Writer.AddSourceLocation(D->getRBraceLoc(), Record);
   Writer.AddSourceLocation(D->getTagKeywordLoc(), Record);
-  // FIXME: maybe write optional qualifier and its range.
-  Writer.AddDeclRef(D->getTypedefForAnonDecl(), Record);
+  Record.push_back(D->hasExtInfo());
+  if (D->hasExtInfo())
+    Writer.AddQualifierInfo(*D->getExtInfo(), Record);
+  else
+    Writer.AddDeclRef(D->getTypedefForAnonDecl(), Record);
 }
 
 void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) {
   VisitTagDecl(D);
-  Writer.AddTypeRef(D->getIntegerType(), Record);
+  Writer.AddTypeSourceInfo(D->getIntegerTypeSourceInfo(), Record);
+  if (!D->getIntegerTypeSourceInfo())
+    Writer.AddTypeRef(D->getIntegerType(), Record);
   Writer.AddTypeRef(D->getPromotionType(), Record);
   Record.push_back(D->getNumPositiveBits());
   Record.push_back(D->getNumNegativeBits());
+  Record.push_back(D->isScoped());
+  Record.push_back(D->isScopedUsingClassTag());
+  Record.push_back(D->isFixed());
   Writer.AddDeclRef(D->getInstantiatedFromMemberEnum(), Record);
   Code = serialization::DECL_ENUM;
 }
@@ -208,14 +221,17 @@ void ASTDeclWriter::VisitEnumConstantDecl(EnumConstantDecl *D) {
 
 void ASTDeclWriter::VisitDeclaratorDecl(DeclaratorDecl *D) {
   VisitValueDecl(D);
+  Record.push_back(D->hasExtInfo());
+  if (D->hasExtInfo())
+    Writer.AddQualifierInfo(*D->getExtInfo(), Record);
   Writer.AddTypeSourceInfo(D->getTypeSourceInfo(), Record);
-  // FIXME: write optional qualifier and its range.
 }
 
 void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   VisitDeclaratorDecl(D);
-  // FIXME: write DeclarationNameLoc.
+  VisitRedeclarable(D);
 
+  Writer.AddDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record);
   Record.push_back(D->getIdentifierNamespace());
   Record.push_back(D->getTemplatedKind());
   switch (D->getTemplatedKind()) {
@@ -236,8 +252,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   case FunctionDecl::TK_FunctionTemplateSpecialization: {
     FunctionTemplateSpecializationInfo *
       FTSInfo = D->getTemplateSpecializationInfo();
-    // We want it canonical to guarantee that it has a Common*.
-    Writer.AddDeclRef(FTSInfo->getTemplate()->getCanonicalDecl(), Record);
+    Writer.AddDeclRef(FTSInfo->getTemplate(), Record);
     Record.push_back(FTSInfo->getTemplateSpecializationKind());
     
     // Template arguments.
@@ -257,6 +272,12 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
     }
     
     Writer.AddSourceLocation(FTSInfo->getPointOfInstantiation(), Record);
+
+    if (D->isCanonicalDecl()) {
+      // Write the template that contains the specializations set. We will
+      // add a FunctionTemplateSpecializationInfo to it when reading.
+      Writer.AddDeclRef(FTSInfo->getTemplate()->getCanonicalDecl(), Record);
+    }
     break;
   }
   case FunctionDecl::TK_DependentFunctionTemplateSpecialization: {
@@ -281,9 +302,9 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   // FunctionDecl's body is handled last at ASTWriterDecl::Visit,
   // after everything else is written.
 
-  VisitRedeclarable(D);
   Record.push_back(D->getStorageClass()); // FIXME: stable encoding
   Record.push_back(D->getStorageClassAsWritten());
+  Record.push_back(D->IsInline);
   Record.push_back(D->isInlineSpecified());
   Record.push_back(D->isVirtualAsWritten());
   Record.push_back(D->isPure());
@@ -291,7 +312,6 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   Record.push_back(D->hasWrittenPrototype());
   Record.push_back(D->isDeleted());
   Record.push_back(D->isTrivial());
-  Record.push_back(D->isCopyAssignment());
   Record.push_back(D->hasImplicitReturnZero());
   Writer.AddSourceLocation(D->getLocEnd(), Record);
 
@@ -484,8 +504,8 @@ void ASTDeclWriter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
 void ASTDeclWriter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   VisitObjCImplDecl(D);
   Writer.AddDeclRef(D->getSuperClass(), Record);
-  Writer.AddCXXBaseOrMemberInitializers(D->IvarInitializers,
-                                        D->NumIvarInitializers, Record);
+  Writer.AddCXXCtorInitializers(D->IvarInitializers, D->NumIvarInitializers,
+                                Record);
   Record.push_back(D->hasSynthBitfield());
   Code = serialization::DECL_OBJC_IMPLEMENTATION;
 }
@@ -495,6 +515,7 @@ void ASTDeclWriter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
   Writer.AddSourceLocation(D->getLocStart(), Record);
   Writer.AddDeclRef(D->getPropertyDecl(), Record);
   Writer.AddDeclRef(D->getPropertyIvarDecl(), Record);
+  Writer.AddSourceLocation(D->getPropertyIvarDeclLoc(), Record);
   Writer.AddStmt(D->getGetterCXXConstructor());
   Writer.AddStmt(D->getSetterCXXAssignment());
   Code = serialization::DECL_OBJC_PROPERTY_IMPL;
@@ -511,15 +532,26 @@ void ASTDeclWriter::VisitFieldDecl(FieldDecl *D) {
   Code = serialization::DECL_FIELD;
 }
 
+void ASTDeclWriter::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
+  VisitValueDecl(D);
+  Record.push_back(D->getChainingSize());
+
+  for (IndirectFieldDecl::chain_iterator
+       P = D->chain_begin(),
+       PEnd = D->chain_end(); P != PEnd; ++P)
+    Writer.AddDeclRef(*P, Record);
+  Code = serialization::DECL_INDIRECTFIELD;
+}
+
 void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
   VisitDeclaratorDecl(D);
+  VisitRedeclarable(D);
   Record.push_back(D->getStorageClass()); // FIXME: stable encoding
   Record.push_back(D->getStorageClassAsWritten());
   Record.push_back(D->isThreadSpecified());
   Record.push_back(D->hasCXXDirectInitializer());
   Record.push_back(D->isExceptionVariable());
   Record.push_back(D->isNRVOVariable());
-  VisitRedeclarable(D);
   Record.push_back(D->getInit() ? 1 : 0);
   if (D->getInit())
     Writer.AddStmt(D->getInit());
@@ -592,6 +624,22 @@ void ASTDeclWriter::VisitBlockDecl(BlockDecl *D) {
   for (FunctionDecl::param_iterator P = D->param_begin(), PEnd = D->param_end();
        P != PEnd; ++P)
     Writer.AddDeclRef(*P, Record);
+  Record.push_back(D->capturesCXXThis());
+  Record.push_back(D->getNumCaptures());
+  for (BlockDecl::capture_iterator
+         i = D->capture_begin(), e = D->capture_end(); i != e; ++i) {
+    const BlockDecl::Capture &capture = *i;
+    Writer.AddDeclRef(capture.getVariable(), Record);
+
+    unsigned flags = 0;
+    if (capture.isByRef()) flags |= 1;
+    if (capture.isNested()) flags |= 2;
+    if (capture.hasCopyExpr()) flags |= 4;
+    Record.push_back(flags);
+
+    if (capture.hasCopyExpr()) Writer.AddStmt(capture.getCopyExpr());
+  }
+
   Code = serialization::DECL_BLOCK;
 }
 
@@ -604,8 +652,15 @@ void ASTDeclWriter::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
   Code = serialization::DECL_LINKAGE_SPEC;
 }
 
+void ASTDeclWriter::VisitLabelDecl(LabelDecl *D) {
+  VisitNamedDecl(D);
+  Code = serialization::DECL_LABEL;
+}
+
+
 void ASTDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
   VisitNamedDecl(D);
+  Record.push_back(D->isInline());
   Writer.AddSourceLocation(D->getLBracLoc(), Record);
   Writer.AddSourceLocation(D->getRBracLoc(), Record);
   Writer.AddDeclRef(D->getNextNamespace(), Record);
@@ -620,7 +675,22 @@ void ASTDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
 
   if (Writer.hasChain() && !D->isOriginalNamespace() &&
       D->getOriginalNamespace()->getPCHLevel() > 0) {
-    Writer.AddUpdatedNamespace(D->getOriginalNamespace());
+    NamespaceDecl *NS = D->getOriginalNamespace();
+    Writer.AddUpdatedDeclContext(NS);
+
+    // Make sure all visible decls are written. They will be recorded later.
+    NS->lookup(DeclarationName());
+    StoredDeclsMap *Map = static_cast<StoredDeclsMap*>(NS->getLookupPtr());
+    if (Map) {
+      for (StoredDeclsMap::iterator D = Map->begin(), DEnd = Map->end();
+           D != DEnd; ++D) {
+        DeclContext::lookup_result Result = D->second.getLookupResult();
+        while (Result.first != Result.second) {
+          Writer.GetDeclRef(*Result.first);
+          ++Result.first;
+        }
+      }
+    }
   }
 }
 
@@ -639,10 +709,8 @@ void ASTDeclWriter::VisitUsingDecl(UsingDecl *D) {
   Writer.AddSourceRange(D->getNestedNameRange(), Record);
   Writer.AddSourceLocation(D->getUsingLocation(), Record);
   Writer.AddNestedNameSpecifier(D->getTargetNestedNameDecl(), Record);
-  Record.push_back(D->getNumShadowDecls());
-  for (UsingDecl::shadow_iterator P = D->shadow_begin(),
-       PEnd = D->shadow_end(); P != PEnd; ++P)
-    Writer.AddDeclRef(*P, Record);
+  Writer.AddDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record);
+  Writer.AddDeclRef(D->FirstUsingShadow, Record);
   Record.push_back(D->isTypeName());
   Writer.AddDeclRef(Context.getInstantiatedFromUsingDecl(D), Record);
   Code = serialization::DECL_USING;
@@ -651,7 +719,7 @@ void ASTDeclWriter::VisitUsingDecl(UsingDecl *D) {
 void ASTDeclWriter::VisitUsingShadowDecl(UsingShadowDecl *D) {
   VisitNamedDecl(D);
   Writer.AddDeclRef(D->getTargetDecl(), Record);
-  Writer.AddDeclRef(D->getUsingDecl(), Record);
+  Writer.AddDeclRef(D->UsingOrNextShadow, Record);
   Writer.AddDeclRef(Context.getInstantiatedFromUsingShadowDecl(D), Record);
   Code = serialization::DECL_USING_SHADOW;
 }
@@ -672,6 +740,7 @@ void ASTDeclWriter::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
   Writer.AddSourceRange(D->getTargetNestedNameRange(), Record);
   Writer.AddSourceLocation(D->getUsingLoc(), Record);
   Writer.AddNestedNameSpecifier(D->getTargetNestedNameSpecifier(), Record);
+  Writer.AddDeclarationNameLoc(D->DNLoc, D->getDeclName(), Record);
   Code = serialization::DECL_UNRESOLVED_USING_VALUE;
 }
 
@@ -686,64 +755,14 @@ void ASTDeclWriter::VisitUnresolvedUsingTypenameDecl(
 }
 
 void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
-  // See comments at ASTDeclReader::VisitCXXRecordDecl about why this happens
-  // before VisitRecordDecl.
-  enum { Data_NoDefData, Data_Owner, Data_NotOwner };
-  bool OwnsDefinitionData = false;
-  if (D->DefinitionData) {
-    assert(D->DefinitionData->Definition &&
-           "DefinitionData don't point to a definition decl!");
-    OwnsDefinitionData = D->DefinitionData->Definition == D;
-    if (OwnsDefinitionData) {
-      Record.push_back(Data_Owner);
-    } else {
-      Record.push_back(Data_NotOwner);
-      Writer.AddDeclRef(D->DefinitionData->Definition, Record);
-    }
-  } else
-    Record.push_back(Data_NoDefData);
-
   VisitRecordDecl(D);
 
-  if (OwnsDefinitionData) {
-    assert(D->DefinitionData);
-    struct CXXRecordDecl::DefinitionData &Data = *D->DefinitionData;
-
-    Record.push_back(Data.UserDeclaredConstructor);
-    Record.push_back(Data.UserDeclaredCopyConstructor);
-    Record.push_back(Data.UserDeclaredCopyAssignment);
-    Record.push_back(Data.UserDeclaredDestructor);
-    Record.push_back(Data.Aggregate);
-    Record.push_back(Data.PlainOldData);
-    Record.push_back(Data.Empty);
-    Record.push_back(Data.Polymorphic);
-    Record.push_back(Data.Abstract);
-    Record.push_back(Data.HasTrivialConstructor);
-    Record.push_back(Data.HasTrivialCopyConstructor);
-    Record.push_back(Data.HasTrivialCopyAssignment);
-    Record.push_back(Data.HasTrivialDestructor);
-    Record.push_back(Data.ComputedVisibleConversions);
-    Record.push_back(Data.DeclaredDefaultConstructor);
-    Record.push_back(Data.DeclaredCopyConstructor);
-    Record.push_back(Data.DeclaredCopyAssignment);
-    Record.push_back(Data.DeclaredDestructor);
-
-    Record.push_back(D->getNumBases());
-    for (CXXRecordDecl::base_class_iterator I = D->bases_begin(),
-           E = D->bases_end(); I != E; ++I)
-      Writer.AddCXXBaseSpecifier(*I, Record);
-
-    // FIXME: Make VBases lazily computed when needed to avoid storing them.
-    Record.push_back(D->getNumVBases());
-    for (CXXRecordDecl::base_class_iterator I = D->vbases_begin(),
-           E = D->vbases_end(); I != E; ++I)
-      Writer.AddCXXBaseSpecifier(*I, Record);
-
-    Writer.AddUnresolvedSet(Data.Conversions, Record);
-    Writer.AddUnresolvedSet(Data.VisibleConversions, Record);
-    // Data.Definition is written at the top. 
-    Writer.AddDeclRef(Data.FirstFriend, Record);
-  }
+  CXXRecordDecl *DefinitionDecl = 0;
+  if (D->DefinitionData)
+    DefinitionDecl = D->DefinitionData->Definition;
+  Writer.AddDeclRef(DefinitionDecl, Record);
+  if (D == DefinitionDecl)
+    Writer.AddCXXDefinitionData(D, Record);
 
   enum {
     CXXRecNotTemplate = 0, CXXRecTemplate, CXXRecMemberSpecialization
@@ -761,6 +780,11 @@ void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
     Record.push_back(CXXRecNotTemplate);
   }
 
+  // Store the key function to avoid deserializing every method so we can
+  // compute it.
+  if (D->IsDefinition)
+    Writer.AddDeclRef(Context.getKeyFunction(D), Record);
+
   Code = serialization::DECL_CXX_RECORD;
 }
 
@@ -779,8 +803,8 @@ void ASTDeclWriter::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
 
   Record.push_back(D->IsExplicitSpecified);
   Record.push_back(D->ImplicitlyDefined);
-  Writer.AddCXXBaseOrMemberInitializers(D->BaseOrMemberInitializers,
-                                        D->NumBaseOrMemberInitializers, Record);
+  Writer.AddCXXCtorInitializers(D->CtorInitializers, D->NumCtorInitializers,
+                                Record);
 
   Code = serialization::DECL_CXX_CONSTRUCTOR;
 }
@@ -813,7 +837,8 @@ void ASTDeclWriter::VisitFriendDecl(FriendDecl *D) {
     Writer.AddTypeSourceInfo(D->Friend.get<TypeSourceInfo*>(), Record);
   else
     Writer.AddDeclRef(D->Friend.get<NamedDecl*>(), Record);
-  Writer.AddDeclRef(D->NextFriend, Record);
+  Writer.AddDeclRef(D->getNextFriend(), Record);
+  Record.push_back(D->UnsupportedFriend);
   Writer.AddSourceLocation(D->FriendLoc, Record);
   Code = serialization::DECL_FRIEND;
 }
@@ -840,10 +865,13 @@ void ASTDeclWriter::VisitTemplateDecl(TemplateDecl *D) {
 }
 
 void ASTDeclWriter::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
-  VisitTemplateDecl(D);
+  // Emit data to initialize CommonOrPrev before VisitTemplateDecl so that
+  // getCommonPtr() can be used while this is still initializing.
 
-  Record.push_back(D->getIdentifierNamespace());
   Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
+  if (D->getPreviousDeclaration())
+    Writer.AddDeclRef(D->getFirstDeclaration(), Record);
+
   if (D->getPreviousDeclaration() == 0) {
     // This TemplateDecl owns the CommonPtr; write it.
     assert(D->isCanonicalDecl());
@@ -866,6 +894,9 @@ void ASTDeclWriter::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
       Writer.FirstLatestDecls[First] = D;
     }
   }
+
+  VisitTemplateDecl(D);
+  Record.push_back(D->getIdentifierNamespace());
 }
 
 void ASTDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
@@ -911,10 +942,6 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
     InstFromD = cast<ClassTemplatePartialSpecializationDecl>(InstFromD)->
                     getSpecializedTemplate();
   }
-  // Is this a specialization of an already-serialized template?
-  if (InstFromD->getCanonicalDecl()->getPCHLevel() != 0)
-    Writer.AddAdditionalTemplateSpecialization(Writer.getDeclID(InstFromD),
-                                               Writer.getDeclID(D));
 
   // Explicit info.
   Writer.AddTypeSourceInfo(D->getTypeAsWritten(), Record);
@@ -987,17 +1014,33 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
 }
 
 void ASTDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
-  VisitVarDecl(D);
+  // For an expanded parameter pack, record the number of expansion types here
+  // so that it's easier for 
+  if (D->isExpandedParameterPack())
+    Record.push_back(D->getNumExpansionTypes());
+  
+  VisitDeclaratorDecl(D);
   // TemplateParmPosition.
   Record.push_back(D->getDepth());
   Record.push_back(D->getPosition());
-  // Rest of NonTypeTemplateParmDecl.
-  Record.push_back(D->getDefaultArgument() != 0);
-  if (D->getDefaultArgument()) {
-    Writer.AddStmt(D->getDefaultArgument());
-    Record.push_back(D->defaultArgumentWasInherited());
+  
+  if (D->isExpandedParameterPack()) {
+    for (unsigned I = 0, N = D->getNumExpansionTypes(); I != N; ++I) {
+      Writer.AddTypeRef(D->getExpansionType(I), Record);
+      Writer.AddTypeSourceInfo(D->getExpansionTypeSourceInfo(I), Record);
+    }
+      
+    Code = serialization::DECL_EXPANDED_NON_TYPE_TEMPLATE_PARM_PACK;
+  } else {
+    // Rest of NonTypeTemplateParmDecl.
+    Record.push_back(D->isParameterPack());
+    Record.push_back(D->getDefaultArgument() != 0);
+    if (D->getDefaultArgument()) {
+      Writer.AddStmt(D->getDefaultArgument());
+      Record.push_back(D->defaultArgumentWasInherited());
+    }
+    Code = serialization::DECL_NON_TYPE_TEMPLATE_PARM;
   }
-  Code = serialization::DECL_NON_TYPE_TEMPLATE_PARM;
 }
 
 void ASTDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
@@ -1008,6 +1051,7 @@ void ASTDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   // Rest of TemplateTemplateParmDecl.
   Writer.AddTemplateArgumentLoc(D->getDefaultArgument(), Record);
   Record.push_back(D->defaultArgumentWasInherited());
+  Record.push_back(D->isParameterPack());
   Code = serialization::DECL_TEMPLATE_TEMPLATE_PARM;
 }
 
@@ -1041,9 +1085,14 @@ void ASTDeclWriter::VisitRedeclarable(Redeclarable<T> *D) {
   if (D->RedeclLink.getNext() == D) {
     Record.push_back(NoRedeclaration);
   } else {
-    Record.push_back(D->RedeclLink.NextIsPrevious() ? PointsToPrevious
-                                                    : PointsToLatest);
-    Writer.AddDeclRef(D->RedeclLink.getPointer(), Record);
+    if (D->RedeclLink.NextIsPrevious()) {
+      Record.push_back(PointsToPrevious);
+      Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
+      Writer.AddDeclRef(D->getFirstDeclaration(), Record);
+    } else {
+      Record.push_back(PointsToLatest);
+      Writer.AddDeclRef(D->RedeclLink.getPointer(), Record);
+    }
   }
 
   T *First = D->getFirstDeclaration();
@@ -1086,15 +1135,16 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   // ValueDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type
   // DeclaratorDecl
+  Abv->Add(BitCodeAbbrevOp(0));                       // hasExtInfo
   Abv->Add(BitCodeAbbrevOp(serialization::PREDEF_TYPE_NULL_ID)); // InfoType
   // VarDecl
+  Abv->Add(BitCodeAbbrevOp(0));                       // No redeclaration
   Abv->Add(BitCodeAbbrevOp(0));                       // StorageClass
   Abv->Add(BitCodeAbbrevOp(0));                       // StorageClassAsWritten
   Abv->Add(BitCodeAbbrevOp(0));                       // isThreadSpecified
   Abv->Add(BitCodeAbbrevOp(0));                       // hasCXXDirectInitializer
   Abv->Add(BitCodeAbbrevOp(0));                       // isExceptionVariable
   Abv->Add(BitCodeAbbrevOp(0));                       // isNRVOVariable
-  Abv->Add(BitCodeAbbrevOp(0));                       // PrevDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // HasInit
   Abv->Add(BitCodeAbbrevOp(0));                   // HasMemberSpecializationInfo
   // ParmVarDecl
@@ -1137,6 +1187,9 @@ static bool isRequiredDecl(const Decl *D, ASTContext &Context) {
 }
 
 void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) {
+  // Switch case IDs are per Decl.
+  ClearSwitchCaseIDs();
+
   RecordData Record;
   ASTDeclWriter W(*this, Context, Record);
 
@@ -1186,13 +1239,12 @@ void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) {
                             D->getDeclKindName() + "'");
   Stream.EmitRecord(W.Code, Record, W.AbbrevToUse);
 
-  // If the declaration had any attributes, write them now.
-  if (D->hasAttrs())
-    WriteAttributeRecord(D->getAttrs());
-
   // Flush any expressions that were written as part of this declaration.
   FlushStmts();
-
+  
+  // Flush C++ base specifiers, if there are any.
+  FlushCXXBaseSpecifiers();
+  
   // Note "external" declarations so that we can add them to a record in the
   // AST file later.
   //
diff --git a/lib/Serialization/ASTWriterStmt.cpp b/lib/Serialization/ASTWriterStmt.cpp
index 7f2da6c225a3..8a5ffe96db17 100644
--- a/lib/Serialization/ASTWriterStmt.cpp
+++ b/lib/Serialization/ASTWriterStmt.cpp
@@ -14,6 +14,7 @@
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/StmtVisitor.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 using namespace clang;
@@ -75,6 +76,7 @@ namespace clang {
     void VisitBinaryOperator(BinaryOperator *E);
     void VisitCompoundAssignOperator(CompoundAssignOperator *E);
     void VisitConditionalOperator(ConditionalOperator *E);
+    void VisitBinaryConditionalOperator(BinaryConditionalOperator *E);
     void VisitImplicitCastExpr(ImplicitCastExpr *E);
     void VisitExplicitCastExpr(ExplicitCastExpr *E);
     void VisitCStyleCastExpr(CStyleCastExpr *E);
@@ -86,7 +88,6 @@ namespace clang {
     void VisitVAArgExpr(VAArgExpr *E);
     void VisitAddrLabelExpr(AddrLabelExpr *E);
     void VisitStmtExpr(StmtExpr *E);
-    void VisitTypesCompatibleExpr(TypesCompatibleExpr *E);
     void VisitChooseExpr(ChooseExpr *E);
     void VisitGNUNullExpr(GNUNullExpr *E);
     void VisitShuffleVectorExpr(ShuffleVectorExpr *E);
@@ -100,10 +101,7 @@ namespace clang {
     void VisitObjCProtocolExpr(ObjCProtocolExpr *E);
     void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E);
     void VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E);
-    void VisitObjCImplicitSetterGetterRefExpr(
-                        ObjCImplicitSetterGetterRefExpr *E);
     void VisitObjCMessageExpr(ObjCMessageExpr *E);
-    void VisitObjCSuperExpr(ObjCSuperExpr *E);
     void VisitObjCIsaExpr(ObjCIsaExpr *E);
 
     // Objective-C Statements
@@ -131,6 +129,7 @@ namespace clang {
     void VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E);
     void VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E);
     void VisitCXXTypeidExpr(CXXTypeidExpr *E);
+    void VisitCXXUuidofExpr(CXXUuidofExpr *E);
     void VisitCXXThisExpr(CXXThisExpr *E);
     void VisitCXXThrowExpr(CXXThrowExpr *E);
     void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E);
@@ -141,7 +140,7 @@ namespace clang {
     void VisitCXXDeleteExpr(CXXDeleteExpr *E);
     void VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E);
 
-    void VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E);
+    void VisitExprWithCleanups(ExprWithCleanups *E);
     void VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E);
     void VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E);
     void VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E);
@@ -151,6 +150,16 @@ namespace clang {
     void VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E);
 
     void VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E);
+    void VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *E);
+    void VisitCXXNoexceptExpr(CXXNoexceptExpr *E);
+    void VisitPackExpansionExpr(PackExpansionExpr *E);
+    void VisitSizeOfPackExpr(SizeOfPackExpr *E);
+    void VisitSubstNonTypeTemplateParmPackExpr(
+                                           SubstNonTypeTemplateParmPackExpr *E);
+    void VisitOpaqueValueExpr(OpaqueValueExpr *E);
+
+    // CUDA Expressions
+    void VisitCUDAKernelCallExpr(CUDAKernelCallExpr *E);
   };
 }
 
@@ -168,6 +177,7 @@ void ASTStmtWriter::VisitStmt(Stmt *S) {
 void ASTStmtWriter::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getSemiLoc(), Record);
+  Record.push_back(S->LeadingEmptyMacro);
   Code = serialization::STMT_NULL;
 }
 
@@ -208,10 +218,9 @@ void ASTStmtWriter::VisitDefaultStmt(DefaultStmt *S) {
 
 void ASTStmtWriter::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
-  Writer.AddIdentifierRef(S->getID(), Record);
+  Writer.AddDeclRef(S->getDecl(), Record);
   Writer.AddStmt(S->getSubStmt());
   Writer.AddSourceLocation(S->getIdentLoc(), Record);
-  Record.push_back(Writer.GetLabelID(S));
   Code = serialization::STMT_LABEL;
 }
 
@@ -232,6 +241,7 @@ void ASTStmtWriter::VisitSwitchStmt(SwitchStmt *S) {
   Writer.AddStmt(S->getCond());
   Writer.AddStmt(S->getBody());
   Writer.AddSourceLocation(S->getSwitchLoc(), Record);
+  Record.push_back(S->isAllEnumCasesCovered());
   for (SwitchCase *SC = S->getSwitchCaseList(); SC;
        SC = SC->getNextSwitchCase())
     Record.push_back(Writer.RecordSwitchCaseID(SC));
@@ -272,7 +282,7 @@ void ASTStmtWriter::VisitForStmt(ForStmt *S) {
 
 void ASTStmtWriter::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
-  Record.push_back(Writer.GetLabelID(S->getLabel()));
+  Writer.AddDeclRef(S->getLabel(), Record);
   Writer.AddSourceLocation(S->getGotoLoc(), Record);
   Writer.AddSourceLocation(S->getLabelLoc(), Record);
   Code = serialization::STMT_GOTO;
@@ -354,6 +364,9 @@ void ASTStmtWriter::VisitExpr(Expr *E) {
   Writer.AddTypeRef(E->getType(), Record);
   Record.push_back(E->isTypeDependent());
   Record.push_back(E->isValueDependent());
+  Record.push_back(E->containsUnexpandedParameterPack());
+  Record.push_back(E->getValueKind());
+  Record.push_back(E->getObjectKind());
 }
 
 void ASTStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) {
@@ -367,22 +380,22 @@ void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) {
   VisitExpr(E);
 
   Record.push_back(E->hasQualifier());
-  unsigned NumTemplateArgs = E->getNumTemplateArgs();
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
-         "Template args list with no args ?");
-  Record.push_back(NumTemplateArgs);
+  Record.push_back(E->hasExplicitTemplateArgs());
 
   if (E->hasQualifier()) {
     Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
     Writer.AddSourceRange(E->getQualifierRange(), Record);
   }
 
-  if (NumTemplateArgs)
+  if (E->hasExplicitTemplateArgs()) {
+    unsigned NumTemplateArgs = E->getNumTemplateArgs();
+    Record.push_back(NumTemplateArgs);
     AddExplicitTemplateArgumentList(E->getExplicitTemplateArgs());
+  }
 
   Writer.AddDeclRef(E->getDecl(), Record);
-  // FIXME: write DeclarationNameLoc.
   Writer.AddSourceLocation(E->getLocation(), Record);
+  Writer.AddDeclarationNameLoc(E->DNLoc, E->getDecl()->getDeclName(), Record);
   Code = serialization::EXPR_DECL_REF;
 }
 
@@ -533,11 +546,10 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) {
     Writer.AddSourceRange(E->getQualifierRange(), Record);
   }
 
-  unsigned NumTemplateArgs = E->getNumTemplateArgs();
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
-         "Template args list with no args ?");
-  Record.push_back(NumTemplateArgs);
-  if (NumTemplateArgs) {
+  Record.push_back(E->hasExplicitTemplateArgs());
+  if (E->hasExplicitTemplateArgs()) {
+    unsigned NumTemplateArgs = E->getNumTemplateArgs();
+    Record.push_back(NumTemplateArgs);
     Writer.AddSourceLocation(E->getLAngleLoc(), Record);
     Writer.AddSourceLocation(E->getRAngleLoc(), Record);
     for (unsigned i=0; i != NumTemplateArgs; ++i)
@@ -549,11 +561,14 @@ void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) {
   Record.push_back(FoundDecl.getAccess());
 
   Writer.AddTypeRef(E->getType(), Record);
+  Record.push_back(E->getValueKind());
+  Record.push_back(E->getObjectKind());
   Writer.AddStmt(E->getBase());
   Writer.AddDeclRef(E->getMemberDecl(), Record);
-  // FIXME: write DeclarationNameLoc.
   Writer.AddSourceLocation(E->getMemberLoc(), Record);
   Record.push_back(E->isArrow());
+  Writer.AddDeclarationNameLoc(E->MemberDNLoc,
+                               E->getMemberDecl()->getDeclName(), Record);
   Code = serialization::EXPR_MEMBER;
 }
 
@@ -597,15 +612,26 @@ void ASTStmtWriter::VisitConditionalOperator(ConditionalOperator *E) {
   Writer.AddStmt(E->getCond());
   Writer.AddStmt(E->getLHS());
   Writer.AddStmt(E->getRHS());
-  Writer.AddStmt(E->getSAVE());
   Writer.AddSourceLocation(E->getQuestionLoc(), Record);
   Writer.AddSourceLocation(E->getColonLoc(), Record);
   Code = serialization::EXPR_CONDITIONAL_OPERATOR;
 }
 
+void
+ASTStmtWriter::VisitBinaryConditionalOperator(BinaryConditionalOperator *E) {
+  VisitExpr(E);
+  Writer.AddStmt(E->getOpaqueValue());
+  Writer.AddStmt(E->getCommon());
+  Writer.AddStmt(E->getCond());
+  Writer.AddStmt(E->getTrueExpr());
+  Writer.AddStmt(E->getFalseExpr());
+  Writer.AddSourceLocation(E->getQuestionLoc(), Record);
+  Writer.AddSourceLocation(E->getColonLoc(), Record);
+  Code = serialization::EXPR_BINARY_CONDITIONAL_OPERATOR;
+}
+
 void ASTStmtWriter::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   VisitCastExpr(E);
-  Record.push_back(E->getValueKind());
   Code = serialization::EXPR_IMPLICIT_CAST;
 }
 
@@ -706,7 +732,7 @@ void ASTStmtWriter::VisitAddrLabelExpr(AddrLabelExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getAmpAmpLoc(), Record);
   Writer.AddSourceLocation(E->getLabelLoc(), Record);
-  Record.push_back(Writer.GetLabelID(E->getLabel()));
+  Writer.AddDeclRef(E->getLabel(), Record);
   Code = serialization::EXPR_ADDR_LABEL;
 }
 
@@ -718,15 +744,6 @@ void ASTStmtWriter::VisitStmtExpr(StmtExpr *E) {
   Code = serialization::EXPR_STMT;
 }
 
-void ASTStmtWriter::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
-  VisitExpr(E);
-  Writer.AddTypeSourceInfo(E->getArgTInfo1(), Record);
-  Writer.AddTypeSourceInfo(E->getArgTInfo2(), Record);
-  Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
-  Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = serialization::EXPR_TYPES_COMPATIBLE;
-}
-
 void ASTStmtWriter::VisitChooseExpr(ChooseExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getCond());
@@ -756,7 +773,6 @@ void ASTStmtWriter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
 void ASTStmtWriter::VisitBlockExpr(BlockExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getBlockDecl(), Record);
-  Record.push_back(E->hasBlockDeclRefExprs());
   Code = serialization::EXPR_BLOCK;
 }
 
@@ -766,7 +782,6 @@ void ASTStmtWriter::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->isByRef());
   Record.push_back(E->isConstQualAdded());
-  Writer.AddStmt(E->getCopyConstructorExpr());
   Code = serialization::EXPR_BLOCK_DECL_REF;
 }
 
@@ -817,26 +832,29 @@ void ASTStmtWriter::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
 
 void ASTStmtWriter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   VisitExpr(E);
-  Writer.AddDeclRef(E->getProperty(), Record);
+  Record.push_back(E->isImplicitProperty());
+  if (E->isImplicitProperty()) {
+    Writer.AddDeclRef(E->getImplicitPropertyGetter(), Record);
+    Writer.AddDeclRef(E->getImplicitPropertySetter(), Record);
+  } else {
+    Writer.AddDeclRef(E->getExplicitProperty(), Record);
+  }
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Writer.AddStmt(E->getBase());
+  Writer.AddSourceLocation(E->getReceiverLocation(), Record);
+  if (E->isObjectReceiver()) {
+    Record.push_back(0);
+    Writer.AddStmt(E->getBase());
+  } else if (E->isSuperReceiver()) {
+    Record.push_back(1);
+    Writer.AddTypeRef(E->getSuperReceiverType(), Record);
+  } else {
+    Record.push_back(2);
+    Writer.AddDeclRef(E->getClassReceiver(), Record);
+  }
+  
   Code = serialization::EXPR_OBJC_PROPERTY_REF_EXPR;
 }
 
-void ASTStmtWriter::VisitObjCImplicitSetterGetterRefExpr(
-                                  ObjCImplicitSetterGetterRefExpr *E) {
-  VisitExpr(E);
-  Writer.AddDeclRef(E->getGetterMethod(), Record);
-  Writer.AddDeclRef(E->getSetterMethod(), Record);
-
-  // NOTE: InterfaceDecl and Base are mutually exclusive.
-  Writer.AddDeclRef(E->getInterfaceDecl(), Record);
-  Writer.AddStmt(E->getBase());
-  Writer.AddSourceLocation(E->getLocation(), Record);
-  Writer.AddSourceLocation(E->getClassLoc(), Record);
-  Code = serialization::EXPR_OBJC_KVC_REF_EXPR;
-}
-
 void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
@@ -867,6 +885,7 @@ void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
     
   Writer.AddSourceLocation(E->getLeftLoc(), Record);
   Writer.AddSourceLocation(E->getRightLoc(), Record);
+  Writer.AddSourceLocation(E->getSelectorLoc(), Record);
 
   for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg)
@@ -874,12 +893,6 @@ void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   Code = serialization::EXPR_OBJC_MESSAGE_EXPR;
 }
 
-void ASTStmtWriter::VisitObjCSuperExpr(ObjCSuperExpr *E) {
-  VisitExpr(E);
-  Writer.AddSourceLocation(E->getLoc(), Record);
-  Code = serialization::EXPR_OBJC_SUPER_EXPR;
-}
-
 void ASTStmtWriter::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   VisitStmt(S);
   Writer.AddStmt(S->getElement());
@@ -972,19 +985,20 @@ void ASTStmtWriter::VisitCXXConstructExpr(CXXConstructExpr *E) {
   Record.push_back(E->isElidable());
   Record.push_back(E->requiresZeroInitialization());
   Record.push_back(E->getConstructionKind()); // FIXME: stable encoding
+  Writer.AddSourceRange(E->getParenRange(), Record);
   Code = serialization::EXPR_CXX_CONSTRUCT;
 }
 
 void ASTStmtWriter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
   VisitCXXConstructExpr(E);
-  Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
-  Writer.AddSourceLocation(E->getRParenLoc(), Record);
+  Writer.AddTypeSourceInfo(E->getTypeSourceInfo(), Record);
   Code = serialization::EXPR_CXX_TEMPORARY_OBJECT;
 }
 
 void ASTStmtWriter::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
   VisitExplicitCastExpr(E);
-  Writer.AddSourceLocation(E->getOperatorLoc(), Record);
+  Writer.AddSourceRange(SourceRange(E->getOperatorLoc(), E->getRParenLoc()),
+                        Record);
 }
 
 void ASTStmtWriter::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
@@ -1039,6 +1053,18 @@ void ASTStmtWriter::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
   }
 }
 
+void ASTStmtWriter::VisitCXXUuidofExpr(CXXUuidofExpr *E) {
+  VisitExpr(E);
+  Writer.AddSourceRange(E->getSourceRange(), Record);
+  if (E->isTypeOperand()) {
+    Writer.AddTypeSourceInfo(E->getTypeOperandSourceInfo(), Record);
+    Code = serialization::EXPR_CXX_UUIDOF_TYPE;
+  } else {
+    Writer.AddStmt(E->getExprOperand());
+    Code = serialization::EXPR_CXX_UUIDOF_EXPR;
+  }
+}
+
 void ASTStmtWriter::VisitCXXThisExpr(CXXThisExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLocation(), Record);
@@ -1076,7 +1102,7 @@ void ASTStmtWriter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
 
 void ASTStmtWriter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
   VisitExpr(E);
-  Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
+  Writer.AddTypeSourceInfo(E->getTypeSourceInfo(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
   Code = serialization::EXPR_CXX_SCALAR_VALUE_INIT;
 }
@@ -1085,15 +1111,19 @@ void ASTStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
   VisitExpr(E);
   Record.push_back(E->isGlobalNew());
   Record.push_back(E->hasInitializer());
+  Record.push_back(E->doesUsualArrayDeleteWantSize());
   Record.push_back(E->isArray());
   Record.push_back(E->getNumPlacementArgs());
   Record.push_back(E->getNumConstructorArgs());
   Writer.AddDeclRef(E->getOperatorNew(), Record);
   Writer.AddDeclRef(E->getOperatorDelete(), Record);
   Writer.AddDeclRef(E->getConstructor(), Record);
+  Writer.AddTypeSourceInfo(E->getAllocatedTypeSourceInfo(), Record);
   Writer.AddSourceRange(E->getTypeIdParens(), Record);
   Writer.AddSourceLocation(E->getStartLoc(), Record);
   Writer.AddSourceLocation(E->getEndLoc(), Record);
+  Writer.AddSourceLocation(E->getConstructorLParen(), Record);
+  Writer.AddSourceLocation(E->getConstructorRParen(), Record);
   for (CXXNewExpr::arg_iterator I = E->raw_arg_begin(), e = E->raw_arg_end();
        I != e; ++I)
     Writer.AddStmt(*I);
@@ -1105,6 +1135,8 @@ void ASTStmtWriter::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   VisitExpr(E);
   Record.push_back(E->isGlobalDelete());
   Record.push_back(E->isArrayForm());
+  Record.push_back(E->isArrayFormAsWritten());
+  Record.push_back(E->doesUsualArrayDeleteWantSize());
   Writer.AddDeclRef(E->getOperatorDelete(), Record);
   Writer.AddStmt(E->getArgument());
   Writer.AddSourceLocation(E->getSourceRange().getBegin(), Record);
@@ -1134,30 +1166,28 @@ void ASTStmtWriter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   Code = serialization::EXPR_CXX_PSEUDO_DESTRUCTOR;
 }
 
-void ASTStmtWriter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
+void ASTStmtWriter::VisitExprWithCleanups(ExprWithCleanups *E) {
   VisitExpr(E);
   Record.push_back(E->getNumTemporaries());
   for (unsigned i = 0, e = E->getNumTemporaries(); i != e; ++i)
     Writer.AddCXXTemporary(E->getTemporary(i), Record);
   
   Writer.AddStmt(E->getSubExpr());
-  Code = serialization::EXPR_CXX_EXPR_WITH_TEMPORARIES;
+  Code = serialization::EXPR_EXPR_WITH_CLEANUPS;
 }
 
 void
 ASTStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   VisitExpr(E);
   
-  // Don't emit anything here, NumTemplateArgs must be emitted first.
+  // Don't emit anything here, hasExplicitTemplateArgs() must be
+  // emitted first.
 
+  Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
     const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
-    assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
-  } else {
-    Record.push_back(0);
   }
   
   if (!E->isImplicitAccess())
@@ -1170,9 +1200,7 @@ ASTStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
   Writer.AddDeclRef(E->getFirstQualifierFoundInScope(), Record);
-  // FIXME: write whole DeclarationNameInfo.
-  Writer.AddDeclarationName(E->getMember(), Record);
-  Writer.AddSourceLocation(E->getMemberLoc(), Record);
+  Writer.AddDeclarationNameInfo(E->MemberNameInfo, Record);
   Code = serialization::EXPR_CXX_DEPENDENT_SCOPE_MEMBER;
 }
 
@@ -1180,21 +1208,16 @@ void
 ASTStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   VisitExpr(E);
   
-  // Don't emit anything here, NumTemplateArgs must be emitted first.
-
+  // Don't emit anything here, hasExplicitTemplateArgs() must be
+  // emitted first.
+  Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
     const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
-    assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
-  } else {
-    Record.push_back(0);
   }
 
-  // FIXME: write whole DeclarationNameInfo.
-  Writer.AddDeclarationName(E->getDeclName(), Record);
-  Writer.AddSourceLocation(E->getLocation(), Record);
+  Writer.AddDeclarationNameInfo(E->NameInfo, Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
   Code = serialization::EXPR_CXX_DEPENDENT_SCOPE_DECL_REF;
@@ -1207,8 +1230,7 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   for (CXXUnresolvedConstructExpr::arg_iterator
          ArgI = E->arg_begin(), ArgE = E->arg_end(); ArgI != ArgE; ++ArgI)
     Writer.AddStmt(*ArgI);
-  Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
-  Writer.AddTypeRef(E->getTypeAsWritten(), Record);
+  Writer.AddTypeSourceInfo(E->getTypeSourceInfo(), Record);
   Writer.AddSourceLocation(E->getLParenLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
   Code = serialization::EXPR_CXX_UNRESOLVED_CONSTRUCT;
@@ -1217,16 +1239,12 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
 void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
   VisitExpr(E);
   
-  // Don't emit anything here, NumTemplateArgs must be emitted first.
-
+  // Don't emit anything here, hasExplicitTemplateArgs() must be emitted first.
+  Record.push_back(E->hasExplicitTemplateArgs());
   if (E->hasExplicitTemplateArgs()) {
     const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
-    assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
-  } else {
-    Record.push_back(0);
   }
 
   Record.push_back(E->getNumDecls());
@@ -1236,11 +1254,9 @@ void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
     Record.push_back(OvI.getAccess());
   }
 
-  // FIXME: write whole DeclarationNameInfo.
-  Writer.AddDeclarationName(E->getName(), Record);
+  Writer.AddDeclarationNameInfo(E->NameInfo, Record);
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
-  Writer.AddSourceLocation(E->getNameLoc(), Record);
 }
 
 void ASTStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
@@ -1264,11 +1280,74 @@ void ASTStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
 void ASTStmtWriter::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getTrait());
+  Record.push_back(E->getValue());
   Writer.AddSourceRange(E->getSourceRange(), Record);
-  Writer.AddTypeRef(E->getQueriedType(), Record);
+  Writer.AddTypeSourceInfo(E->getQueriedTypeSourceInfo(), Record);
   Code = serialization::EXPR_CXX_UNARY_TYPE_TRAIT;
 }
 
+void ASTStmtWriter::VisitBinaryTypeTraitExpr(BinaryTypeTraitExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getTrait());
+  Record.push_back(E->getValue());
+  Writer.AddSourceRange(E->getSourceRange(), Record);
+  Writer.AddTypeSourceInfo(E->getLhsTypeSourceInfo(), Record);
+  Writer.AddTypeSourceInfo(E->getRhsTypeSourceInfo(), Record);
+  Code = serialization::EXPR_BINARY_TYPE_TRAIT;
+}
+
+void ASTStmtWriter::VisitCXXNoexceptExpr(CXXNoexceptExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getValue());
+  Writer.AddSourceRange(E->getSourceRange(), Record);
+  Writer.AddStmt(E->getOperand());
+  Code = serialization::EXPR_CXX_NOEXCEPT;
+}
+
+void ASTStmtWriter::VisitPackExpansionExpr(PackExpansionExpr *E) {
+  VisitExpr(E);
+  Writer.AddSourceLocation(E->getEllipsisLoc(), Record);
+  Record.push_back(E->NumExpansions);
+  Writer.AddStmt(E->getPattern());
+  Code = serialization::EXPR_PACK_EXPANSION;
+}
+
+void ASTStmtWriter::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
+  VisitExpr(E);
+  Writer.AddSourceLocation(E->OperatorLoc, Record);
+  Writer.AddSourceLocation(E->PackLoc, Record);
+  Writer.AddSourceLocation(E->RParenLoc, Record);
+  Record.push_back(E->Length);
+  Writer.AddDeclRef(E->Pack, Record);
+  Code = serialization::EXPR_SIZEOF_PACK;
+}
+
+void ASTStmtWriter::VisitSubstNonTypeTemplateParmPackExpr(
+                                          SubstNonTypeTemplateParmPackExpr *E) {
+  VisitExpr(E);
+  Writer.AddDeclRef(E->Param, Record);
+  Writer.AddTemplateArgument(E->getArgumentPack(), Record);
+  Writer.AddSourceLocation(E->NameLoc, Record);
+  Code = serialization::EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK;
+}
+
+void ASTStmtWriter::VisitOpaqueValueExpr(OpaqueValueExpr *E) {
+  VisitExpr(E);
+  Record.push_back(Writer.getOpaqueValueID(E));
+  Writer.AddSourceLocation(E->getLocation(), Record);
+  Code = serialization::EXPR_OPAQUE_VALUE;
+}
+
+//===----------------------------------------------------------------------===//
+// CUDA Expressions and Statements.
+//===----------------------------------------------------------------------===//
+
+void ASTStmtWriter::VisitCUDAKernelCallExpr(CUDAKernelCallExpr *E) {
+  VisitCallExpr(E);
+  Writer.AddStmt(E->getConfig());
+  Code = serialization::EXPR_CUDA_KERNEL_CALL;
+}
+
 //===----------------------------------------------------------------------===//
 // ASTWriter Implementation
 //===----------------------------------------------------------------------===//
@@ -1287,16 +1366,14 @@ unsigned ASTWriter::getSwitchCaseID(SwitchCase *S) {
   return SwitchCaseIDs[S];
 }
 
-/// \brief Retrieve the ID for the given label statement, which may
-/// or may not have been emitted yet.
-unsigned ASTWriter::GetLabelID(LabelStmt *S) {
-  std::map<LabelStmt *, unsigned>::iterator Pos = LabelIDs.find(S);
-  if (Pos != LabelIDs.end())
-    return Pos->second;
+void ASTWriter::ClearSwitchCaseIDs() {
+  SwitchCaseIDs.clear();
+}
 
-  unsigned NextID = LabelIDs.size();
-  LabelIDs[S] = NextID;
-  return NextID;
+unsigned ASTWriter::getOpaqueValueID(OpaqueValueExpr *e) {
+  unsigned &entry = OpaqueValues[e];
+  if (!entry) entry = OpaqueValues.size();
+  return entry;
 }
 
 /// \brief Write the given substatement or subexpression to the
diff --git a/lib/Serialization/CMakeLists.txt b/lib/Serialization/CMakeLists.txt
index d863c179bed2..10c8904b7aa5 100644
--- a/lib/Serialization/CMakeLists.txt
+++ b/lib/Serialization/CMakeLists.txt
@@ -1,4 +1,5 @@
-set(LLVM_NO_RTTI 1)
+# TODO: This must need some dependencies, but it builds fine without them.
+#set(LLVM_USED_LIBS ???)
 
 add_clang_library(clangSerialization
   GeneratePCH.cpp
diff --git a/lib/Serialization/GeneratePCH.cpp b/lib/Serialization/GeneratePCH.cpp
index 5329b6cbd4d0..b8833ceacc7e 100644
--- a/lib/Serialization/GeneratePCH.cpp
+++ b/lib/Serialization/GeneratePCH.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/ASTConsumer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
@@ -26,15 +27,15 @@
 using namespace clang;
 
 PCHGenerator::PCHGenerator(const Preprocessor &PP,
+                           const std::string &OutputFile,
                            bool Chaining,
                            const char *isysroot,
                            llvm::raw_ostream *OS)
-  : PP(PP), isysroot(isysroot), Out(OS), SemaPtr(0),
-    StatCalls(0), Stream(Buffer), Writer(Stream) {
-
+  : PP(PP), OutputFile(OutputFile), isysroot(isysroot), Out(OS), SemaPtr(0),
+    StatCalls(0), Stream(Buffer), Writer(Stream), Chaining(Chaining) {
   // Install a stat() listener to keep track of all of the stat()
   // calls.
-  StatCalls = new MemorizeStatCalls;
+  StatCalls = new MemorizeStatCalls();
   // If we have a chain, we want new stat calls only, so install the memorizer
   // *after* the already installed ASTReader's stat cache.
   PP.getFileManager().addStatCache(StatCalls,
@@ -45,9 +46,12 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   if (PP.getDiagnostics().hasErrorOccurred())
     return;
 
+  // Set up the serialization listener.
+  Writer.SetSerializationListener(GetASTSerializationListener());
+  
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
-  Writer.WriteAST(*SemaPtr, StatCalls, isysroot);
+  Writer.WriteAST(*SemaPtr, StatCalls, OutputFile, isysroot);
 
   // Write the generated bitstream to "Out".
   Out->write((char *)&Buffer.front(), Buffer.size());
@@ -59,6 +63,16 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   Buffer.clear();
 }
 
+ASTMutationListener *PCHGenerator::GetASTMutationListener() {
+  if (Chaining)
+    return &Writer;
+  return 0;
+}
+
+ASTSerializationListener *PCHGenerator::GetASTSerializationListener() {
+  return 0;
+}
+
 ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() {
   return &Writer;
 }
diff --git a/lib/StaticAnalyzer/CMakeLists.txt b/lib/StaticAnalyzer/CMakeLists.txt
new file mode 100644
index 000000000000..3d1509254f52
--- /dev/null
+++ b/lib/StaticAnalyzer/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_subdirectory(Core)
+add_subdirectory(Checkers)
+add_subdirectory(Frontend)
diff --git a/lib/Checker/AdjustedReturnValueChecker.cpp b/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
index 0ed04fb14aba..8832b053db00 100644
--- a/lib/Checker/AdjustedReturnValueChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
@@ -13,12 +13,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class AdjustedReturnValueChecker : 
@@ -34,7 +35,7 @@ public:
 };
 }
 
-void clang::RegisterAdjustedReturnValueChecker(GRExprEngine &Eng) {
+void ento::RegisterAdjustedReturnValueChecker(ExprEngine &Eng) {
   Eng.registerCheck(new AdjustedReturnValueChecker());
 }
 
@@ -54,7 +55,7 @@ void AdjustedReturnValueChecker::PostVisitCallExpr(CheckerContext &C,
   
   // Casting to void?  Discard the value.
   if (expectedResultTy->isVoidType()) {
-    C.GenerateNode(state->BindExpr(CE, UnknownVal()));
+    C.generateNode(state->BindExpr(CE, UnknownVal()));
     return;
   }                   
 
@@ -88,8 +89,8 @@ void AdjustedReturnValueChecker::PostVisitCallExpr(CheckerContext &C,
   if (expectedResultTy != actualResultTy) {
     // FIXME: Do more checking and actual emit an error. At least performing
     // the cast avoids some assertion failures elsewhere.
-    SValuator &SVator = C.getSValuator();
-    V = SVator.EvalCast(V, expectedResultTy, actualResultTy);
-    C.GenerateNode(state->BindExpr(CE, V));
+    SValBuilder &svalBuilder = C.getSValBuilder();
+    V = svalBuilder.evalCast(V, expectedResultTy, actualResultTy);
+    C.generateNode(state->BindExpr(CE, V));
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp b/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
new file mode 100644
index 000000000000..7b68887f6640
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/AnalyzerStatsChecker.cpp
@@ -0,0 +1,123 @@
+//==--AnalyzerStatsChecker.cpp - Analyzer visitation statistics --*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file reports various statistics about analyzer visitation.
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+
+// FIXME: Restructure checker registration.
+#include "ExperimentalChecks.h"
+
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/SmallPtrSet.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class AnalyzerStatsChecker : public CheckerVisitor<AnalyzerStatsChecker> {
+public:
+  static void *getTag();
+  void VisitEndAnalysis(ExplodedGraph &G, BugReporter &B, ExprEngine &Eng);
+
+private:
+  llvm::SmallPtrSet<const CFGBlock*, 256> reachable;
+};
+}
+
+void *AnalyzerStatsChecker::getTag() {
+  static int x = 0;
+  return &x;
+}
+
+void ento::RegisterAnalyzerStatsChecker(ExprEngine &Eng) {
+  Eng.registerCheck(new AnalyzerStatsChecker());
+}
+
+void AnalyzerStatsChecker::VisitEndAnalysis(ExplodedGraph &G,
+                                            BugReporter &B,
+                                            ExprEngine &Eng) {
+  const CFG *C  = 0;
+  const Decl *D = 0;
+  const LocationContext *LC = 0;
+  const SourceManager &SM = B.getSourceManager();
+
+  // Iterate over explodedgraph
+  for (ExplodedGraph::node_iterator I = G.nodes_begin();
+      I != G.nodes_end(); ++I) {
+    const ProgramPoint &P = I->getLocation();
+    // Save the LocationContext if we don't have it already
+    if (!LC)
+      LC = P.getLocationContext();
+
+    if (const BlockEntrance *BE = dyn_cast<BlockEntrance>(&P)) {
+      const CFGBlock *CB = BE->getBlock();
+      reachable.insert(CB);
+    }
+  }
+
+  // Get the CFG and the Decl of this block
+  C = LC->getCFG();
+  D = LC->getAnalysisContext()->getDecl();
+
+  unsigned total = 0, unreachable = 0;
+
+  // Find CFGBlocks that were not covered by any node
+  for (CFG::const_iterator I = C->begin(); I != C->end(); ++I) {
+    const CFGBlock *CB = *I;
+    ++total;
+    // Check if the block is unreachable
+    if (!reachable.count(CB)) {
+      ++unreachable;
+    }
+  }
+
+  // We never 'reach' the entry block, so correct the unreachable count
+  unreachable--;
+
+  // Generate the warning string
+  llvm::SmallString<128> buf;
+  llvm::raw_svector_ostream output(buf);
+  PresumedLoc Loc = SM.getPresumedLoc(D->getLocation());
+  if (Loc.isValid()) {
+    output << Loc.getFilename() << " : ";
+
+    if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
+      const NamedDecl *ND = cast<NamedDecl>(D);
+      output << ND;
+    }
+    else if (isa<BlockDecl>(D)) {
+      output << "block(line:" << Loc.getLine() << ":col:" << Loc.getColumn();
+    }
+  }
+  
+  output << " -> Total CFGBlocks: " << total << " | Unreachable CFGBlocks: "
+      << unreachable << " | Aborted Block: "
+      << (Eng.wasBlockAborted() ? "yes" : "no")
+      << " | Empty WorkList: "
+      << (Eng.hasEmptyWorkList() ? "yes" : "no");
+
+  B.EmitBasicReport("Analyzer Statistics", "Internal Statistics", output.str(),
+      D->getLocation());
+
+  // Emit warning for each block we bailed out on
+  typedef CoreEngine::BlocksAborted::const_iterator AbortedIterator;
+  const CoreEngine &CE = Eng.getCoreEngine();
+  for (AbortedIterator I = CE.blocks_aborted_begin(),
+      E = CE.blocks_aborted_end(); I != E; ++I) {
+    const BlockEdge &BE =  I->first;
+    const CFGBlock *Exit = BE.getDst();
+    const CFGElement &CE = Exit->front();
+    if (const CFGStmt *CS = dyn_cast<CFGStmt>(&CE))
+      B.EmitBasicReport("Bailout Point", "Internal Statistics", "The analyzer "
+          "stopped analyzing at this point", CS->getStmt()->getLocStart());
+  }
+}
diff --git a/lib/Checker/ArrayBoundChecker.cpp b/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index 98345bd70375..9194791fc075 100644
--- a/lib/Checker/ArrayBoundChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -12,46 +12,47 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class ArrayBoundChecker : 
     public CheckerVisitor<ArrayBoundChecker> {      
   BuiltinBug *BT;
 public:
-    ArrayBoundChecker() : BT(0) {}
-    static void *getTag();
-    void VisitLocation(CheckerContext &C, const Stmt *S, SVal l);
+  ArrayBoundChecker() : BT(0) {}
+  static void *getTag() { static int x = 0; return &x; }
+  void visitLocation(CheckerContext &C, const Stmt *S, SVal l, bool isLoad);
 };
 }
 
-void clang::RegisterArrayBoundChecker(GRExprEngine &Eng) {
+void ento::RegisterArrayBoundChecker(ExprEngine &Eng) {
   Eng.registerCheck(new ArrayBoundChecker());
 }
 
-void *ArrayBoundChecker::getTag() {
-  static int x = 0; return &x;
-}
-
-void ArrayBoundChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l){
+void ArrayBoundChecker::visitLocation(CheckerContext &C, const Stmt *S, SVal l,
+                                      bool isLoad) {
   // Check for out of bound array element access.
   const MemRegion *R = l.getAsRegion();
   if (!R)
     return;
 
-  R = R->StripCasts();
-
   const ElementRegion *ER = dyn_cast<ElementRegion>(R);
   if (!ER)
     return;
 
   // Get the index of the accessed element.
-  DefinedOrUnknownSVal &Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
+  DefinedOrUnknownSVal Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
+
+  // Zero index is always in bound, this also passes ElementRegions created for
+  // pointer casts.
+  if (Idx.isZeroConstant())
+    return;
 
   const GRState *state = C.getState();
 
@@ -60,10 +61,10 @@ void ArrayBoundChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l){
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(), 
                                             ER->getValueType());
 
-  const GRState *StInBound = state->AssumeInBound(Idx, NumElements, true);
-  const GRState *StOutBound = state->AssumeInBound(Idx, NumElements, false);
+  const GRState *StInBound = state->assumeInBound(Idx, NumElements, true);
+  const GRState *StOutBound = state->assumeInBound(Idx, NumElements, false);
   if (StOutBound && !StInBound) {
-    ExplodedNode *N = C.GenerateSink(StOutBound);
+    ExplodedNode *N = C.generateSink(StOutBound);
     if (!N)
       return;
   
diff --git a/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
new file mode 100644
index 000000000000..f803d27a2678
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -0,0 +1,277 @@
+//== ArrayBoundCheckerV2.cpp ------------------------------------*- C++ -*--==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines ArrayBoundCheckerV2, which is a path-sensitive check
+// which looks for an out-of-bound array element access.
+//
+//===----------------------------------------------------------------------===//
+
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/AST/CharUnits.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class ArrayBoundCheckerV2 : 
+    public CheckerVisitor<ArrayBoundCheckerV2> {      
+  BuiltinBug *BT;
+      
+  enum OOB_Kind { OOB_Precedes, OOB_Excedes };
+  
+  void reportOOB(CheckerContext &C, const GRState *errorState,
+                 OOB_Kind kind);
+      
+public:
+  ArrayBoundCheckerV2() : BT(0) {}
+  static void *getTag() { static int x = 0; return &x; }
+  void visitLocation(CheckerContext &C, const Stmt *S, SVal l, bool isLoad);
+};
+
+// FIXME: Eventually replace RegionRawOffset with this class.
+class RegionRawOffsetV2 {
+private:
+  const SubRegion *baseRegion;
+  SVal byteOffset;
+  
+  RegionRawOffsetV2()
+    : baseRegion(0), byteOffset(UnknownVal()) {}
+
+public:
+  RegionRawOffsetV2(const SubRegion* base, SVal offset)
+    : baseRegion(base), byteOffset(offset) {}
+
+  NonLoc getByteOffset() const { return cast<NonLoc>(byteOffset); }
+  const SubRegion *getRegion() const { return baseRegion; }
+  
+  static RegionRawOffsetV2 computeOffset(const GRState *state,
+                                         SValBuilder &svalBuilder,
+                                         SVal location);
+
+  void dump() const;
+  void dumpToStream(llvm::raw_ostream& os) const;
+};
+}
+
+void ento::RegisterArrayBoundCheckerV2(ExprEngine &Eng) {
+  Eng.registerCheck(new ArrayBoundCheckerV2());
+}
+
+void ArrayBoundCheckerV2::visitLocation(CheckerContext &checkerContext,
+                                        const Stmt *S,
+                                        SVal location, bool isLoad) {
+
+  // NOTE: Instead of using GRState::assumeInBound(), we are prototyping
+  // some new logic here that reasons directly about memory region extents.
+  // Once that logic is more mature, we can bring it back to assumeInBound()
+  // for all clients to use.
+  //
+  // The algorithm we are using here for bounds checking is to see if the
+  // memory access is within the extent of the base region.  Since we
+  // have some flexibility in defining the base region, we can achieve
+  // various levels of conservatism in our buffer overflow checking.
+  const GRState *state = checkerContext.getState();  
+  const GRState *originalState = state;
+
+  SValBuilder &svalBuilder = checkerContext.getSValBuilder();
+  const RegionRawOffsetV2 &rawOffset = 
+    RegionRawOffsetV2::computeOffset(state, svalBuilder, location);
+
+  if (!rawOffset.getRegion())
+    return;
+
+  // CHECK LOWER BOUND: Is byteOffset < 0?  If so, we are doing a load/store
+  //  before the first valid offset in the memory region.
+
+  SVal lowerBound
+    = svalBuilder.evalBinOpNN(state, BO_LT, rawOffset.getByteOffset(),
+                              svalBuilder.makeZeroArrayIndex(),
+                              svalBuilder.getConditionType());
+
+  NonLoc *lowerBoundToCheck = dyn_cast<NonLoc>(&lowerBound);
+  if (!lowerBoundToCheck)
+    return;
+    
+  const GRState *state_precedesLowerBound, *state_withinLowerBound;
+  llvm::tie(state_precedesLowerBound, state_withinLowerBound) =
+      state->assume(*lowerBoundToCheck);
+
+  // Are we constrained enough to definitely precede the lower bound?
+  if (state_precedesLowerBound && !state_withinLowerBound) {
+    reportOOB(checkerContext, state_precedesLowerBound, OOB_Precedes);
+    return;
+  }
+  
+  // Otherwise, assume the constraint of the lower bound.
+  assert(state_withinLowerBound);
+  state = state_withinLowerBound;
+  
+  do {
+    // CHECK UPPER BOUND: Is byteOffset >= extent(baseRegion)?  If so,
+    // we are doing a load/store after the last valid offset.
+    DefinedOrUnknownSVal extentVal =
+      rawOffset.getRegion()->getExtent(svalBuilder);
+    if (!isa<NonLoc>(extentVal))
+      break;
+
+    SVal upperbound
+      = svalBuilder.evalBinOpNN(state, BO_GE, rawOffset.getByteOffset(),
+                                cast<NonLoc>(extentVal),
+                                svalBuilder.getConditionType());
+  
+    NonLoc *upperboundToCheck = dyn_cast<NonLoc>(&upperbound);
+    if (!upperboundToCheck)
+      break;
+  
+    const GRState *state_exceedsUpperBound, *state_withinUpperBound;
+    llvm::tie(state_exceedsUpperBound, state_withinUpperBound) =
+      state->assume(*upperboundToCheck);
+  
+    // Are we constrained enough to definitely exceed the upper bound?
+    if (state_exceedsUpperBound && !state_withinUpperBound) {
+      reportOOB(checkerContext, state_exceedsUpperBound, OOB_Excedes);
+      return;
+    }
+  
+    assert(state_withinUpperBound);
+    state = state_withinUpperBound;
+  }
+  while (false);
+  
+  if (state != originalState)
+    checkerContext.generateNode(state);
+}
+
+void ArrayBoundCheckerV2::reportOOB(CheckerContext &checkerContext,
+                                    const GRState *errorState,
+                                    OOB_Kind kind) {
+  
+  ExplodedNode *errorNode = checkerContext.generateSink(errorState);
+  if (!errorNode)
+    return;
+
+  if (!BT)
+    BT = new BuiltinBug("Out-of-bound access");
+
+  // FIXME: This diagnostics are preliminary.  We should get far better
+  // diagnostics for explaining buffer overruns.
+
+  llvm::SmallString<256> buf;
+  llvm::raw_svector_ostream os(buf);
+  os << "Out of bound memory access "
+     << (kind == OOB_Precedes ? "(accessed memory precedes memory block)"
+                              : "(access exceeds upper limit of memory block)");
+
+  checkerContext.EmitReport(new RangedBugReport(*BT, os.str(), errorNode));
+}
+
+void RegionRawOffsetV2::dump() const {
+  dumpToStream(llvm::errs());
+}
+
+void RegionRawOffsetV2::dumpToStream(llvm::raw_ostream& os) const {
+  os << "raw_offset_v2{" << getRegion() << ',' << getByteOffset() << '}';
+}
+
+// FIXME: Merge with the implementation of the same method in Store.cpp
+static bool IsCompleteType(ASTContext &Ctx, QualType Ty) {
+  if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    const RecordDecl *D = RT->getDecl();
+    if (!D->getDefinition())
+      return false;
+  }
+
+  return true;
+}
+
+
+// Lazily computes a value to be used by 'computeOffset'.  If 'val'
+// is unknown or undefined, we lazily substitute '0'.  Otherwise,
+// return 'val'.
+static inline SVal getValue(SVal val, SValBuilder &svalBuilder) {
+  return isa<UndefinedVal>(val) ? svalBuilder.makeArrayIndex(0) : val;
+}
+
+// Scale a base value by a scaling factor, and return the scaled
+// value as an SVal.  Used by 'computeOffset'.
+static inline SVal scaleValue(const GRState *state,
+                              NonLoc baseVal, CharUnits scaling,
+                              SValBuilder &sb) {
+  return sb.evalBinOpNN(state, BO_Mul, baseVal,
+                        sb.makeArrayIndex(scaling.getQuantity()),
+                        sb.getArrayIndexType());
+}
+
+// Add an SVal to another, treating unknown and undefined values as
+// summing to UnknownVal.  Used by 'computeOffset'.
+static SVal addValue(const GRState *state, SVal x, SVal y,
+                     SValBuilder &svalBuilder) {
+  // We treat UnknownVals and UndefinedVals the same here because we
+  // only care about computing offsets.
+  if (x.isUnknownOrUndef() || y.isUnknownOrUndef())
+    return UnknownVal();
+  
+  return svalBuilder.evalBinOpNN(state, BO_Add,                                 
+                                 cast<NonLoc>(x), cast<NonLoc>(y),
+                                 svalBuilder.getArrayIndexType());
+}
+
+/// Compute a raw byte offset from a base region.  Used for array bounds
+/// checking.
+RegionRawOffsetV2 RegionRawOffsetV2::computeOffset(const GRState *state,
+                                                   SValBuilder &svalBuilder,
+                                                   SVal location)
+{
+  const MemRegion *region = location.getAsRegion();
+  SVal offset = UndefinedVal();
+  
+  while (region) {
+    switch (region->getKind()) {
+      default: {
+        if (const SubRegion *subReg = dyn_cast<SubRegion>(region))
+          if (!offset.isUnknownOrUndef())
+            return RegionRawOffsetV2(subReg, offset);
+        return RegionRawOffsetV2();
+      }
+      case MemRegion::ElementRegionKind: {
+        const ElementRegion *elemReg = cast<ElementRegion>(region);
+        SVal index = elemReg->getIndex();
+        if (!isa<NonLoc>(index))
+          return RegionRawOffsetV2();
+        QualType elemType = elemReg->getElementType();
+        // If the element is an incomplete type, go no further.
+        ASTContext &astContext = svalBuilder.getContext();
+        if (!IsCompleteType(astContext, elemType))
+          return RegionRawOffsetV2();
+        
+        // Update the offset.
+        offset = addValue(state,
+                          getValue(offset, svalBuilder),
+                          scaleValue(state,
+                                     cast<NonLoc>(index),
+                                     astContext.getTypeSizeInChars(elemType),
+                                     svalBuilder),
+                          svalBuilder);
+
+        if (offset.isUnknownOrUndef())
+          return RegionRawOffsetV2();
+
+        region = elemReg->getSuperRegion();
+        continue;
+      }
+    }
+  }
+  return RegionRawOffsetV2();
+}
+
+
+
diff --git a/lib/Checker/AttrNonNullChecker.cpp b/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
index d0bccb27b405..e4865b142c0c 100644
--- a/lib/Checker/AttrNonNullChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
@@ -7,16 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines AttrNonNullChecker, a builtin check in GRExprEngine that 
+// This defines AttrNonNullChecker, a builtin check in ExprEngine that 
 // performs checks for arguments declared to have nonnull attribute.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class AttrNonNullChecker
@@ -32,7 +33,7 @@ public:
 };
 } // end anonymous namespace
 
-void clang::RegisterAttrNonNullChecker(GRExprEngine &Eng) {
+void ento::RegisterAttrNonNullChecker(ExprEngine &Eng) {
   Eng.registerCheck(new AttrNonNullChecker());
 }
 
@@ -67,14 +68,36 @@ void AttrNonNullChecker::PreVisitCallExpr(CheckerContext &C,
     if (!DV)
       continue;
 
+    if (!isa<Loc>(*DV)) {
+      // If the argument is a union type, we want to handle a potential
+      // transparent_unoin GCC extension.
+      QualType T = (*I)->getType();
+      const RecordType *UT = T->getAsUnionType();
+      if (!UT || !UT->getDecl()->hasAttr<TransparentUnionAttr>())
+        continue;
+      if (nonloc::CompoundVal *CSV = dyn_cast<nonloc::CompoundVal>(DV)) {
+        nonloc::CompoundVal::iterator CSV_I = CSV->begin();
+        assert(CSV_I != CSV->end());
+        V = *CSV_I;
+        DV = dyn_cast<DefinedSVal>(&V);
+        assert(++CSV_I == CSV->end());
+        if (!DV)
+          continue;        
+      }
+      else {
+        // FIXME: Handle LazyCompoundVals?
+        continue;
+      }
+    }
+
     ConstraintManager &CM = C.getConstraintManager();
     const GRState *stateNotNull, *stateNull;
-    llvm::tie(stateNotNull, stateNull) = CM.AssumeDual(state, *DV);
+    llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, *DV);
 
     if (stateNull && !stateNotNull) {
       // Generate an error node.  Check for a null node in case
       // we cache out.
-      if (ExplodedNode *errorNode = C.GenerateSink(stateNull)) {
+      if (ExplodedNode *errorNode = C.generateSink(stateNull)) {
 
         // Lazily allocate the BugType object if it hasn't already been
         // created. Ownership is transferred to the BugReporter object once
diff --git a/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
new file mode 100644
index 000000000000..1b6c528c2f16
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
@@ -0,0 +1,521 @@
+//== BasicObjCFoundationChecks.cpp - Simple Apple-Foundation checks -*- C++ -*--
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines BasicObjCFoundationChecks, a class that encapsulates
+//  a set of simple checks to run on Objective-C code using Apple's Foundation
+//  classes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "BasicObjCFoundationChecks.h"
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/ASTContext.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class APIMisuse : public BugType {
+public:
+  APIMisuse(const char* name) : BugType(name, "API Misuse (Apple)") {}
+};
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// Utility functions.
+//===----------------------------------------------------------------------===//
+
+static const ObjCInterfaceType* GetReceiverType(const ObjCMessage &msg) {
+  if (const ObjCInterfaceDecl *ID = msg.getReceiverInterface())
+    return ID->getTypeForDecl()->getAs<ObjCInterfaceType>();
+  return NULL;
+}
+
+static const char* GetReceiverNameType(const ObjCMessage &msg) {
+  if (const ObjCInterfaceType *ReceiverType = GetReceiverType(msg))
+    return ReceiverType->getDecl()->getIdentifier()->getNameStart();
+  return NULL;
+}
+
+static bool isNSString(llvm::StringRef ClassName) {
+  return ClassName == "NSString" || ClassName == "NSMutableString";
+}
+
+static inline bool isNil(SVal X) {
+  return isa<loc::ConcreteInt>(X);
+}
+
+//===----------------------------------------------------------------------===//
+// NilArgChecker - Check for prohibited nil arguments to ObjC method calls.
+//===----------------------------------------------------------------------===//
+
+namespace {
+  class NilArgChecker : public CheckerVisitor<NilArgChecker> {
+    APIMisuse *BT;
+    void WarnNilArg(CheckerContext &C, const ObjCMessage &msg, unsigned Arg);
+  public:
+    NilArgChecker() : BT(0) {}
+    static void *getTag() { static int x = 0; return &x; }
+    void preVisitObjCMessage(CheckerContext &C, ObjCMessage msg);
+  };
+}
+
+void NilArgChecker::WarnNilArg(CheckerContext &C,
+                               const ObjCMessage &msg,
+                               unsigned int Arg)
+{
+  if (!BT)
+    BT = new APIMisuse("nil argument");
+  
+  if (ExplodedNode *N = C.generateSink()) {
+    llvm::SmallString<128> sbuf;
+    llvm::raw_svector_ostream os(sbuf);
+    os << "Argument to '" << GetReceiverNameType(msg) << "' method '"
+       << msg.getSelector().getAsString() << "' cannot be nil";
+
+    RangedBugReport *R = new RangedBugReport(*BT, os.str(), N);
+    R->addRange(msg.getArgSourceRange(Arg));
+    C.EmitReport(R);
+  }
+}
+
+void NilArgChecker::preVisitObjCMessage(CheckerContext &C,
+                                        ObjCMessage msg)
+{
+  const ObjCInterfaceType *ReceiverType = GetReceiverType(msg);
+  if (!ReceiverType)
+    return;
+  
+  if (isNSString(ReceiverType->getDecl()->getIdentifier()->getName())) {
+    Selector S = msg.getSelector();
+    
+    if (S.isUnarySelector())
+      return;
+    
+    // FIXME: This is going to be really slow doing these checks with
+    //  lexical comparisons.
+    
+    std::string NameStr = S.getAsString();
+    llvm::StringRef Name(NameStr);
+    assert(!Name.empty());
+    
+    // FIXME: Checking for initWithFormat: will not work in most cases
+    //  yet because [NSString alloc] returns id, not NSString*.  We will
+    //  need support for tracking expected-type information in the analyzer
+    //  to find these errors.
+    if (Name == "caseInsensitiveCompare:" ||
+        Name == "compare:" ||
+        Name == "compare:options:" ||
+        Name == "compare:options:range:" ||
+        Name == "compare:options:range:locale:" ||
+        Name == "componentsSeparatedByCharactersInSet:" ||
+        Name == "initWithFormat:") {
+      if (isNil(msg.getArgSVal(0, C.getState())))
+        WarnNilArg(C, msg, 0);
+    }
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Error reporting.
+//===----------------------------------------------------------------------===//
+
+namespace {
+class CFNumberCreateChecker : public CheckerVisitor<CFNumberCreateChecker> {
+  APIMisuse* BT;
+  IdentifierInfo* II;
+public:
+  CFNumberCreateChecker() : BT(0), II(0) {}
+  ~CFNumberCreateChecker() {}
+  static void *getTag() { static int x = 0; return &x; }
+  void PreVisitCallExpr(CheckerContext &C, const CallExpr *CE);
+private:
+  void EmitError(const TypedRegion* R, const Expr* Ex,
+                uint64_t SourceSize, uint64_t TargetSize, uint64_t NumberKind);
+};
+} // end anonymous namespace
+
+enum CFNumberType {
+  kCFNumberSInt8Type = 1,
+  kCFNumberSInt16Type = 2,
+  kCFNumberSInt32Type = 3,
+  kCFNumberSInt64Type = 4,
+  kCFNumberFloat32Type = 5,
+  kCFNumberFloat64Type = 6,
+  kCFNumberCharType = 7,
+  kCFNumberShortType = 8,
+  kCFNumberIntType = 9,
+  kCFNumberLongType = 10,
+  kCFNumberLongLongType = 11,
+  kCFNumberFloatType = 12,
+  kCFNumberDoubleType = 13,
+  kCFNumberCFIndexType = 14,
+  kCFNumberNSIntegerType = 15,
+  kCFNumberCGFloatType = 16
+};
+
+namespace {
+  template<typename T>
+  class Optional {
+    bool IsKnown;
+    T Val;
+  public:
+    Optional() : IsKnown(false), Val(0) {}
+    Optional(const T& val) : IsKnown(true), Val(val) {}
+
+    bool isKnown() const { return IsKnown; }
+
+    const T& getValue() const {
+      assert (isKnown());
+      return Val;
+    }
+
+    operator const T&() const {
+      return getValue();
+    }
+  };
+}
+
+static Optional<uint64_t> GetCFNumberSize(ASTContext& Ctx, uint64_t i) {
+  static const unsigned char FixedSize[] = { 8, 16, 32, 64, 32, 64 };
+
+  if (i < kCFNumberCharType)
+    return FixedSize[i-1];
+
+  QualType T;
+
+  switch (i) {
+    case kCFNumberCharType:     T = Ctx.CharTy;     break;
+    case kCFNumberShortType:    T = Ctx.ShortTy;    break;
+    case kCFNumberIntType:      T = Ctx.IntTy;      break;
+    case kCFNumberLongType:     T = Ctx.LongTy;     break;
+    case kCFNumberLongLongType: T = Ctx.LongLongTy; break;
+    case kCFNumberFloatType:    T = Ctx.FloatTy;    break;
+    case kCFNumberDoubleType:   T = Ctx.DoubleTy;   break;
+    case kCFNumberCFIndexType:
+    case kCFNumberNSIntegerType:
+    case kCFNumberCGFloatType:
+      // FIXME: We need a way to map from names to Type*.
+    default:
+      return Optional<uint64_t>();
+  }
+
+  return Ctx.getTypeSize(T);
+}
+
+#if 0
+static const char* GetCFNumberTypeStr(uint64_t i) {
+  static const char* Names[] = {
+    "kCFNumberSInt8Type",
+    "kCFNumberSInt16Type",
+    "kCFNumberSInt32Type",
+    "kCFNumberSInt64Type",
+    "kCFNumberFloat32Type",
+    "kCFNumberFloat64Type",
+    "kCFNumberCharType",
+    "kCFNumberShortType",
+    "kCFNumberIntType",
+    "kCFNumberLongType",
+    "kCFNumberLongLongType",
+    "kCFNumberFloatType",
+    "kCFNumberDoubleType",
+    "kCFNumberCFIndexType",
+    "kCFNumberNSIntegerType",
+    "kCFNumberCGFloatType"
+  };
+
+  return i <= kCFNumberCGFloatType ? Names[i-1] : "Invalid CFNumberType";
+}
+#endif
+
+void CFNumberCreateChecker::PreVisitCallExpr(CheckerContext &C,
+                                             const CallExpr *CE)
+{
+  const Expr* Callee = CE->getCallee();
+  const GRState *state = C.getState();
+  SVal CallV = state->getSVal(Callee);
+  const FunctionDecl* FD = CallV.getAsFunctionDecl();
+
+  if (!FD)
+    return;
+  
+  ASTContext &Ctx = C.getASTContext();
+  if (!II)
+    II = &Ctx.Idents.get("CFNumberCreate");
+
+  if (FD->getIdentifier() != II || CE->getNumArgs() != 3)
+    return;
+
+  // Get the value of the "theType" argument.
+  SVal TheTypeVal = state->getSVal(CE->getArg(1));
+
+  // FIXME: We really should allow ranges of valid theType values, and
+  //   bifurcate the state appropriately.
+  nonloc::ConcreteInt* V = dyn_cast<nonloc::ConcreteInt>(&TheTypeVal);
+  if (!V)
+    return;
+
+  uint64_t NumberKind = V->getValue().getLimitedValue();
+  Optional<uint64_t> TargetSize = GetCFNumberSize(Ctx, NumberKind);
+
+  // FIXME: In some cases we can emit an error.
+  if (!TargetSize.isKnown())
+    return;
+
+  // Look at the value of the integer being passed by reference.  Essentially
+  // we want to catch cases where the value passed in is not equal to the
+  // size of the type being created.
+  SVal TheValueExpr = state->getSVal(CE->getArg(2));
+
+  // FIXME: Eventually we should handle arbitrary locations.  We can do this
+  //  by having an enhanced memory model that does low-level typing.
+  loc::MemRegionVal* LV = dyn_cast<loc::MemRegionVal>(&TheValueExpr);
+  if (!LV)
+    return;
+
+  const TypedRegion* R = dyn_cast<TypedRegion>(LV->stripCasts());
+  if (!R)
+    return;
+
+  QualType T = Ctx.getCanonicalType(R->getValueType());
+
+  // FIXME: If the pointee isn't an integer type, should we flag a warning?
+  //  People can do weird stuff with pointers.
+
+  if (!T->isIntegerType())
+    return;
+
+  uint64_t SourceSize = Ctx.getTypeSize(T);
+
+  // CHECK: is SourceSize == TargetSize
+  if (SourceSize == TargetSize)
+    return;
+
+  // Generate an error.  Only generate a sink if 'SourceSize < TargetSize';
+  // otherwise generate a regular node.
+  //
+  // FIXME: We can actually create an abstract "CFNumber" object that has
+  //  the bits initialized to the provided values.
+  //
+  if (ExplodedNode *N = SourceSize < TargetSize ? C.generateSink() 
+                                                : C.generateNode()) {
+    llvm::SmallString<128> sbuf;
+    llvm::raw_svector_ostream os(sbuf);
+    
+    os << (SourceSize == 8 ? "An " : "A ")
+       << SourceSize << " bit integer is used to initialize a CFNumber "
+                        "object that represents "
+       << (TargetSize == 8 ? "an " : "a ")
+       << TargetSize << " bit integer. ";
+    
+    if (SourceSize < TargetSize)
+      os << (TargetSize - SourceSize)
+      << " bits of the CFNumber value will be garbage." ;
+    else
+      os << (SourceSize - TargetSize)
+      << " bits of the input integer will be lost.";
+
+    if (!BT)
+      BT = new APIMisuse("Bad use of CFNumberCreate");
+    
+    RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
+    report->addRange(CE->getArg(2)->getSourceRange());
+    C.EmitReport(report);
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// CFRetain/CFRelease checking for null arguments.
+//===----------------------------------------------------------------------===//
+
+namespace {
+class CFRetainReleaseChecker : public CheckerVisitor<CFRetainReleaseChecker> {
+  APIMisuse *BT;
+  IdentifierInfo *Retain, *Release;
+public:
+  CFRetainReleaseChecker(): BT(0), Retain(0), Release(0) {}
+  static void *getTag() { static int x = 0; return &x; }
+  void PreVisitCallExpr(CheckerContext& C, const CallExpr* CE);
+};
+} // end anonymous namespace
+
+
+void CFRetainReleaseChecker::PreVisitCallExpr(CheckerContext& C,
+                                              const CallExpr* CE) {
+  // If the CallExpr doesn't have exactly 1 argument just give up checking.
+  if (CE->getNumArgs() != 1)
+    return;
+
+  // Get the function declaration of the callee.
+  const GRState* state = C.getState();
+  SVal X = state->getSVal(CE->getCallee());
+  const FunctionDecl* FD = X.getAsFunctionDecl();
+
+  if (!FD)
+    return;
+  
+  if (!BT) {
+    ASTContext &Ctx = C.getASTContext();
+    Retain = &Ctx.Idents.get("CFRetain");
+    Release = &Ctx.Idents.get("CFRelease");
+    BT = new APIMisuse("null passed to CFRetain/CFRelease");
+  }
+
+  // Check if we called CFRetain/CFRelease.
+  const IdentifierInfo *FuncII = FD->getIdentifier();
+  if (!(FuncII == Retain || FuncII == Release))
+    return;
+
+  // FIXME: The rest of this just checks that the argument is non-null.
+  // It should probably be refactored and combined with AttrNonNullChecker.
+
+  // Get the argument's value.
+  const Expr *Arg = CE->getArg(0);
+  SVal ArgVal = state->getSVal(Arg);
+  DefinedSVal *DefArgVal = dyn_cast<DefinedSVal>(&ArgVal);
+  if (!DefArgVal)
+    return;
+
+  // Get a NULL value.
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  DefinedSVal zero = cast<DefinedSVal>(svalBuilder.makeZeroVal(Arg->getType()));
+
+  // Make an expression asserting that they're equal.
+  DefinedOrUnknownSVal ArgIsNull = svalBuilder.evalEQ(state, zero, *DefArgVal);
+
+  // Are they equal?
+  const GRState *stateTrue, *stateFalse;
+  llvm::tie(stateTrue, stateFalse) = state->assume(ArgIsNull);
+
+  if (stateTrue && !stateFalse) {
+    ExplodedNode *N = C.generateSink(stateTrue);
+    if (!N)
+      return;
+
+    const char *description = (FuncII == Retain)
+                            ? "Null pointer argument in call to CFRetain"
+                            : "Null pointer argument in call to CFRelease";
+
+    EnhancedBugReport *report = new EnhancedBugReport(*BT, description, N);
+    report->addRange(Arg->getSourceRange());
+    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Arg);
+    C.EmitReport(report);
+    return;
+  }
+
+  // From here on, we know the argument is non-null.
+  C.addTransition(stateFalse);
+}
+
+//===----------------------------------------------------------------------===//
+// Check for sending 'retain', 'release', or 'autorelease' directly to a Class.
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ClassReleaseChecker : public CheckerVisitor<ClassReleaseChecker> {
+  Selector releaseS;
+  Selector retainS;
+  Selector autoreleaseS;
+  Selector drainS;
+  BugType *BT;
+public:
+  ClassReleaseChecker()
+    : BT(0) {}
+
+  static void *getTag() { static int x = 0; return &x; }
+      
+  void preVisitObjCMessage(CheckerContext &C, ObjCMessage msg);
+};
+}
+
+void ClassReleaseChecker::preVisitObjCMessage(CheckerContext &C,
+                                              ObjCMessage msg) {
+  
+  if (!BT) {
+    BT = new APIMisuse("message incorrectly sent to class instead of class "
+                       "instance");
+  
+    ASTContext &Ctx = C.getASTContext();
+    releaseS = GetNullarySelector("release", Ctx);
+    retainS = GetNullarySelector("retain", Ctx);
+    autoreleaseS = GetNullarySelector("autorelease", Ctx);
+    drainS = GetNullarySelector("drain", Ctx);
+  }
+  
+  if (msg.isInstanceMessage())
+    return;
+  const ObjCInterfaceDecl *Class = msg.getReceiverInterface();
+  assert(Class);
+
+  Selector S = msg.getSelector();
+  if (!(S == releaseS || S == retainS || S == autoreleaseS || S == drainS))
+    return;
+  
+  if (ExplodedNode *N = C.generateNode()) {
+    llvm::SmallString<200> buf;
+    llvm::raw_svector_ostream os(buf);
+
+    os << "The '" << S.getAsString() << "' message should be sent to instances "
+          "of class '" << Class->getName()
+       << "' and not the class directly";
+  
+    RangedBugReport *report = new RangedBugReport(*BT, os.str(), N);
+    report->addRange(msg.getSourceRange());
+    C.EmitReport(report);
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Check registration.
+//===----------------------------------------------------------------------===//
+
+static void RegisterNilArgChecker(ExprEngine& Eng) {
+  Eng.registerCheck(new NilArgChecker());
+}
+
+void ento::registerNilArgChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterNilArgChecker);
+}
+
+static void RegisterCFNumberCreateChecker(ExprEngine& Eng) {
+  Eng.registerCheck(new CFNumberCreateChecker());
+}
+
+void ento::registerCFNumberCreateChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterCFNumberCreateChecker);
+}
+
+static void RegisterCFRetainReleaseChecker(ExprEngine& Eng) {
+  Eng.registerCheck(new CFRetainReleaseChecker());
+}
+
+void ento::registerCFRetainReleaseChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterCFRetainReleaseChecker);
+}
+
+static void RegisterClassReleaseChecker(ExprEngine& Eng) {
+  Eng.registerCheck(new ClassReleaseChecker());
+}
+
+void ento::registerClassReleaseChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterClassReleaseChecker);
+}
diff --git a/lib/Checker/BasicObjCFoundationChecks.h b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.h
index 8fb057087086..92cfb1ae556c 100644
--- a/lib/Checker/BasicObjCFoundationChecks.h
+++ b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.h
@@ -13,25 +13,22 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_ANALYSIS_BASICOBJCFOUNDATIONCHECKS
-#define LLVM_CLANG_ANALYSIS_BASICOBJCFOUNDATIONCHECKS
+#ifndef LLVM_CLANG_GR_BASICOBJCFOUNDATIONCHECKS
+#define LLVM_CLANG_GR_BASICOBJCFOUNDATIONCHECKS
 
 namespace clang {
 
 class ASTContext;
-class BugReporter;
 class Decl;
-class GRExprEngine;
-class GRSimpleAPICheck;
 
-GRSimpleAPICheck *CreateBasicObjCFoundationChecks(ASTContext& Ctx,
-                                                  BugReporter& BR);
+namespace ento {
+
+class BugReporter;
+class ExprEngine;
 
-GRSimpleAPICheck *CreateAuditCFNumberCreate(ASTContext& Ctx,
-                                            BugReporter& BR);
+void RegisterNSErrorChecks(BugReporter& BR, ExprEngine &Eng, const Decl &D);
 
-void RegisterNSErrorChecks(BugReporter& BR, GRExprEngine &Eng, const Decl &D);
-void RegisterNSAutoreleasePoolChecks(GRExprEngine &Eng);
+} // end GR namespace
 
 } // end clang namespace
 
diff --git a/lib/Checker/BuiltinFunctionChecker.cpp b/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 057e474626f6..417b015ca39a 100644
--- a/lib/Checker/BuiltinFunctionChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -11,27 +11,28 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/PathSensitive/Checker.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 #include "clang/Basic/Builtins.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
 class BuiltinFunctionChecker : public Checker {
 public:
   static void *getTag() { static int tag = 0; return &tag; }
-  virtual bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
+  virtual bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
 };
 
 }
 
-void clang::RegisterBuiltinFunctionChecker(GRExprEngine &Eng) {
+void ento::RegisterBuiltinFunctionChecker(ExprEngine &Eng) {
   Eng.registerCheck(new BuiltinFunctionChecker());
 }
 
-bool BuiltinFunctionChecker::EvalCallExpr(CheckerContext &C,const CallExpr *CE){
+bool BuiltinFunctionChecker::evalCallExpr(CheckerContext &C,const CallExpr *CE){
   const GRState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
@@ -50,7 +51,7 @@ bool BuiltinFunctionChecker::EvalCallExpr(CheckerContext &C,const CallExpr *CE){
     // For __builtin_expect, just return the value of the subexpression.
     assert (CE->arg_begin() != CE->arg_end());
     SVal X = state->getSVal(*(CE->arg_begin()));
-    C.GenerateNode(state->BindExpr(CE, X));
+    C.generateNode(state->BindExpr(CE, X));
     return true;
   }
 
@@ -67,15 +68,13 @@ bool BuiltinFunctionChecker::EvalCallExpr(CheckerContext &C,const CallExpr *CE){
     DefinedOrUnknownSVal Size =
       cast<DefinedOrUnknownSVal>(state->getSVal(*(CE->arg_begin())));
 
-    ValueManager& ValMgr = C.getValueManager();
-    DefinedOrUnknownSVal Extent = R->getExtent(ValMgr);
+    SValBuilder& svalBuilder = C.getSValBuilder();
+    DefinedOrUnknownSVal Extent = R->getExtent(svalBuilder);
+    DefinedOrUnknownSVal extentMatchesSizeArg =
+      svalBuilder.evalEQ(state, Extent, Size);
+    state = state->assume(extentMatchesSizeArg, true);
 
-    SValuator& SVator = ValMgr.getSValuator();
-    DefinedOrUnknownSVal ExtentMatchesSizeArg =
-      SVator.EvalEQ(state, Extent, Size);
-    state = state->Assume(ExtentMatchesSizeArg, true);
-
-    C.GenerateNode(state->BindExpr(CE, loc::MemRegionVal(R)));
+    C.generateNode(state->BindExpr(CE, loc::MemRegionVal(R)));
     return true;
   }
   }
diff --git a/lib/Checker/CMakeLists.txt b/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 5b54f0d12d5d..e172a529d768 100644
--- a/lib/Checker/CMakeLists.txt
+++ b/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -1,74 +1,55 @@
-set(LLVM_NO_RTTI 1)
+set(LLVM_TARGET_DEFINITIONS Checkers.td)
+tablegen(Checkers.inc
+         -gen-clang-sa-checkers
+         -I ${CMAKE_CURRENT_SOURCE_DIR}/../../../include)
+add_custom_target(ClangSACheckers
+  DEPENDS Checkers.inc)
 
-add_clang_library(clangChecker
+set(LLVM_USED_LIBS clangBasic clangAST)
+
+add_clang_library(clangStaticAnalyzerCheckers
   AdjustedReturnValueChecker.cpp
-  AggExprVisitor.cpp
-  AnalysisConsumer.cpp
-  AnalysisManager.cpp
+  AnalyzerStatsChecker.cpp
   ArrayBoundChecker.cpp
+  ArrayBoundCheckerV2.cpp
   AttrNonNullChecker.cpp
-  BasicConstraintManager.cpp
   BasicObjCFoundationChecks.cpp
-  BasicStore.cpp
-  BasicValueFactory.cpp
-  BugReporter.cpp
-  BugReporterVisitors.cpp
   BuiltinFunctionChecker.cpp
-  CFRefCount.cpp
+  CStringChecker.cpp
   CallAndMessageChecker.cpp
   CastSizeChecker.cpp
   CastToStructChecker.cpp
-  CheckDeadStores.cpp
   CheckObjCDealloc.cpp
   CheckObjCInstMethSignature.cpp
   CheckSecuritySyntaxOnly.cpp
   CheckSizeofPointer.cpp
-  Checker.cpp
-  CheckerHelpers.cpp
-  CocoaConventions.cpp
-  CStringChecker.cpp
+  ChrootChecker.cpp
+  ClangSACheckerProvider.cpp
+  DeadStoresChecker.cpp
+  DebugCheckers.cpp
   DereferenceChecker.cpp
   DivZeroChecker.cpp
-  Environment.cpp
-  ExplodedGraph.cpp
+  ExprEngine.cpp
+  ExperimentalChecks.cpp
   FixedAddressChecker.cpp
-  FlatStore.cpp
-  FrontendActions.cpp
-  GRBlockCounter.cpp
-  GRCXXExprEngine.cpp
-  GRCoreEngine.cpp
-  GRExprEngine.cpp
-  GRExprEngineExperimentalChecks.cpp
-  GRState.cpp
-  HTMLDiagnostics.cpp
   IdempotentOperationChecker.cpp
   LLVMConventionsChecker.cpp
   MacOSXAPIChecker.cpp
   MallocChecker.cpp
-  ManagerRegistry.cpp
-  MemRegion.cpp
   NSAutoreleasePoolChecker.cpp
   NSErrorChecker.cpp
   NoReturnFunctionChecker.cpp
   OSAtomicChecker.cpp
+  ObjCAtSyncChecker.cpp
+  ObjCSelfInitChecker.cpp
   ObjCUnusedIVarsChecker.cpp
-  PathDiagnostic.cpp
-  PlistDiagnostics.cpp
   PointerArithChecker.cpp
   PointerSubChecker.cpp
   PthreadLockChecker.cpp
-  RangeConstraintManager.cpp
-  RegionStore.cpp
   ReturnPointerRangeChecker.cpp
   ReturnUndefChecker.cpp
-  SVals.cpp
-  SValuator.cpp
-  SimpleConstraintManager.cpp
-  SimpleSValuator.cpp
   StackAddrLeakChecker.cpp
-  Store.cpp
   StreamChecker.cpp
-  SymbolManager.cpp
   UndefBranchChecker.cpp
   UndefCapturedBlockVarChecker.cpp
   UndefResultChecker.cpp
@@ -77,8 +58,13 @@ add_clang_library(clangChecker
   UnixAPIChecker.cpp
   UnreachableCodeChecker.cpp
   VLASizeChecker.cpp
-  ValueManager.cpp
   )
 
-add_dependencies(clangChecker ClangAttrClasses ClangAttrList ClangDeclNodes
-                 ClangStmtNodes)
+add_dependencies(clangStaticAnalyzerCheckers
+  clangStaticAnalyzerCore
+  ClangAttrClasses
+  ClangAttrList
+  ClangDeclNodes
+  ClangStmtNodes
+  ClangSACheckers
+  )
diff --git a/lib/Checker/CStringChecker.cpp b/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 9ea572f90dfb..03f9047e960c 100644
--- a/lib/Checker/CStringChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -12,13 +12,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineExperimentalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class CStringChecker : public CheckerVisitor<CStringChecker> {
@@ -29,11 +31,11 @@ public:
   {}
   static void *getTag() { static int tag; return &tag; }
 
-  bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
+  bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
   void PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS);
   void MarkLiveSymbols(const GRState *state, SymbolReaper &SR);
-  void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SR);
-  bool WantsRegionChangeUpdate(const GRState *state);
+  void evalDeadSymbols(CheckerContext &C, SymbolReaper &SR);
+  bool wantsRegionChangeUpdate(const GRState *state);
 
   const GRState *EvalRegionChanges(const GRState *state,
                                    const MemRegion * const *Begin,
@@ -42,30 +44,30 @@ public:
 
   typedef void (CStringChecker::*FnCheck)(CheckerContext &, const CallExpr *);
 
-  void EvalMemcpy(CheckerContext &C, const CallExpr *CE);
-  void EvalMemmove(CheckerContext &C, const CallExpr *CE);
-  void EvalBcopy(CheckerContext &C, const CallExpr *CE);
-  void EvalCopyCommon(CheckerContext &C, const GRState *state,
+  void evalMemcpy(CheckerContext &C, const CallExpr *CE);
+  void evalMemmove(CheckerContext &C, const CallExpr *CE);
+  void evalBcopy(CheckerContext &C, const CallExpr *CE);
+  void evalCopyCommon(CheckerContext &C, const GRState *state,
                       const Expr *Size, const Expr *Source, const Expr *Dest,
                       bool Restricted = false);
 
-  void EvalMemcmp(CheckerContext &C, const CallExpr *CE);
+  void evalMemcmp(CheckerContext &C, const CallExpr *CE);
 
-  void EvalStrlen(CheckerContext &C, const CallExpr *CE);
+  void evalstrLength(CheckerContext &C, const CallExpr *CE);
 
-  void EvalStrcpy(CheckerContext &C, const CallExpr *CE);
-  void EvalStpcpy(CheckerContext &C, const CallExpr *CE);
-  void EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE, bool ReturnEnd);
+  void evalStrcpy(CheckerContext &C, const CallExpr *CE);
+  void evalStpcpy(CheckerContext &C, const CallExpr *CE);
+  void evalStrcpyCommon(CheckerContext &C, const CallExpr *CE, bool returnEnd);
 
   // Utility methods
   std::pair<const GRState*, const GRState*>
-  AssumeZero(CheckerContext &C, const GRState *state, SVal V, QualType Ty);
+  assumeZero(CheckerContext &C, const GRState *state, SVal V, QualType Ty);
 
-  const GRState *SetCStringLength(const GRState *state, const MemRegion *MR,
-                                  SVal StrLen);
-  SVal GetCStringLengthForRegion(CheckerContext &C, const GRState *&state,
+  const GRState *setCStringLength(const GRState *state, const MemRegion *MR,
+                                  SVal strLength);
+  SVal getCStringLengthForRegion(CheckerContext &C, const GRState *&state,
                                  const Expr *Ex, const MemRegion *MR);
-  SVal GetCStringLength(CheckerContext &C, const GRState *&state,
+  SVal getCStringLength(CheckerContext &C, const GRState *&state,
                         const Expr *Ex, SVal Buf);
 
   const GRState *InvalidateBuffer(CheckerContext &C, const GRState *state,
@@ -75,7 +77,7 @@ public:
                        const MemRegion *MR);
 
   // Re-usable checks
-  const GRState *CheckNonNull(CheckerContext &C, const GRState *state,
+  const GRState *checkNonNull(CheckerContext &C, const GRState *state,
                                const Expr *S, SVal l);
   const GRState *CheckLocation(CheckerContext &C, const GRState *state,
                                const Expr *S, SVal l,
@@ -88,7 +90,7 @@ public:
   const GRState *CheckOverlap(CheckerContext &C, const GRState *state,
                               const Expr *Size, const Expr *First,
                               const Expr *Second);
-  void EmitOverlapBug(CheckerContext &C, const GRState *state,
+  void emitOverlapBug(CheckerContext &C, const GRState *state,
                       const Stmt *First, const Stmt *Second);
 };
 
@@ -99,38 +101,40 @@ public:
 } //end anonymous namespace
 
 namespace clang {
+namespace ento {
   template <>
   struct GRStateTrait<CStringLength> 
     : public GRStatePartialTrait<CStringLength::EntryMap> {
     static void *GDMIndex() { return CStringChecker::getTag(); }
   };
 }
+}
 
-void clang::RegisterCStringChecker(GRExprEngine &Eng) {
+static void RegisterCStringChecker(ExprEngine &Eng) {
   Eng.registerCheck(new CStringChecker());
 }
 
+void ento::registerCStringChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterCStringChecker);
+}
+
 //===----------------------------------------------------------------------===//
 // Individual checks and utility methods.
 //===----------------------------------------------------------------------===//
 
 std::pair<const GRState*, const GRState*>
-CStringChecker::AssumeZero(CheckerContext &C, const GRState *state, SVal V,
+CStringChecker::assumeZero(CheckerContext &C, const GRState *state, SVal V,
                            QualType Ty) {
-  DefinedSVal *Val = dyn_cast<DefinedSVal>(&V);
-  if (!Val)
+  DefinedSVal *val = dyn_cast<DefinedSVal>(&V);
+  if (!val)
     return std::pair<const GRState*, const GRState *>(state, state);
 
-  ValueManager &ValMgr = C.getValueManager();
-  SValuator &SV = ValMgr.getSValuator();
-
-  DefinedOrUnknownSVal Zero = ValMgr.makeZeroVal(Ty);
-  DefinedOrUnknownSVal ValIsZero = SV.EvalEQ(state, *Val, Zero);
-
-  return state->Assume(ValIsZero);
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  DefinedOrUnknownSVal zero = svalBuilder.makeZeroVal(Ty);
+  return state->assume(svalBuilder.evalEQ(state, *val, zero));
 }
 
-const GRState *CStringChecker::CheckNonNull(CheckerContext &C,
+const GRState *CStringChecker::checkNonNull(CheckerContext &C,
                                             const GRState *state,
                                             const Expr *S, SVal l) {
   // If a previous check has failed, propagate the failure.
@@ -138,10 +142,10 @@ const GRState *CStringChecker::CheckNonNull(CheckerContext &C,
     return NULL;
 
   const GRState *stateNull, *stateNonNull;
-  llvm::tie(stateNull, stateNonNull) = AssumeZero(C, state, l, S->getType());
+  llvm::tie(stateNull, stateNonNull) = assumeZero(C, state, l, S->getType());
 
   if (stateNull && !stateNonNull) {
-    ExplodedNode *N = C.GenerateSink(stateNull);
+    ExplodedNode *N = C.generateSink(stateNull);
     if (!N)
       return NULL;
 
@@ -187,18 +191,18 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
     "CheckLocation should only be called with char* ElementRegions");
 
   // Get the size of the array.
-  const SubRegion *Super = cast<SubRegion>(ER->getSuperRegion());
-  ValueManager &ValMgr = C.getValueManager();
-  SVal Extent = ValMgr.convertToArrayIndex(Super->getExtent(ValMgr));
+  const SubRegion *superReg = cast<SubRegion>(ER->getSuperRegion());
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  SVal Extent = svalBuilder.convertToArrayIndex(superReg->getExtent(svalBuilder));
   DefinedOrUnknownSVal Size = cast<DefinedOrUnknownSVal>(Extent);
 
   // Get the index of the accessed element.
-  DefinedOrUnknownSVal &Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
+  DefinedOrUnknownSVal Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
 
-  const GRState *StInBound = state->AssumeInBound(Idx, Size, true);
-  const GRState *StOutBound = state->AssumeInBound(Idx, Size, false);
+  const GRState *StInBound = state->assumeInBound(Idx, Size, true);
+  const GRState *StOutBound = state->assumeInBound(Idx, Size, false);
   if (StOutBound && !StInBound) {
-    ExplodedNode *N = C.GenerateSink(StOutBound);
+    ExplodedNode *N = C.generateSink(StOutBound);
     if (!N)
       return NULL;
 
@@ -244,16 +248,15 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
   if (!state)
     return NULL;
 
-  ValueManager &VM = C.getValueManager();
-  SValuator &SV = VM.getSValuator();
+  SValBuilder &svalBuilder = C.getSValBuilder();
   ASTContext &Ctx = C.getASTContext();
 
-  QualType SizeTy = Size->getType();
+  QualType sizeTy = Size->getType();
   QualType PtrTy = Ctx.getPointerType(Ctx.CharTy);
 
   // Check that the first buffer is non-null.
   SVal BufVal = state->getSVal(FirstBuf);
-  state = CheckNonNull(C, state, FirstBuf, BufVal);
+  state = checkNonNull(C, state, FirstBuf, BufVal);
   if (!state)
     return NULL;
 
@@ -264,15 +267,15 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
     return state;
 
   // Compute the offset of the last element to be accessed: size-1.
-  NonLoc One = cast<NonLoc>(VM.makeIntVal(1, SizeTy));
-  NonLoc LastOffset = cast<NonLoc>(SV.EvalBinOpNN(state, BO_Sub,
-                                                  *Length, One, SizeTy));
+  NonLoc One = cast<NonLoc>(svalBuilder.makeIntVal(1, sizeTy));
+  NonLoc LastOffset = cast<NonLoc>(svalBuilder.evalBinOpNN(state, BO_Sub,
+                                                    *Length, One, sizeTy));
 
   // Check that the first buffer is sufficently long.
-  SVal BufStart = SV.EvalCast(BufVal, PtrTy, FirstBuf->getType());
+  SVal BufStart = svalBuilder.evalCast(BufVal, PtrTy, FirstBuf->getType());
   if (Loc *BufLoc = dyn_cast<Loc>(&BufStart)) {
-    SVal BufEnd = SV.EvalBinOpLN(state, BO_Add, *BufLoc,
-                                 LastOffset, PtrTy);
+    SVal BufEnd = svalBuilder.evalBinOpLN(state, BO_Add, *BufLoc,
+                                          LastOffset, PtrTy);
     state = CheckLocation(C, state, FirstBuf, BufEnd, FirstIsDestination);
 
     // If the buffer isn't large enough, abort.
@@ -283,14 +286,14 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
   // If there's a second buffer, check it as well.
   if (SecondBuf) {
     BufVal = state->getSVal(SecondBuf);
-    state = CheckNonNull(C, state, SecondBuf, BufVal);
+    state = checkNonNull(C, state, SecondBuf, BufVal);
     if (!state)
       return NULL;
 
-    BufStart = SV.EvalCast(BufVal, PtrTy, SecondBuf->getType());
+    BufStart = svalBuilder.evalCast(BufVal, PtrTy, SecondBuf->getType());
     if (Loc *BufLoc = dyn_cast<Loc>(&BufStart)) {
-      SVal BufEnd = SV.EvalBinOpLN(state, BO_Add, *BufLoc,
-                                   LastOffset, PtrTy);
+      SVal BufEnd = svalBuilder.evalBinOpLN(state, BO_Add, *BufLoc,
+                                            LastOffset, PtrTy);
       state = CheckLocation(C, state, SecondBuf, BufEnd);
     }
   }
@@ -312,56 +315,54 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
   if (!state)
     return NULL;
 
-  ValueManager &VM = state->getStateManager().getValueManager();
-  SValuator &SV = VM.getSValuator();
-  ASTContext &Ctx = VM.getContext();
   const GRState *stateTrue, *stateFalse;
 
   // Get the buffer values and make sure they're known locations.
-  SVal FirstVal = state->getSVal(First);
-  SVal SecondVal = state->getSVal(Second);
+  SVal firstVal = state->getSVal(First);
+  SVal secondVal = state->getSVal(Second);
 
-  Loc *FirstLoc = dyn_cast<Loc>(&FirstVal);
-  if (!FirstLoc)
+  Loc *firstLoc = dyn_cast<Loc>(&firstVal);
+  if (!firstLoc)
     return state;
 
-  Loc *SecondLoc = dyn_cast<Loc>(&SecondVal);
-  if (!SecondLoc)
+  Loc *secondLoc = dyn_cast<Loc>(&secondVal);
+  if (!secondLoc)
     return state;
 
   // Are the two values the same?
-  DefinedOrUnknownSVal EqualTest = SV.EvalEQ(state, *FirstLoc, *SecondLoc);
-  llvm::tie(stateTrue, stateFalse) = state->Assume(EqualTest);
+  SValBuilder &svalBuilder = C.getSValBuilder();  
+  llvm::tie(stateTrue, stateFalse) =
+    state->assume(svalBuilder.evalEQ(state, *firstLoc, *secondLoc));
 
   if (stateTrue && !stateFalse) {
     // If the values are known to be equal, that's automatically an overlap.
-    EmitOverlapBug(C, stateTrue, First, Second);
+    emitOverlapBug(C, stateTrue, First, Second);
     return NULL;
   }
 
-  // Assume the two expressions are not equal.
+  // assume the two expressions are not equal.
   assert(stateFalse);
   state = stateFalse;
 
   // Which value comes first?
-  QualType CmpTy = Ctx.IntTy;
-  SVal Reverse = SV.EvalBinOpLL(state, BO_GT,
-                                *FirstLoc, *SecondLoc, CmpTy);
-  DefinedOrUnknownSVal *ReverseTest = dyn_cast<DefinedOrUnknownSVal>(&Reverse);
-  if (!ReverseTest)
+  ASTContext &Ctx = svalBuilder.getContext();
+  QualType cmpTy = Ctx.IntTy;
+  SVal reverse = svalBuilder.evalBinOpLL(state, BO_GT,
+                                         *firstLoc, *secondLoc, cmpTy);
+  DefinedOrUnknownSVal *reverseTest = dyn_cast<DefinedOrUnknownSVal>(&reverse);
+  if (!reverseTest)
     return state;
 
-  llvm::tie(stateTrue, stateFalse) = state->Assume(*ReverseTest);
-
+  llvm::tie(stateTrue, stateFalse) = state->assume(*reverseTest);
   if (stateTrue) {
     if (stateFalse) {
       // If we don't know which one comes first, we can't perform this test.
       return state;
     } else {
-      // Switch the values so that FirstVal is before SecondVal.
-      Loc *tmpLoc = FirstLoc;
-      FirstLoc = SecondLoc;
-      SecondLoc = tmpLoc;
+      // Switch the values so that firstVal is before secondVal.
+      Loc *tmpLoc = firstLoc;
+      firstLoc = secondLoc;
+      secondLoc = tmpLoc;
 
       // Switch the Exprs as well, so that they still correspond.
       const Expr *tmpExpr = First;
@@ -379,41 +380,41 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
   // Convert the first buffer's start address to char*.
   // Bail out if the cast fails.
   QualType CharPtrTy = Ctx.getPointerType(Ctx.CharTy);
-  SVal FirstStart = SV.EvalCast(*FirstLoc, CharPtrTy, First->getType());
+  SVal FirstStart = svalBuilder.evalCast(*firstLoc, CharPtrTy, First->getType());
   Loc *FirstStartLoc = dyn_cast<Loc>(&FirstStart);
   if (!FirstStartLoc)
     return state;
 
   // Compute the end of the first buffer. Bail out if THAT fails.
-  SVal FirstEnd = SV.EvalBinOpLN(state, BO_Add,
+  SVal FirstEnd = svalBuilder.evalBinOpLN(state, BO_Add,
                                  *FirstStartLoc, *Length, CharPtrTy);
   Loc *FirstEndLoc = dyn_cast<Loc>(&FirstEnd);
   if (!FirstEndLoc)
     return state;
 
   // Is the end of the first buffer past the start of the second buffer?
-  SVal Overlap = SV.EvalBinOpLL(state, BO_GT,
-                                *FirstEndLoc, *SecondLoc, CmpTy);
+  SVal Overlap = svalBuilder.evalBinOpLL(state, BO_GT,
+                                *FirstEndLoc, *secondLoc, cmpTy);
   DefinedOrUnknownSVal *OverlapTest = dyn_cast<DefinedOrUnknownSVal>(&Overlap);
   if (!OverlapTest)
     return state;
 
-  llvm::tie(stateTrue, stateFalse) = state->Assume(*OverlapTest);
+  llvm::tie(stateTrue, stateFalse) = state->assume(*OverlapTest);
 
   if (stateTrue && !stateFalse) {
     // Overlap!
-    EmitOverlapBug(C, stateTrue, First, Second);
+    emitOverlapBug(C, stateTrue, First, Second);
     return NULL;
   }
 
-  // Assume the two expressions don't overlap.
+  // assume the two expressions don't overlap.
   assert(stateFalse);
   return stateFalse;
 }
 
-void CStringChecker::EmitOverlapBug(CheckerContext &C, const GRState *state,
+void CStringChecker::emitOverlapBug(CheckerContext &C, const GRState *state,
                                     const Stmt *First, const Stmt *Second) {
-  ExplodedNode *N = C.GenerateSink(state);
+  ExplodedNode *N = C.generateSink(state);
   if (!N)
     return;
 
@@ -430,11 +431,11 @@ void CStringChecker::EmitOverlapBug(CheckerContext &C, const GRState *state,
   C.EmitReport(report);
 }
 
-const GRState *CStringChecker::SetCStringLength(const GRState *state,
+const GRState *CStringChecker::setCStringLength(const GRState *state,
                                                 const MemRegion *MR,
-                                                SVal StrLen) {
-  assert(!StrLen.isUndef() && "Attempt to set an undefined string length");
-  if (StrLen.isUnknown())
+                                                SVal strLength) {
+  assert(!strLength.isUndef() && "Attempt to set an undefined string length");
+  if (strLength.isUnknown())
     return state;
 
   MR = MR->StripCasts();
@@ -450,7 +451,7 @@ const GRState *CStringChecker::SetCStringLength(const GRState *state,
   case MemRegion::VarRegionKind:
   case MemRegion::FieldRegionKind:
   case MemRegion::ObjCIvarRegionKind:
-    return state->set<CStringLength>(MR, StrLen);
+    return state->set<CStringLength>(MR, strLength);
 
   case MemRegion::ElementRegionKind:
     // FIXME: Handle element regions by upper-bounding the parent region's
@@ -466,7 +467,7 @@ const GRState *CStringChecker::SetCStringLength(const GRState *state,
   }
 }
 
-SVal CStringChecker::GetCStringLengthForRegion(CheckerContext &C,
+SVal CStringChecker::getCStringLengthForRegion(CheckerContext &C,
                                                const GRState *&state,
                                                const Expr *Ex,
                                                const MemRegion *MR) {
@@ -477,15 +478,14 @@ SVal CStringChecker::GetCStringLengthForRegion(CheckerContext &C,
   
   // Otherwise, get a new symbol and update the state.
   unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-  ValueManager &ValMgr = C.getValueManager();
-  QualType SizeTy = ValMgr.getContext().getSizeType();
-  SVal Strlen = ValMgr.getMetadataSymbolVal(getTag(), MR, Ex, SizeTy, Count);
-  
-  state = state->set<CStringLength>(MR, Strlen);
-  return Strlen;
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  QualType sizeTy = svalBuilder.getContext().getSizeType();
+  SVal strLength = svalBuilder.getMetadataSymbolVal(getTag(), MR, Ex, sizeTy, Count);
+  state = state->set<CStringLength>(MR, strLength);
+  return strLength;
 }
 
-SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
+SVal CStringChecker::getCStringLength(CheckerContext &C, const GRState *&state,
                                       const Expr *Ex, SVal Buf) {
   const MemRegion *MR = Buf.getAsRegion();
   if (!MR) {
@@ -493,7 +493,7 @@ SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
     // C string. In the context of locations, the only time we can issue such
     // a warning is for labels.
     if (loc::GotoLabel *Label = dyn_cast<loc::GotoLabel>(&Buf)) {
-      if (ExplodedNode *N = C.GenerateNode(state)) {
+      if (ExplodedNode *N = C.generateNode(state)) {
         if (!BT_NotCString)
           BT_NotCString = new BuiltinBug("API",
             "Argument is not a null-terminated string.");
@@ -501,7 +501,7 @@ SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
         llvm::SmallString<120> buf;
         llvm::raw_svector_ostream os(buf);
         os << "Argument to byte string function is the address of the label '"
-           << Label->getLabel()->getID()->getName()
+           << Label->getLabel()->getName()
            << "', which is not a null-terminated string";
 
         // Generate a report for this bug.
@@ -527,17 +527,17 @@ SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
   case MemRegion::StringRegionKind: {
     // Modifying the contents of string regions is undefined [C99 6.4.5p6],
     // so we can assume that the byte length is the correct C string length.
-    ValueManager &ValMgr = C.getValueManager();
-    QualType SizeTy = ValMgr.getContext().getSizeType();
-    const StringLiteral *Str = cast<StringRegion>(MR)->getStringLiteral();
-    return ValMgr.makeIntVal(Str->getByteLength(), SizeTy);
+    SValBuilder &svalBuilder = C.getSValBuilder();
+    QualType sizeTy = svalBuilder.getContext().getSizeType();
+    const StringLiteral *strLit = cast<StringRegion>(MR)->getStringLiteral();
+    return svalBuilder.makeIntVal(strLit->getByteLength(), sizeTy);
   }
   case MemRegion::SymbolicRegionKind:
   case MemRegion::AllocaRegionKind:
   case MemRegion::VarRegionKind:
   case MemRegion::FieldRegionKind:
   case MemRegion::ObjCIvarRegionKind:
-    return GetCStringLengthForRegion(C, state, Ex, MR);
+    return getCStringLengthForRegion(C, state, Ex, MR);
   case MemRegion::CompoundLiteralRegionKind:
     // FIXME: Can we track this? Is it necessary?
     return UnknownVal();
@@ -549,7 +549,7 @@ SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
     // Other regions (mostly non-data) can't have a reliable C string length.
     // In this case, an error is emitted and UndefinedVal is returned.
     // The caller should always be prepared to handle this case.
-    if (ExplodedNode *N = C.GenerateNode(state)) {
+    if (ExplodedNode *N = C.generateNode(state)) {
       if (!BT_NotCString)
         BT_NotCString = new BuiltinBug("API",
           "Argument is not a null-terminated string.");
@@ -598,7 +598,7 @@ const GRState *CStringChecker::InvalidateBuffer(CheckerContext &C,
 
     // Invalidate this region.
     unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-    return state->InvalidateRegion(R, E, Count, NULL);
+    return state->invalidateRegion(R, E, Count, NULL);
   }
 
   // If we have a non-region value by chance, just remove the binding.
@@ -629,8 +629,8 @@ bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
     os << "a block";
     return true;
   case MemRegion::CXXThisRegionKind:
-  case MemRegion::CXXObjectRegionKind:
-    os << "a C++ object of type " << TR->getValueType().getAsString();
+  case MemRegion::CXXTempObjectRegionKind:
+    os << "a C++ temp object of type " << TR->getValueType().getAsString();
     return true;
   case MemRegion::VarRegionKind:
     os << "a variable of type" << TR->getValueType().getAsString();
@@ -647,26 +647,26 @@ bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
 }
 
 //===----------------------------------------------------------------------===//
-// Evaluation of individual function calls.
+// evaluation of individual function calls.
 //===----------------------------------------------------------------------===//
 
-void CStringChecker::EvalCopyCommon(CheckerContext &C, const GRState *state,
+void CStringChecker::evalCopyCommon(CheckerContext &C, const GRState *state,
                                     const Expr *Size, const Expr *Dest,
                                     const Expr *Source, bool Restricted) {
   // See if the size argument is zero.
-  SVal SizeVal = state->getSVal(Size);
-  QualType SizeTy = Size->getType();
+  SVal sizeVal = state->getSVal(Size);
+  QualType sizeTy = Size->getType();
 
-  const GRState *StZeroSize, *StNonZeroSize;
-  llvm::tie(StZeroSize, StNonZeroSize) = AssumeZero(C, state, SizeVal, SizeTy);
+  const GRState *stateZeroSize, *stateNonZeroSize;
+  llvm::tie(stateZeroSize, stateNonZeroSize) = assumeZero(C, state, sizeVal, sizeTy);
 
   // If the size is zero, there won't be any actual memory access.
-  if (StZeroSize)
-    C.addTransition(StZeroSize);
+  if (stateZeroSize)
+    C.addTransition(stateZeroSize);
 
   // If the size can be nonzero, we have to check the other arguments.
-  if (StNonZeroSize) {
-    state = StNonZeroSize;
+  if (stateNonZeroSize) {
+    state = stateNonZeroSize;
     state = CheckBufferAccess(C, state, Size, Dest, Source,
                               /* FirstIsDst = */ true);
     if (Restricted)
@@ -685,58 +685,57 @@ void CStringChecker::EvalCopyCommon(CheckerContext &C, const GRState *state,
 }
 
 
-void CStringChecker::EvalMemcpy(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalMemcpy(CheckerContext &C, const CallExpr *CE) {
   // void *memcpy(void *restrict dst, const void *restrict src, size_t n);
   // The return value is the address of the destination buffer.
   const Expr *Dest = CE->getArg(0);
   const GRState *state = C.getState();
   state = state->BindExpr(CE, state->getSVal(Dest));
-  EvalCopyCommon(C, state, CE->getArg(2), Dest, CE->getArg(1), true);
+  evalCopyCommon(C, state, CE->getArg(2), Dest, CE->getArg(1), true);
 }
 
-void CStringChecker::EvalMemmove(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalMemmove(CheckerContext &C, const CallExpr *CE) {
   // void *memmove(void *dst, const void *src, size_t n);
   // The return value is the address of the destination buffer.
   const Expr *Dest = CE->getArg(0);
   const GRState *state = C.getState();
   state = state->BindExpr(CE, state->getSVal(Dest));
-  EvalCopyCommon(C, state, CE->getArg(2), Dest, CE->getArg(1));
+  evalCopyCommon(C, state, CE->getArg(2), Dest, CE->getArg(1));
 }
 
-void CStringChecker::EvalBcopy(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalBcopy(CheckerContext &C, const CallExpr *CE) {
   // void bcopy(const void *src, void *dst, size_t n);
-  EvalCopyCommon(C, C.getState(), CE->getArg(2), CE->getArg(1), CE->getArg(0));
+  evalCopyCommon(C, C.getState(), CE->getArg(2), CE->getArg(1), CE->getArg(0));
 }
 
-void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalMemcmp(CheckerContext &C, const CallExpr *CE) {
   // int memcmp(const void *s1, const void *s2, size_t n);
   const Expr *Left = CE->getArg(0);
   const Expr *Right = CE->getArg(1);
   const Expr *Size = CE->getArg(2);
 
   const GRState *state = C.getState();
-  ValueManager &ValMgr = C.getValueManager();
-  SValuator &SV = ValMgr.getSValuator();
+  SValBuilder &svalBuilder = C.getSValBuilder();
 
   // See if the size argument is zero.
-  SVal SizeVal = state->getSVal(Size);
-  QualType SizeTy = Size->getType();
+  SVal sizeVal = state->getSVal(Size);
+  QualType sizeTy = Size->getType();
 
-  const GRState *StZeroSize, *StNonZeroSize;
-  llvm::tie(StZeroSize, StNonZeroSize) = AssumeZero(C, state, SizeVal, SizeTy);
+  const GRState *stateZeroSize, *stateNonZeroSize;
+  llvm::tie(stateZeroSize, stateNonZeroSize) =
+    assumeZero(C, state, sizeVal, sizeTy);
 
   // If the size can be zero, the result will be 0 in that case, and we don't
   // have to check either of the buffers.
-  if (StZeroSize) {
-    state = StZeroSize;
-    state = state->BindExpr(CE, ValMgr.makeZeroVal(CE->getType()));
+  if (stateZeroSize) {
+    state = stateZeroSize;
+    state = state->BindExpr(CE, svalBuilder.makeZeroVal(CE->getType()));
     C.addTransition(state);
   }
 
   // If the size can be nonzero, we have to check the other arguments.
-  if (StNonZeroSize) {
-    state = StNonZeroSize;
-
+  if (stateNonZeroSize) {
+    state = stateNonZeroSize;
     // If we know the two buffers are the same, we know the result is 0.
     // First, get the two buffers' addresses. Another checker will have already
     // made sure they're not undefined.
@@ -744,9 +743,9 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
     DefinedOrUnknownSVal RV = cast<DefinedOrUnknownSVal>(state->getSVal(Right));
 
     // See if they are the same.
-    DefinedOrUnknownSVal SameBuf = SV.EvalEQ(state, LV, RV);
+    DefinedOrUnknownSVal SameBuf = svalBuilder.evalEQ(state, LV, RV);
     const GRState *StSameBuf, *StNotSameBuf;
-    llvm::tie(StSameBuf, StNotSameBuf) = state->Assume(SameBuf);
+    llvm::tie(StSameBuf, StNotSameBuf) = state->assume(SameBuf);
 
     // If the two arguments might be the same buffer, we know the result is zero,
     // and we only need to check one size.
@@ -754,7 +753,7 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
       state = StSameBuf;
       state = CheckBufferAccess(C, state, Size, Left);
       if (state) {
-        state = StSameBuf->BindExpr(CE, ValMgr.makeZeroVal(CE->getType()));
+        state = StSameBuf->BindExpr(CE, svalBuilder.makeZeroVal(CE->getType()));
         C.addTransition(state); 
       }
     }
@@ -767,7 +766,7 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
       if (state) {
         // The return value is the comparison result, which we don't know.
         unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-        SVal CmpV = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
+        SVal CmpV = svalBuilder.getConjuredSymbolVal(NULL, CE, Count);
         state = state->BindExpr(CE, CmpV);
         C.addTransition(state);
       }
@@ -775,94 +774,90 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
   }
 }
 
-void CStringChecker::EvalStrlen(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalstrLength(CheckerContext &C, const CallExpr *CE) {
   // size_t strlen(const char *s);
   const GRState *state = C.getState();
   const Expr *Arg = CE->getArg(0);
   SVal ArgVal = state->getSVal(Arg);
 
   // Check that the argument is non-null.
-  state = CheckNonNull(C, state, Arg, ArgVal);
+  state = checkNonNull(C, state, Arg, ArgVal);
 
   if (state) {
-    SVal StrLen = GetCStringLength(C, state, Arg, ArgVal);
+    SVal strLength = getCStringLength(C, state, Arg, ArgVal);
 
     // If the argument isn't a valid C string, there's no valid state to
     // transition to.
-    if (StrLen.isUndef())
+    if (strLength.isUndef())
       return;
 
-    // If GetCStringLength couldn't figure out the length, conjure a return
+    // If getCStringLength couldn't figure out the length, conjure a return
     // value, so it can be used in constraints, at least.
-    if (StrLen.isUnknown()) {
-      ValueManager &ValMgr = C.getValueManager();
+    if (strLength.isUnknown()) {
       unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-      StrLen = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
+      strLength = C.getSValBuilder().getConjuredSymbolVal(NULL, CE, Count);
     }
 
     // Bind the return value.
-    state = state->BindExpr(CE, StrLen);
+    state = state->BindExpr(CE, strLength);
     C.addTransition(state);
   }
 }
 
-void CStringChecker::EvalStrcpy(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalStrcpy(CheckerContext &C, const CallExpr *CE) {
   // char *strcpy(char *restrict dst, const char *restrict src);
-  EvalStrcpyCommon(C, CE, /* ReturnEnd = */ false);
+  evalStrcpyCommon(C, CE, /* returnEnd = */ false);
 }
 
-void CStringChecker::EvalStpcpy(CheckerContext &C, const CallExpr *CE) {
+void CStringChecker::evalStpcpy(CheckerContext &C, const CallExpr *CE) {
   // char *stpcpy(char *restrict dst, const char *restrict src);
-  EvalStrcpyCommon(C, CE, /* ReturnEnd = */ true);
+  evalStrcpyCommon(C, CE, /* returnEnd = */ true);
 }
 
-void CStringChecker::EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
-                                      bool ReturnEnd) {
+void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
+                                      bool returnEnd) {
   const GRState *state = C.getState();
 
   // Check that the destination is non-null
   const Expr *Dst = CE->getArg(0);
   SVal DstVal = state->getSVal(Dst);
 
-  state = CheckNonNull(C, state, Dst, DstVal);
+  state = checkNonNull(C, state, Dst, DstVal);
   if (!state)
     return;
 
   // Check that the source is non-null.
-  const Expr *Src = CE->getArg(1);
-  SVal SrcVal = state->getSVal(Src);
-
-  state = CheckNonNull(C, state, Src, SrcVal);
+  const Expr *srcExpr = CE->getArg(1);
+  SVal srcVal = state->getSVal(srcExpr);
+  state = checkNonNull(C, state, srcExpr, srcVal);
   if (!state)
     return;
 
   // Get the string length of the source.
-  SVal StrLen = GetCStringLength(C, state, Src, SrcVal);
+  SVal strLength = getCStringLength(C, state, srcExpr, srcVal);
 
   // If the source isn't a valid C string, give up.
-  if (StrLen.isUndef())
+  if (strLength.isUndef())
     return;
 
-  SVal Result = (ReturnEnd ? UnknownVal() : DstVal);
+  SVal Result = (returnEnd ? UnknownVal() : DstVal);
 
   // If the destination is a MemRegion, try to check for a buffer overflow and
   // record the new string length.
-  if (loc::MemRegionVal *DstRegVal = dyn_cast<loc::MemRegionVal>(&DstVal)) {
+  if (loc::MemRegionVal *dstRegVal = dyn_cast<loc::MemRegionVal>(&DstVal)) {
     // If the length is known, we can check for an overflow.
-    if (NonLoc *KnownStrLen = dyn_cast<NonLoc>(&StrLen)) {
-      SValuator &SV = C.getSValuator();
+    if (NonLoc *knownStrLength = dyn_cast<NonLoc>(&strLength)) {
+      SVal lastElement =
+        C.getSValBuilder().evalBinOpLN(state, BO_Add, *dstRegVal,
+                                       *knownStrLength, Dst->getType());
 
-      SVal LastElement = SV.EvalBinOpLN(state, BO_Add,
-                                        *DstRegVal, *KnownStrLen,
-                                        Dst->getType());
-
-      state = CheckLocation(C, state, Dst, LastElement, /* IsDst = */ true);
+      state = CheckLocation(C, state, Dst, lastElement, /* IsDst = */ true);
       if (!state)
         return;
 
       // If this is a stpcpy-style copy, the last element is the return value.
-      if (ReturnEnd)
-        Result = LastElement;
+      if (returnEnd)
+        Result = lastElement;
     }
 
     // Invalidate the destination. This must happen before we set the C string
@@ -871,18 +866,18 @@ void CStringChecker::EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
     // can use LazyCompoundVals to copy the source values into the destination.
     // This would probably remove any existing bindings past the end of the
     // string, but that's still an improvement over blank invalidation.
-    state = InvalidateBuffer(C, state, Dst, *DstRegVal);
+    state = InvalidateBuffer(C, state, Dst, *dstRegVal);
 
     // Set the C string length of the destination.
-    state = SetCStringLength(state, DstRegVal->getRegion(), StrLen);
+    state = setCStringLength(state, dstRegVal->getRegion(), strLength);
   }
 
   // If this is a stpcpy-style copy, but we were unable to check for a buffer
   // overflow, we still need a result. Conjure a return value.
-  if (ReturnEnd && Result.isUnknown()) {
-    ValueManager &ValMgr = C.getValueManager();
+  if (returnEnd && Result.isUnknown()) {
+    SValBuilder &svalBuilder = C.getSValBuilder();
     unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-    StrLen = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
+    strLength = svalBuilder.getConjuredSymbolVal(NULL, CE, Count);
   }
 
   // Set the return value.
@@ -894,7 +889,7 @@ void CStringChecker::EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
 // The driver method, and other Checker callbacks.
 //===----------------------------------------------------------------------===//
 
-bool CStringChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
+bool CStringChecker::evalCallExpr(CheckerContext &C, const CallExpr *CE) {
   // Get the callee.  All the functions we care about are C functions
   // with simple identifiers.
   const GRState *state = C.getState();
@@ -905,26 +900,29 @@ bool CStringChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
     return false;
 
   // Get the name of the callee. If it's a builtin, strip off the prefix.
-  llvm::StringRef Name = FD->getName();
+  IdentifierInfo *II = FD->getIdentifier();
+  if (!II)   // if no identifier, not a simple C function
+    return false;
+  llvm::StringRef Name = II->getName();
   if (Name.startswith("__builtin_"))
     Name = Name.substr(10);
 
-  FnCheck EvalFunction = llvm::StringSwitch<FnCheck>(Name)
-    .Cases("memcpy", "__memcpy_chk", &CStringChecker::EvalMemcpy)
-    .Cases("memcmp", "bcmp", &CStringChecker::EvalMemcmp)
-    .Cases("memmove", "__memmove_chk", &CStringChecker::EvalMemmove)
-    .Cases("strcpy", "__strcpy_chk", &CStringChecker::EvalStrcpy)
-    .Cases("stpcpy", "__stpcpy_chk", &CStringChecker::EvalStpcpy)
-    .Case("strlen", &CStringChecker::EvalStrlen)
-    .Case("bcopy", &CStringChecker::EvalBcopy)
+  FnCheck evalFunction = llvm::StringSwitch<FnCheck>(Name)
+    .Cases("memcpy", "__memcpy_chk", &CStringChecker::evalMemcpy)
+    .Cases("memcmp", "bcmp", &CStringChecker::evalMemcmp)
+    .Cases("memmove", "__memmove_chk", &CStringChecker::evalMemmove)
+    .Cases("strcpy", "__strcpy_chk", &CStringChecker::evalStrcpy)
+    .Cases("stpcpy", "__stpcpy_chk", &CStringChecker::evalStpcpy)
+    .Case("strlen", &CStringChecker::evalstrLength)
+    .Case("bcopy", &CStringChecker::evalBcopy)
     .Default(NULL);
 
   // If the callee isn't a string function, let another checker handle it.
-  if (!EvalFunction)
+  if (!evalFunction)
     return false;
 
   // Check and evaluate the call.
-  (this->*EvalFunction)(C, CE);
+  (this->*evalFunction)(C, CE);
   return true;
 }
 
@@ -955,16 +953,16 @@ void CStringChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
 
     SVal StrVal = state->getSVal(Init);
     assert(StrVal.isValid() && "Initializer string is unknown or undefined");
-    DefinedOrUnknownSVal StrLen
-      = cast<DefinedOrUnknownSVal>(GetCStringLength(C, state, Init, StrVal));
+    DefinedOrUnknownSVal strLength
+      = cast<DefinedOrUnknownSVal>(getCStringLength(C, state, Init, StrVal));
 
-    state = state->set<CStringLength>(MR, StrLen);
+    state = state->set<CStringLength>(MR, strLength);
   }
 
   C.addTransition(state);
 }
 
-bool CStringChecker::WantsRegionChangeUpdate(const GRState *state) {
+bool CStringChecker::wantsRegionChangeUpdate(const GRState *state) {
   CStringLength::EntryMap Entries = state->get<CStringLength>();
   return !Entries.isEmpty();
 }
@@ -1001,7 +999,7 @@ const GRState *CStringChecker::EvalRegionChanges(const GRState *state,
 
     // Is this entry for a super-region of a changed region?
     if (SuperRegions.count(MR)) {
-      Entries = F.Remove(Entries, MR);
+      Entries = F.remove(Entries, MR);
       continue;
     }
 
@@ -1010,7 +1008,7 @@ const GRState *CStringChecker::EvalRegionChanges(const GRState *state,
     while (const SubRegion *SR = dyn_cast<SubRegion>(Super)) {
       Super = SR->getSuperRegion();
       if (Invalidated.count(Super)) {
-        Entries = F.Remove(Entries, MR);
+        Entries = F.remove(Entries, MR);
         break;
       }
     }
@@ -1031,7 +1029,7 @@ void CStringChecker::MarkLiveSymbols(const GRState *state, SymbolReaper &SR) {
   }
 }
 
-void CStringChecker::EvalDeadSymbols(CheckerContext &C, SymbolReaper &SR) {
+void CStringChecker::evalDeadSymbols(CheckerContext &C, SymbolReaper &SR) {
   if (!SR.hasDeadSymbols())
     return;
 
@@ -1046,10 +1044,10 @@ void CStringChecker::EvalDeadSymbols(CheckerContext &C, SymbolReaper &SR) {
     SVal Len = I.getData();
     if (SymbolRef Sym = Len.getAsSymbol()) {
       if (SR.isDead(Sym))
-        Entries = F.Remove(Entries, I.getKey());
+        Entries = F.remove(Entries, I.getKey());
     }
   }
 
   state = state->set<CStringLength>(Entries);
-  C.GenerateNode(state);
+  C.generateNode(state);
 }
diff --git a/lib/Checker/CallAndMessageChecker.cpp b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 3c9ddce9f664..415900eb000e 100644
--- a/lib/Checker/CallAndMessageChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -12,13 +12,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
+#include "InternalChecks.h"
 #include "clang/AST/ParentMap.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class CallAndMessageChecker
@@ -40,19 +41,21 @@ public:
   }
 
   void PreVisitCallExpr(CheckerContext &C, const CallExpr *CE);
-  void PreVisitObjCMessageExpr(CheckerContext &C, const ObjCMessageExpr *ME);
-  bool EvalNilReceiver(CheckerContext &C, const ObjCMessageExpr *ME);
+  void preVisitObjCMessage(CheckerContext &C, ObjCMessage msg);
+  bool evalNilReceiver(CheckerContext &C, ObjCMessage msg);
 
 private:
-  bool PreVisitProcessArg(CheckerContext &C, const Expr *Ex,
-                          const char *BT_desc, BugType *&BT);
+  void PreVisitProcessArgs(CheckerContext &C, CallOrObjCMessage callOrMsg,
+                           const char *BT_desc, BugType *&BT);
+  bool PreVisitProcessArg(CheckerContext &C, SVal V, SourceRange argRange,
+                          const Expr *argEx, const char *BT_desc, BugType *&BT);
 
   void EmitBadCall(BugType *BT, CheckerContext &C, const CallExpr *CE);
-  void EmitNilReceiverBug(CheckerContext &C, const ObjCMessageExpr *ME,
+  void emitNilReceiverBug(CheckerContext &C, const ObjCMessage &msg,
                           ExplodedNode *N);
 
   void HandleNilReceiver(CheckerContext &C, const GRState *state,
-                         const ObjCMessageExpr *ME);
+                         ObjCMessage msg);
 
   void LazyInit_BT(const char *desc, BugType *&BT) {
     if (!BT)
@@ -61,13 +64,13 @@ private:
 };
 } // end anonymous namespace
 
-void clang::RegisterCallAndMessageChecker(GRExprEngine &Eng) {
+void ento::RegisterCallAndMessageChecker(ExprEngine &Eng) {
   Eng.registerCheck(new CallAndMessageChecker());
 }
 
 void CallAndMessageChecker::EmitBadCall(BugType *BT, CheckerContext &C,
                                         const CallExpr *CE) {
-  ExplodedNode *N = C.GenerateSink();
+  ExplodedNode *N = C.generateSink();
   if (!N)
     return;
 
@@ -77,21 +80,32 @@ void CallAndMessageChecker::EmitBadCall(BugType *BT, CheckerContext &C,
   C.EmitReport(R);
 }
 
+void CallAndMessageChecker::PreVisitProcessArgs(CheckerContext &C,
+                                                CallOrObjCMessage callOrMsg,
+                                                const char *BT_desc,
+                                                BugType *&BT) {
+  for (unsigned i = 0, e = callOrMsg.getNumArgs(); i != e; ++i)
+    if (PreVisitProcessArg(C, callOrMsg.getArgSVal(i),
+                           callOrMsg.getArgSourceRange(i), callOrMsg.getArg(i),
+                           BT_desc, BT))
+      return;
+}
+
 bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
-                                               const Expr *Ex,
+                                               SVal V, SourceRange argRange,
+                                               const Expr *argEx,
                                                const char *BT_desc,
                                                BugType *&BT) {
 
-  const SVal &V = C.getState()->getSVal(Ex);
-
   if (V.isUndef()) {
-    if (ExplodedNode *N = C.GenerateSink()) {
+    if (ExplodedNode *N = C.generateSink()) {
       LazyInit_BT(BT_desc, BT);
 
       // Generate a report for this bug.
       EnhancedBugReport *R = new EnhancedBugReport(*BT, BT->getName(), N);
-      R->addRange(Ex->getSourceRange());
-      R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Ex);
+      R->addRange(argRange);
+      if (argEx)
+        R->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, argEx);
       C.EmitReport(R);
     }
     return true;
@@ -143,11 +157,11 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
     const LazyCompoundValData *D = LV->getCVData();
     FindUninitializedField F(C.getASTContext(),
                              C.getState()->getStateManager().getStoreManager(),
-                             C.getValueManager().getRegionManager(),
+                             C.getSValBuilder().getRegionManager(),
                              D->getStore());
 
     if (F.Find(D->getRegion())) {
-      if (ExplodedNode *N = C.GenerateSink()) {
+      if (ExplodedNode *N = C.generateSink()) {
         LazyInit_BT(BT_desc, BT);
         llvm::SmallString<512> Str;
         llvm::raw_svector_ostream os(Str);
@@ -171,7 +185,7 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
 
         // Generate a report for this bug.
         EnhancedBugReport *R = new EnhancedBugReport(*BT, os.str(), N);
-        R->addRange(Ex->getSourceRange());
+        R->addRange(argRange);
 
         // FIXME: enhance track back for uninitialized value for arbitrary
         // memregions
@@ -193,7 +207,7 @@ void CallAndMessageChecker::PreVisitCallExpr(CheckerContext &C,
   if (L.isUndef()) {
     if (!BT_call_undef)
       BT_call_undef =
-        new BuiltinBug("Called function pointer is an undefined pointer value");
+        new BuiltinBug("Called function pointer is an uninitalized pointer value");
     EmitBadCall(BT_call_undef, C, CE);
     return;
   }
@@ -205,26 +219,23 @@ void CallAndMessageChecker::PreVisitCallExpr(CheckerContext &C,
     EmitBadCall(BT_call_null, C, CE);
   }
 
-  for (CallExpr::const_arg_iterator I = CE->arg_begin(), E = CE->arg_end();
-       I != E; ++I)
-    if (PreVisitProcessArg(C, *I,
-                           "Pass-by-value argument in function call is"
-                           " undefined", BT_call_arg))
-      return;
+  PreVisitProcessArgs(C, CallOrObjCMessage(CE, C.getState()),
+                      "Function call argument is an uninitialized value",
+                      BT_call_arg);
 }
 
-void CallAndMessageChecker::PreVisitObjCMessageExpr(CheckerContext &C,
-                                                    const ObjCMessageExpr *ME) {
+void CallAndMessageChecker::preVisitObjCMessage(CheckerContext &C,
+                                                ObjCMessage msg) {
 
   const GRState *state = C.getState();
 
   // FIXME: Handle 'super'?
-  if (const Expr *receiver = ME->getInstanceReceiver())
+  if (const Expr *receiver = msg.getInstanceReceiver())
     if (state->getSVal(receiver).isUndef()) {
-      if (ExplodedNode *N = C.GenerateSink()) {
+      if (ExplodedNode *N = C.generateSink()) {
         if (!BT_msg_undef)
           BT_msg_undef =
-            new BuiltinBug("Receiver in message expression is a garbage value");
+            new BuiltinBug("Receiver in message expression is an uninitialized value");
         EnhancedBugReport *R =
           new EnhancedBugReport(*BT_msg_undef, BT_msg_undef->getName(), N);
         R->addRange(receiver->getSourceRange());
@@ -235,23 +246,21 @@ void CallAndMessageChecker::PreVisitObjCMessageExpr(CheckerContext &C,
       return;
     }
 
+  const char *bugDesc = msg.isPropertySetter() ?
+                     "Argument for property setter is an uninitialized value"
+                   : "Argument in message expression is an uninitialized value";
   // Check for any arguments that are uninitialized/undefined.
-  for (ObjCMessageExpr::const_arg_iterator I = ME->arg_begin(),
-         E = ME->arg_end(); I != E; ++I)
-    if (PreVisitProcessArg(C, *I,
-                           "Pass-by-value argument in message expression "
-                           "is undefined", BT_msg_arg))
-        return;
+  PreVisitProcessArgs(C, CallOrObjCMessage(msg, state), bugDesc, BT_msg_arg);
 }
 
-bool CallAndMessageChecker::EvalNilReceiver(CheckerContext &C,
-                                            const ObjCMessageExpr *ME) {
-  HandleNilReceiver(C, C.getState(), ME);
+bool CallAndMessageChecker::evalNilReceiver(CheckerContext &C,
+                                            ObjCMessage msg) {
+  HandleNilReceiver(C, C.getState(), msg);
   return true; // Nil receiver is not handled elsewhere.
 }
 
-void CallAndMessageChecker::EmitNilReceiverBug(CheckerContext &C,
-                                               const ObjCMessageExpr *ME,
+void CallAndMessageChecker::emitNilReceiverBug(CheckerContext &C,
+                                               const ObjCMessage &msg,
                                                ExplodedNode *N) {
 
   if (!BT_msg_ret)
@@ -261,12 +270,12 @@ void CallAndMessageChecker::EmitNilReceiverBug(CheckerContext &C,
 
   llvm::SmallString<200> buf;
   llvm::raw_svector_ostream os(buf);
-  os << "The receiver of message '" << ME->getSelector().getAsString()
+  os << "The receiver of message '" << msg.getSelector().getAsString()
      << "' is nil and returns a value of type '"
-     << ME->getType().getAsString() << "' that will be garbage";
+     << msg.getType(C.getASTContext()).getAsString() << "' that will be garbage";
 
   EnhancedBugReport *report = new EnhancedBugReport(*BT_msg_ret, os.str(), N);
-  if (const Expr *receiver = ME->getInstanceReceiver()) {
+  if (const Expr *receiver = msg.getInstanceReceiver()) {
     report->addRange(receiver->getSourceRange());
     report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
                               receiver);
@@ -274,29 +283,31 @@ void CallAndMessageChecker::EmitNilReceiverBug(CheckerContext &C,
   C.EmitReport(report);
 }
 
-static bool SupportsNilWithFloatRet(const llvm::Triple &triple) {
+static bool supportsNilWithFloatRet(const llvm::Triple &triple) {
   return triple.getVendor() == llvm::Triple::Apple &&
-         triple.getDarwinMajorNumber() >= 9;
+         (triple.getDarwinMajorNumber() >= 9 || 
+          triple.getArch() == llvm::Triple::arm || 
+          triple.getArch() == llvm::Triple::thumb);
 }
 
 void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
                                               const GRState *state,
-                                              const ObjCMessageExpr *ME) {
+                                              ObjCMessage msg) {
+  ASTContext &Ctx = C.getASTContext();
 
   // Check the return type of the message expression.  A message to nil will
   // return different values depending on the return type and the architecture.
-  QualType RetTy = ME->getType();
+  QualType RetTy = msg.getType(Ctx);
 
-  ASTContext &Ctx = C.getASTContext();
   CanQualType CanRetTy = Ctx.getCanonicalType(RetTy);
 
   if (CanRetTy->isStructureOrClassType()) {
     // FIXME: At some point we shouldn't rely on isConsumedExpr(), but instead
     // have the "use of undefined value" be smarter about where the
     // undefined value came from.
-    if (C.getPredecessor()->getParentMap().isConsumedExpr(ME)) {
-      if (ExplodedNode* N = C.GenerateSink(state))
-        EmitNilReceiverBug(C, ME, N);
+    if (C.getPredecessor()->getParentMap().isConsumedExpr(msg.getOriginExpr())){
+      if (ExplodedNode* N = C.generateSink(state))
+        emitNilReceiverBug(C, msg, N);
       return;
     }
 
@@ -308,19 +319,20 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
 
   // Other cases: check if the return type is smaller than void*.
   if (CanRetTy != Ctx.VoidTy &&
-      C.getPredecessor()->getParentMap().isConsumedExpr(ME)) {
+      C.getPredecessor()->getParentMap().isConsumedExpr(msg.getOriginExpr())) {
     // Compute: sizeof(void *) and sizeof(return type)
     const uint64_t voidPtrSize = Ctx.getTypeSize(Ctx.VoidPtrTy);
     const uint64_t returnTypeSize = Ctx.getTypeSize(CanRetTy);
 
     if (voidPtrSize < returnTypeSize &&
-        !(SupportsNilWithFloatRet(Ctx.Target.getTriple()) &&
+        !(supportsNilWithFloatRet(Ctx.Target.getTriple()) &&
           (Ctx.FloatTy == CanRetTy ||
            Ctx.DoubleTy == CanRetTy ||
            Ctx.LongDoubleTy == CanRetTy ||
-           Ctx.LongLongTy == CanRetTy))) {
-      if (ExplodedNode* N = C.GenerateSink(state))
-        EmitNilReceiverBug(C, ME, N);
+           Ctx.LongLongTy == CanRetTy ||
+           Ctx.UnsignedLongLongTy == CanRetTy))) {
+      if (ExplodedNode* N = C.generateSink(state))
+        emitNilReceiverBug(C, msg, N);
       return;
     }
 
@@ -337,8 +349,8 @@ void CallAndMessageChecker::HandleNilReceiver(CheckerContext &C,
     // it most likely isn't nil.  We should assume the semantics
     // of this case unless we have *a lot* more knowledge.
     //
-    SVal V = C.getValueManager().makeZeroVal(ME->getType());
-    C.GenerateNode(state->BindExpr(ME, V));
+    SVal V = C.getSValBuilder().makeZeroVal(msg.getType(Ctx));
+    C.generateNode(state->BindExpr(msg.getOriginExpr(), V));
     return;
   }
 
diff --git a/lib/Checker/CastSizeChecker.cpp b/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index 6676fe5e7a36..518cf963bef4 100644
--- a/lib/Checker/CastSizeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -12,11 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/AST/CharUnits.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "GRExprEngineInternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class CastSizeChecker : public CheckerVisitor<CastSizeChecker> {
@@ -37,7 +38,7 @@ void CastSizeChecker::PreVisitCastExpr(CheckerContext &C, const CastExpr *CE) {
   const Expr *E = CE->getSubExpr();
   ASTContext &Ctx = C.getASTContext();
   QualType ToTy = Ctx.getCanonicalType(CE->getType());
-  PointerType *ToPTy = dyn_cast<PointerType>(ToTy.getTypePtr());
+  const PointerType *ToPTy = dyn_cast<PointerType>(ToTy.getTypePtr());
 
   if (!ToPTy)
     return;
@@ -57,28 +58,27 @@ void CastSizeChecker::PreVisitCastExpr(CheckerContext &C, const CastExpr *CE) {
   if (SR == 0)
     return;
 
-  ValueManager &ValMgr = C.getValueManager();
-  SVal Extent = SR->getExtent(ValMgr);
-
-  SValuator &SVator = ValMgr.getSValuator();
-  const llvm::APSInt *ExtentInt = SVator.getKnownValue(state, Extent);
-  if (!ExtentInt)
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  SVal extent = SR->getExtent(svalBuilder);
+  const llvm::APSInt *extentInt = svalBuilder.getKnownValue(state, extent);
+  if (!extentInt)
     return;
 
-  CharUnits RegionSize = CharUnits::fromQuantity(ExtentInt->getSExtValue());
-  CharUnits TypeSize = C.getASTContext().getTypeSizeInChars(ToPointeeTy);
-  
+  CharUnits regionSize = CharUnits::fromQuantity(extentInt->getSExtValue());
+  CharUnits typeSize = C.getASTContext().getTypeSizeInChars(ToPointeeTy);
+
   // Ignore void, and a few other un-sizeable types.
-  if (TypeSize.isZero())
+  if (typeSize.isZero())
     return;
-  
-  if (RegionSize % TypeSize != 0) {
-    if (ExplodedNode *N = C.GenerateSink()) {
+
+  if (regionSize % typeSize != 0) {
+    if (ExplodedNode *errorNode = C.generateSink()) {
       if (!BT)
         BT = new BuiltinBug("Cast region with wrong size.",
                             "Cast a region whose size is not a multiple of the"
                             " destination type size.");
-      RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(), N);
+      RangedBugReport *R = new RangedBugReport(*BT, BT->getDescription(),
+                                               errorNode);
       R->addRange(CE->getSourceRange());
       C.EmitReport(R);
     }
@@ -86,6 +86,6 @@ void CastSizeChecker::PreVisitCastExpr(CheckerContext &C, const CastExpr *CE) {
 }
 
 
-void clang::RegisterCastSizeChecker(GRExprEngine &Eng) {
+void ento::RegisterCastSizeChecker(ExprEngine &Eng) {
   Eng.registerCheck(new CastSizeChecker());
 }
diff --git a/lib/Checker/CastToStructChecker.cpp b/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
index eeaed970b533..8ec226abf6a9 100644
--- a/lib/Checker/CastToStructChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
@@ -13,11 +13,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "GRExprEngineInternalChecks.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class CastToStructChecker 
@@ -42,8 +44,8 @@ void CastToStructChecker::PreVisitCastExpr(CheckerContext &C,
   QualType OrigTy = Ctx.getCanonicalType(E->getType());
   QualType ToTy = Ctx.getCanonicalType(CE->getType());
 
-  PointerType *OrigPTy = dyn_cast<PointerType>(OrigTy.getTypePtr());
-  PointerType *ToPTy = dyn_cast<PointerType>(ToTy.getTypePtr());
+  const PointerType *OrigPTy = dyn_cast<PointerType>(OrigTy.getTypePtr());
+  const PointerType *ToPTy = dyn_cast<PointerType>(ToTy.getTypePtr());
 
   if (!ToPTy || !OrigPTy)
     return;
@@ -60,7 +62,7 @@ void CastToStructChecker::PreVisitCastExpr(CheckerContext &C,
 
   // Now the cast-to-type is struct pointer, the original type is not void*.
   if (!OrigPointeeTy->isRecordType()) {
-    if (ExplodedNode *N = C.GenerateNode()) {
+    if (ExplodedNode *N = C.generateNode()) {
       if (!BT)
         BT = new BuiltinBug("Cast from non-struct type to struct type",
                             "Casting a non-structure type to a structure type "
@@ -73,6 +75,10 @@ void CastToStructChecker::PreVisitCastExpr(CheckerContext &C,
   }
 }
 
-void clang::RegisterCastToStructChecker(GRExprEngine &Eng) {
+static void RegisterCastToStructChecker(ExprEngine &Eng) {
   Eng.registerCheck(new CastToStructChecker());
 }
+
+void ento::registerCastToStructChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterCastToStructChecker);
+}
diff --git a/lib/Checker/CheckObjCDealloc.cpp b/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index 11ddaca9d6fc..ad3bab6f7ea9 100644
--- a/lib/Checker/CheckObjCDealloc.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -13,9 +13,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/DeclObjC.h"
@@ -23,6 +25,7 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 static bool scan_dealloc(Stmt* S, Selector Dealloc) {
 
@@ -76,8 +79,8 @@ static bool scan_ivar_release(Stmt* S, ObjCIvarDecl* ID,
   if (BinaryOperator* BO = dyn_cast<BinaryOperator>(S))
     if (BO->isAssignmentOp())
       if (ObjCPropertyRefExpr* PRE =
-         dyn_cast<ObjCPropertyRefExpr>(BO->getLHS()->IgnoreParenCasts()))
-          if (PRE->getProperty() == PD)
+           dyn_cast<ObjCPropertyRefExpr>(BO->getLHS()->IgnoreParenCasts()))
+        if (PRE->isExplicitProperty() && PRE->getExplicitProperty() == PD)
             if (BO->getRHS()->isNullPointerConstant(Ctx, 
                                             Expr::NPC_ValueDependentIsNull)) {
               // This is only a 'release' if the property kind is not
@@ -93,7 +96,7 @@ static bool scan_ivar_release(Stmt* S, ObjCIvarDecl* ID,
   return false;
 }
 
-void clang::CheckObjCDealloc(const ObjCImplementationDecl* D,
+static void checkObjCDealloc(const ObjCImplementationDecl* D,
                              const LangOptions& LOpts, BugReporter& BR) {
 
   assert (LOpts.getGCMode() != LangOptions::GCOnly);
@@ -259,3 +262,23 @@ void clang::CheckObjCDealloc(const ObjCImplementationDecl* D,
   }
 }
 
+//===----------------------------------------------------------------------===//
+// ObjCDeallocChecker
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ObjCDeallocChecker : public CheckerV2<
+                                      check::ASTDecl<ObjCImplementationDecl> > {
+public:
+  void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager& mgr,
+                    BugReporter &BR) const {
+    if (mgr.getLangOptions().getGCMode() == LangOptions::GCOnly)
+      return;
+    checkObjCDealloc(cast<ObjCImplementationDecl>(D), mgr.getLangOptions(), BR);
+  }
+};
+}
+
+void ento::registerObjCDeallocChecker(CheckerManager &mgr) {
+  mgr.registerChecker<ObjCDeallocChecker>();
+}
diff --git a/lib/Checker/CheckObjCInstMethSignature.cpp b/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
index 76a092393bb4..369ba0bbb2ac 100644
--- a/lib/Checker/CheckObjCInstMethSignature.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckObjCInstMethSignature.cpp
@@ -13,9 +13,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/ASTContext.h"
@@ -24,6 +25,7 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 static bool AreTypesCompatible(QualType Derived, QualType Ancestor,
                                ASTContext& C) {
@@ -69,7 +71,7 @@ static void CompareReturnTypes(const ObjCMethodDecl *MethDerived,
   }
 }
 
-void clang::CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
+static void CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
                                        BugReporter& BR) {
 
   const ObjCInterfaceDecl* D = ID->getClassInterface();
@@ -117,3 +119,22 @@ void clang::CheckObjCInstMethSignature(const ObjCImplementationDecl* ID,
     C = C->getSuperClass();
   }
 }
+
+//===----------------------------------------------------------------------===//
+// ObjCMethSigsChecker
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ObjCMethSigsChecker : public CheckerV2<
+                                      check::ASTDecl<ObjCImplementationDecl> > {
+public:
+  void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager& mgr,
+                    BugReporter &BR) const {
+    CheckObjCInstMethSignature(D, BR);
+  }
+};
+}
+
+void ento::registerObjCMethSigsChecker(CheckerManager &mgr) {
+  mgr.registerChecker<ObjCMethSigsChecker>();
+}
diff --git a/lib/Checker/CheckSecuritySyntaxOnly.cpp b/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
index 9a2ac45fa2e1..185520c8df18 100644
--- a/lib/Checker/CheckSecuritySyntaxOnly.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
@@ -11,18 +11,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
 #include "clang/AST/StmtVisitor.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 static bool isArc4RandomAvailable(const ASTContext &Ctx) {
   const llvm::Triple &T = Ctx.Target.getTriple();
   return T.getVendor() == llvm::Triple::Apple ||
-         T.getOS() == llvm::Triple::FreeBSD;
+         T.getOS() == llvm::Triple::FreeBSD ||
+         T.getOS() == llvm::Triple::NetBSD ||
+         T.getOS() == llvm::Triple::OpenBSD ||
+         T.getOS() == llvm::Triple::DragonFly;
 }
 
 namespace {
@@ -187,8 +192,10 @@ void WalkAST::CheckLoopConditionForFloat(const ForStmt *FS) {
     return;
 
   // Are we comparing variables?
-  const DeclRefExpr *drLHS = dyn_cast<DeclRefExpr>(B->getLHS()->IgnoreParens());
-  const DeclRefExpr *drRHS = dyn_cast<DeclRefExpr>(B->getRHS()->IgnoreParens());
+  const DeclRefExpr *drLHS =
+    dyn_cast<DeclRefExpr>(B->getLHS()->IgnoreParenLValueCasts());
+  const DeclRefExpr *drRHS =
+    dyn_cast<DeclRefExpr>(B->getRHS()->IgnoreParenLValueCasts());
 
   // Does at least one of the variables have a floating point type?
   drLHS = drLHS && drLHS->getType()->isRealFloatingType() ? drLHS : NULL;
@@ -240,7 +247,8 @@ void WalkAST::CheckCall_gets(const CallExpr *CE, const FunctionDecl *FD) {
   if (FD->getIdentifier() != GetIdentifier(II_gets, "gets"))
     return;
 
-  const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FPT
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if (!FPT)
     return;
 
@@ -274,7 +282,8 @@ void WalkAST::CheckCall_getpw(const CallExpr *CE, const FunctionDecl *FD) {
   if (FD->getIdentifier() != GetIdentifier(II_getpw, "getpw"))
     return;
 
-  const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FPT
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if (!FPT)
     return;
 
@@ -312,7 +321,8 @@ void WalkAST::CheckCall_mktemp(const CallExpr *CE, const FunctionDecl *FD) {
   if (FD->getIdentifier() != GetIdentifier(II_mktemp, "mktemp"))
     return;
 
-  const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FPT
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if(!FPT)
     return;
 
@@ -367,7 +377,8 @@ void WalkAST::CheckCall_rand(const CallExpr *CE, const FunctionDecl *FD) {
   if (identifierid >= num_rands)
     return;
 
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FTP
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if (!FTP)
     return;
 
@@ -408,7 +419,8 @@ void WalkAST::CheckCall_random(const CallExpr *CE, const FunctionDecl *FD) {
   if (FD->getIdentifier() != GetIdentifier(II_random, "random"))
     return;
 
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FTP
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if (!FTP)
     return;
 
@@ -455,7 +467,8 @@ void WalkAST::CheckUncheckedReturnValue(CallExpr *CE) {
   if (identifierid >= num_setids)
     return;
 
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FD->getType());
+  const FunctionProtoType *FTP
+    = dyn_cast<FunctionProtoType>(FD->getType().IgnoreParens());
   if (!FTP)
     return;
 
@@ -485,10 +498,20 @@ void WalkAST::CheckUncheckedReturnValue(CallExpr *CE) {
 }
 
 //===----------------------------------------------------------------------===//
-// Entry point for check.
+// SecuritySyntaxChecker
 //===----------------------------------------------------------------------===//
 
-void clang::CheckSecuritySyntaxOnly(const Decl *D, BugReporter &BR) {
-  WalkAST walker(BR);
-  walker.Visit(D->getBody());
+namespace {
+class SecuritySyntaxChecker : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    WalkAST walker(BR);
+    walker.Visit(D->getBody());
+  }
+};
+}
+
+void ento::registerSecuritySyntaxChecker(CheckerManager &mgr) {
+  mgr.registerChecker<SecuritySyntaxChecker>();
 }
diff --git a/lib/Checker/CheckSizeofPointer.cpp b/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
index bbe494c99da5..d46ac8121b5a 100644
--- a/lib/Checker/CheckSizeofPointer.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckSizeofPointer.cpp
@@ -12,11 +12,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/BugReporter/BugReporter.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/StmtVisitor.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class WalkAST : public StmtVisitor<WalkAST> {
@@ -65,7 +67,21 @@ void WalkAST::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   }
 }
 
-void clang::CheckSizeofPointer(const Decl *D, BugReporter &BR) {
-  WalkAST walker(BR);
-  walker.Visit(D->getBody());
+//===----------------------------------------------------------------------===//
+// SizeofPointerChecker
+//===----------------------------------------------------------------------===//
+
+namespace {
+class SizeofPointerChecker : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    WalkAST walker(BR);
+    walker.Visit(D->getBody());
+  }
+};
+}
+
+void ento::registerSizeofPointerChecker(CheckerManager &mgr) {
+  mgr.registerChecker<SizeofPointerChecker>();
 }
diff --git a/lib/StaticAnalyzer/Checkers/Checkers.td b/lib/StaticAnalyzer/Checkers/Checkers.td
new file mode 100644
index 000000000000..1dc748666441
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/Checkers.td
@@ -0,0 +1,197 @@
+//===--- Checkers.td - Static Analyzer Checkers -===-----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+include "clang/StaticAnalyzer/Checkers/CheckerBase.td"
+
+//===----------------------------------------------------------------------===//
+// Packages.
+//===----------------------------------------------------------------------===//
+
+def Core : Package<"core">;
+def Cocoa : Package<"cocoa">;
+def Unix : Package<"unix">;
+def MacOSX : Package<"macosx">;
+
+def CoreExperimental : Package<"experimental">,
+  InPackage<Core>, Hidden;
+
+def CocoaExperimental : Package<"experimental">,
+  InPackage<Cocoa>, Hidden;
+
+def UnixExperimental : Package<"experimental">,
+  InPackage<Unix>, Hidden;
+
+def LLVM : Package<"llvm">;
+def Debug : Package<"debug">;
+
+//===----------------------------------------------------------------------===//
+// Groups.
+//===----------------------------------------------------------------------===//
+
+def AllExperimental : CheckerGroup<"all-experimental">,
+  Hidden;
+
+//===----------------------------------------------------------------------===//
+// Checkers.
+//===----------------------------------------------------------------------===//
+
+let ParentPackage = Cocoa in {
+
+def ObjCSelfInitChecker : Checker<"SelfInit">,
+  HelpText<"Check that 'self' is propely initialized inside an initializer method">,
+  DescFile<"ObjCSelfInitChecker.cpp">;
+
+def ObjCAtSyncChecker : Checker<"AtSync">,
+  HelpText<"Check for null pointers used as mutexes for @synchronized">,
+  DescFile<"ObjCAtSyncChecker.cpp">;
+
+def NilArgChecker : Checker<"NilArg">,
+  HelpText<"Check for prohibited nil arguments to ObjC method calls">,
+  DescFile<"BasicObjCFoundationChecks.cpp">;
+
+def ClassReleaseChecker : Checker<"ClassRelease">,
+  HelpText<"Check for sending 'retain', 'release', or 'autorelease' directly to a Class">,
+  DescFile<"BasicObjCFoundationChecks.cpp">;
+
+def NSAutoreleasePoolChecker : Checker<"NSAutoreleasePool">,
+  HelpText<"Warn for subpar uses of NSAutoreleasePool">,
+  DescFile<"NSAutoreleasePoolChecker.cpp">;
+
+def ObjCMethSigsChecker : Checker<"MethodSigs">,
+  HelpText<"Warn about Objective-C method signatures with type incompatibilities">,
+  DescFile<"CheckObjCInstMethSignature.cpp">;
+
+def ObjCUnusedIvarsChecker : Checker<"UnusedIvars">,
+  HelpText<"Warn about private ivars that are never used">,
+  DescFile<"ObjCUnusedIVarsChecker.cpp">;
+
+}
+
+def StackAddrLeakChecker : Checker<"StackAddrLeak">,
+  InPackage<Core>,
+  HelpText<"Check that addresses to stack memory are not leaked outside the function">,
+  DescFile<"StackAddrLeakChecker.cpp">;
+
+def DeadStoresChecker : Checker<"DeadStores">,
+  InPackage<Core>,
+  HelpText<"Check for stores to dead variables">,
+  DescFile<"DeadStoresChecker.cpp">;
+
+def UnixAPIChecker : Checker<"API">,
+  InPackage<Unix>,
+  HelpText<"Check calls to various UNIX/Posix functions">,
+  DescFile<"UnixAPIChecker.cpp">;
+
+def MacOSXAPIChecker : Checker<"API">,
+  InPackage<MacOSX>,
+  HelpText<"Check calls to various MacOSXAPIChecker">,
+  DescFile<"MacOSXAPIChecker.cpp">;
+
+def CFNumberCreateChecker : Checker<"CFNumber">,
+  InPackage<MacOSX>,
+  HelpText<"Check for CFNumberCreate">,
+  DescFile<"BasicObjCFoundationChecks.cpp">;
+
+def CFRetainReleaseChecker : Checker<"CFRetainRelease">,
+  InPackage<MacOSX>,
+  HelpText<"Check for null arguments to CFRetain/CFRelease">,
+  DescFile<"BasicObjCFoundationChecks.cpp">;
+
+def LLVMConventionsChecker : Checker<"Conventions">,
+  InPackage<LLVM>,
+  HelpText<"Check code for LLVM codebase conventions">,
+  DescFile<"LLVMConventionsChecker.cpp">;
+
+def LiveVariablesDumper : Checker<"DumpLiveVars">,
+  InPackage<Debug>,
+  HelpText<"Print results of live variable analysis">,
+  DescFile<"DebugCheckers.cpp">;
+
+def CFGViewer : Checker<"ViewCFG">,
+  InPackage<Debug>,
+  HelpText<"View Control-Flow Graphs using GraphViz">,
+  DescFile<"DebugCheckers.cpp">;
+
+def CFGDumper : Checker<"DumpCFG">,
+  InPackage<Debug>,
+  HelpText<"Display Control-Flow Graphs">,
+  DescFile<"DebugCheckers.cpp">;
+
+//===----------------------------------------------------------------------===//
+// Hidden experimental checkers.
+//===----------------------------------------------------------------------===//
+
+let Group = AllExperimental in {
+
+def CStringChecker : Checker<"CString">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check calls to functions in <string.h>">,
+  DescFile<"CStringChecker.cpp">;
+
+def UnreachableCodeChecker : Checker<"UnreachableCode">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check unreachable code">,
+  DescFile<"UnreachableCodeChecker.cpp">;
+
+def IdempotentOperationChecker : Checker<"IdempotentOps">,
+  InPackage<CoreExperimental>,
+  HelpText<"Warn about idempotent operations">,
+  DescFile<"IdempotentOperationChecker.cpp">;
+
+def CastToStructChecker : Checker<"CastToStruct">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check for cast from non-struct pointer to struct pointer">,
+  DescFile<"CastToStructChecker.cpp">;
+
+def FixedAddressChecker : Checker<"FixedAddr">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check for assignment of a fixed address to a pointer">,
+  DescFile<"FixedAddressChecker.cpp">;
+
+def PointerArithChecker : Checker<"PointerArithm">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check for pointer arithmetic on locations other than array elements">,
+  DescFile<"PointerArithChecker">;
+
+def PointerSubChecker : Checker<"PointerSub">,
+  InPackage<CoreExperimental>,
+  HelpText<"Check for pointer subtractions on two pointers pointing to different memory chunks">,
+  DescFile<"PointerSubChecker">;
+
+def SizeofPointerChecker : Checker<"SizeofPtr">,
+  InPackage<CoreExperimental>,
+  HelpText<"Warn about unintended use of sizeof() on pointer expressions">,
+  DescFile<"CheckSizeofPointer.cpp">;
+
+def SecuritySyntaxChecker : Checker<"SecuritySyntactic">,
+  InPackage<CoreExperimental>,
+  HelpText<"Perform quick security checks that require no data flow">,
+  DescFile<"CheckSecuritySyntaxOnly.cpp">;
+
+def ObjCDeallocChecker : Checker<"Dealloc">,
+  InPackage<CocoaExperimental>,
+  HelpText<"Warn about Objective-C classes that lack a correct implementation of -dealloc">,
+  DescFile<"CheckObjCDealloc.cpp">;
+
+def ChrootChecker : Checker<"Chroot">,
+  InPackage<UnixExperimental>,
+  HelpText<"Check improper use of chroot">,
+  DescFile<"ChrootChecker.cpp">;
+
+def PthreadLockChecker : Checker<"PthreadLock">,
+  InPackage<UnixExperimental>,
+  HelpText<"Simple lock -> unlock checker">,
+  DescFile<"PthreadLockChecker.cpp">;
+
+def StreamChecker : Checker<"Stream">,
+  InPackage<UnixExperimental>,
+  HelpText<"Check stream handling functions">,
+  DescFile<"StreamChecker.cpp">;
+
+}
diff --git a/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp b/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
new file mode 100644
index 000000000000..36e76d0d3ce2
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
@@ -0,0 +1,167 @@
+//===- Chrootchecker.cpp -------- Basic security checks ----------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines chroot checker, which checks improper use of chroot.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
+#include "llvm/ADT/ImmutableMap.h"
+using namespace clang;
+using namespace ento;
+
+namespace {
+
+// enum value that represent the jail state
+enum Kind { NO_CHROOT, ROOT_CHANGED, JAIL_ENTERED };
+  
+bool isRootChanged(intptr_t k) { return k == ROOT_CHANGED; }
+//bool isJailEntered(intptr_t k) { return k == JAIL_ENTERED; }
+
+// This checker checks improper use of chroot.
+// The state transition:
+// NO_CHROOT ---chroot(path)--> ROOT_CHANGED ---chdir(/) --> JAIL_ENTERED
+//                                  |                               |
+//         ROOT_CHANGED<--chdir(..)--      JAIL_ENTERED<--chdir(..)--
+//                                  |                               |
+//                      bug<--foo()--          JAIL_ENTERED<--foo()--
+class ChrootChecker : public CheckerVisitor<ChrootChecker> {
+  IdentifierInfo *II_chroot, *II_chdir;
+  // This bug refers to possibly break out of a chroot() jail.
+  BuiltinBug *BT_BreakJail;
+
+public:
+  ChrootChecker() : II_chroot(0), II_chdir(0), BT_BreakJail(0) {}
+  
+  static void *getTag() {
+    static int x;
+    return &x;
+  }
+  
+  virtual bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
+  virtual void PreVisitCallExpr(CheckerContext &C, const CallExpr *CE);
+
+private:
+  void Chroot(CheckerContext &C, const CallExpr *CE);
+  void Chdir(CheckerContext &C, const CallExpr *CE);
+};
+
+} // end anonymous namespace
+
+static void RegisterChrootChecker(ExprEngine &Eng) {
+  Eng.registerCheck(new ChrootChecker());
+}
+
+void ento::registerChrootChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterChrootChecker);
+}
+
+bool ChrootChecker::evalCallExpr(CheckerContext &C, const CallExpr *CE) {
+  const GRState *state = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = state->getSVal(Callee);
+  const FunctionDecl *FD = L.getAsFunctionDecl();
+  if (!FD)
+    return false;
+
+  ASTContext &Ctx = C.getASTContext();
+  if (!II_chroot)
+    II_chroot = &Ctx.Idents.get("chroot");
+  if (!II_chdir)
+    II_chdir = &Ctx.Idents.get("chdir");
+
+  if (FD->getIdentifier() == II_chroot) {
+    Chroot(C, CE);
+    return true;
+  }
+  if (FD->getIdentifier() == II_chdir) {
+    Chdir(C, CE);
+    return true;
+  }
+
+  return false;
+}
+
+void ChrootChecker::Chroot(CheckerContext &C, const CallExpr *CE) {
+  const GRState *state = C.getState();
+  GRStateManager &Mgr = state->getStateManager();
+  
+  // Once encouter a chroot(), set the enum value ROOT_CHANGED directly in 
+  // the GDM.
+  state = Mgr.addGDM(state, ChrootChecker::getTag(), (void*) ROOT_CHANGED);
+  C.addTransition(state);
+}
+
+void ChrootChecker::Chdir(CheckerContext &C, const CallExpr *CE) {
+  const GRState *state = C.getState();
+  GRStateManager &Mgr = state->getStateManager();
+
+  // If there are no jail state in the GDM, just return.
+  const void* k = state->FindGDM(ChrootChecker::getTag());
+  if (!k)
+    return;
+
+  // After chdir("/"), enter the jail, set the enum value JAIL_ENTERED.
+  const Expr *ArgExpr = CE->getArg(0);
+  SVal ArgVal = state->getSVal(ArgExpr);
+  
+  if (const MemRegion *R = ArgVal.getAsRegion()) {
+    R = R->StripCasts();
+    if (const StringRegion* StrRegion= dyn_cast<StringRegion>(R)) {
+      const StringLiteral* Str = StrRegion->getStringLiteral();
+      if (Str->getString() == "/")
+        state = Mgr.addGDM(state, ChrootChecker::getTag(),
+                           (void*) JAIL_ENTERED);
+    }
+  }
+
+  C.addTransition(state);
+}
+
+// Check the jail state before any function call except chroot and chdir().
+void ChrootChecker::PreVisitCallExpr(CheckerContext &C, const CallExpr *CE) {
+  const GRState *state = C.getState();
+  const Expr *Callee = CE->getCallee();
+  SVal L = state->getSVal(Callee);
+  const FunctionDecl *FD = L.getAsFunctionDecl();
+  if (!FD)
+    return;
+
+  ASTContext &Ctx = C.getASTContext();
+  if (!II_chroot)
+    II_chroot = &Ctx.Idents.get("chroot");
+  if (!II_chdir)
+    II_chdir = &Ctx.Idents.get("chdir");
+
+  // Ingnore chroot and chdir.
+  if (FD->getIdentifier() == II_chroot || FD->getIdentifier() == II_chdir)
+    return;
+  
+  // If jail state is ROOT_CHANGED, generate BugReport.
+  void* const* k = state->FindGDM(ChrootChecker::getTag());
+  if (k)
+    if (isRootChanged((intptr_t) *k))
+      if (ExplodedNode *N = C.generateNode()) {
+        if (!BT_BreakJail)
+          BT_BreakJail = new BuiltinBug("Break out of jail",
+                                        "No call of chdir(\"/\") immediately "
+                                        "after chroot");
+        BugReport *R = new BugReport(*BT_BreakJail, 
+                                     BT_BreakJail->getDescription(), N);
+        C.EmitReport(R);
+      }
+  
+  return;
+}
diff --git a/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp b/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp
new file mode 100644
index 000000000000..94f200f3e892
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.cpp
@@ -0,0 +1,137 @@
+//===--- ClangSACheckerProvider.cpp - Clang SA Checkers Provider ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the CheckerProvider for the checkers defined in
+// libclangStaticAnalyzerCheckers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckerProvider.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
+#include "llvm/ADT/DenseSet.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+
+/// \brief Provider for all the checkers in libclangStaticAnalyzerCheckers.
+class ClangSACheckerProvider : public CheckerProvider {
+public:
+  virtual void registerCheckers(CheckerManager &checkerMgr,
+                              CheckerOptInfo *checkOpts, unsigned numCheckOpts);
+};
+
+}
+
+CheckerProvider *ento::createClangSACheckerProvider() {
+  return new ClangSACheckerProvider();
+}
+
+namespace {
+
+struct StaticCheckerInfoRec {
+  const char *FullName;
+  void (*RegFunc)(CheckerManager &mgr);
+  bool Hidden;
+};
+
+} // end anonymous namespace.
+
+static const StaticCheckerInfoRec StaticCheckerInfo[] = {
+#define GET_CHECKERS
+#define CHECKER(FULLNAME,CLASS,DESCFILE,HELPTEXT,HIDDEN)    \
+  { FULLNAME, register##CLASS, HIDDEN },
+#include "Checkers.inc"
+  { 0, 0, 0}
+#undef CHECKER
+#undef GET_CHECKERS
+};
+
+namespace {
+
+struct CheckNameOption {
+  const char  *Name;
+  const short *Members;
+  const short *SubGroups;
+  bool Hidden;
+};
+
+} // end anonymous namespace.
+
+#define GET_MEMBER_ARRAYS
+#include "Checkers.inc"
+#undef GET_MEMBER_ARRAYS
+
+// The table of check name options, sorted by name for fast binary lookup.
+static const CheckNameOption CheckNameTable[] = {
+#define GET_CHECKNAME_TABLE
+#include "Checkers.inc"
+#undef GET_CHECKNAME_TABLE
+};
+static const size_t
+        CheckNameTableSize = sizeof(CheckNameTable) / sizeof(CheckNameTable[0]);
+
+static bool CheckNameOptionCompare(const CheckNameOption &LHS,
+                                   const CheckNameOption &RHS) {
+  return strcmp(LHS.Name, RHS.Name) < 0;
+}
+
+static void collectCheckers(const CheckNameOption *checkName,
+                            bool enable,
+                         llvm::DenseSet<const StaticCheckerInfoRec *> &checkers,
+                            bool collectHidden) {
+  if (checkName->Hidden && !collectHidden)
+    return;
+
+  if (const short *member = checkName->Members) {
+    if (enable) {
+      if (collectHidden || !StaticCheckerInfo[*member].Hidden)
+        checkers.insert(&StaticCheckerInfo[*member]);
+    } else {
+      for (; *member != -1; ++member)
+        checkers.erase(&StaticCheckerInfo[*member]);
+    }
+  }
+
+  // Enable/disable all subgroups along with this one.
+  if (const short *subGroups = checkName->SubGroups) {
+    for (; *subGroups != -1; ++subGroups)
+      collectCheckers(&CheckNameTable[*subGroups], enable, checkers,
+                      collectHidden && checkName->Hidden);
+  }
+}
+
+static void collectCheckers(CheckerOptInfo &opt,
+                       llvm::DenseSet<const StaticCheckerInfoRec *> &checkers) {
+  const char *optName = opt.getName();
+  CheckNameOption key = { optName, 0, 0, false };
+  const CheckNameOption *found =
+  std::lower_bound(CheckNameTable, CheckNameTable + CheckNameTableSize, key,
+                   CheckNameOptionCompare);
+  if (found == CheckNameTable + CheckNameTableSize ||
+      strcmp(found->Name, optName) != 0)
+    return;  // Check name not found.
+
+  opt.claim();
+  collectCheckers(found, opt.isEnabled(), checkers, /*collectHidden=*/true);
+}
+
+void ClangSACheckerProvider::registerCheckers(CheckerManager &checkerMgr,
+                             CheckerOptInfo *checkOpts, unsigned numCheckOpts) {
+  llvm::DenseSet<const StaticCheckerInfoRec *> enabledCheckers;
+  for (unsigned i = 0; i != numCheckOpts; ++i)
+    collectCheckers(checkOpts[i], enabledCheckers);
+  for (llvm::DenseSet<const StaticCheckerInfoRec *>::iterator
+         I = enabledCheckers.begin(), E = enabledCheckers.end(); I != E; ++I) {
+    (*I)->RegFunc(checkerMgr);
+  }
+}
diff --git a/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h b/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h
new file mode 100644
index 000000000000..f6c80119fdc0
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ClangSACheckerProvider.h
@@ -0,0 +1,29 @@
+//===--- ClangSACheckerProvider.h - Clang SA Checkers Provider --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the entry point for creating the provider for the checkers defined
+// in libclangStaticAnalyzerCheckers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SA_CHECKERS_CLANGSACHECKERPROVIDER_H
+#define LLVM_CLANG_SA_CHECKERS_CLANGSACHECKERPROVIDER_H
+
+namespace clang {
+
+namespace ento {
+  class CheckerProvider;
+
+CheckerProvider *createClangSACheckerProvider();
+
+} // end ento namespace
+
+} // end clang namespace
+
+#endif
diff --git a/lib/StaticAnalyzer/Checkers/ClangSACheckers.h b/lib/StaticAnalyzer/Checkers/ClangSACheckers.h
new file mode 100644
index 000000000000..73239f55b4e4
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ClangSACheckers.h
@@ -0,0 +1,34 @@
+//===--- ClangSACheckers.h - Registration functions for Checkers *- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Declares the registation functions for the checkers defined in
+// libclangStaticAnalyzerCheckers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SA_LIB_CHECKERS_CLANGSACHECKERS_H
+#define LLVM_CLANG_SA_LIB_CHECKERS_CLANGSACHECKERS_H
+
+namespace clang {
+
+namespace ento {
+class CheckerManager;
+
+#define GET_CHECKERS
+#define CHECKER(FULLNAME,CLASS,CXXFILE,HELPTEXT,HIDDEN)    \
+  void register##CLASS(CheckerManager &mgr);
+#include "Checkers.inc"
+#undef CHECKER
+#undef GET_CHECKERS
+
+} // end ento namespace
+
+} // end clang namespace
+
+#endif
diff --git a/lib/Checker/CheckDeadStores.cpp b/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index 38961000d3fd..3b39372725a4 100644
--- a/lib/Checker/CheckDeadStores.cpp
+++ b/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -1,4 +1,4 @@
-//==- DeadStores.cpp - Check for stores to dead variables --------*- C++ -*-==//
+//==- DeadStoresChecker.cpp - Check for stores to dead variables -*- C++ -*-==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,11 +12,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/Checkers/LocalCheckers.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/Visitors/CFGRecStmtVisitor.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/AST/ASTContext.h"
@@ -24,29 +26,84 @@
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
+// FIXME: Eventually migrate into its own file, and have it managed by
+// AnalysisManager.
+class ReachableCode {
+  const CFG &cfg;
+  llvm::BitVector reachable;
+public:
+  ReachableCode(const CFG &cfg)
+    : cfg(cfg), reachable(cfg.getNumBlockIDs(), false) {}
+  
+  void computeReachableBlocks();
+  
+  bool isReachable(const CFGBlock *block) const {
+    return reachable[block->getBlockID()];
+  }
+};
+}
+
+void ReachableCode::computeReachableBlocks() {
+  if (!cfg.getNumBlockIDs())
+    return;
+  
+  llvm::SmallVector<const CFGBlock*, 10> worklist;
+  worklist.push_back(&cfg.getEntry());
+  
+  while (!worklist.empty()) {
+    const CFGBlock *block = worklist.back();
+    worklist.pop_back();
+    llvm::BitVector::reference isReachable = reachable[block->getBlockID()];
+    if (isReachable)
+      continue;
+    isReachable = true;
+    for (CFGBlock::const_succ_iterator i = block->succ_begin(),
+                                       e = block->succ_end(); i != e; ++i)
+      if (const CFGBlock *succ = *i)
+        worklist.push_back(succ);
+  }
+}
+
+namespace {
 class DeadStoreObs : public LiveVariables::ObserverTy {
+  const CFG &cfg;
   ASTContext &Ctx;
   BugReporter& BR;
   ParentMap& Parents;
   llvm::SmallPtrSet<VarDecl*, 20> Escaped;
+  llvm::OwningPtr<ReachableCode> reachableCode;
+  const CFGBlock *currentBlock;
 
   enum DeadStoreKind { Standard, Enclosing, DeadIncrement, DeadInit };
 
 public:
-  DeadStoreObs(ASTContext &ctx, BugReporter& br, ParentMap& parents,
+  DeadStoreObs(const CFG &cfg, ASTContext &ctx,
+               BugReporter& br, ParentMap& parents,
                llvm::SmallPtrSet<VarDecl*, 20> &escaped)
-    : Ctx(ctx), BR(br), Parents(parents), Escaped(escaped) {}
+    : cfg(cfg), Ctx(ctx), BR(br), Parents(parents),
+      Escaped(escaped), currentBlock(0) {}
 
   virtual ~DeadStoreObs() {}
 
   void Report(VarDecl* V, DeadStoreKind dsk, SourceLocation L, SourceRange R) {
     if (Escaped.count(V))
       return;
+    
+    // Compute reachable blocks within the CFG for trivial cases
+    // where a bogus dead store can be reported because itself is unreachable.
+    if (!reachableCode.get()) {
+      reachableCode.reset(new ReachableCode(cfg));
+      reachableCode->computeReachableBlocks();
+    }
+    
+    if (!reachableCode->isReachable(currentBlock))
+      return;
 
-    std::string name = V->getNameAsString();
+    const std::string &name = V->getNameAsString();
 
     const char* BugType = 0;
     std::string msg;
@@ -69,11 +126,10 @@ public:
         break;
 
       case Enclosing:
-        BugType = "Dead nested assignment";
-        msg = "Although the value stored to '" + name +
-          "' is used in the enclosing expression, the value is never actually"
-          " read from '" + name + "'";
-        break;
+        // Don't report issues in this case, e.g.: "if (x = foo())",
+        // where 'x' is unused later.  We have yet to see a case where 
+        // this is a real bug.
+        return;
     }
 
     BR.EmitBasicReport(BugType, "Dead store", msg, L, R);
@@ -127,14 +183,18 @@ public:
     return false;
   }
 
-  virtual void ObserveStmt(Stmt* S,
+  virtual void ObserveStmt(Stmt* S, const CFGBlock *block,
                            const LiveVariables::AnalysisDataTy& AD,
                            const LiveVariables::ValTy& Live) {
 
+    currentBlock = block;
+    
     // Skip statements in macros.
     if (S->getLocStart().isMacroID())
       return;
 
+    // Only cover dead stores from regular assignments.  ++/-- dead stores
+    // have never flagged a real bug.
     if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
       if (!B->isAssignmentOp()) return; // Skip non-assignments.
 
@@ -165,14 +225,11 @@ public:
         }
     }
     else if (UnaryOperator* U = dyn_cast<UnaryOperator>(S)) {
-      if (!U->isIncrementOp())
+      if (!U->isIncrementOp() || U->isPrefix())
         return;
 
-      // Handle: ++x within a subexpression.  The solution is not warn
-      //  about preincrements to dead variables when the preincrement occurs
-      //  as a subexpression.  This can lead to false negatives, e.g. "(++x);"
-      //  A generalized dead code checker should find such issues.
-      if (U->isPrefix() && Parents.isConsumedExpr(U))
+      Stmt *parent = Parents.getParentIgnoreParenCasts(U);
+      if (!parent || !isa<ReturnStmt>(parent))
         return;
 
       Expr *Ex = U->getSubExpr()->IgnoreParenCasts();
@@ -203,7 +260,7 @@ public:
             if (isa<CXXConstructExpr>(E))
               return;
 
-            if (isa<CXXExprWithTemporaries>(E))
+            if (isa<ExprWithCleanups>(E))
               return;
             
             // A dead initialization is a variable that is dead after it
@@ -280,10 +337,27 @@ public:
 } // end anonymous namespace
 
 
-void clang::CheckDeadStores(CFG &cfg, LiveVariables &L, ParentMap &pmap, 
-                            BugReporter& BR) {
-  FindEscaped FS(&cfg);
-  FS.getCFG().VisitBlockStmts(FS);
-  DeadStoreObs A(BR.getContext(), BR, pmap, FS.Escaped);
-  L.runOnAllBlocks(cfg, &A);
+//===----------------------------------------------------------------------===//
+// DeadStoresChecker
+//===----------------------------------------------------------------------===//
+
+namespace {
+class DeadStoresChecker : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    if (LiveVariables *L = mgr.getLiveVariables(D)) {
+      CFG &cfg = *mgr.getCFG(D);
+      ParentMap &pmap = mgr.getParentMap(D);
+      FindEscaped FS(&cfg);
+      FS.getCFG().VisitBlockStmts(FS);
+      DeadStoreObs A(cfg, BR.getContext(), BR, pmap, FS.Escaped);
+      L->runOnAllBlocks(cfg, &A);
+    }
+  }
+};
+}
+
+void ento::registerDeadStoresChecker(CheckerManager &mgr) {
+  mgr.registerChecker<DeadStoresChecker>();
 }
diff --git a/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp b/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
new file mode 100644
index 000000000000..091d99b1352c
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
@@ -0,0 +1,80 @@
+//==- DebugCheckers.cpp - Debugging Checkers ---------------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines a checkers that display debugging information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/Analysis/Analyses/LiveVariables.h"
+
+using namespace clang;
+using namespace ento;
+
+//===----------------------------------------------------------------------===//
+// LiveVariablesDumper
+//===----------------------------------------------------------------------===//
+
+namespace {
+class LiveVariablesDumper : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    if (LiveVariables* L = mgr.getLiveVariables(D)) {
+      L->dumpBlockLiveness(mgr.getSourceManager());
+    }
+  }
+};
+}
+
+void ento::registerLiveVariablesDumper(CheckerManager &mgr) {
+  mgr.registerChecker<LiveVariablesDumper>();
+}
+
+//===----------------------------------------------------------------------===//
+// CFGViewer
+//===----------------------------------------------------------------------===//
+
+namespace {
+class CFGViewer : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    if (CFG *cfg = mgr.getCFG(D)) {
+      cfg->viewCFG(mgr.getLangOptions());
+    }
+  }
+};
+}
+
+void ento::registerCFGViewer(CheckerManager &mgr) {
+  mgr.registerChecker<CFGViewer>();
+}
+
+//===----------------------------------------------------------------------===//
+// CFGDumper
+//===----------------------------------------------------------------------===//
+
+namespace {
+class CFGDumper : public CheckerV2<check::ASTCodeBody> {
+public:
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    if (CFG *cfg = mgr.getCFG(D)) {
+      cfg->dump(mgr.getLangOptions());
+    }
+  }
+};
+}
+
+void ento::registerCFGDumper(CheckerManager &mgr) {
+  mgr.registerChecker<CFGDumper>();
+}
diff --git a/lib/Checker/DereferenceChecker.cpp b/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index af74c79558d5..606ac4ab4e4e 100644
--- a/lib/Checker/DereferenceChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -7,18 +7,19 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines NullDerefChecker, a builtin check in GRExprEngine that performs
+// This defines NullDerefChecker, a builtin check in ExprEngine that performs
 // checks for null pointers at loads and stores.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/Checkers/DereferenceChecker.h"
-#include "clang/Checker/PathSensitive/Checker.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Checkers/DereferenceChecker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class DereferenceChecker : public Checker {
@@ -28,7 +29,8 @@ class DereferenceChecker : public Checker {
 public:
   DereferenceChecker() : BT_null(0), BT_undef(0) {}
   static void *getTag() { static int tag = 0; return &tag; }
-  void VisitLocation(CheckerContext &C, const Stmt *S, SVal location);
+  void visitLocation(CheckerContext &C, const Stmt *S, SVal location,
+                     bool isLoad);
 
   std::pair<ExplodedNode * const*, ExplodedNode * const*>
   getImplicitNodes() const {
@@ -36,15 +38,18 @@ public:
                           ImplicitNullDerefNodes.data() +
                           ImplicitNullDerefNodes.size());
   }
+  void AddDerefSource(llvm::raw_ostream &os,
+                      llvm::SmallVectorImpl<SourceRange> &Ranges,
+                      const Expr *Ex, bool loadedFrom = false);
 };
 } // end anonymous namespace
 
-void clang::RegisterDereferenceChecker(GRExprEngine &Eng) {
+void ento::RegisterDereferenceChecker(ExprEngine &Eng) {
   Eng.registerCheck(new DereferenceChecker());
 }
 
 std::pair<ExplodedNode * const *, ExplodedNode * const *>
-clang::GetImplicitNullDereferences(GRExprEngine &Eng) {
+ento::GetImplicitNullDereferences(ExprEngine &Eng) {
   DereferenceChecker *checker = Eng.getChecker<DereferenceChecker>();
   if (!checker)
     return std::make_pair((ExplodedNode * const *) 0,
@@ -52,11 +57,39 @@ clang::GetImplicitNullDereferences(GRExprEngine &Eng) {
   return checker->getImplicitNodes();
 }
 
-void DereferenceChecker::VisitLocation(CheckerContext &C, const Stmt *S,
-                                       SVal l) {
+void DereferenceChecker::AddDerefSource(llvm::raw_ostream &os,
+                                     llvm::SmallVectorImpl<SourceRange> &Ranges,
+                                        const Expr *Ex,
+                                        bool loadedFrom) {
+  Ex = Ex->IgnoreParenLValueCasts();
+  switch (Ex->getStmtClass()) {
+    default:
+      return;
+    case Stmt::DeclRefExprClass: {
+      const DeclRefExpr *DR = cast<DeclRefExpr>(Ex);
+      if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
+        os << " (" << (loadedFrom ? "loaded from" : "from")
+           << " variable '" <<  VD->getName() << "')";
+        Ranges.push_back(DR->getSourceRange());
+      }
+      return;
+    }
+    case Stmt::MemberExprClass: {
+      const MemberExpr *ME = cast<MemberExpr>(Ex);
+      os << " (" << (loadedFrom ? "loaded from" : "via")
+         << " field '" << ME->getMemberNameInfo() << "')";
+      SourceLocation L = ME->getMemberLoc();
+      Ranges.push_back(SourceRange(L, L));
+      break;
+    }
+  }
+}
+
+void DereferenceChecker::visitLocation(CheckerContext &C, const Stmt *S,
+                                       SVal l, bool isLoad) {
   // Check for dereference of an undefined value.
   if (l.isUndef()) {
-    if (ExplodedNode *N = C.GenerateSink()) {
+    if (ExplodedNode *N = C.generateSink()) {
       if (!BT_undef)
         BT_undef = new BuiltinBug("Dereference of undefined pointer value");
 
@@ -77,13 +110,13 @@ void DereferenceChecker::VisitLocation(CheckerContext &C, const Stmt *S,
 
   const GRState *state = C.getState();
   const GRState *notNullState, *nullState;
-  llvm::tie(notNullState, nullState) = state->Assume(location);
+  llvm::tie(notNullState, nullState) = state->assume(location);
 
   // The explicit NULL case.
   if (nullState) {
     if (!notNullState) {
       // Generate an error node.
-      ExplodedNode *N = C.GenerateSink(nullState);
+      ExplodedNode *N = C.generateSink(nullState);
       if (!N)
         return;
 
@@ -94,33 +127,49 @@ void DereferenceChecker::VisitLocation(CheckerContext &C, const Stmt *S,
 
       llvm::SmallString<100> buf;
       llvm::SmallVector<SourceRange, 2> Ranges;
+      
+      // Walk through lvalue casts to get the original expression
+      // that syntactically caused the load.
+      if (const Expr *expr = dyn_cast<Expr>(S))
+        S = expr->IgnoreParenLValueCasts();
 
       switch (S->getStmtClass()) {
+        case Stmt::ArraySubscriptExprClass: {
+          llvm::raw_svector_ostream os(buf);
+          os << "Array access";
+          const ArraySubscriptExpr *AE = cast<ArraySubscriptExpr>(S);
+          AddDerefSource(os, Ranges, AE->getBase()->IgnoreParenCasts());
+          os << " results in a null pointer dereference";
+          break;
+        }
         case Stmt::UnaryOperatorClass: {
+          llvm::raw_svector_ostream os(buf);
+          os << "Dereference of null pointer";
           const UnaryOperator *U = cast<UnaryOperator>(S);
-          const Expr *SU = U->getSubExpr()->IgnoreParens();
-          if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(SU)) {
-            if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
-              llvm::raw_svector_ostream os(buf);
-              os << "Dereference of null pointer (loaded from variable '"
-                 << VD->getName() << "')";
-              Ranges.push_back(DR->getSourceRange());
-            }
-          }
+          AddDerefSource(os, Ranges, U->getSubExpr()->IgnoreParens(), true);
           break;
         }
         case Stmt::MemberExprClass: {
           const MemberExpr *M = cast<MemberExpr>(S);
-          if (M->isArrow())
-            if (DeclRefExpr *DR =
-                dyn_cast<DeclRefExpr>(M->getBase()->IgnoreParenCasts())) {
-              if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
-                llvm::raw_svector_ostream os(buf);
-                os << "Field access results in a dereference of a null pointer "
-                      "(loaded from variable '" << VD->getName() << "')";
-                Ranges.push_back(M->getBase()->getSourceRange());
-              }
+          if (M->isArrow()) {
+            llvm::raw_svector_ostream os(buf);
+            os << "Access to field '" << M->getMemberNameInfo()
+               << "' results in a dereference of a null pointer";
+            AddDerefSource(os, Ranges, M->getBase()->IgnoreParenCasts(), true);
+          }
+          break;
+        }
+        case Stmt::ObjCIvarRefExprClass: {
+          const ObjCIvarRefExpr *IV = cast<ObjCIvarRefExpr>(S);
+          if (const DeclRefExpr *DR =
+              dyn_cast<DeclRefExpr>(IV->getBase()->IgnoreParenCasts())) {
+            if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
+              llvm::raw_svector_ostream os(buf);
+              os << "Instance variable access (via '" << VD->getName()
+                 << "') results in a null pointer dereference";
             }
+          }
+          Ranges.push_back(IV->getSourceRange());
           break;
         }
         default:
@@ -146,7 +195,7 @@ void DereferenceChecker::VisitLocation(CheckerContext &C, const Stmt *S,
       // Otherwise, we have the case where the location could either be
       // null or not-null.  Record the error node as an "implicit" null
       // dereference.
-      if (ExplodedNode *N = C.GenerateSink(nullState))
+      if (ExplodedNode *N = C.generateSink(nullState))
         ImplicitNullDerefNodes.push_back(N);
     }
   }
diff --git a/lib/Checker/DivZeroChecker.cpp b/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
index 32e2a1782d10..20cc90492246 100644
--- a/lib/Checker/DivZeroChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
@@ -7,16 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines DivZeroChecker, a builtin check in GRExprEngine that performs
+// This defines DivZeroChecker, a builtin check in ExprEngine that performs
 // checks for division by zeros.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class DivZeroChecker : public CheckerVisitor<DivZeroChecker> {
@@ -28,7 +29,7 @@ public:
 };  
 } // end anonymous namespace
 
-void clang::RegisterDivZeroChecker(GRExprEngine &Eng) {
+void ento::RegisterDivZeroChecker(ExprEngine &Eng) {
   Eng.registerCheck(new DivZeroChecker());
 }
 
@@ -61,10 +62,10 @@ void DivZeroChecker::PreVisitBinaryOperator(CheckerContext &C,
   // Check for divide by zero.
   ConstraintManager &CM = C.getConstraintManager();
   const GRState *stateNotZero, *stateZero;
-  llvm::tie(stateNotZero, stateZero) = CM.AssumeDual(C.getState(), *DV);
+  llvm::tie(stateNotZero, stateZero) = CM.assumeDual(C.getState(), *DV);
 
   if (stateZero && !stateNotZero) {
-    if (ExplodedNode *N = C.GenerateSink(stateZero)) {
+    if (ExplodedNode *N = C.generateSink(stateZero)) {
       if (!BT)
         BT = new BuiltinBug("Division by zero");
 
diff --git a/lib/Checker/GRExprEngineExperimentalChecks.cpp b/lib/StaticAnalyzer/Checkers/ExperimentalChecks.cpp
index 84262b004341..d9bb4801c381 100644
--- a/lib/Checker/GRExprEngineExperimentalChecks.cpp
+++ b/lib/StaticAnalyzer/Checkers/ExperimentalChecks.cpp
@@ -1,4 +1,4 @@
-//=-- GRExprEngineExperimentalChecks.h ------------------------------*- C++ -*-=
+//=-- ExperimentalChecks.h ----------------------------------------*- C++ -*-=//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -8,27 +8,24 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines functions to instantiate and register experimental
-//  checks in GRExprEngine.
+//  checks in ExprEngine.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "GRExprEngineExperimentalChecks.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
+#include "InternalChecks.h"
+#include "ExperimentalChecks.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
 
 using namespace clang;
+using namespace ento;
 
-void clang::RegisterExperimentalChecks(GRExprEngine &Eng) {
+void ento::RegisterExperimentalChecks(ExprEngine &Eng) {
   // These are checks that never belong as internal checks
-  // within GRExprEngine.
-  RegisterCStringChecker(Eng);
-  RegisterMallocChecker(Eng);
-  RegisterPthreadLockChecker(Eng);
-  RegisterStreamChecker(Eng);
-  RegisterUnreachableCodeChecker(Eng);
+  // within ExprEngine.
+  RegisterMallocChecker(Eng); // ArrayBoundChecker depends on this.
 }
 
-void clang::RegisterExperimentalInternalChecks(GRExprEngine &Eng) {
+void ento::RegisterExperimentalInternalChecks(ExprEngine &Eng) {
   // These are internal checks that should eventually migrate to
   // RegisterInternalChecks() once they have been further tested.
   
@@ -37,8 +34,4 @@ void clang::RegisterExperimentalInternalChecks(GRExprEngine &Eng) {
   
   RegisterArrayBoundChecker(Eng);
   RegisterCastSizeChecker(Eng);
-  RegisterCastToStructChecker(Eng);
-  RegisterFixedAddressChecker(Eng);
-  RegisterPointerArithChecker(Eng);
-  RegisterPointerSubChecker(Eng);
 }
diff --git a/lib/StaticAnalyzer/Checkers/ExperimentalChecks.h b/lib/StaticAnalyzer/Checkers/ExperimentalChecks.h
new file mode 100644
index 000000000000..1f38ad77ebde
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ExperimentalChecks.h
@@ -0,0 +1,31 @@
+//=-- ExperimentalChecks.h ----------------------------------------*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines functions to instantiate and register experimental
+//  checks in ExprEngine.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_GR_ExprEngine_EXPERIMENTAL_CHECKS
+#define LLVM_CLANG_GR_ExprEngine_EXPERIMENTAL_CHECKS
+
+namespace clang {
+
+namespace ento {
+
+class ExprEngine;
+
+void RegisterAnalyzerStatsChecker(ExprEngine &Eng);
+void RegisterMallocChecker(ExprEngine &Eng);
+
+} // end GR namespace
+
+} // end clang namespace
+
+#endif
diff --git a/lib/Checker/GRExprEngine.cpp b/lib/StaticAnalyzer/Checkers/ExprEngine.cpp
index feb826e4a640..ab8d56471c55 100644
--- a/lib/Checker/GRExprEngine.cpp
+++ b/lib/StaticAnalyzer/Checkers/ExprEngine.cpp
@@ -1,4 +1,4 @@
-//=-- GRExprEngine.cpp - Path-Sensitive Expression-Level Dataflow ---*- C++ -*-=
+//=-- ExprEngine.cpp - Path-Sensitive Expression-Level Dataflow ---*- C++ -*-=
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,12 +12,16 @@
 //  functions and build the ExplodedGraph at the expression level.
 //
 //===----------------------------------------------------------------------===//
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/AnalysisManager.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/GRExprEngineBuilders.h"
-#include "clang/Checker/PathSensitive/Checker.h"
+
+// FIXME: Restructure checker registration.
+#include "InternalChecks.h"
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtObjC.h"
@@ -34,6 +38,7 @@
 #endif
 
 using namespace clang;
+using namespace ento;
 using llvm::dyn_cast;
 using llvm::dyn_cast_or_null;
 using llvm::cast;
@@ -57,119 +62,11 @@ static inline Selector GetNullarySelector(const char* name, ASTContext& Ctx) {
   return Ctx.Selectors.getSelector(0, &II);
 }
 
-
-static QualType GetCalleeReturnType(const CallExpr *CE) {
-  const Expr *Callee = CE->getCallee();
-  QualType T = Callee->getType();
-  if (const PointerType *PT = T->getAs<PointerType>()) {
-    const FunctionType *FT = PT->getPointeeType()->getAs<FunctionType>();
-    T = FT->getResultType();
-  }
-  else {
-    const BlockPointerType *BT = T->getAs<BlockPointerType>();
-    T = BT->getPointeeType()->getAs<FunctionType>()->getResultType();
-  }
-  return T;
-}
-
-static bool CalleeReturnsReference(const CallExpr *CE) {
-  return (bool) GetCalleeReturnType(CE)->getAs<ReferenceType>();
-}
-
-static bool ReceiverReturnsReference(const ObjCMessageExpr *ME) {
-  const ObjCMethodDecl *MD = ME->getMethodDecl();
-  if (!MD)
-    return false;
-  return MD->getResultType()->getAs<ReferenceType>();
-}
-
-#ifndef NDEBUG
-static bool ReceiverReturnsReferenceOrRecord(const ObjCMessageExpr *ME) {
-  const ObjCMethodDecl *MD = ME->getMethodDecl();
-  if (!MD)
-    return false;
-  QualType T = MD->getResultType();
-  return T->getAs<RecordType>() || T->getAs<ReferenceType>();
-}
-
-static bool CalleeReturnsReferenceOrRecord(const CallExpr *CE) {
-  QualType T = GetCalleeReturnType(CE);
-  return T->getAs<ReferenceType>() || T->getAs<RecordType>();
-}
-#endif
-
-//===----------------------------------------------------------------------===//
-// Batch auditor.  DEPRECATED.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class MappedBatchAuditor : public GRSimpleAPICheck {
-  typedef llvm::ImmutableList<GRSimpleAPICheck*> Checks;
-  typedef llvm::DenseMap<void*,Checks> MapTy;
-
-  MapTy M;
-  Checks::Factory F;
-  Checks AllStmts;
-
-public:
-  MappedBatchAuditor(llvm::BumpPtrAllocator& Alloc) :
-    F(Alloc), AllStmts(F.GetEmptyList()) {}
-
-  virtual ~MappedBatchAuditor() {
-    llvm::DenseSet<GRSimpleAPICheck*> AlreadyVisited;
-
-    for (MapTy::iterator MI = M.begin(), ME = M.end(); MI != ME; ++MI)
-      for (Checks::iterator I=MI->second.begin(), E=MI->second.end(); I!=E;++I){
-
-        GRSimpleAPICheck* check = *I;
-
-        if (AlreadyVisited.count(check))
-          continue;
-
-        AlreadyVisited.insert(check);
-        delete check;
-      }
-  }
-
-  void AddCheck(GRSimpleAPICheck *A, Stmt::StmtClass C) {
-    assert (A && "Check cannot be null.");
-    void* key = reinterpret_cast<void*>((uintptr_t) C);
-    MapTy::iterator I = M.find(key);
-    M[key] = F.Concat(A, I == M.end() ? F.GetEmptyList() : I->second);
-  }
-
-  void AddCheck(GRSimpleAPICheck *A) {
-    assert (A && "Check cannot be null.");
-    AllStmts = F.Concat(A, AllStmts);
-  }
-
-  virtual bool Audit(ExplodedNode* N, GRStateManager& VMgr) {
-    // First handle the auditors that accept all statements.
-    bool isSink = false;
-    for (Checks::iterator I = AllStmts.begin(), E = AllStmts.end(); I!=E; ++I)
-      isSink |= (*I)->Audit(N, VMgr);
-
-    // Next handle the auditors that accept only specific statements.
-    const Stmt* S = cast<PostStmt>(N->getLocation()).getStmt();
-    void* key = reinterpret_cast<void*>((uintptr_t) S->getStmtClass());
-    MapTy::iterator MI = M.find(key);
-    if (MI != M.end()) {
-      for (Checks::iterator I=MI->second.begin(), E=MI->second.end(); I!=E; ++I)
-        isSink |= (*I)->Audit(N, VMgr);
-    }
-
-    return isSink;
-  }
-};
-
-} // end anonymous namespace
-
 //===----------------------------------------------------------------------===//
 // Checker worklist routines.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::CheckerVisit(const Stmt *S, ExplodedNodeSet &Dst,
+void ExprEngine::CheckerVisit(const Stmt *S, ExplodedNodeSet &Dst,
                                 ExplodedNodeSet &Src, CallbackKind Kind) {
 
   // Determine if we already have a cached 'CheckersOrdered' vector
@@ -249,27 +146,66 @@ void GRExprEngine::CheckerVisit(const Stmt *S, ExplodedNodeSet &Dst,
   // automatically.
 }
 
-void GRExprEngine::CheckerEvalNilReceiver(const ObjCMessageExpr *ME,
-                                          ExplodedNodeSet &Dst,
-                                          const GRState *state,
-                                          ExplodedNode *Pred) {
-  bool Evaluated = false;
+void ExprEngine::CheckerVisitObjCMessage(const ObjCMessage &msg,
+                                         ExplodedNodeSet &Dst,
+                                         ExplodedNodeSet &Src,
+                                         bool isPrevisit) {
+
+  if (Checkers.empty()) {
+    Dst.insert(Src);
+    return;
+  }
+
+  ExplodedNodeSet Tmp;
+  ExplodedNodeSet *PrevSet = &Src;
+
+  for (CheckersOrdered::iterator I=Checkers.begin(),E=Checkers.end(); I!=E; ++I)
+  {
+    ExplodedNodeSet *CurrSet = 0;
+    if (I+1 == E)
+      CurrSet = &Dst;
+    else {
+      CurrSet = (PrevSet == &Tmp) ? &Src : &Tmp;
+      CurrSet->clear();
+    }
+
+    void *tag = I->first;
+    Checker *checker = I->second;
+
+    for (ExplodedNodeSet::iterator NI = PrevSet->begin(), NE = PrevSet->end();
+         NI != NE; ++NI)
+      checker->GR_visitObjCMessage(*CurrSet, *Builder, *this, msg,
+                                   *NI, tag, isPrevisit);
+
+    // Update which NodeSet is the current one.
+    PrevSet = CurrSet;
+  }
+
+  // Don't autotransition.  The CheckerContext objects should do this
+  // automatically.
+}
+
+void ExprEngine::CheckerEvalNilReceiver(const ObjCMessage &msg,
+                                        ExplodedNodeSet &Dst,
+                                        const GRState *state,
+                                        ExplodedNode *Pred) {
+  bool evaluated = false;
   ExplodedNodeSet DstTmp;
 
   for (CheckersOrdered::iterator I=Checkers.begin(),E=Checkers.end();I!=E;++I) {
     void *tag = I->first;
     Checker *checker = I->second;
 
-    if (checker->GR_EvalNilReceiver(DstTmp, *Builder, *this, ME, Pred, state,
+    if (checker->GR_evalNilReceiver(DstTmp, *Builder, *this, msg, Pred, state,
                                     tag)) {
-      Evaluated = true;
+      evaluated = true;
       break;
     } else
       // The checker didn't evaluate the expr. Restore the Dst.
       DstTmp.clear();
   }
 
-  if (Evaluated)
+  if (evaluated)
     Dst.insert(DstTmp);
   else
     Dst.insert(Pred);
@@ -278,35 +214,35 @@ void GRExprEngine::CheckerEvalNilReceiver(const ObjCMessageExpr *ME,
 // CheckerEvalCall returns true if one of the checkers processed the node.
 // This may return void when all call evaluation logic goes to some checker
 // in the future.
-bool GRExprEngine::CheckerEvalCall(const CallExpr *CE,
+bool ExprEngine::CheckerEvalCall(const CallExpr *CE,
                                    ExplodedNodeSet &Dst,
                                    ExplodedNode *Pred) {
-  bool Evaluated = false;
+  bool evaluated = false;
   ExplodedNodeSet DstTmp;
 
   for (CheckersOrdered::iterator I=Checkers.begin(),E=Checkers.end();I!=E;++I) {
     void *tag = I->first;
     Checker *checker = I->second;
 
-    if (checker->GR_EvalCallExpr(DstTmp, *Builder, *this, CE, Pred, tag)) {
-      Evaluated = true;
+    if (checker->GR_evalCallExpr(DstTmp, *Builder, *this, CE, Pred, tag)) {
+      evaluated = true;
       break;
     } else
       // The checker didn't evaluate the expr. Restore the DstTmp set.
       DstTmp.clear();
   }
 
-  if (Evaluated)
+  if (evaluated)
     Dst.insert(DstTmp);
   else
     Dst.insert(Pred);
 
-  return Evaluated;
+  return evaluated;
 }
 
 // FIXME: This is largely copy-paste from CheckerVisit().  Need to
 // unify.
-void GRExprEngine::CheckerVisitBind(const Stmt *StoreE, ExplodedNodeSet &Dst,
+void ExprEngine::CheckerVisitBind(const Stmt *StoreE, ExplodedNodeSet &Dst,
                                     ExplodedNodeSet &Src, SVal location,
                                     SVal val, bool isPrevisit) {
 
@@ -347,16 +283,16 @@ void GRExprEngine::CheckerVisitBind(const Stmt *StoreE, ExplodedNodeSet &Dst,
 // Engine construction and deletion.
 //===----------------------------------------------------------------------===//
 
-static void RegisterInternalChecks(GRExprEngine &Eng) {
+static void RegisterInternalChecks(ExprEngine &Eng) {
   // Register internal "built-in" BugTypes with the BugReporter. These BugTypes
   // are different than what probably many checks will do since they don't
-  // create BugReports on-the-fly but instead wait until GRExprEngine finishes
+  // create BugReports on-the-fly but instead wait until ExprEngine finishes
   // analyzing a function.  Generation of BugReport objects is done via a call
   // to 'FlushReports' from BugReporter.
   // The following checks do not need to have their associated BugTypes
   // explicitly registered with the BugReporter.  If they issue any BugReports,
   // their associated BugType will get registered with the BugReporter
-  // automatically.  Note that the check itself is owned by the GRExprEngine
+  // automatically.  Note that the check itself is owned by the ExprEngine
   // object.
   RegisterAdjustedReturnValueChecker(Eng);
   // CallAndMessageChecker should be registered before AttrNonNullChecker,
@@ -372,28 +308,24 @@ static void RegisterInternalChecks(GRExprEngine &Eng) {
   RegisterUndefBranchChecker(Eng);
   RegisterUndefCapturedBlockVarChecker(Eng);
   RegisterUndefResultChecker(Eng);
-  RegisterStackAddrLeakChecker(Eng);
 
   // This is not a checker yet.
   RegisterNoReturnFunctionChecker(Eng);
   RegisterBuiltinFunctionChecker(Eng);
   RegisterOSAtomicChecker(Eng);
-  RegisterUnixAPIChecker(Eng);
-  RegisterMacOSXAPIChecker(Eng);
 }
 
-GRExprEngine::GRExprEngine(AnalysisManager &mgr, GRTransferFuncs *tf)
+ExprEngine::ExprEngine(AnalysisManager &mgr, TransferFuncs *tf)
   : AMgr(mgr),
-    CoreEngine(*this),
-    G(CoreEngine.getGraph()),
+    Engine(*this),
+    G(Engine.getGraph()),
     Builder(NULL),
     StateMgr(getContext(), mgr.getStoreManagerCreator(),
              mgr.getConstraintManagerCreator(), G.getAllocator(),
              *this),
     SymMgr(StateMgr.getSymbolManager()),
-    ValMgr(StateMgr.getValueManager()),
-    SVator(ValMgr.getSValuator()),
-    CurrentStmt(NULL),
+    svalBuilder(StateMgr.getSValBuilder()),
+    EntryNode(NULL), currentStmt(NULL),
     NSExceptionII(NULL), NSExceptionInstanceRaiseSelectors(NULL),
     RaiseSel(GetNullarySelector("raise", getContext())),
     BR(mgr, *this), TF(tf) {
@@ -403,9 +335,16 @@ GRExprEngine::GRExprEngine(AnalysisManager &mgr, GRTransferFuncs *tf)
   // FIXME: Eventually remove the TF object entirely.
   TF->RegisterChecks(*this);
   TF->RegisterPrinters(getStateManager().Printers);
+
+  mgr.getCheckerManager()->registerCheckersToEngine(*this);
+  
+  if (mgr.shouldEagerlyTrimExplodedGraph()) {
+    // Enable eager node reclaimation when constructing the ExplodedGraph.  
+    G.enableNodeReclamation();
+  }
 }
 
-GRExprEngine::~GRExprEngine() {
+ExprEngine::~ExprEngine() {
   BR.FlushReports();
   delete [] NSExceptionInstanceRaiseSelectors;
   
@@ -422,21 +361,7 @@ GRExprEngine::~GRExprEngine() {
 // Utility methods.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::AddCheck(GRSimpleAPICheck* A, Stmt::StmtClass C) {
-  if (!BatchAuditor)
-    BatchAuditor.reset(new MappedBatchAuditor(getGraph().getAllocator()));
-
-  ((MappedBatchAuditor*) BatchAuditor.get())->AddCheck(A, C);
-}
-
-void GRExprEngine::AddCheck(GRSimpleAPICheck *A) {
-  if (!BatchAuditor)
-    BatchAuditor.reset(new MappedBatchAuditor(getGraph().getAllocator()));
-
-  ((MappedBatchAuditor*) BatchAuditor.get())->AddCheck(A);
-}
-
-const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
+const GRState* ExprEngine::getInitialState(const LocationContext *InitLoc) {
   const GRState *state = StateMgr.getInitialState(InitLoc);
 
   // Preconditions.
@@ -462,8 +387,8 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
         break;
 
       SVal V = state->getSVal(loc::MemRegionVal(R));
-      SVal Constraint_untested = EvalBinOp(state, BO_GT, V,
-                                           ValMgr.makeZeroVal(T),
+      SVal Constraint_untested = evalBinOp(state, BO_GT, V,
+                                           svalBuilder.makeZeroVal(T),
                                            getContext().IntTy);
 
       DefinedOrUnknownSVal *Constraint =
@@ -472,7 +397,7 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
       if (!Constraint)
         break;
 
-      if (const GRState *newState = state->Assume(*Constraint, true))
+      if (const GRState *newState = state->assume(*Constraint, true))
         state = newState;
 
       break;
@@ -487,7 +412,7 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
 
       if (const Loc *LV = dyn_cast<Loc>(&V)) {
         // Assume that the pointer value in 'self' is non-null.
-        state = state->Assume(*LV, true);
+        state = state->assume(*LV, true);
         assert(state && "'self' cannot be null");
       }
     }
@@ -500,9 +425,9 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
 // Top-level transfer function logic (Dispatcher).
 //===----------------------------------------------------------------------===//
 
-/// EvalAssume - Called by ConstraintManager. Used to call checker-specific
+/// evalAssume - Called by ConstraintManager. Used to call checker-specific
 ///  logic for handling assumptions on symbolic values.
-const GRState *GRExprEngine::ProcessAssume(const GRState *state, SVal cond,
+const GRState *ExprEngine::processAssume(const GRState *state, SVal cond,
                                            bool assumption) {
   // Determine if we already have a cached 'CheckersOrdered' vector
   // specifically tailored for processing assumptions.  This
@@ -510,7 +435,7 @@ const GRState *GRExprEngine::ProcessAssume(const GRState *state, SVal cond,
   CheckersOrdered *CO = &Checkers;
   llvm::OwningPtr<CheckersOrdered> NewCO;
 
-  CallbackTag K = GetCallbackTag(ProcessAssumeCallback);
+  CallbackTag K = GetCallbackTag(processAssumeCallback);
   CheckersOrdered *& CO_Ref = COCache[K];
 
   if (!CO_Ref) {
@@ -536,15 +461,16 @@ const GRState *GRExprEngine::ProcessAssume(const GRState *state, SVal cond,
       Checker *C = I->second;
       bool respondsToCallback = true;
 
-      state = C->EvalAssume(state, cond, assumption, &respondsToCallback);
+      state = C->evalAssume(state, cond, assumption, &respondsToCallback);
 
-      // Check if we're building the cache of checkers that care about Assumes.
+      // Check if we're building the cache of checkers that care about
+      // assumptions.
       if (NewCO.get() && respondsToCallback)
         NewCO->push_back(*I);
     }
 
     // If we got through all the checkers, and we built a list of those that
-    // care about Assumes, save it.
+    // care about assumptions, save it.
     if (NewCO.get())
       CO_Ref = NewCO.take();
   }
@@ -553,10 +479,10 @@ const GRState *GRExprEngine::ProcessAssume(const GRState *state, SVal cond,
   if (!state)
     return NULL;
 
-  return TF->EvalAssume(state, cond, assumption);
+  return TF->evalAssume(state, cond, assumption);
 }
 
-bool GRExprEngine::WantsRegionChangeUpdate(const GRState* state) {
+bool ExprEngine::wantsRegionChangeUpdate(const GRState* state) {
   CallbackTag K = GetCallbackTag(EvalRegionChangesCallback);
   CheckersOrdered *CO = COCache[K];
 
@@ -565,7 +491,7 @@ bool GRExprEngine::WantsRegionChangeUpdate(const GRState* state) {
 
   for (CheckersOrdered::iterator I = CO->begin(), E = CO->end(); I != E; ++I) {
     Checker *C = I->second;
-    if (C->WantsRegionChangeUpdate(state))
+    if (C->wantsRegionChangeUpdate(state))
       return true;
   }
 
@@ -573,10 +499,10 @@ bool GRExprEngine::WantsRegionChangeUpdate(const GRState* state) {
 }
 
 const GRState *
-GRExprEngine::ProcessRegionChanges(const GRState *state,
+ExprEngine::processRegionChanges(const GRState *state,
                                    const MemRegion * const *Begin,
                                    const MemRegion * const *End) {
-  // FIXME: Most of this method is copy-pasted from ProcessAssume.
+  // FIXME: Most of this method is copy-pasted from processAssume.
 
   // Determine if we already have a cached 'CheckersOrdered' vector
   // specifically tailored for processing region changes.  This
@@ -625,29 +551,48 @@ GRExprEngine::ProcessRegionChanges(const GRState *state,
   return state;
 }
 
-void GRExprEngine::ProcessEndWorklist(bool hasWorkRemaining) {
+void ExprEngine::processEndWorklist(bool hasWorkRemaining) {
   for (CheckersOrdered::iterator I = Checkers.begin(), E = Checkers.end();
        I != E; ++I) {
     I->second->VisitEndAnalysis(G, BR, *this);
   }
 }
 
-void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
-  CurrentStmt = CE.getStmt();
+void ExprEngine::processCFGElement(const CFGElement E, 
+                                  StmtNodeBuilder& builder) {
+  switch (E.getKind()) {
+  case CFGElement::Statement:
+    ProcessStmt(E.getAs<CFGStmt>(), builder);
+    break;
+  case CFGElement::Initializer:
+    ProcessInitializer(E.getAs<CFGInitializer>(), builder);
+    break;
+  case CFGElement::ImplicitDtor:
+    ProcessImplicitDtor(E.getAs<CFGImplicitDtor>(), builder);
+    break;
+  default:
+    // Suppress compiler warning.
+    llvm_unreachable("Unexpected CFGElement kind.");
+  }
+}
+
+void ExprEngine::ProcessStmt(const CFGStmt S, StmtNodeBuilder& builder) {
+  // Reclaim any unnecessary nodes in the ExplodedGraph.
+  G.reclaimRecentlyAllocatedNodes();
+  // Recycle any unused states in the GRStateManager.
+  StateMgr.recycleUnusedStates();
+  
+  currentStmt = S.getStmt();
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
-                                CurrentStmt->getLocStart(),
+                                currentStmt->getLocStart(),
                                 "Error evaluating statement");
 
   Builder = &builder;
-  EntryNode = builder.getBasePredecessor();
-
-  // Set up our simple checks.
-  if (BatchAuditor)
-    Builder->setAuditor(BatchAuditor.get());
+  EntryNode = builder.getPredecessor();
 
   // Create the cleaned state.
   const LocationContext *LC = EntryNode->getLocationContext();
-  SymbolReaper SymReaper(LC, CurrentStmt, SymMgr);
+  SymbolReaper SymReaper(LC, currentStmt, SymMgr);
 
   if (AMgr.shouldPurgeDead()) {
     const GRState *St = EntryNode->getState();
@@ -659,7 +604,7 @@ void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
     }
 
     const StackFrameContext *SFC = LC->getCurrentStackFrame();
-    CleanedState = StateMgr.RemoveDeadBindings(St, SFC, SymReaper);
+    CleanedState = StateMgr.removeDeadBindings(St, SFC, SymReaper);
   } else {
     CleanedState = EntryNode->getState();
   }
@@ -671,14 +616,14 @@ void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
     Tmp.Add(EntryNode);
   else {
     SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-    SaveOr OldHasGen(Builder->HasGeneratedNode);
+    SaveOr OldHasGen(Builder->hasGeneratedNode);
 
     SaveAndRestore<bool> OldPurgeDeadSymbols(Builder->PurgingDeadSymbols);
     Builder->PurgingDeadSymbols = true;
 
     // FIXME: This should soon be removed.
     ExplodedNodeSet Tmp2;
-    getTF().EvalDeadSymbols(Tmp2, *this, *Builder, EntryNode,
+    getTF().evalDeadSymbols(Tmp2, *this, *Builder, EntryNode,
                             CleanedState, SymReaper);
 
     if (Checkers.empty())
@@ -700,38 +645,34 @@ void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
         Checker *checker = I->second;
         for (ExplodedNodeSet::iterator NI = SrcSet->begin(), NE = SrcSet->end();
              NI != NE; ++NI)
-          checker->GR_EvalDeadSymbols(*DstSet, *Builder, *this, CurrentStmt,
+          checker->GR_evalDeadSymbols(*DstSet, *Builder, *this, currentStmt,
                                       *NI, SymReaper, tag);
         SrcSet = DstSet;
       }
     }
 
-    if (!Builder->BuildSinks && !Builder->HasGeneratedNode)
+    if (!Builder->BuildSinks && !Builder->hasGeneratedNode)
       Tmp.Add(EntryNode);
   }
 
   bool HasAutoGenerated = false;
 
   for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-
     ExplodedNodeSet Dst;
 
     // Set the cleaned state.
     Builder->SetCleanedState(*I == EntryNode ? CleanedState : GetState(*I));
 
     // Visit the statement.
-    if (CE.asLValue())
-      VisitLValue(cast<Expr>(CurrentStmt), *I, Dst);
-    else
-      Visit(CurrentStmt, *I, Dst);
+    Visit(currentStmt, *I, Dst);
 
     // Do we need to auto-generate a node?  We only need to do this to generate
-    // a node with a "cleaned" state; GRCoreEngine will actually handle
+    // a node with a "cleaned" state; CoreEngine will actually handle
     // auto-transitions for other cases.
     if (Dst.size() == 1 && *Dst.begin() == EntryNode
-        && !Builder->HasGeneratedNode && !HasAutoGenerated) {
+        && !Builder->hasGeneratedNode && !HasAutoGenerated) {
       HasAutoGenerated = true;
-      builder.generateNode(CurrentStmt, GetState(EntryNode), *I);
+      builder.generateNode(currentStmt, GetState(EntryNode), *I);
     }
   }
 
@@ -739,22 +680,135 @@ void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
   CleanedState = NULL;
   EntryNode = NULL;
 
-  CurrentStmt = 0;
+  currentStmt = 0;
 
   Builder = NULL;
 }
 
-void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred, 
+void ExprEngine::ProcessInitializer(const CFGInitializer Init,
+                                    StmtNodeBuilder &builder) {
+  // We don't set EntryNode and currentStmt. And we don't clean up state.
+  const CXXCtorInitializer *BMI = Init.getInitializer();
+
+  ExplodedNode *pred = builder.getPredecessor();
+
+  const StackFrameContext *stackFrame = cast<StackFrameContext>(pred->getLocationContext());
+  const CXXConstructorDecl *decl = cast<CXXConstructorDecl>(stackFrame->getDecl());
+  const CXXThisRegion *thisReg = getCXXThisRegion(decl, stackFrame);
+
+  SVal thisVal = pred->getState()->getSVal(thisReg);
+
+  if (BMI->isAnyMemberInitializer()) {
+    ExplodedNodeSet Dst;
+
+    // Evaluate the initializer.
+    Visit(BMI->getInit(), pred, Dst);
+
+    for (ExplodedNodeSet::iterator I = Dst.begin(), E = Dst.end(); I != E; ++I){
+      ExplodedNode *Pred = *I;
+      const GRState *state = Pred->getState();
+
+      const FieldDecl *FD = BMI->getAnyMember();
+
+      SVal FieldLoc = state->getLValue(FD, thisVal);
+      SVal InitVal = state->getSVal(BMI->getInit());
+      state = state->bindLoc(FieldLoc, InitVal);
+
+      // Use a custom node building process.
+      PostInitializer PP(BMI, stackFrame);
+      // Builder automatically add the generated node to the deferred set,
+      // which are processed in the builder's dtor.
+      builder.generateNode(PP, state, Pred);
+    }
+    return;
+  }
+
+  assert(BMI->isBaseInitializer());
+
+  // Get the base class declaration.
+  const CXXConstructExpr *ctorExpr = cast<CXXConstructExpr>(BMI->getInit());
+
+  // Create the base object region.
+  SVal baseVal = 
+    getStoreManager().evalDerivedToBase(thisVal, ctorExpr->getType());
+  const MemRegion *baseReg = baseVal.getAsRegion();
+  assert(baseReg);
+  Builder = &builder;
+  ExplodedNodeSet dst;
+  VisitCXXConstructExpr(ctorExpr, baseReg, pred, dst);
+}
+
+void ExprEngine::ProcessImplicitDtor(const CFGImplicitDtor D,
+                                       StmtNodeBuilder &builder) {
+  Builder = &builder;
+
+  switch (D.getDtorKind()) {
+  case CFGElement::AutomaticObjectDtor:
+    ProcessAutomaticObjDtor(cast<CFGAutomaticObjDtor>(D), builder);
+    break;
+  case CFGElement::BaseDtor:
+    ProcessBaseDtor(cast<CFGBaseDtor>(D), builder);
+    break;
+  case CFGElement::MemberDtor:
+    ProcessMemberDtor(cast<CFGMemberDtor>(D), builder);
+    break;
+  case CFGElement::TemporaryDtor:
+    ProcessTemporaryDtor(cast<CFGTemporaryDtor>(D), builder);
+    break;
+  default:
+    llvm_unreachable("Unexpected dtor kind.");
+  }
+}
+
+void ExprEngine::ProcessAutomaticObjDtor(const CFGAutomaticObjDtor dtor,
+                                           StmtNodeBuilder &builder) {
+  ExplodedNode *pred = builder.getPredecessor();
+  const GRState *state = pred->getState();
+  const VarDecl *varDecl = dtor.getVarDecl();
+
+  QualType varType = varDecl->getType();
+
+  if (const ReferenceType *refType = varType->getAs<ReferenceType>())
+    varType = refType->getPointeeType();
+
+  const CXXRecordDecl *recordDecl = varType->getAsCXXRecordDecl();
+  assert(recordDecl && "get CXXRecordDecl fail");
+  const CXXDestructorDecl *dtorDecl = recordDecl->getDestructor();
+
+  Loc dest = state->getLValue(varDecl, pred->getLocationContext());
+
+  ExplodedNodeSet dstSet;
+  VisitCXXDestructor(dtorDecl, cast<loc::MemRegionVal>(dest).getRegion(),
+                     dtor.getTriggerStmt(), pred, dstSet);
+}
+
+void ExprEngine::ProcessBaseDtor(const CFGBaseDtor D,
+                                   StmtNodeBuilder &builder) {
+}
+
+void ExprEngine::ProcessMemberDtor(const CFGMemberDtor D,
+                                     StmtNodeBuilder &builder) {
+}
+
+void ExprEngine::ProcessTemporaryDtor(const CFGTemporaryDtor D,
+                                        StmtNodeBuilder &builder) {
+}
+
+void ExprEngine::Visit(const Stmt* S, ExplodedNode* Pred, 
                          ExplodedNodeSet& Dst) {
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 S->getLocStart(),
                                 "Error evaluating statement");
 
+  // Expressions to ignore.
+  if (const Expr *Ex = dyn_cast<Expr>(S))
+    S = Ex->IgnoreParens();
+  
   // FIXME: add metadata to the CFG so that we can disable
   //  this check when we KNOW that there is no block-level subexpression.
   //  The motivation is that this check requires a hashtable lookup.
 
-  if (S != CurrentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(S)) {
+  if (S != currentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(S)) {
     Dst.Add(Pred);
     return;
   }
@@ -763,29 +817,35 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     // C++ stuff we don't support yet.
     case Stmt::CXXBindTemporaryExprClass:
     case Stmt::CXXCatchStmtClass:
-    case Stmt::CXXConstructExprClass:
     case Stmt::CXXDefaultArgExprClass:
     case Stmt::CXXDependentScopeMemberExprClass:
-    case Stmt::CXXExprWithTemporariesClass:
+    case Stmt::ExprWithCleanupsClass:
     case Stmt::CXXNullPtrLiteralExprClass:
     case Stmt::CXXPseudoDestructorExprClass:
     case Stmt::CXXTemporaryObjectExprClass:
     case Stmt::CXXThrowExprClass:
     case Stmt::CXXTryStmtClass:
     case Stmt::CXXTypeidExprClass:
+    case Stmt::CXXUuidofExprClass:
     case Stmt::CXXUnresolvedConstructExprClass:
     case Stmt::CXXScalarValueInitExprClass:
     case Stmt::DependentScopeDeclRefExprClass:
     case Stmt::UnaryTypeTraitExprClass:
+    case Stmt::BinaryTypeTraitExprClass:
     case Stmt::UnresolvedLookupExprClass:
     case Stmt::UnresolvedMemberExprClass:
+    case Stmt::CXXNoexceptExprClass:
+    case Stmt::PackExpansionExprClass:
+    case Stmt::SubstNonTypeTemplateParmPackExprClass:
     {
       SaveAndRestore<bool> OldSink(Builder->BuildSinks);
       Builder->BuildSinks = true;
       MakeNode(Dst, S, Pred, GetState(Pred));
       break;
     }
-
+      
+    case Stmt::ParenExprClass:
+      llvm_unreachable("ParenExprs already handled.");
     // Cases that should never be evaluated simply because they shouldn't
     // appear in the CFG.
     case Stmt::BreakStmtClass:
@@ -799,15 +859,22 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::LabelStmtClass:
     case Stmt::NoStmtClass:
     case Stmt::NullStmtClass:
-    case Stmt::SwitchCaseClass:
       llvm_unreachable("Stmt should not be in analyzer evaluation loop");
       break;
 
     case Stmt::GNUNullExprClass: {
-      MakeNode(Dst, S, Pred, GetState(Pred)->BindExpr(S, ValMgr.makeNull()));
+      MakeNode(Dst, S, Pred, GetState(Pred)->BindExpr(S, svalBuilder.makeNull()));
       break;
     }
 
+    case Stmt::ObjCAtSynchronizedStmtClass:
+      VisitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(S), Pred, Dst);
+      break;
+
+    case Stmt::ObjCPropertyRefExprClass:
+      VisitObjCPropertyRefExpr(cast<ObjCPropertyRefExpr>(S), Pred, Dst);
+      break;
+
     // Cases not handled yet; but will handle some day.
     case Stmt::DesignatedInitExprClass:
     case Stmt::ExtVectorElementExprClass:
@@ -815,21 +882,18 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::ImplicitValueInitExprClass:
     case Stmt::ObjCAtCatchStmtClass:
     case Stmt::ObjCAtFinallyStmtClass:
-    case Stmt::ObjCAtSynchronizedStmtClass:
     case Stmt::ObjCAtTryStmtClass:
     case Stmt::ObjCEncodeExprClass:
-    case Stmt::ObjCImplicitSetterGetterRefExprClass:
     case Stmt::ObjCIsaExprClass:
-    case Stmt::ObjCPropertyRefExprClass:
     case Stmt::ObjCProtocolExprClass:
     case Stmt::ObjCSelectorExprClass:
     case Stmt::ObjCStringLiteralClass:
-    case Stmt::ObjCSuperExprClass:
     case Stmt::ParenListExprClass:
     case Stmt::PredefinedExprClass:
     case Stmt::ShuffleVectorExprClass:
-    case Stmt::TypesCompatibleExprClass:
     case Stmt::VAArgExprClass:
+    case Stmt::CUDAKernelCallExprClass:
+    case Stmt::OpaqueValueExprClass:
         // Fall through.
 
     // Cases we intentionally don't evaluate, since they don't need
@@ -839,20 +903,23 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CharacterLiteralClass:
     case Stmt::CXXBoolLiteralExprClass:
     case Stmt::FloatingLiteralClass:
+    case Stmt::SizeOfPackExprClass:
       Dst.Add(Pred); // No-op. Simply propagate the current state unchanged.
       break;
 
     case Stmt::ArraySubscriptExprClass:
-      VisitArraySubscriptExpr(cast<ArraySubscriptExpr>(S), Pred, Dst, false);
+      VisitLvalArraySubscriptExpr(cast<ArraySubscriptExpr>(S), Pred, Dst);
       break;
 
     case Stmt::AsmStmtClass:
       VisitAsmStmt(cast<AsmStmt>(S), Pred, Dst);
       break;
 
-    case Stmt::BlockDeclRefExprClass:
-      VisitBlockDeclRefExpr(cast<BlockDeclRefExpr>(S), Pred, Dst, false);
+    case Stmt::BlockDeclRefExprClass: {
+      const BlockDeclRefExpr *BE = cast<BlockDeclRefExpr>(S);
+      VisitCommonDeclRefExpr(BE, BE->getDecl(), Pred, Dst);
       break;
+    }
 
     case Stmt::BlockExprClass:
       VisitBlockExpr(cast<BlockExpr>(S), Pred, Dst);
@@ -860,7 +927,6 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
 
     case Stmt::BinaryOperatorClass: {
       const BinaryOperator* B = cast<BinaryOperator>(S);
-
       if (B->isLogicalOp()) {
         VisitLogicalExpr(B, Pred, Dst);
         break;
@@ -874,19 +940,26 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       if (AMgr.shouldEagerlyAssume() &&
           (B->isRelationalOp() || B->isEqualityOp())) {
         ExplodedNodeSet Tmp;
-        VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Tmp, false);
-        EvalEagerlyAssume(Dst, Tmp, cast<Expr>(S));
+        VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Tmp);
+        evalEagerlyAssume(Dst, Tmp, cast<Expr>(S));
       }
       else
-        VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Dst, false);
+        VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Dst);
 
       break;
     }
 
-    case Stmt::CallExprClass:
-    case Stmt::CXXOperatorCallExprClass: {
+    case Stmt::CallExprClass: {
       const CallExpr* C = cast<CallExpr>(S);
-      VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst, false);
+      VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst);
+      break;
+    }
+
+    case Stmt::CXXConstructExprClass: {
+      const CXXConstructExpr *C = cast<CXXConstructExpr>(S);
+      // For block-level CXXConstructExpr, we don't have a destination region.
+      // Let VisitCXXConstructExpr() create one.
+      VisitCXXConstructExpr(C, 0, Pred, Dst);
       break;
     }
 
@@ -896,6 +969,12 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
     }
 
+    case Stmt::CXXOperatorCallExprClass: {
+      const CXXOperatorCallExpr *C = cast<CXXOperatorCallExpr>(S);
+      VisitCXXOperatorCallExpr(C, Pred, Dst);
+      break;
+    }
+
     case Stmt::CXXNewExprClass: {
       const CXXNewExpr *NE = cast<CXXNewExpr>(S);
       VisitCXXNewExpr(NE, Pred, Dst);
@@ -917,16 +996,18 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     }
 
     case Stmt::CompoundAssignOperatorClass:
-      VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Dst, false);
+      VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Dst);
       break;
 
     case Stmt::CompoundLiteralExprClass:
-      VisitCompoundLiteralExpr(cast<CompoundLiteralExpr>(S), Pred, Dst, false);
+      VisitCompoundLiteralExpr(cast<CompoundLiteralExpr>(S), Pred, Dst);
       break;
 
+    case Stmt::BinaryConditionalOperatorClass:
     case Stmt::ConditionalOperatorClass: { // '?' operator
-      const ConditionalOperator* C = cast<ConditionalOperator>(S);
-      VisitGuardedExpr(C, C->getLHS(), C->getRHS(), Pred, Dst);
+      const AbstractConditionalOperator *C
+        = cast<AbstractConditionalOperator>(S);
+      VisitGuardedExpr(C, C->getTrueExpr(), C->getFalseExpr(), Pred, Dst);
       break;
     }
 
@@ -934,9 +1015,11 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       VisitCXXThisExpr(cast<CXXThisExpr>(S), Pred, Dst);
       break;
 
-    case Stmt::DeclRefExprClass:
-      VisitDeclRefExpr(cast<DeclRefExpr>(S), Pred, Dst, false);
+    case Stmt::DeclRefExprClass: {
+      const DeclRefExpr *DE = cast<DeclRefExpr>(S);
+      VisitCommonDeclRefExpr(DE, DE->getDecl(), Pred, Dst);
       break;
+    }
 
     case Stmt::DeclStmtClass:
       VisitDeclStmt(cast<DeclStmt>(S), Pred, Dst);
@@ -956,7 +1039,7 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
     case Stmt::CXXConstCastExprClass:
     case Stmt::CXXFunctionalCastExprClass: {
       const CastExpr* C = cast<CastExpr>(S);
-      VisitCast(C, C->getSubExpr(), Pred, Dst, false);
+      VisitCast(C, C->getSubExpr(), Pred, Dst);
       break;
     }
 
@@ -971,11 +1054,10 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
 
     case Stmt::MemberExprClass:
-      VisitMemberExpr(cast<MemberExpr>(S), Pred, Dst, false);
+      VisitMemberExpr(cast<MemberExpr>(S), Pred, Dst);
       break;
-
     case Stmt::ObjCIvarRefExprClass:
-      VisitObjCIvarRefExpr(cast<ObjCIvarRefExpr>(S), Pred, Dst, false);
+      VisitLvalObjCIvarRefExpr(cast<ObjCIvarRefExpr>(S), Pred, Dst);
       break;
 
     case Stmt::ObjCForCollectionStmtClass:
@@ -983,7 +1065,7 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
 
     case Stmt::ObjCMessageExprClass:
-      VisitObjCMessageExpr(cast<ObjCMessageExpr>(S), Pred, Dst, false);
+      VisitObjCMessageExpr(cast<ObjCMessageExpr>(S), Pred, Dst);
       break;
 
     case Stmt::ObjCAtThrowStmtClass: {
@@ -995,10 +1077,6 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
     }
 
-    case Stmt::ParenExprClass:
-      Visit(cast<ParenExpr>(S)->getSubExpr()->IgnoreParens(), Pred, Dst);
-      break;
-
     case Stmt::ReturnStmtClass:
       VisitReturnStmt(cast<ReturnStmt>(S), Pred, Dst);
       break;
@@ -1032,9 +1110,12 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       break;
     }
 
-    case Stmt::StringLiteralClass:
-      VisitLValue(cast<StringLiteral>(S), Pred, Dst);
-      break;
+    case Stmt::StringLiteralClass: {
+      const GRState* state = GetState(Pred);
+      SVal V = state->getLValue(cast<StringLiteral>(S));
+      MakeNode(Dst, S, Pred, state->BindExpr(S, V));
+      return;
+    }
 
     case Stmt::SwitchStmtClass:
       // This case isn't for branch processing, but for handling the
@@ -1046,11 +1127,11 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
       const UnaryOperator *U = cast<UnaryOperator>(S);
       if (AMgr.shouldEagerlyAssume()&&(U->getOpcode() == UO_LNot)) {
         ExplodedNodeSet Tmp;
-        VisitUnaryOperator(U, Pred, Tmp, false);
-        EvalEagerlyAssume(Dst, Tmp, U);
+        VisitUnaryOperator(U, Pred, Tmp);
+        evalEagerlyAssume(Dst, Tmp, U);
       }
       else
-        VisitUnaryOperator(U, Pred, Dst, false);
+        VisitUnaryOperator(U, Pred, Dst);
       break;
     }
 
@@ -1062,162 +1143,34 @@ void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred,
   }
 }
 
-void GRExprEngine::VisitLValue(const Expr* Ex, ExplodedNode* Pred,
-                               ExplodedNodeSet& Dst) {
-
-  PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
-                                Ex->getLocStart(),
-                                "Error evaluating statement");
-
-
-  Ex = Ex->IgnoreParens();
-
-  if (Ex != CurrentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(Ex)){
-    Dst.Add(Pred);
-    return;
-  }
-
-  switch (Ex->getStmtClass()) {
-    // C++ stuff we don't support yet.
-    case Stmt::CXXExprWithTemporariesClass:
-    case Stmt::CXXMemberCallExprClass:
-    case Stmt::CXXScalarValueInitExprClass: {
-      SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-      Builder->BuildSinks = true;
-      MakeNode(Dst, Ex, Pred, GetState(Pred));
-      break;
-    }
-
-    case Stmt::ArraySubscriptExprClass:
-      VisitArraySubscriptExpr(cast<ArraySubscriptExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::BinaryOperatorClass:
-    case Stmt::CompoundAssignOperatorClass:
-      VisitBinaryOperator(cast<BinaryOperator>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::BlockDeclRefExprClass:
-      VisitBlockDeclRefExpr(cast<BlockDeclRefExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::CallExprClass:
-    case Stmt::CXXOperatorCallExprClass: {
-      const CallExpr *C = cast<CallExpr>(Ex);
-      assert(CalleeReturnsReferenceOrRecord(C));
-      VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst, true);
-      break;
-    }
-
-    case Stmt::CompoundLiteralExprClass:
-      VisitCompoundLiteralExpr(cast<CompoundLiteralExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::DeclRefExprClass:
-      VisitDeclRefExpr(cast<DeclRefExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::ImplicitCastExprClass:
-    case Stmt::CStyleCastExprClass: {
-      const CastExpr *C = cast<CastExpr>(Ex);
-      QualType T = Ex->getType();
-      VisitCast(C, C->getSubExpr(), Pred, Dst, true);
-      break;
-    }
-
-    case Stmt::MemberExprClass:
-      VisitMemberExpr(cast<MemberExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::ObjCIvarRefExprClass:
-      VisitObjCIvarRefExpr(cast<ObjCIvarRefExpr>(Ex), Pred, Dst, true);
-      return;
-
-    case Stmt::ObjCMessageExprClass: {
-      const ObjCMessageExpr *ME = cast<ObjCMessageExpr>(Ex);
-      assert(ReceiverReturnsReferenceOrRecord(ME));
-      VisitObjCMessageExpr(ME, Pred, Dst, true);
-      return;
-    }
-
-    case Stmt::ObjCIsaExprClass:
-      // FIXME: Do something more intelligent with 'x->isa = ...'.
-      //  For now, just ignore the assignment.
-      return;
-
-    case Stmt::ObjCPropertyRefExprClass:
-    case Stmt::ObjCImplicitSetterGetterRefExprClass:
-      // FIXME: Property assignments are lvalues, but not really "locations".
-      //  e.g.:  self.x = something;
-      //  Here the "self.x" really can translate to a method call (setter) when
-      //  the assignment is made.  Moreover, the entire assignment expression
-      //  evaluate to whatever "something" is, not calling the "getter" for
-      //  the property (which would make sense since it can have side effects).
-      //  We'll probably treat this as a location, but not one that we can
-      //  take the address of.  Perhaps we need a new SVal class for cases
-      //  like thsis?
-      //  Note that we have a similar problem for bitfields, since they don't
-      //  have "locations" in the sense that we can take their address.
-      Dst.Add(Pred);
-      return;
-
-    case Stmt::StringLiteralClass: {
-      const GRState* state = GetState(Pred);
-      SVal V = state->getLValue(cast<StringLiteral>(Ex));
-      MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V));
-      return;
-    }
-
-    case Stmt::UnaryOperatorClass:
-      VisitUnaryOperator(cast<UnaryOperator>(Ex), Pred, Dst, true);
-      return;
-
-    // In C++, binding an rvalue to a reference requires to create an object.
-    case Stmt::CXXBoolLiteralExprClass:
-    case Stmt::IntegerLiteralClass:
-    case Stmt::CharacterLiteralClass:
-    case Stmt::FloatingLiteralClass:
-    case Stmt::ImaginaryLiteralClass:
-      CreateCXXTemporaryObject(Ex, Pred, Dst);
-      return;
-
-    default: {
-      // Arbitrary subexpressions can return aggregate temporaries that
-      // can be used in a lvalue context.  We need to enhance our support
-      // of such temporaries in both the environment and the store, so right
-      // now we just do a regular visit.
-
-      // NOTE: Do not use 'isAggregateType()' here as CXXRecordDecls that
-      //  are non-pod are not aggregates.
-      assert ((isa<RecordType>(Ex->getType().getDesugaredType()) ||
-               isa<ArrayType>(Ex->getType().getDesugaredType())) &&
-              "Other kinds of expressions with non-aggregate/union/class types"
-              " do not have lvalues.");
-
-      Visit(Ex, Pred, Dst);
-    }
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Block entrance.  (Update counters).
 //===----------------------------------------------------------------------===//
 
-bool GRExprEngine::ProcessBlockEntrance(const CFGBlock* B, 
-                                        const ExplodedNode *Pred,
-                                        GRBlockCounter BC) {
-  return BC.getNumVisited(Pred->getLocationContext()->getCurrentStackFrame(), 
-                          B->getBlockID()) < AMgr.getMaxLoop();
+void ExprEngine::processCFGBlockEntrance(ExplodedNodeSet &dstNodes,
+                               GenericNodeBuilder<BlockEntrance> &nodeBuilder){
+  
+  // FIXME: Refactor this into a checker.
+  const CFGBlock *block = nodeBuilder.getProgramPoint().getBlock();
+  ExplodedNode *pred = nodeBuilder.getPredecessor();
+  
+  if (nodeBuilder.getBlockCounter().getNumVisited(
+                       pred->getLocationContext()->getCurrentStackFrame(), 
+                       block->getBlockID()) >= AMgr.getMaxVisit()) {
+
+    static int tag = 0;
+    nodeBuilder.generateNode(pred->getState(), pred, &tag, true);
+  }
 }
 
 //===----------------------------------------------------------------------===//
 // Generic node creation.
 //===----------------------------------------------------------------------===//
 
-ExplodedNode* GRExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
+ExplodedNode* ExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
                                      ExplodedNode* Pred, const GRState* St,
                                      ProgramPoint::Kind K, const void *tag) {
-  assert (Builder && "GRStmtNodeBuilder not present.");
+  assert (Builder && "StmtNodeBuilder not present.");
   SaveAndRestore<const void*> OldTag(Builder->Tag);
   Builder->Tag = tag;
   return Builder->MakeNode(Dst, S, Pred, St, K);
@@ -1227,7 +1180,7 @@ ExplodedNode* GRExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
 // Branch processing.
 //===----------------------------------------------------------------------===//
 
-const GRState* GRExprEngine::MarkBranch(const GRState* state,
+const GRState* ExprEngine::MarkBranch(const GRState* state,
                                         const Stmt* Terminator,
                                         bool branchTaken) {
 
@@ -1255,9 +1208,10 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
       return state->BindExpr(B, UndefinedVal(Ex));
     }
 
+    case Stmt::BinaryConditionalOperatorClass:
     case Stmt::ConditionalOperatorClass: { // ?:
-
-      const ConditionalOperator* C = cast<ConditionalOperator>(Terminator);
+      const AbstractConditionalOperator* C
+        = cast<AbstractConditionalOperator>(Terminator);
 
       // For ?, if branchTaken == true then the value is either the LHS or
       // the condition itself. (GNU extension).
@@ -1265,9 +1219,9 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
       const Expr* Ex;
 
       if (branchTaken)
-        Ex = C->getLHS() ? C->getLHS() : C->getCond();
+        Ex = C->getTrueExpr();
       else
-        Ex = C->getRHS();
+        Ex = C->getFalseExpr();
 
       return state->BindExpr(C, UndefinedVal(Ex));
     }
@@ -1321,8 +1275,8 @@ static SVal RecoverCastedSymbol(GRStateManager& StateMgr, const GRState* state,
   return state->getSVal(Ex);
 }
 
-void GRExprEngine::ProcessBranch(const Stmt* Condition, const Stmt* Term,
-                                 GRBranchNodeBuilder& builder) {
+void ExprEngine::processBranch(const Stmt* Condition, const Stmt* Term,
+                                 BranchNodeBuilder& builder) {
 
   // Check for NULL conditions; e.g. "for(;;)"
   if (!Condition) {
@@ -1376,7 +1330,7 @@ void GRExprEngine::ProcessBranch(const Stmt* Condition, const Stmt* Term,
 
   // Process the true branch.
   if (builder.isFeasible(true)) {
-    if (const GRState *state = PrevState->Assume(V, true))
+    if (const GRState *state = PrevState->assume(V, true))
       builder.generateNode(MarkBranch(state, Term, true), true);
     else
       builder.markInfeasible(true);
@@ -1384,16 +1338,16 @@ void GRExprEngine::ProcessBranch(const Stmt* Condition, const Stmt* Term,
 
   // Process the false branch.
   if (builder.isFeasible(false)) {
-    if (const GRState *state = PrevState->Assume(V, false))
+    if (const GRState *state = PrevState->assume(V, false))
       builder.generateNode(MarkBranch(state, Term, false), false);
     else
       builder.markInfeasible(false);
   }
 }
 
-/// ProcessIndirectGoto - Called by GRCoreEngine.  Used to generate successor
+/// processIndirectGoto - Called by CoreEngine.  Used to generate successor
 ///  nodes by processing the 'effects' of a computed goto jump.
-void GRExprEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& builder) {
+void ExprEngine::processIndirectGoto(IndirectGotoNodeBuilder &builder) {
 
   const GRState *state = builder.getState();
   SVal V = state->getSVal(builder.getTarget());
@@ -1405,19 +1359,19 @@ void GRExprEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& builder) {
   //   (3) We have no clue about the label.  Dispatch to all targets.
   //
 
-  typedef GRIndirectGotoNodeBuilder::iterator iterator;
+  typedef IndirectGotoNodeBuilder::iterator iterator;
 
   if (isa<loc::GotoLabel>(V)) {
-    const LabelStmt* L = cast<loc::GotoLabel>(V).getLabel();
+    const LabelDecl *L = cast<loc::GotoLabel>(V).getLabel();
 
-    for (iterator I=builder.begin(), E=builder.end(); I != E; ++I) {
+    for (iterator I = builder.begin(), E = builder.end(); I != E; ++I) {
       if (I.getLabel() == L) {
         builder.generateNode(I, state);
         return;
       }
     }
 
-    assert (false && "No block with label.");
+    assert(false && "No block with label.");
     return;
   }
 
@@ -1437,11 +1391,11 @@ void GRExprEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& builder) {
 }
 
 
-void GRExprEngine::VisitGuardedExpr(const Expr* Ex, const Expr* L, 
+void ExprEngine::VisitGuardedExpr(const Expr* Ex, const Expr* L, 
                                     const Expr* R,
                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
-  assert(Ex == CurrentStmt &&
+  assert(Ex == currentStmt &&
          Pred->getLocationContext()->getCFG()->isBlkExpr(Ex));
 
   const GRState* state = GetState(Pred);
@@ -1457,22 +1411,22 @@ void GRExprEngine::VisitGuardedExpr(const Expr* Ex, const Expr* L,
   MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, X, true));
 }
 
-/// ProcessEndPath - Called by GRCoreEngine.  Used to generate end-of-path
+/// ProcessEndPath - Called by CoreEngine.  Used to generate end-of-path
 ///  nodes when the control reaches the end of a function.
-void GRExprEngine::ProcessEndPath(GREndPathNodeBuilder& builder) {
-  getTF().EvalEndPath(*this, builder);
+void ExprEngine::processEndOfFunction(EndOfFunctionNodeBuilder& builder) {
+  getTF().evalEndPath(*this, builder);
   StateMgr.EndPath(builder.getState());
   for (CheckersOrdered::iterator I=Checkers.begin(),E=Checkers.end(); I!=E;++I){
     void *tag = I->first;
     Checker *checker = I->second;
-    checker->EvalEndPath(builder, tag, *this);
+    checker->evalEndPath(builder, tag, *this);
   }
 }
 
-/// ProcessSwitch - Called by GRCoreEngine.  Used to generate successor
+/// ProcessSwitch - Called by CoreEngine.  Used to generate successor
 ///  nodes by processing the 'effects' of a switch statement.
-void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
-  typedef GRSwitchNodeBuilder::iterator iterator;
+void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
+  typedef SwitchNodeBuilder::iterator iterator;
   const GRState* state = builder.getState();
   const Expr* CondE = builder.getCondition();
   SVal  CondV_untested = state->getSVal(CondE);
@@ -1501,7 +1455,7 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
     // Sanity checks.  These go away in Release builds.
     assert(b && V1.Val.isInt() && !V1.HasSideEffects
              && "Case condition must evaluate to an integer constant.");
-    b = b; // silence unused variable warning
+    (void)b; // silence unused variable warning
     assert(V1.Val.getInt().getBitWidth() ==
            getContext().getTypeSize(CondE->getType()));
 
@@ -1512,7 +1466,7 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
       b = E->Evaluate(V2, getContext());
       assert(b && V2.Val.isInt() && !V2.HasSideEffects
              && "Case condition must evaluate to an integer constant.");
-      b = b; // silence unused variable warning
+      (void)b; // silence unused variable warning
     }
     else
       V2 = V1;
@@ -1523,11 +1477,11 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
 
     do {
       nonloc::ConcreteInt CaseVal(getBasicVals().getValue(V1.Val.getInt()));
-      DefinedOrUnknownSVal Res = SVator.EvalEQ(DefaultSt ? DefaultSt : state,
+      DefinedOrUnknownSVal Res = svalBuilder.evalEQ(DefaultSt ? DefaultSt : state,
                                                CondV, CaseVal);
 
       // Now "assume" that the case matches.
-      if (const GRState* stateNew = state->Assume(Res, true)) {
+      if (const GRState* stateNew = state->assume(Res, true)) {
         builder.generateCaseStmtNode(I, stateNew);
 
         // If CondV evaluates to a constant, then we know that this
@@ -1540,7 +1494,7 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
       // Now "assume" that the case doesn't match.  Add this state
       // to the default state (if it is feasible).
       if (DefaultSt) {
-        if (const GRState *stateNew = DefaultSt->Assume(Res, false)) {
+        if (const GRState *stateNew = DefaultSt->assume(Res, false)) {
           defaultIsFeasible = true;
           DefaultSt = stateNew;
         }
@@ -1560,30 +1514,37 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
     } while (true);
   }
 
-  // If we reach here, than we know that the default branch is
-  // possible.
-  if (defaultIsFeasible) builder.generateDefaultCaseNode(DefaultSt);
-}
+  if (!defaultIsFeasible)
+    return;
 
-void GRExprEngine::ProcessCallEnter(GRCallEnterNodeBuilder &B) {
-  const StackFrameContext *LocCtx 
-    = AMgr.getStackFrame(B.getCalleeContext(),
-                         B.getLocationContext(),
-                         B.getCallExpr(),
-                         B.getBlock(),
-                         B.getIndex());
+  // If we have switch(enum value), the default branch is not
+  // feasible if all of the enum constants not covered by 'case:' statements
+  // are not feasible values for the switch condition.
+  //
+  // Note that this isn't as accurate as it could be.  Even if there isn't
+  // a case for a particular enum value as long as that enum value isn't
+  // feasible then it shouldn't be considered for making 'default:' reachable.
+  const SwitchStmt *SS = builder.getSwitch();
+  const Expr *CondExpr = SS->getCond()->IgnoreParenImpCasts();
+  if (CondExpr->getType()->getAs<EnumType>()) {
+    if (SS->isAllEnumCasesCovered())
+      return;
+  }
 
-  const GRState *state = B.getState()->EnterStackFrame(LocCtx);
+  builder.generateDefaultCaseNode(DefaultSt);
+}
 
-  B.GenerateNode(state, LocCtx);
+void ExprEngine::processCallEnter(CallEnterNodeBuilder &B) {
+  const GRState *state = B.getState()->enterStackFrame(B.getCalleeContext());
+  B.generateNode(state);
 }
 
-void GRExprEngine::ProcessCallExit(GRCallExitNodeBuilder &B) {
+void ExprEngine::processCallExit(CallExitNodeBuilder &B) {
   const GRState *state = B.getState();
   const ExplodedNode *Pred = B.getPredecessor();
-  const StackFrameContext *LocCtx = 
+  const StackFrameContext *calleeCtx = 
                             cast<StackFrameContext>(Pred->getLocationContext());
-  const Stmt *CE = LocCtx->getCallSite();
+  const Stmt *CE = calleeCtx->getCallSite();
 
   // If the callee returns an expression, bind its value to CallExpr.
   const Stmt *ReturnedExpr = state->get<ReturnExpr>();
@@ -1596,32 +1557,28 @@ void GRExprEngine::ProcessCallExit(GRCallExitNodeBuilder &B) {
 
   // Bind the constructed object value to CXXConstructExpr.
   if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(CE)) {
-    const CXXThisRegion *ThisR = getCXXThisRegion(CCE->getConstructor(),LocCtx);
-    // We might not have 'this' region in the binding if we didn't inline
-    // the ctor call.
+    const CXXThisRegion *ThisR =
+      getCXXThisRegion(CCE->getConstructor()->getParent(), calleeCtx);
+
     SVal ThisV = state->getSVal(ThisR);
-    loc::MemRegionVal *V = dyn_cast<loc::MemRegionVal>(&ThisV);
-    if (V) {
-      SVal ObjVal = state->getSVal(V->getRegion());
-      assert(isa<nonloc::LazyCompoundVal>(ObjVal));
-      state = state->BindExpr(CCE, ObjVal);
-    }
+    // Always bind the region to the CXXConstructExpr.
+    state = state->BindExpr(CCE, ThisV);
   }
 
-  B.GenerateNode(state);
+  B.generateNode(state);
 }
 
 //===----------------------------------------------------------------------===//
 // Transfer functions: logical operations ('&&', '||').
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
+void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
                                     ExplodedNodeSet& Dst) {
 
   assert(B->getOpcode() == BO_LAnd ||
          B->getOpcode() == BO_LOr);
 
-  assert(B==CurrentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(B));
+  assert(B==currentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(B));
 
   const GRState* state = GetState(Pred);
   SVal X = state->getSVal(B);
@@ -1647,19 +1604,19 @@ void GRExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
     // value later when necessary.  We don't have the machinery in place for
     // this right now, and since most logical expressions are used for branches,
     // the payoff is not likely to be large.  Instead, we do eager evaluation.
-    if (const GRState *newState = state->Assume(XD, true))
+    if (const GRState *newState = state->assume(XD, true))
       MakeNode(Dst, B, Pred,
-               newState->BindExpr(B, ValMgr.makeIntVal(1U, B->getType())));
+               newState->BindExpr(B, svalBuilder.makeIntVal(1U, B->getType())));
 
-    if (const GRState *newState = state->Assume(XD, false))
+    if (const GRState *newState = state->assume(XD, false))
       MakeNode(Dst, B, Pred,
-               newState->BindExpr(B, ValMgr.makeIntVal(0U, B->getType())));
+               newState->BindExpr(B, svalBuilder.makeIntVal(0U, B->getType())));
   }
   else {
     // We took the LHS expression.  Depending on whether we are '&&' or
     // '||' we know what the value of the expression is via properties of
     // the short-circuiting.
-    X = ValMgr.makeIntVal(B->getOpcode() == BO_LAnd ? 0U : 1U,
+    X = svalBuilder.makeIntVal(B->getOpcode() == BO_LAnd ? 0U : 1U,
                           B->getType());
     MakeNode(Dst, B, Pred, state->BindExpr(B, X));
   }
@@ -1669,13 +1626,13 @@ void GRExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
 // Transfer functions: Loads and stores.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
+void ExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
                                   ExplodedNodeSet &Dst) {
 
   ExplodedNodeSet Tmp;
 
   CanQualType T = getContext().getCanonicalType(BE->getType());
-  SVal V = ValMgr.getBlockPointer(BE->getBlockDecl(), T,
+  SVal V = svalBuilder.getBlockPointer(BE->getBlockDecl(), T,
                                   Pred->getLocationContext());
 
   MakeNode(Tmp, BE, Pred, GetState(Pred)->BindExpr(BE, V),
@@ -1685,86 +1642,59 @@ void GRExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
   CheckerVisit(BE, Dst, Tmp, PostVisitStmtCallback);
 }
 
-void GRExprEngine::VisitDeclRefExpr(const DeclRefExpr *Ex, ExplodedNode *Pred,
-                                    ExplodedNodeSet &Dst, bool asLValue) {
-  VisitCommonDeclRefExpr(Ex, Ex->getDecl(), Pred, Dst, asLValue);
-}
-
-void GRExprEngine::VisitBlockDeclRefExpr(const BlockDeclRefExpr *Ex,
-                                         ExplodedNode *Pred,
-                                    ExplodedNodeSet &Dst, bool asLValue) {
-  VisitCommonDeclRefExpr(Ex, Ex->getDecl(), Pred, Dst, asLValue);
-}
-
-void GRExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
-                                          ExplodedNode *Pred,
-                                          ExplodedNodeSet &Dst, bool asLValue) {
-
+void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
+                                        ExplodedNode *Pred,
+                                        ExplodedNodeSet &Dst) {
   const GRState *state = GetState(Pred);
 
   if (const VarDecl* VD = dyn_cast<VarDecl>(D)) {
-
+    assert(Ex->isLValue());
     SVal V = state->getLValue(VD, Pred->getLocationContext());
 
-    if (asLValue) {
-      // For references, the 'lvalue' is the pointer address stored in the
-      // reference region.
-      if (VD->getType()->isReferenceType()) {
-        if (const MemRegion *R = V.getAsRegion())
-          V = state->getSVal(R);
-        else
-          V = UnknownVal();
-      }
-
-      MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V),
-               ProgramPoint::PostLValueKind);
+    // For references, the 'lvalue' is the pointer address stored in the
+    // reference region.
+    if (VD->getType()->isReferenceType()) {
+      if (const MemRegion *R = V.getAsRegion())
+        V = state->getSVal(R);
+      else
+        V = UnknownVal();
     }
-    else
-      EvalLoad(Dst, Ex, Pred, state, V);
 
+    MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V),
+             ProgramPoint::PostLValueKind);
     return;
-  } else if (const EnumConstantDecl* ED = dyn_cast<EnumConstantDecl>(D)) {
-    assert(!asLValue && "EnumConstantDecl does not have lvalue.");
-
-    SVal V = ValMgr.makeIntVal(ED->getInitVal());
+  }
+  if (const EnumConstantDecl* ED = dyn_cast<EnumConstantDecl>(D)) {
+    assert(!Ex->isLValue());
+    SVal V = svalBuilder.makeIntVal(ED->getInitVal());
     MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V));
     return;
-
-  } else if (const FunctionDecl* FD = dyn_cast<FunctionDecl>(D)) {
-    // This code is valid regardless of the value of 'isLValue'.
-    SVal V = ValMgr.getFunctionPointer(FD);
+  }
+  if (const FunctionDecl* FD = dyn_cast<FunctionDecl>(D)) {
+    SVal V = svalBuilder.getFunctionPointer(FD);
     MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, V),
              ProgramPoint::PostLValueKind);
     return;
   }
-
   assert (false &&
           "ValueDecl support for this ValueDecl not implemented.");
 }
 
 /// VisitArraySubscriptExpr - Transfer function for array accesses
-void GRExprEngine::VisitArraySubscriptExpr(const ArraySubscriptExpr* A,
-                                           ExplodedNode* Pred,
-                                           ExplodedNodeSet& Dst, bool asLValue){
+void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr* A,
+                                             ExplodedNode* Pred,
+                                             ExplodedNodeSet& Dst){
 
   const Expr* Base = A->getBase()->IgnoreParens();
   const Expr* Idx  = A->getIdx()->IgnoreParens();
+  
+  // Evaluate the base.
   ExplodedNodeSet Tmp;
-
-  if (Base->getType()->isVectorType()) {
-    // For vector types get its lvalue.
-    // FIXME: This may not be correct.  Is the rvalue of a vector its location?
-    //  In fact, I think this is just a hack.  We need to get the right
-    // semantics.
-    VisitLValue(Base, Pred, Tmp);
-  }
-  else
-    Visit(Base, Pred, Tmp);   // Get Base's rvalue, which should be an LocVal.
+  Visit(Base, Pred, Tmp);
 
   for (ExplodedNodeSet::iterator I1=Tmp.begin(), E1=Tmp.end(); I1!=E1; ++I1) {
     ExplodedNodeSet Tmp2;
     Visit(Idx, *I1, Tmp2);     // Evaluate the index.
-
     ExplodedNodeSet Tmp3;
     CheckerVisit(A, Tmp3, Tmp2, PreVisitStmtCallback);
 
@@ -1772,49 +1702,54 @@ void GRExprEngine::VisitArraySubscriptExpr(const ArraySubscriptExpr* A,
       const GRState* state = GetState(*I2);
       SVal V = state->getLValue(A->getType(), state->getSVal(Idx),
                                 state->getSVal(Base));
-
-      if (asLValue)
-        MakeNode(Dst, A, *I2, state->BindExpr(A, V),
-                 ProgramPoint::PostLValueKind);
-      else
-        EvalLoad(Dst, A, *I2, state, V);
+      assert(A->isLValue());
+      MakeNode(Dst, A, *I2, state->BindExpr(A, V), ProgramPoint::PostLValueKind);
     }
   }
 }
 
 /// VisitMemberExpr - Transfer function for member expressions.
-void GRExprEngine::VisitMemberExpr(const MemberExpr* M, ExplodedNode* Pred,
-                                   ExplodedNodeSet& Dst, bool asLValue) {
-
-  Expr* Base = M->getBase()->IgnoreParens();
-  ExplodedNodeSet Tmp;
+void ExprEngine::VisitMemberExpr(const MemberExpr* M, ExplodedNode* Pred,
+                                 ExplodedNodeSet& Dst) {
 
-  if (M->isArrow())
-    Visit(Base, Pred, Tmp);        // p->f = ...  or   ... = p->f
-  else
-    VisitLValue(Base, Pred, Tmp);  // x.f = ...   or   ... = x.f
+  Expr *baseExpr = M->getBase()->IgnoreParens();
+  ExplodedNodeSet dstBase;
+  Visit(baseExpr, Pred, dstBase);
 
-  FieldDecl *Field = dyn_cast<FieldDecl>(M->getMemberDecl());
-  if (!Field) // FIXME: skipping member expressions for non-fields
+  FieldDecl *field = dyn_cast<FieldDecl>(M->getMemberDecl());
+  if (!field) // FIXME: skipping member expressions for non-fields
     return;
 
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E; ++I) {
+  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
+    I != E; ++I) {
     const GRState* state = GetState(*I);
+    SVal baseExprVal = state->getSVal(baseExpr);
+    if (isa<nonloc::LazyCompoundVal>(baseExprVal) ||
+        isa<nonloc::CompoundVal>(baseExprVal) ||
+        // FIXME: This can originate by conjuring a symbol for an unknown
+        // temporary struct object, see test/Analysis/fields.c:
+        // (p = getit()).x
+        isa<nonloc::SymbolVal>(baseExprVal)) {
+      MakeNode(Dst, M, *I, state->BindExpr(M, UnknownVal()));
+      continue;
+    }
+
     // FIXME: Should we insert some assumption logic in here to determine
     // if "Base" is a valid piece of memory?  Before we put this assumption
     // later when using FieldOffset lvals (which we no longer have).
-    SVal L = state->getLValue(Field, state->getSVal(Base));
 
-    if (asLValue)
+    // For all other cases, compute an lvalue.    
+    SVal L = state->getLValue(field, baseExprVal);
+    if (M->isLValue())
       MakeNode(Dst, M, *I, state->BindExpr(M, L), ProgramPoint::PostLValueKind);
     else
-      EvalLoad(Dst, M, *I, state, L);
+      evalLoad(Dst, M, *I, state, L);
   }
 }
 
-/// EvalBind - Handle the semantics of binding a value to a specific location.
-///  This method is used by EvalStore and (soon) VisitDeclStmt, and others.
-void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
+/// evalBind - Handle the semantics of binding a value to a specific location.
+///  This method is used by evalStore and (soon) VisitDeclStmt, and others.
+void ExprEngine::evalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
                             ExplodedNode* Pred, const GRState* state,
                             SVal location, SVal Val, bool atDeclInit) {
 
@@ -1852,21 +1787,21 @@ void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
       }
     }
 
-    // The next thing to do is check if the GRTransferFuncs object wants to
+    // The next thing to do is check if the TransferFuncs object wants to
     // update the state based on the new binding.  If the GRTransferFunc object
     // doesn't do anything, just auto-propagate the current state.
     
     // NOTE: We use 'AssignE' for the location of the PostStore if 'AssignE'
     // is non-NULL.  Checkers typically care about 
     
-    GRStmtNodeBuilderRef BuilderRef(Dst, *Builder, *this, *I, newState, StoreE,
-                                    newState != state);
+    StmtNodeBuilderRef BuilderRef(Dst, *Builder, *this, *I, newState, StoreE,
+                                    true);
 
-    getTF().EvalBind(BuilderRef, location, Val);
+    getTF().evalBind(BuilderRef, location, Val);
   }
 }
 
-/// EvalStore - Handle the semantics of a store via an assignment.
+/// evalStore - Handle the semantics of a store via an assignment.
 ///  @param Dst The node set to store generated state nodes
 ///  @param AssignE The assignment expression if the store happens in an 
 ///         assignment.
@@ -1874,17 +1809,28 @@ void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
 ///  @param state The current simulation state
 ///  @param location The location to store the value
 ///  @param Val The value to be stored
-void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
+void ExprEngine::evalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
                              const Expr* LocationE,
                              ExplodedNode* Pred,
                              const GRState* state, SVal location, SVal Val,
                              const void *tag) {
 
-  assert(Builder && "GRStmtNodeBuilder must be defined.");
+  assert(Builder && "StmtNodeBuilder must be defined.");
+
+  // Proceed with the store.  We use AssignE as the anchor for the PostStore
+  // ProgramPoint if it is non-NULL, and LocationE otherwise.
+  const Expr *StoreE = AssignE ? AssignE : LocationE;
+
+  if (isa<loc::ObjCPropRef>(location)) {
+    loc::ObjCPropRef prop = cast<loc::ObjCPropRef>(location);
+    ExplodedNodeSet src = Pred;
+    return VisitObjCMessage(ObjCPropertySetter(prop.getPropRefExpr(),
+                                               StoreE, Val), src, Dst);
+  }
 
   // Evaluate the location (checks for bad dereferences).
   ExplodedNodeSet Tmp;
-  EvalLocation(Tmp, LocationE, Pred, state, location, tag, false);
+  evalLocation(Tmp, LocationE, Pred, state, location, tag, false);
 
   if (Tmp.empty())
     return;
@@ -1893,20 +1839,23 @@ void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
 
   SaveAndRestore<ProgramPoint::Kind> OldSPointKind(Builder->PointKind,
                                                    ProgramPoint::PostStoreKind);
-  SaveAndRestore<const void*> OldTag(Builder->Tag, tag);
-
-  // Proceed with the store.  We use AssignE as the anchor for the PostStore
-  // ProgramPoint if it is non-NULL, and LocationE otherwise.
-  const Expr *StoreE = AssignE ? AssignE : LocationE;
 
   for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI)
-    EvalBind(Dst, StoreE, *NI, GetState(*NI), location, Val);
+    evalBind(Dst, StoreE, *NI, GetState(*NI), location, Val);
 }
 
-void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, const Expr *Ex, 
+void ExprEngine::evalLoad(ExplodedNodeSet& Dst, const Expr *Ex,
                             ExplodedNode* Pred,
                             const GRState* state, SVal location,
                             const void *tag, QualType LoadTy) {
+  assert(!isa<NonLoc>(location) && "location cannot be a NonLoc.");
+
+  if (isa<loc::ObjCPropRef>(location)) {
+    loc::ObjCPropRef prop = cast<loc::ObjCPropRef>(location);
+    ExplodedNodeSet src = Pred;
+    return VisitObjCMessage(ObjCPropertyGetter(prop.getPropRefExpr(), Ex),
+                            src, Dst);
+  }
 
   // Are we loading from a region?  This actually results in two loads; one
   // to fetch the address of the referenced value and one to fetch the
@@ -1918,30 +1867,30 @@ void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, const Expr *Ex,
     if (const ReferenceType *RT = ValTy->getAs<ReferenceType>()) {
       static int loadReferenceTag = 0;
       ExplodedNodeSet Tmp;
-      EvalLoadCommon(Tmp, Ex, Pred, state, location, &loadReferenceTag,
+      evalLoadCommon(Tmp, Ex, Pred, state, location, &loadReferenceTag,
                      getContext().getPointerType(RT->getPointeeType()));
 
       // Perform the load from the referenced value.
       for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end() ; I!=E; ++I) {
         state = GetState(*I);
         location = state->getSVal(Ex);
-        EvalLoadCommon(Dst, Ex, *I, state, location, tag, LoadTy);
+        evalLoadCommon(Dst, Ex, *I, state, location, tag, LoadTy);
       }
       return;
     }
   }
 
-  EvalLoadCommon(Dst, Ex, Pred, state, location, tag, LoadTy);
+  evalLoadCommon(Dst, Ex, Pred, state, location, tag, LoadTy);
 }
 
-void GRExprEngine::EvalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
+void ExprEngine::evalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
                                   ExplodedNode* Pred,
                                   const GRState* state, SVal location,
                                   const void *tag, QualType LoadTy) {
 
   // Evaluate the location (checks for bad dereferences).
   ExplodedNodeSet Tmp;
-  EvalLocation(Tmp, Ex, Pred, state, location, tag, true);
+  evalLocation(Tmp, Ex, Pred, state, location, tag, true);
 
   if (Tmp.empty())
     return;
@@ -1949,26 +1898,27 @@ void GRExprEngine::EvalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
   assert(!location.isUndef());
 
   SaveAndRestore<ProgramPoint::Kind> OldSPointKind(Builder->PointKind);
-  SaveAndRestore<const void*> OldTag(Builder->Tag);
 
   // Proceed with the load.
   for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI) {
     state = GetState(*NI);
+
     if (location.isUnknown()) {
       // This is important.  We must nuke the old binding.
       MakeNode(Dst, Ex, *NI, state->BindExpr(Ex, UnknownVal()),
                ProgramPoint::PostLoadKind, tag);
     }
     else {
-      SVal V = state->getSVal(cast<Loc>(location), LoadTy.isNull() ?
-                                                     Ex->getType() : LoadTy);
-      MakeNode(Dst, Ex, *NI, state->BindExpr(Ex, V), ProgramPoint::PostLoadKind,
-               tag);
+      if (LoadTy.isNull())
+        LoadTy = Ex->getType();
+      SVal V = state->getSVal(cast<Loc>(location), LoadTy);
+      MakeNode(Dst, Ex, *NI, state->bindExprAndLocation(Ex, location, V),
+               ProgramPoint::PostLoadKind, tag);
     }
   }
 }
 
-void GRExprEngine::EvalLocation(ExplodedNodeSet &Dst, const Stmt *S,
+void ExprEngine::evalLocation(ExplodedNodeSet &Dst, const Stmt *S,
                                 ExplodedNode* Pred,
                                 const GRState* state, SVal location,
                                 const void *tag, bool isLoad) {
@@ -1999,7 +1949,7 @@ void GRExprEngine::EvalLocation(ExplodedNodeSet &Dst, const Stmt *S,
          NI != NE; ++NI) {
       // Use the 'state' argument only when the predecessor node is the
       // same as Pred.  This allows us to catch updates to the state.
-      checker->GR_VisitLocation(*CurrSet, *Builder, *this, S, *NI,
+      checker->GR_visitLocation(*CurrSet, *Builder, *this, S, *NI,
                                 *NI == Pred ? state : GetState(*NI),
                                 location, tag, isLoad);
     }
@@ -2009,7 +1959,7 @@ void GRExprEngine::EvalLocation(ExplodedNodeSet &Dst, const Stmt *S,
   }
 }
 
-bool GRExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE, 
+bool ExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE, 
                               ExplodedNode *Pred) {
   const GRState *state = GetState(Pred);
   const Expr *Callee = CE->getCallee();
@@ -2021,8 +1971,12 @@ bool GRExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
 
   // Check if the function definition is in the same translation unit.
   if (FD->hasBody(FD)) {
+    const StackFrameContext *stackFrame = 
+      AMgr.getStackFrame(AMgr.getAnalysisContext(FD), 
+                         Pred->getLocationContext(),
+                         CE, Builder->getBlock(), Builder->getIndex());
     // Now we have the definition of the callee, create a CallEnter node.
-    CallEnter Loc(CE, AMgr.getAnalysisContext(FD), Pred->getLocationContext());
+    CallEnter Loc(CE, stackFrame, Pred->getLocationContext());
 
     ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
     Dst.Add(N);
@@ -2031,11 +1985,13 @@ bool GRExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
 
   // Check if we can find the function definition in other translation units.
   if (AMgr.hasIndexer()) {
-    const AnalysisContext *C = AMgr.getAnalysisContextInAnotherTU(FD);
+    AnalysisContext *C = AMgr.getAnalysisContextInAnotherTU(FD);
     if (C == 0)
       return false;
-
-    CallEnter Loc(CE, C, Pred->getLocationContext());
+    const StackFrameContext *stackFrame = 
+      AMgr.getStackFrame(C, Pred->getLocationContext(),
+                         CE, Builder->getBlock(), Builder->getIndex());
+    CallEnter Loc(CE, stackFrame, Pred->getLocationContext());
     ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
     Dst.Add(N);
     return true;
@@ -2044,10 +2000,10 @@ bool GRExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
   return false;
 }
 
-void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
+void ExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
                              CallExpr::const_arg_iterator AI,
                              CallExpr::const_arg_iterator AE,
-                             ExplodedNodeSet& Dst, bool asLValue) {
+                             ExplodedNodeSet& Dst) {
 
   // Determine the type of function we're calling (if available).
   const FunctionProtoType *Proto = NULL;
@@ -2055,41 +2011,9 @@ void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
   if (const PointerType *FnTypePtr = FnType->getAs<PointerType>())
     Proto = FnTypePtr->getPointeeType()->getAs<FunctionProtoType>();
 
-  // Create a worklist to process the arguments.
-  llvm::SmallVector<CallExprWLItem, 20> WorkList;
-  WorkList.reserve(AE - AI);
-  WorkList.push_back(CallExprWLItem(AI, Pred));
-
+  // Evaluate the arguments.
   ExplodedNodeSet ArgsEvaluated;
-
-  while (!WorkList.empty()) {
-    CallExprWLItem Item = WorkList.back();
-    WorkList.pop_back();
-
-    if (Item.I == AE) {
-      ArgsEvaluated.insert(Item.N);
-      continue;
-    }
-
-    // Evaluate the argument.
-    ExplodedNodeSet Tmp;
-    const unsigned ParamIdx = Item.I - AI;
-
-    bool VisitAsLvalue = false;
-    if (Proto && ParamIdx < Proto->getNumArgs())
-      VisitAsLvalue = Proto->getArgType(ParamIdx)->isReferenceType();
-
-    if (VisitAsLvalue)
-      VisitLValue(*Item.I, Item.N, Tmp);
-    else
-      Visit(*Item.I, Item.N, Tmp);
-
-    // Enqueue evaluating the next argument on the worklist.
-    ++(Item.I);
-
-    for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI)
-      WorkList.push_back(CallExprWLItem(Item.I, *NI));
-  }
+  evalArguments(CE->arg_begin(), CE->arg_end(), Proto, Pred, ArgsEvaluated);
 
   // Now process the call itself.
   ExplodedNodeSet DstTmp;
@@ -2108,7 +2032,6 @@ void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
   // to see if the can evaluate the function call.
   ExplodedNodeSet DstTmp3;
 
-
   for (ExplodedNodeSet::iterator DI = DstTmp.begin(), DE = DstTmp.end();
        DI != DE; ++DI) {
 
@@ -2133,19 +2056,19 @@ void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
            DI_Checker != DE_Checker; ++DI_Checker) {
 
         // Dispatch to the plug-in transfer function.
-        unsigned OldSize = DstTmp3.size();
-        SaveOr OldHasGen(Builder->HasGeneratedNode);
+        unsigned oldSize = DstTmp3.size();
+        SaveOr OldHasGen(Builder->hasGeneratedNode);
         Pred = *DI_Checker;
 
         // Dispatch to transfer function logic to handle the call itself.
         // FIXME: Allow us to chain together transfer functions.
-        assert(Builder && "GRStmtNodeBuilder must be defined.");
-        getTF().EvalCall(DstTmp3, *this, *Builder, CE, L, Pred);
+        assert(Builder && "StmtNodeBuilder must be defined.");
+        getTF().evalCall(DstTmp3, *this, *Builder, CE, L, Pred);
 
         // Handle the case where no nodes where generated.  Auto-generate that
         // contains the updated state if we aren't generating sinks.
-        if (!Builder->BuildSinks && DstTmp3.size() == OldSize &&
-            !Builder->HasGeneratedNode)
+        if (!Builder->BuildSinks && DstTmp3.size() == oldSize &&
+            !Builder->hasGeneratedNode)
           MakeNode(DstTmp3, CE, Pred, state);
       }
     }
@@ -2153,29 +2076,35 @@ void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
 
   // Finally, perform the post-condition check of the CallExpr and store
   // the created nodes in 'Dst'.
-  // If the callee returns a reference and we want an rvalue, skip this check
-  // and do the load.
-  if (!(!asLValue && CalleeReturnsReference(CE))) {
-    CheckerVisit(CE, Dst, DstTmp3, PostVisitStmtCallback);
-    return;
-  }
+  CheckerVisit(CE, Dst, DstTmp3, PostVisitStmtCallback);
+}
+
+//===----------------------------------------------------------------------===//
+// Transfer function: Objective-C dot-syntax to access a property.
+//===----------------------------------------------------------------------===//
+
+void ExprEngine::VisitObjCPropertyRefExpr(const ObjCPropertyRefExpr *Ex,
+                                          ExplodedNode *Pred,
+                                          ExplodedNodeSet &Dst) {
+  ExplodedNodeSet dstBase;
+
+  // Visit the receiver (if any).
+  if (Ex->isObjectReceiver())
+    Visit(Ex->getBase(), Pred, dstBase);
+  else
+    dstBase = Pred;
 
-  // Handle the case where the called function returns a reference but
-  // we expect an rvalue.  For such cases, convert the reference to
-  // an rvalue.
-  // FIXME: This conversion doesn't actually happen unless the result
-  //  of CallExpr is consumed by another expression.
-  ExplodedNodeSet DstTmp4;
-  CheckerVisit(CE, DstTmp4, DstTmp3, PostVisitStmtCallback);
-  QualType LoadTy = CE->getType();
+  ExplodedNodeSet dstPropRef;
 
-  static int *ConvertToRvalueTag = 0;
-  for (ExplodedNodeSet::iterator NI = DstTmp4.begin(), NE = DstTmp4.end();
-       NI!=NE; ++NI) {
-    const GRState *state = GetState(*NI);
-    EvalLoad(Dst, CE, *NI, state, state->getSVal(CE),
-             &ConvertToRvalueTag, LoadTy);
+  // Using the base, compute the lvalue of the instance variable.
+  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
+       I!=E; ++I) {
+    ExplodedNode *nodeBase = *I;
+    const GRState *state = GetState(nodeBase);
+    MakeNode(dstPropRef, Ex, *I, state->BindExpr(Ex, loc::ObjCPropRef(Ex)));
   }
+  
+  Dst.insert(dstPropRef);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2185,7 +2114,7 @@ void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
 static std::pair<const void*,const void*> EagerlyAssumeTag
   = std::pair<const void*,const void*>(&EagerlyAssumeTag,static_cast<void*>(0));
 
-void GRExprEngine::EvalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
+void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
                                      const Expr *Ex) {
   for (ExplodedNodeSet::iterator I=Src.begin(), E=Src.end(); I!=E; ++I) {
     ExplodedNode *Pred = *I;
@@ -2203,18 +2132,18 @@ void GRExprEngine::EvalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
     SVal V = state->getSVal(Ex);
     if (nonloc::SymExprVal *SEV = dyn_cast<nonloc::SymExprVal>(&V)) {
       // First assume that the condition is true.
-      if (const GRState *stateTrue = state->Assume(*SEV, true)) {
+      if (const GRState *stateTrue = state->assume(*SEV, true)) {
         stateTrue = stateTrue->BindExpr(Ex,
-                                        ValMgr.makeIntVal(1U, Ex->getType()));
+                                        svalBuilder.makeIntVal(1U, Ex->getType()));
         Dst.Add(Builder->generateNode(PostStmtCustom(Ex,
                                 &EagerlyAssumeTag, Pred->getLocationContext()),
                                       stateTrue, Pred));
       }
 
       // Next, assume that the condition is false.
-      if (const GRState *stateFalse = state->Assume(*SEV, false)) {
+      if (const GRState *stateFalse = state->assume(*SEV, false)) {
         stateFalse = stateFalse->BindExpr(Ex,
-                                          ValMgr.makeIntVal(0U, Ex->getType()));
+                                          svalBuilder.makeIntVal(0U, Ex->getType()));
         Dst.Add(Builder->generateNode(PostStmtCustom(Ex, &EagerlyAssumeTag,
                                                    Pred->getLocationContext()),
                                       stateFalse, Pred));
@@ -2226,34 +2155,58 @@ void GRExprEngine::EvalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
 }
 
 //===----------------------------------------------------------------------===//
-// Transfer function: Objective-C ivar references.
+// Transfer function: Objective-C @synchronized.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitObjCIvarRefExpr(const ObjCIvarRefExpr* Ex, 
-                                        ExplodedNode* Pred,
-                                        ExplodedNodeSet& Dst, bool asLValue) {
+void ExprEngine::VisitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt *S,
+                                               ExplodedNode *Pred,
+                                               ExplodedNodeSet &Dst) {
 
-  const Expr* Base = cast<Expr>(Ex->getBase());
+  // The mutex expression is a CFGElement, so we don't need to explicitly
+  // visit it since it will already be processed.
+
+  // Pre-visit the ObjCAtSynchronizedStmt.
   ExplodedNodeSet Tmp;
-  Visit(Base, Pred, Tmp);
+  Tmp.Add(Pred);
+  CheckerVisit(S, Dst, Tmp, PreVisitStmtCallback);
+}
 
-  for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-    const GRState* state = GetState(*I);
-    SVal BaseVal = state->getSVal(Base);
-    SVal location = state->getLValue(Ex->getDecl(), BaseVal);
+//===----------------------------------------------------------------------===//
+// Transfer function: Objective-C ivar references.
+//===----------------------------------------------------------------------===//
 
-    if (asLValue)
-      MakeNode(Dst, Ex, *I, state->BindExpr(Ex, location));
-    else
-      EvalLoad(Dst, Ex, *I, state, location);
+void ExprEngine::VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr* Ex, 
+                                          ExplodedNode* Pred,
+                                          ExplodedNodeSet& Dst) {
+
+  // Visit the base expression, which is needed for computing the lvalue
+  // of the ivar.
+  ExplodedNodeSet dstBase;
+  const Expr *baseExpr = Ex->getBase();
+  Visit(baseExpr, Pred, dstBase);
+
+  ExplodedNodeSet dstIvar;
+
+  // Using the base, compute the lvalue of the instance variable.
+  for (ExplodedNodeSet::iterator I = dstBase.begin(), E = dstBase.end();
+       I!=E; ++I) {
+    ExplodedNode *nodeBase = *I;
+    const GRState *state = GetState(nodeBase);
+    SVal baseVal = state->getSVal(baseExpr);
+    SVal location = state->getLValue(Ex->getDecl(), baseVal);
+    MakeNode(dstIvar, Ex, *I, state->BindExpr(Ex, location));
   }
+
+  // Perform the post-condition check of the ObjCIvarRefExpr and store
+  // the created nodes in 'Dst'.
+  CheckerVisit(Ex, Dst, dstIvar, PostVisitStmtCallback);
 }
 
 //===----------------------------------------------------------------------===//
 // Transfer function: Objective-C fast enumeration 'for' statements.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S,
+void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S,
                                      ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
   // ObjCForCollectionStmts are processed in two places.  This method
@@ -2293,22 +2246,21 @@ void GRExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S,
   }
 
   ExplodedNodeSet Tmp;
-  VisitLValue(cast<Expr>(elem), Pred, Tmp);
-
+  Visit(cast<Expr>(elem), Pred, Tmp);
   for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
     const GRState* state = GetState(*I);
     VisitObjCForCollectionStmtAux(S, *I, Dst, state->getSVal(elem));
   }
 }
 
-void GRExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
+void ExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
                                        ExplodedNode* Pred, ExplodedNodeSet& Dst,
                                                  SVal ElementV) {
 
   // Check if the location we are writing back to is a null pointer.
   const Stmt* elem = S->getElement();
   ExplodedNodeSet Tmp;
-  EvalLocation(Tmp, elem, Pred, GetState(Pred), ElementV, NULL, false);
+  evalLocation(Tmp, elem, Pred, GetState(Pred), ElementV, NULL, false);
 
   if (Tmp.empty())
     return;
@@ -2318,11 +2270,11 @@ void GRExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
     const GRState *state = GetState(Pred);
 
     // Handle the case where the container still has elements.
-    SVal TrueV = ValMgr.makeTruthVal(1);
+    SVal TrueV = svalBuilder.makeTruthVal(1);
     const GRState *hasElems = state->BindExpr(S, TrueV);
 
     // Handle the case where the container has no elements.
-    SVal FalseV = ValMgr.makeTruthVal(0);
+    SVal FalseV = svalBuilder.makeTruthVal(0);
     const GRState *noElems = state->BindExpr(S, FalseV);
 
     if (loc::MemRegionVal* MV = dyn_cast<loc::MemRegionVal>(&ElementV))
@@ -2331,14 +2283,14 @@ void GRExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
         //  container.  We will do this with dispatch logic to the store.
         //  For now, just 'conjure' up a symbolic value.
         QualType T = R->getValueType();
-        assert(Loc::IsLocType(T));
+        assert(Loc::isLocType(T));
         unsigned Count = Builder->getCurrentBlockCount();
         SymbolRef Sym = SymMgr.getConjuredSymbol(elem, T, Count);
-        SVal V = ValMgr.makeLoc(Sym);
+        SVal V = svalBuilder.makeLoc(Sym);
         hasElems = hasElems->bindLoc(ElementV, V);
 
         // Bind the location to 'nil' on the false branch.
-        SVal nilV = ValMgr.makeIntVal(0, T);
+        SVal nilV = svalBuilder.makeIntVal(0, T);
         noElems = noElems->bindLoc(ElementV, nilV);
       }
 
@@ -2363,9 +2315,9 @@ public:
 };
 } // end anonymous namespace
 
-void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME, 
+void ExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME, 
                                         ExplodedNode* Pred,
-                                        ExplodedNodeSet& Dst, bool asLValue){
+                                        ExplodedNodeSet& Dst){
 
   // Create a worklist to process both the arguments.
   llvm::SmallVector<ObjCMsgWLItem, 20> WL;
@@ -2408,23 +2360,30 @@ void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME,
       WL.push_back(ObjCMsgWLItem(Item.I, *NI));
   }
 
-  // Now that the arguments are processed, handle the previsits checks.
+  // Now that the arguments are processed, handle the ObjC message.
+  VisitObjCMessage(ME, ArgsEvaluated, Dst);
+}
+
+void ExprEngine::VisitObjCMessage(const ObjCMessage &msg,
+                                  ExplodedNodeSet &Src, ExplodedNodeSet& Dst) {
+
+  // Handle the previsits checks.
   ExplodedNodeSet DstPrevisit;
-  CheckerVisit(ME, DstPrevisit, ArgsEvaluated, PreVisitStmtCallback);
+  CheckerVisitObjCMessage(msg, DstPrevisit, Src, /*isPreVisit=*/true);
 
   // Proceed with evaluate the message expression.
-  ExplodedNodeSet DstEval;
+  ExplodedNodeSet dstEval;
 
   for (ExplodedNodeSet::iterator DI = DstPrevisit.begin(),
                                  DE = DstPrevisit.end(); DI != DE; ++DI) {
 
-    Pred = *DI;
+    ExplodedNode *Pred = *DI;
     bool RaisesException = false;
-    unsigned OldSize = DstEval.size();
+    unsigned oldSize = dstEval.size();
     SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-    SaveOr OldHasGen(Builder->HasGeneratedNode);
+    SaveOr OldHasGen(Builder->hasGeneratedNode);
 
-    if (const Expr *Receiver = ME->getInstanceReceiver()) {
+    if (const Expr *Receiver = msg.getInstanceReceiver()) {
       const GRState *state = GetState(Pred);
 
       // Bifurcate the state into nil and non-nil ones.
@@ -2432,18 +2391,18 @@ void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME,
         cast<DefinedOrUnknownSVal>(state->getSVal(Receiver));
 
       const GRState *notNilState, *nilState;
-      llvm::tie(notNilState, nilState) = state->Assume(receiverVal);
+      llvm::tie(notNilState, nilState) = state->assume(receiverVal);
 
       // There are three cases: can be nil or non-nil, must be nil, must be
       // non-nil. We handle must be nil, and merge the rest two into non-nil.
       if (nilState && !notNilState) {
-        CheckerEvalNilReceiver(ME, DstEval, nilState, Pred);
+        CheckerEvalNilReceiver(msg, dstEval, nilState, Pred);
         continue;
       }
 
       // Check if the "raise" message was sent.
       assert(notNilState);
-      if (ME->getSelector() == RaiseSel)
+      if (msg.getSelector() == RaiseSel)
         RaisesException = true;
 
       // Check if we raise an exception.  For now treat these as sinks.
@@ -2452,11 +2411,11 @@ void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME,
         Builder->BuildSinks = true;
 
       // Dispatch to plug-in transfer function.
-      EvalObjCMessageExpr(DstEval, ME, Pred, notNilState);
+      evalObjCMessage(dstEval, msg, Pred, notNilState);
     }
-    else if (ObjCInterfaceDecl *Iface = ME->getReceiverInterface()) {
+    else if (const ObjCInterfaceDecl *Iface = msg.getReceiverInterface()) {
       IdentifierInfo* ClsName = Iface->getIdentifier();
-      Selector S = ME->getSelector();
+      Selector S = msg.getSelector();
 
       // Check for special instance methods.
       if (!NSExceptionII) {
@@ -2500,64 +2459,51 @@ void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME,
         Builder->BuildSinks = true;
 
       // Dispatch to plug-in transfer function.
-      EvalObjCMessageExpr(DstEval, ME, Pred, Builder->GetState(Pred));
+      evalObjCMessage(dstEval, msg, Pred, Builder->GetState(Pred));
     }
 
     // Handle the case where no nodes where generated.  Auto-generate that
     // contains the updated state if we aren't generating sinks.
-    if (!Builder->BuildSinks && DstEval.size() == OldSize &&
-        !Builder->HasGeneratedNode)
-      MakeNode(DstEval, ME, Pred, GetState(Pred));
+    if (!Builder->BuildSinks && dstEval.size() == oldSize &&
+        !Builder->hasGeneratedNode)
+      MakeNode(dstEval, msg.getOriginExpr(), Pred, GetState(Pred));
   }
 
   // Finally, perform the post-condition check of the ObjCMessageExpr and store
   // the created nodes in 'Dst'.
-  if (!(!asLValue && ReceiverReturnsReference(ME))) {
-    CheckerVisit(ME, Dst, DstEval, PostVisitStmtCallback);
-    return;
-  }
-
-  // Handle the case where the message expression returns a reference but
-  // we expect an rvalue.  For such cases, convert the reference to
-  // an rvalue.
-  // FIXME: This conversion doesn't actually happen unless the result
-  //  of ObjCMessageExpr is consumed by another expression.
-  ExplodedNodeSet DstRValueConvert;
-  CheckerVisit(ME, DstRValueConvert, DstEval, PostVisitStmtCallback);
-  QualType LoadTy = ME->getType();
-
-  static int *ConvertToRvalueTag = 0;
-  for (ExplodedNodeSet::iterator NI = DstRValueConvert.begin(),
-       NE = DstRValueConvert.end(); NI != NE; ++NI) {
-    const GRState *state = GetState(*NI);
-    EvalLoad(Dst, ME, *NI, state, state->getSVal(ME),
-             &ConvertToRvalueTag, LoadTy);
-  }
+  CheckerVisitObjCMessage(msg, Dst, dstEval, /*isPreVisit=*/false);
 }
 
 //===----------------------------------------------------------------------===//
 // Transfer functions: Miscellaneous statements.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, 
-                             ExplodedNode *Pred, ExplodedNodeSet &Dst, 
-                             bool asLValue) {
+void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, 
+                           ExplodedNode *Pred, ExplodedNodeSet &Dst) {
+  
   ExplodedNodeSet S1;
+  Visit(Ex, Pred, S1);
+  ExplodedNodeSet S2;
+  CheckerVisit(CastE, S2, S1, PreVisitStmtCallback);
+  
+  if (CastE->getCastKind() == CK_LValueToRValue ||
+      CastE->getCastKind() == CK_GetObjCProperty) {
+    for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I!=E; ++I) {
+      ExplodedNode *subExprNode = *I;
+      const GRState *state = GetState(subExprNode);
+      evalLoad(Dst, CastE, subExprNode, state, state->getSVal(Ex));
+    }
+    return;
+  }
+  
+  // All other casts.  
   QualType T = CastE->getType();
   QualType ExTy = Ex->getType();
 
   if (const ExplicitCastExpr *ExCast=dyn_cast_or_null<ExplicitCastExpr>(CastE))
     T = ExCast->getTypeAsWritten();
-
-  if (ExTy->isArrayType() || ExTy->isFunctionType() || T->isReferenceType() ||
-      asLValue)
-    VisitLValue(Ex, Pred, S1);
-  else
-    Visit(Ex, Pred, S1);
-
-  ExplodedNodeSet S2;
-  CheckerVisit(CastE, S2, S1, PreVisitStmtCallback);
-
+ 
+#if 0
   // If we are evaluating the cast in an lvalue context, we implicitly want
   // the cast to evaluate to a location.
   if (asLValue) {
@@ -2565,14 +2511,15 @@ void GRExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
     T = Ctx.getPointerType(Ctx.getCanonicalType(T));
     ExTy = Ctx.getPointerType(Ctx.getCanonicalType(ExTy));
   }
+#endif
 
   switch (CastE->getCastKind()) {
   case CK_ToVoid:
-    assert(!asLValue);
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I)
       Dst.Add(*I);
     return;
 
+  case CK_LValueToRValue:
   case CK_NoOp:
   case CK_FunctionToPointerDecay:
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
@@ -2585,33 +2532,60 @@ void GRExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
     }
     return;
 
-  case CK_Unknown:
+  case CK_GetObjCProperty:
+  case CK_Dependent:
   case CK_ArrayToPointerDecay:
   case CK_BitCast:
   case CK_LValueBitCast:
   case CK_IntegralCast:
+  case CK_NullToPointer:
   case CK_IntegralToPointer:
   case CK_PointerToIntegral:
+  case CK_PointerToBoolean:
+  case CK_IntegralToBoolean:
   case CK_IntegralToFloating:
   case CK_FloatingToIntegral:
+  case CK_FloatingToBoolean:
   case CK_FloatingCast:
+  case CK_FloatingRealToComplex:
+  case CK_FloatingComplexToReal:
+  case CK_FloatingComplexToBoolean:
+  case CK_FloatingComplexCast:
+  case CK_FloatingComplexToIntegralComplex:
+  case CK_IntegralRealToComplex:
+  case CK_IntegralComplexToReal:
+  case CK_IntegralComplexToBoolean:
+  case CK_IntegralComplexCast:
+  case CK_IntegralComplexToFloatingComplex:
   case CK_AnyPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
-  case CK_DerivedToBase:
-  case CK_UncheckedDerivedToBase:
+  
   case CK_ObjCObjectLValueCast: {
-    // Delegate to SValuator to process.
+    // Delegate to SValBuilder to process.
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
       ExplodedNode* N = *I;
       const GRState* state = GetState(N);
       SVal V = state->getSVal(Ex);
-      V = SVator.EvalCast(V, T, ExTy);
+      V = svalBuilder.evalCast(V, T, ExTy);
       state = state->BindExpr(CastE, V);
       MakeNode(Dst, CastE, N, state);
     }
     return;
   }
-  
+
+  case CK_DerivedToBase:
+  case CK_UncheckedDerivedToBase:
+    // For DerivedToBase cast, delegate to the store manager.
+    for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
+      ExplodedNode *node = *I;
+      const GRState *state = GetState(node);
+      SVal val = state->getSVal(Ex);
+      val = getStoreManager().evalDerivedToBase(val, T);
+      state = state->BindExpr(CastE, val);
+      MakeNode(Dst, CastE, node, state);
+    }
+    return;
+
   // Various C++ casts that are not handled yet.
   case CK_Dynamic:  
   case CK_ToUnion:
@@ -2631,10 +2605,9 @@ void GRExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
   }
 }
 
-void GRExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
+void ExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
                                             ExplodedNode* Pred,
-                                            ExplodedNodeSet& Dst,
-                                            bool asLValue) {
+                                            ExplodedNodeSet& Dst) {
   const InitListExpr* ILE 
     = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
   ExplodedNodeSet Tmp;
@@ -2646,7 +2619,7 @@ void GRExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
     const LocationContext *LC = (*I)->getLocationContext();
     state = state->bindCompoundLiteral(CL, LC, ILV);
 
-    if (asLValue) {
+    if (CL->isLValue()) {
       MakeNode(Dst, CL, *I, state->BindExpr(CL, state->getLValue(CL, LC)));
     }
     else
@@ -2654,7 +2627,7 @@ void GRExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
   }
 }
 
-void GRExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
+void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
                                  ExplodedNodeSet& Dst) {
 
   // The CFG has one DeclStmt per Decl.
@@ -2671,18 +2644,16 @@ void GRExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
   ExplodedNodeSet Tmp;
 
   if (InitEx) {
-    QualType InitTy = InitEx->getType();
-    if (getContext().getLangOptions().CPlusPlus && InitTy->isRecordType()) {
-      // Delegate expressions of C++ record type evaluation to AggExprVisitor.
-      VisitAggExpr(InitEx, GetState(Pred)->getLValue(VD,
-                                       Pred->getLocationContext()), Pred, Dst);
-      return;
-    } else if (VD->getType()->isReferenceType())
-      VisitLValue(InitEx, Pred, Tmp);
-    else
+    if (VD->getType()->isReferenceType() && !InitEx->isLValue()) {
+      // If the initializer is C++ record type, it should already has a 
+      // temp object.
+      if (!InitEx->getType()->isRecordType())
+        CreateCXXTemporaryObject(InitEx, Pred, Tmp);
+      else
+        Tmp.Add(Pred);
+    } else
       Visit(InitEx, Pred, Tmp);
-  }
-  else
+  } else
     Tmp.Add(Pred);
 
   ExplodedNodeSet Tmp2;
@@ -2698,16 +2669,24 @@ void GRExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
     if (InitEx) {
       SVal InitVal = state->getSVal(InitEx);
 
+      // We bound the temp obj region to the CXXConstructExpr. Now recover
+      // the lazy compound value when the variable is not a reference.
+      if (AMgr.getLangOptions().CPlusPlus && VD->getType()->isRecordType() && 
+          !VD->getType()->isReferenceType() && isa<loc::MemRegionVal>(InitVal)){
+        InitVal = state->getSVal(cast<loc::MemRegionVal>(InitVal).getRegion());
+        assert(isa<nonloc::LazyCompoundVal>(InitVal));
+      }
+
       // Recover some path-sensitivity if a scalar value evaluated to
       // UnknownVal.
       if ((InitVal.isUnknown() ||
           !getConstraintManager().canReasonAbout(InitVal)) &&
           !VD->getType()->isReferenceType()) {
-        InitVal = ValMgr.getConjuredSymbolVal(NULL, InitEx,
+        InitVal = svalBuilder.getConjuredSymbolVal(NULL, InitEx,
                                                Builder->getCurrentBlockCount());
       }
 
-      EvalBind(Dst, DS, *I, state,
+      evalBind(Dst, DS, *I, state,
                loc::MemRegionVal(state->getRegion(VD, LC)), InitVal, true);
     }
     else {
@@ -2717,7 +2696,7 @@ void GRExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
   }
 }
 
-void GRExprEngine::VisitCondInit(const VarDecl *VD, const Stmt *S,
+void ExprEngine::VisitCondInit(const VarDecl *VD, const Stmt *S,
                                  ExplodedNode *Pred, ExplodedNodeSet& Dst) {
 
   const Expr* InitEx = VD->getInit();
@@ -2735,11 +2714,11 @@ void GRExprEngine::VisitCondInit(const VarDecl *VD, const Stmt *S,
     // UnknownVal.
     if (InitVal.isUnknown() ||
         !getConstraintManager().canReasonAbout(InitVal)) {
-      InitVal = ValMgr.getConjuredSymbolVal(NULL, InitEx,
+      InitVal = svalBuilder.getConjuredSymbolVal(NULL, InitEx,
                                             Builder->getCurrentBlockCount());
     }
 
-    EvalBind(Dst, S, N, state,
+    evalBind(Dst, S, N, state,
              loc::MemRegionVal(state->getRegion(VD, LC)), InitVal, true);
   }
 }
@@ -2760,7 +2739,7 @@ public:
 }
 
 
-void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
+void ExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
                                      ExplodedNodeSet& Dst) {
 
   const GRState* state = GetState(Pred);
@@ -2773,7 +2752,7 @@ void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
     // Handle base case where the initializer has no elements.
     // e.g: static int* myArray[] = {};
     if (NumInitElements == 0) {
-      SVal V = ValMgr.makeCompoundVal(T, StartVals);
+      SVal V = svalBuilder.makeCompoundVal(T, StartVals);
       MakeNode(Dst, E, Pred, state->BindExpr(E, V));
       return;
     }
@@ -2807,7 +2786,7 @@ void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
 
         if (NewItr == ItrEnd) {
           // Now we have a list holding all init values. Make CompoundValData.
-          SVal V = ValMgr.makeCompoundVal(T, NewVals);
+          SVal V = svalBuilder.makeCompoundVal(T, NewVals);
 
           // Make final state and node.
           MakeNode(Dst, E, *NI, state->BindExpr(E, V));
@@ -2822,7 +2801,7 @@ void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
     return;
   }
 
-  if (Loc::IsLocType(T) || T->isIntegerType()) {
+  if (Loc::isLocType(T) || T->isIntegerType()) {
     assert (E->getNumInits() == 1);
     ExplodedNodeSet Tmp;
     const Expr* Init = E->getInit(0);
@@ -2838,7 +2817,7 @@ void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
 }
 
 /// VisitSizeOfAlignOfExpr - Transfer function for sizeof(type).
-void GRExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
+void ExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
                                           ExplodedNode* Pred,
                                           ExplodedNodeSet& Dst) {
   QualType T = Ex->getTypeOfArgument();
@@ -2863,7 +2842,7 @@ void GRExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
       // First, visit the sub-expression to find its region.
       const Expr *Arg = Ex->getArgumentExpr();
       ExplodedNodeSet Tmp;
-      VisitLValue(Arg, Pred, Tmp);
+      Visit(Arg, Pred, Tmp);
 
       for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
         const GRState* state = GetState(*I);
@@ -2877,7 +2856,7 @@ void GRExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
         }
 
         // The result is the extent of the VLA.
-        SVal Extent = cast<SubRegion>(MR)->getExtent(ValMgr);
+        SVal Extent = cast<SubRegion>(MR)->getExtent(svalBuilder);
         MakeNode(Dst, Ex, *I, state->BindExpr(Ex, Extent));
       }
 
@@ -2900,10 +2879,10 @@ void GRExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
 
   MakeNode(Dst, Ex, Pred,
            GetState(Pred)->BindExpr(Ex,
-              ValMgr.makeIntVal(amt.getQuantity(), Ex->getType())));
+              svalBuilder.makeIntVal(amt.getQuantity(), Ex->getType())));
 }
 
-void GRExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE, 
+void ExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE, 
                                      ExplodedNode* Pred, ExplodedNodeSet& Dst) {
   Expr::EvalResult Res;
   if (OOE->Evaluate(Res, getContext()) && Res.Val.isInt()) {
@@ -2911,7 +2890,7 @@ void GRExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE,
     assert(IV.getBitWidth() == getContext().getTypeSize(OOE->getType()));
     assert(OOE->getType()->isIntegerType());
     assert(IV.isSigned() == OOE->getType()->isSignedIntegerType());
-    SVal X = ValMgr.makeIntVal(IV);
+    SVal X = svalBuilder.makeIntVal(IV);
     MakeNode(Dst, OOE, Pred, GetState(Pred)->BindExpr(OOE, X));
     return;
   }
@@ -2919,38 +2898,16 @@ void GRExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE,
   Dst.Add(Pred);
 }
 
-void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
+void ExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
                                       ExplodedNode* Pred,
-                                      ExplodedNodeSet& Dst, bool asLValue) {
+                                      ExplodedNodeSet& Dst) {
 
   switch (U->getOpcode()) {
 
     default:
       break;
 
-    case UO_Deref: {
-
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      Visit(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-
-        const GRState* state = GetState(*I);
-        SVal location = state->getSVal(Ex);
-
-        if (asLValue)
-          MakeNode(Dst, U, *I, state->BindExpr(U, location),
-                   ProgramPoint::PostLValueKind);
-        else
-          EvalLoad(Dst, U, *I, state, location);
-      }
-
-      return;
-    }
-
     case UO_Real: {
-
       const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
@@ -2989,14 +2946,18 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
 
         // For all other types, UO_Imag returns 0.
         const GRState* state = GetState(*I);
-        SVal X = ValMgr.makeZeroVal(Ex->getType());
+        SVal X = svalBuilder.makeZeroVal(Ex->getType());
         MakeNode(Dst, U, *I, state->BindExpr(U, X));
       }
 
       return;
     }
       
-    case UO_Plus: assert(!asLValue);  // FALL-THROUGH.
+    case UO_Plus:
+      assert(!U->isLValue());
+      // FALL-THROUGH.
+    case UO_Deref:
+    case UO_AddrOf:
     case UO_Extension: {
 
       // Unary "+" is a no-op, similar to a parentheses.  We still have places
@@ -3006,11 +2967,7 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
 
       const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
-
-      if (asLValue)
-        VisitLValue(Ex, Pred, Tmp);
-      else
-        Visit(Ex, Pred, Tmp);
+      Visit(Ex, Pred, Tmp);
 
       for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
         const GRState* state = GetState(*I);
@@ -3020,28 +2977,10 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
       return;
     }
 
-    case UO_AddrOf: {
-
-      assert(!asLValue);
-      const Expr* Ex = U->getSubExpr()->IgnoreParens();
-      ExplodedNodeSet Tmp;
-      VisitLValue(Ex, Pred, Tmp);
-
-      for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
-        const GRState* state = GetState(*I);
-        SVal V = state->getSVal(Ex);
-        state = state->BindExpr(U, V);
-        MakeNode(Dst, U, *I, state);
-      }
-
-      return;
-    }
-
     case UO_LNot:
     case UO_Minus:
     case UO_Not: {
-
-      assert (!asLValue);
+      assert (!U->isLValue());
       const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
@@ -3061,7 +3000,7 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
 //        QualType SrcT = getContext().getCanonicalType(Ex->getType());
 //
 //        if (DstT != SrcT) // Perform promotions.
-//          V = EvalCast(V, DstT);
+//          V = evalCast(V, DstT);
 //
 //        if (V.isUnknownOrUndef()) {
 //          MakeNode(Dst, U, *I, BindExpr(St, U, V));
@@ -3075,12 +3014,12 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
 
           case UO_Not:
             // FIXME: Do we need to handle promotions?
-            state = state->BindExpr(U, EvalComplement(cast<NonLoc>(V)));
+            state = state->BindExpr(U, evalComplement(cast<NonLoc>(V)));
             break;
 
           case UO_Minus:
             // FIXME: Do we need to handle promotions?
-            state = state->BindExpr(U, EvalMinus(cast<NonLoc>(V)));
+            state = state->BindExpr(U, evalMinus(cast<NonLoc>(V)));
             break;
 
           case UO_LNot:
@@ -3092,13 +3031,13 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
             SVal Result;
 
             if (isa<Loc>(V)) {
-              Loc X = ValMgr.makeNull();
-              Result = EvalBinOp(state, BO_EQ, cast<Loc>(V), X,
+              Loc X = svalBuilder.makeNull();
+              Result = evalBinOp(state, BO_EQ, cast<Loc>(V), X,
                                  U->getType());
             }
             else {
               nonloc::ConcreteInt X(getBasicVals().getValue(0, Ex->getType()));
-              Result = EvalBinOp(state, BO_EQ, cast<NonLoc>(V), X,
+              Result = evalBinOp(state, BO_EQ, cast<NonLoc>(V), X,
                                  U->getType());
             }
 
@@ -3115,20 +3054,19 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
   }
 
   // Handle ++ and -- (both pre- and post-increment).
-
   assert (U->isIncrementDecrementOp());
   ExplodedNodeSet Tmp;
   const Expr* Ex = U->getSubExpr()->IgnoreParens();
-  VisitLValue(Ex, Pred, Tmp);
+  Visit(Ex, Pred, Tmp);
 
   for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
 
     const GRState* state = GetState(*I);
-    SVal V1 = state->getSVal(Ex);
+    SVal loc = state->getSVal(Ex);
 
     // Perform a load.
     ExplodedNodeSet Tmp2;
-    EvalLoad(Tmp2, Ex, *I, state, V1);
+    evalLoad(Tmp2, Ex, *I, state, loc);
 
     for (ExplodedNodeSet::iterator I2=Tmp2.begin(), E2=Tmp2.end();I2!=E2;++I2) {
 
@@ -3152,53 +3090,58 @@ void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U,
       SVal RHS;
 
       if (U->getType()->isAnyPointerType())
-        RHS = ValMgr.makeIntValWithPtrWidth(1, false);
+        RHS = svalBuilder.makeArrayIndex(1);
       else
-        RHS = ValMgr.makeIntVal(1, U->getType());
+        RHS = svalBuilder.makeIntVal(1, U->getType());
 
-      SVal Result = EvalBinOp(state, Op, V2, RHS, U->getType());
+      SVal Result = evalBinOp(state, Op, V2, RHS, U->getType());
 
       // Conjure a new symbol if necessary to recover precision.
       if (Result.isUnknown() || !getConstraintManager().canReasonAbout(Result)){
         DefinedOrUnknownSVal SymVal =
-          ValMgr.getConjuredSymbolVal(NULL, Ex,
+          svalBuilder.getConjuredSymbolVal(NULL, Ex,
                                       Builder->getCurrentBlockCount());
         Result = SymVal;
 
         // If the value is a location, ++/-- should always preserve
         // non-nullness.  Check if the original value was non-null, and if so
         // propagate that constraint.
-        if (Loc::IsLocType(U->getType())) {
+        if (Loc::isLocType(U->getType())) {
           DefinedOrUnknownSVal Constraint =
-            SVator.EvalEQ(state, V2, ValMgr.makeZeroVal(U->getType()));
+            svalBuilder.evalEQ(state, V2,svalBuilder.makeZeroVal(U->getType()));
 
-          if (!state->Assume(Constraint, true)) {
+          if (!state->assume(Constraint, true)) {
             // It isn't feasible for the original value to be null.
             // Propagate this constraint.
-            Constraint = SVator.EvalEQ(state, SymVal,
-                                       ValMgr.makeZeroVal(U->getType()));
+            Constraint = svalBuilder.evalEQ(state, SymVal,
+                                       svalBuilder.makeZeroVal(U->getType()));
 
 
-            state = state->Assume(Constraint, false);
+            state = state->assume(Constraint, false);
             assert(state);
           }
         }
       }
 
-      state = state->BindExpr(U, U->isPostfix() ? V2 : Result);
+      // Since the lvalue-to-rvalue conversion is explicit in the AST,
+      // we bind an l-value if the operator is prefix and an lvalue (in C++).
+      if (U->isLValue())
+        state = state->BindExpr(U, loc);
+      else
+        state = state->BindExpr(U, V2);
 
       // Perform the store.
-      EvalStore(Dst, NULL, U, *I2, state, V1, Result);
+      evalStore(Dst, NULL, U, *I2, state, loc, Result);
     }
   }
 }
 
-void GRExprEngine::VisitAsmStmt(const AsmStmt* A, ExplodedNode* Pred,
+void ExprEngine::VisitAsmStmt(const AsmStmt* A, ExplodedNode* Pred,
                                 ExplodedNodeSet& Dst) {
   VisitAsmStmtHelperOutputs(A, A->begin_outputs(), A->end_outputs(), Pred, Dst);
 }
 
-void GRExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
+void ExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
                                              AsmStmt::const_outputs_iterator I,
                                              AsmStmt::const_outputs_iterator E,
                                      ExplodedNode* Pred, ExplodedNodeSet& Dst) {
@@ -3208,15 +3151,14 @@ void GRExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
   }
 
   ExplodedNodeSet Tmp;
-  VisitLValue(*I, Pred, Tmp);
-
+  Visit(*I, Pred, Tmp);
   ++I;
 
   for (ExplodedNodeSet::iterator NI = Tmp.begin(), NE = Tmp.end();NI != NE;++NI)
     VisitAsmStmtHelperOutputs(A, I, E, *NI, Dst);
 }
 
-void GRExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
+void ExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
                                             AsmStmt::const_inputs_iterator I,
                                             AsmStmt::const_inputs_iterator E,
                                             ExplodedNode* Pred,
@@ -3255,18 +3197,17 @@ void GRExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
     VisitAsmStmtHelperInputs(A, I, E, *NI, Dst);
 }
 
-void GRExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
+void ExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
                                    ExplodedNodeSet &Dst) {
   ExplodedNodeSet Src;
   if (const Expr *RetE = RS->getRetValue()) {
     // Record the returned expression in the state. It will be used in
-    // ProcessCallExit to bind the return value to the call expr.
+    // processCallExit to bind the return value to the call expr.
     {
-      static int Tag = 0;
-      SaveAndRestore<const void *> OldTag(Builder->Tag, &Tag);
+      static int tag = 0;
       const GRState *state = GetState(Pred);
       state = state->set<ReturnExpr>(RetE);
-      Pred = Builder->generateNode(RetE, state, Pred);
+      Pred = Builder->generateNode(RetE, state, Pred, &tag);
     }
     // We may get a NULL Pred because we generated a cached node.
     if (Pred)
@@ -3282,19 +3223,19 @@ void GRExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
   for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
        I != E; ++I) {
 
-    assert(Builder && "GRStmtNodeBuilder must be defined.");
+    assert(Builder && "StmtNodeBuilder must be defined.");
 
     Pred = *I;
     unsigned size = Dst.size();
 
     SaveAndRestore<bool> OldSink(Builder->BuildSinks);
-    SaveOr OldHasGen(Builder->HasGeneratedNode);
+    SaveOr OldHasGen(Builder->hasGeneratedNode);
 
-    getTF().EvalReturn(Dst, *this, *Builder, RS, Pred);
+    getTF().evalReturn(Dst, *this, *Builder, RS, Pred);
 
     // Handle the case where no nodes where generated.
     if (!Builder->BuildSinks && Dst.size() == size &&
-        !Builder->HasGeneratedNode)
+        !Builder->hasGeneratedNode)
       MakeNode(Dst, RS, Pred, GetState(Pred));
   }
 }
@@ -3303,25 +3244,14 @@ void GRExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
 // Transfer functions: Binary operators.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
+void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
                                        ExplodedNode* Pred,
-                                       ExplodedNodeSet& Dst, bool asLValue) {
-
+                                       ExplodedNodeSet& Dst) {
   ExplodedNodeSet Tmp1;
   Expr* LHS = B->getLHS()->IgnoreParens();
   Expr* RHS = B->getRHS()->IgnoreParens();
 
-  // FIXME: Add proper support for ObjCImplicitSetterGetterRefExpr.
-  if (isa<ObjCImplicitSetterGetterRefExpr>(LHS)) {
-    Visit(RHS, Pred, Dst);
-    return;
-  }
-
-  if (B->isAssignmentOp())
-    VisitLValue(LHS, Pred, Tmp1);
-  else
-    Visit(LHS, Pred, Tmp1);
-
+  Visit(LHS, Pred, Tmp1);
   ExplodedNodeSet Tmp3;
 
   for (ExplodedNodeSet::iterator I1=Tmp1.begin(), E1=Tmp1.end(); I1!=E1; ++I1) {
@@ -3338,7 +3268,6 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
          I2 != E2; ++I2) {
 
       const GRState *state = GetState(*I2);
-      const GRState *OldSt = state;
       SVal RightV = state->getSVal(RHS);
 
       BinaryOperator::Opcode Op = B->getOpcode();
@@ -3346,35 +3275,27 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
       if (Op == BO_Assign) {
         // EXPERIMENTAL: "Conjured" symbols.
         // FIXME: Handle structs.
-        QualType T = RHS->getType();
-
         if (RightV.isUnknown() ||!getConstraintManager().canReasonAbout(RightV))
         {
           unsigned Count = Builder->getCurrentBlockCount();
-          RightV = ValMgr.getConjuredSymbolVal(NULL, B->getRHS(), Count);
+          RightV = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), Count);
         }
 
-        SVal ExprVal = asLValue ? LeftV : RightV;
+        SVal ExprVal = B->isLValue() ? LeftV : RightV;
 
         // Simulate the effects of a "store":  bind the value of the RHS
         // to the L-Value represented by the LHS.
-        EvalStore(Tmp3, B, LHS, *I2, state->BindExpr(B, ExprVal), LeftV,RightV);
+        evalStore(Tmp3, B, LHS, *I2, state->BindExpr(B, ExprVal), LeftV,RightV);
         continue;
       }
 
       if (!B->isAssignmentOp()) {
         // Process non-assignments except commas or short-circuited
         // logical expressions (LAnd and LOr).
-        SVal Result = EvalBinOp(state, Op, LeftV, RightV, B->getType());
+        SVal Result = evalBinOp(state, Op, LeftV, RightV, B->getType());
 
         if (Result.isUnknown()) {
-          if (OldSt != state) {
-            // Generate a new node if we have already created a new state.
-            MakeNode(Tmp3, B, *I2, state);
-          }
-          else
-            Tmp3.Add(*I2);
-
+          MakeNode(Tmp3, B, *I2, state);
           continue;
         }
 
@@ -3405,7 +3326,7 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
       // null dereferences, and so on.
       ExplodedNodeSet Tmp4;
       SVal location = state->getSVal(LHS);
-      EvalLoad(Tmp4, LHS, *I2, state, location);
+      evalLoad(Tmp4, LHS, *I2, state, location);
 
       for (ExplodedNodeSet::iterator I4=Tmp4.begin(), E4=Tmp4.end(); I4!=E4;
            ++I4) {
@@ -3422,13 +3343,12 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
         CLHSTy = getContext().getCanonicalType(CLHSTy);
 
         QualType LTy = getContext().getCanonicalType(LHS->getType());
-        QualType RTy = getContext().getCanonicalType(RHS->getType());
 
         // Promote LHS.
-        V = SVator.EvalCast(V, CLHSTy, LTy);
+        V = svalBuilder.evalCast(V, CLHSTy, LTy);
 
         // Compute the result of the operation.
-        SVal Result = SVator.EvalCast(EvalBinOp(state, Op, V, RightV, CTy),
+        SVal Result = svalBuilder.evalCast(evalBinOp(state, Op, V, RightV, CTy),
                                       B->getType(), CTy);
 
         // EXPERIMENTAL: "Conjured" symbols.
@@ -3444,19 +3364,25 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
           // The symbolic value is actually for the type of the left-hand side
           // expression, not the computation type, as this is the value the
           // LValue on the LHS will bind to.
-          LHSVal = ValMgr.getConjuredSymbolVal(NULL, B->getRHS(), LTy, Count);
+          LHSVal = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), LTy, Count);
 
           // However, we need to convert the symbol to the computation type.
-          Result = SVator.EvalCast(LHSVal, CTy, LTy);
+          Result = svalBuilder.evalCast(LHSVal, CTy, LTy);
         }
         else {
           // The left-hand side may bind to a different value then the
           // computation type.
-          LHSVal = SVator.EvalCast(Result, LTy, CTy);
+          LHSVal = svalBuilder.evalCast(Result, LTy, CTy);
         }
 
-        EvalStore(Tmp3, B, LHS, *I4, state->BindExpr(B, Result),
-                  location, LHSVal);
+        // In C++, assignment and compound assignment operators return an 
+        // lvalue.
+        if (B->isLValue())
+          state = state->BindExpr(B, location);
+        else
+          state = state->BindExpr(B, Result);
+
+        evalStore(Tmp3, B, LHS, *I4, state, location, LHSVal);
       }
     }
   }
@@ -3468,7 +3394,7 @@ void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
 // Checker registration/lookup.
 //===----------------------------------------------------------------------===//
 
-Checker *GRExprEngine::lookupChecker(void *tag) const {
+Checker *ExprEngine::lookupChecker(void *tag) const {
   CheckerMap::const_iterator I = CheckerM.find(tag);
   return (I == CheckerM.end()) ? NULL : Checkers[I->second].second;
 }
@@ -3478,7 +3404,7 @@ Checker *GRExprEngine::lookupChecker(void *tag) const {
 //===----------------------------------------------------------------------===//
 
 #ifndef NDEBUG
-static GRExprEngine* GraphPrintCheckerState;
+static ExprEngine* GraphPrintCheckerState;
 static SourceManager* GraphPrintSourceManager;
 
 namespace llvm {
@@ -3488,7 +3414,7 @@ struct DOTGraphTraits<ExplodedNode*> :
 
   DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {}
 
-  // FIXME: Since we do not cache error nodes in GRExprEngine now, this does not
+  // FIXME: Since we do not cache error nodes in ExprEngine now, this does not
   // work.
   static std::string getNodeAttributes(const ExplodedNode* N, void*) {
 
@@ -3654,11 +3580,10 @@ struct DOTGraphTraits<ExplodedNode*> :
       }
     }
 
-    Out << "\\|StateID: " << (void*) N->getState() << "\\|";
-
     const GRState *state = N->getState();
+    Out << "\\|StateID: " << (void*) state
+        << " NodeID: " << (void*) N << "\\|";
     state->printDOT(Out, *N->getLocationContext()->getCFG());
-
     Out << "\\l";
     return Out.str();
   }
@@ -3677,7 +3602,7 @@ GetGraphNode<llvm::DenseMap<ExplodedNode*, Expr*>::iterator>
 }
 #endif
 
-void GRExprEngine::ViewGraph(bool trim) {
+void ExprEngine::ViewGraph(bool trim) {
 #ifndef NDEBUG
   if (trim) {
     std::vector<ExplodedNode*> Src;
@@ -3693,7 +3618,7 @@ void GRExprEngine::ViewGraph(bool trim) {
            I2!=E2; ++I2) {
         const BugReportEquivClass& EQ = *I2;
         const BugReport &R = **EQ.begin();
-        ExplodedNode *N = const_cast<ExplodedNode*>(R.getEndNode());
+        ExplodedNode *N = const_cast<ExplodedNode*>(R.getErrorNode());
         if (N) Src.push_back(N);
       }
     }
@@ -3704,7 +3629,7 @@ void GRExprEngine::ViewGraph(bool trim) {
     GraphPrintCheckerState = this;
     GraphPrintSourceManager = &getContext().getSourceManager();
 
-    llvm::ViewGraph(*G.roots_begin(), "GRExprEngine");
+    llvm::ViewGraph(*G.roots_begin(), "ExprEngine");
 
     GraphPrintCheckerState = NULL;
     GraphPrintSourceManager = NULL;
@@ -3712,7 +3637,7 @@ void GRExprEngine::ViewGraph(bool trim) {
 #endif
 }
 
-void GRExprEngine::ViewGraph(ExplodedNode** Beg, ExplodedNode** End) {
+void ExprEngine::ViewGraph(ExplodedNode** Beg, ExplodedNode** End) {
 #ifndef NDEBUG
   GraphPrintCheckerState = this;
   GraphPrintSourceManager = &getContext().getSourceManager();
@@ -3722,7 +3647,7 @@ void GRExprEngine::ViewGraph(ExplodedNode** Beg, ExplodedNode** End) {
   if (!TrimmedG.get())
     llvm::errs() << "warning: Trimmed ExplodedGraph is empty.\n";
   else
-    llvm::ViewGraph(*TrimmedG->roots_begin(), "TrimmedGRExprEngine");
+    llvm::ViewGraph(*TrimmedG->roots_begin(), "TrimmedExprEngine");
 
   GraphPrintCheckerState = NULL;
   GraphPrintSourceManager = NULL;
diff --git a/lib/Checker/FixedAddressChecker.cpp b/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index 29a3c3ae3533..fe628a2512b2 100644
--- a/lib/Checker/FixedAddressChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -13,11 +13,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class FixedAddressChecker 
@@ -54,7 +56,7 @@ void FixedAddressChecker::PreVisitBinaryOperator(CheckerContext &C,
   if (!RV.isConstant() || RV.isZeroConstant())
     return;
 
-  if (ExplodedNode *N = C.GenerateNode()) {
+  if (ExplodedNode *N = C.generateNode()) {
     if (!BT)
       BT = new BuiltinBug("Use fixed address", 
                           "Using a fixed address is not portable because that "
@@ -66,6 +68,10 @@ void FixedAddressChecker::PreVisitBinaryOperator(CheckerContext &C,
   }
 }
 
-void clang::RegisterFixedAddressChecker(GRExprEngine &Eng) {
+static void RegisterFixedAddressChecker(ExprEngine &Eng) {
   Eng.registerCheck(new FixedAddressChecker());
 }
+
+void ento::registerFixedAddressChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterFixedAddressChecker);
+}
diff --git a/lib/Checker/IdempotentOperationChecker.cpp b/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
index 6411c790ef7c..f49b125a60a2 100644
--- a/lib/Checker/IdempotentOperationChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
@@ -42,65 +42,89 @@
 // - Finer grained false positive control (levels)
 // - Handling ~0 values
 
-#include "GRExprEngineExperimentalChecks.h"
+#include "ClangSACheckers.h"
 #include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/Analyses/PseudoConstantAnalysis.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerHelpers.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRCoreEngine.h"
-#include "clang/Checker/PathSensitive/SVals.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
 #include "clang/AST/Stmt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <deque>
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class IdempotentOperationChecker
   : public CheckerVisitor<IdempotentOperationChecker> {
+public:
+  static void *getTag();
+  void PreVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
+  void PostVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
+  void VisitEndAnalysis(ExplodedGraph &G, BugReporter &B, ExprEngine &Eng);
+
+private:
+  // Our assumption about a particular operation.
+  enum Assumption { Possible = 0, Impossible, Equal, LHSis1, RHSis1, LHSis0,
+      RHSis0 };
+
+  void UpdateAssumption(Assumption &A, const Assumption &New);
+
+  // False positive reduction methods
+  static bool isSelfAssign(const Expr *LHS, const Expr *RHS);
+  static bool isUnused(const Expr *E, AnalysisContext *AC);
+  static bool isTruncationExtensionAssignment(const Expr *LHS,
+                                              const Expr *RHS);
+  bool pathWasCompletelyAnalyzed(const CFG *cfg,
+                                 const CFGBlock *CB,
+                                 const CFGStmtMap *CBM,
+                                 const CoreEngine &CE);
+  static bool CanVary(const Expr *Ex,
+                      AnalysisContext *AC);
+  static bool isConstantOrPseudoConstant(const DeclRefExpr *DR,
+                                         AnalysisContext *AC);
+  static bool containsNonLocalVarDecl(const Stmt *S);
+
+  // Hash table and related data structures
+  struct BinaryOperatorData {
+    BinaryOperatorData() : assumption(Possible), analysisContext(0) {}
+
+    Assumption assumption;
+    AnalysisContext *analysisContext;
+    ExplodedNodeSet explodedNodes; // Set of ExplodedNodes that refer to a
+                                   // BinaryOperator
+  };
+  typedef llvm::DenseMap<const BinaryOperator *, BinaryOperatorData>
+      AssumptionMap;
+  AssumptionMap hash;
+
+  // A class that performs reachability queries for CFGBlocks. Several internal
+  // checks in this checker require reachability information. The requests all
+  // tend to have a common destination, so we lazily do a predecessor search
+  // from the destination node and cache the results to prevent work
+  // duplication.
+  class CFGReachabilityAnalysis {
+    typedef llvm::BitVector ReachableSet;
+    typedef llvm::DenseMap<unsigned, ReachableSet> ReachableMap;
+    ReachableSet analyzed;
+    ReachableMap reachable;
   public:
-    static void *getTag();
-    void PreVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
-    void PostVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
-    void VisitEndAnalysis(ExplodedGraph &G, BugReporter &B, GRExprEngine &Eng);
-
+    CFGReachabilityAnalysis(const CFG &cfg)
+      : analyzed(cfg.getNumBlockIDs(), false) {}
+    
+    inline bool isReachable(const CFGBlock *Src, const CFGBlock *Dst);
   private:
-    // Our assumption about a particular operation.
-    enum Assumption { Possible = 0, Impossible, Equal, LHSis1, RHSis1, LHSis0,
-        RHSis0 };
-
-    void UpdateAssumption(Assumption &A, const Assumption &New);
-
-    // False positive reduction methods
-    static bool isSelfAssign(const Expr *LHS, const Expr *RHS);
-    static bool isUnused(const Expr *E, AnalysisContext *AC);
-    //static bool isTruncationExtensionAssignment(const Expr *LHS,
-    //                                            const Expr *RHS);
-    static bool PathWasCompletelyAnalyzed(const CFG *C,
-                                          const CFGBlock *CB,
-                                          const GRCoreEngine &CE);
-    static bool CanVary(const Expr *Ex,
-                        AnalysisContext *AC);
-    static bool isConstantOrPseudoConstant(const DeclRefExpr *DR,
-                                           AnalysisContext *AC);
-    static bool containsNonLocalVarDecl(const Stmt *S);
-
-    // Hash table and related data structures
-    struct BinaryOperatorData {
-      BinaryOperatorData() : assumption(Possible), analysisContext(0) {}
-
-      Assumption assumption;
-      AnalysisContext *analysisContext;
-      ExplodedNodeSet explodedNodes; // Set of ExplodedNodes that refer to a
-                                     // BinaryOperator
-    };
-    typedef llvm::DenseMap<const BinaryOperator *, BinaryOperatorData>
-        AssumptionMap;
-    AssumptionMap hash;
+    void MapReachability(const CFGBlock *Dst);
+  };
+  llvm::OwningPtr<CFGReachabilityAnalysis> CRA;
 };
 }
 
@@ -109,10 +133,14 @@ void *IdempotentOperationChecker::getTag() {
   return &x;
 }
 
-void clang::RegisterIdempotentOperationChecker(GRExprEngine &Eng) {
+static void RegisterIdempotentOperationChecker(ExprEngine &Eng) {
   Eng.registerCheck(new IdempotentOperationChecker());
 }
 
+void ento::registerIdempotentOperationChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterIdempotentOperationChecker);
+}
+
 void IdempotentOperationChecker::PreVisitBinaryOperator(
                                                       CheckerContext &C,
                                                       const BinaryOperator *B) {
@@ -181,7 +209,7 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
       A = Impossible;
       return;
     }
-    LHSVal = state->getSVal(cast<Loc>(LHSVal));
+    LHSVal = state->getSVal(cast<Loc>(LHSVal), LHS->getType());
   }
 
 
@@ -196,9 +224,10 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
   case BO_Assign:
     // x Assign x can be used to silence unused variable warnings intentionally.
     // If this is a self assignment and the variable is referenced elsewhere,
-    // then it is a false positive.
+    // and the assignment is not a truncation or extension, then it is a false
+    // positive.
     if (isSelfAssign(LHS, RHS)) {
-      if (!isUnused(LHS, AC)) {
+      if (!isUnused(LHS, AC) && !isTruncationExtensionAssignment(LHS, RHS)) {
         UpdateAssumption(A, Equal);
         return;
       }
@@ -335,12 +364,21 @@ void IdempotentOperationChecker::PostVisitBinaryOperator(
                                                       const BinaryOperator *B) {
   // Add the ExplodedNode we just visited
   BinaryOperatorData &Data = hash[B];
+
+  const Stmt *predStmt 
+    = cast<StmtPoint>(C.getPredecessor()->getLocation()).getStmt();
+  
+  // Ignore implicit calls to setters.
+  if (isa<ObjCPropertyRefExpr>(predStmt))
+    return;
+  
+  assert(isa<BinaryOperator>(predStmt));
   Data.explodedNodes.Add(C.getPredecessor());
 }
 
 void IdempotentOperationChecker::VisitEndAnalysis(ExplodedGraph &G,
                                                   BugReporter &BR,
-                                                  GRExprEngine &Eng) {
+                                                  ExprEngine &Eng) {
   BugType *BT = new BugType("Idempotent operation", "Dead code");
   // Iterate over the hash to see if we have any paths with definite
   // idempotent operations.
@@ -363,8 +401,8 @@ void IdempotentOperationChecker::VisitEndAnalysis(ExplodedGraph &G,
                                                 &AC->getParentMap());
 
       // If we can trace back
-      if (!PathWasCompletelyAnalyzed(AC->getCFG(),
-                                     CBM->getBlock(B),
+      if (!pathWasCompletelyAnalyzed(AC->getCFG(),
+                                     CBM->getBlock(B), CBM,
                                      Eng.getCoreEngine()))
         continue;
 
@@ -500,7 +538,6 @@ bool IdempotentOperationChecker::isUnused(const Expr *E,
   return true;
 }
 
-#if 0
 // Check for self casts truncating/extending a variable
 bool IdempotentOperationChecker::isTruncationExtensionAssignment(
                                                               const Expr *LHS,
@@ -521,59 +558,78 @@ bool IdempotentOperationChecker::isTruncationExtensionAssignment(
   if (VD != RHS_DR->getDecl())
      return false;
 
-  return dyn_cast<DeclRefExpr>(RHS->IgnoreParens()) == NULL;
+  return dyn_cast<DeclRefExpr>(RHS->IgnoreParenLValueCasts()) == NULL;
 }
-#endif
 
 // Returns false if a path to this block was not completely analyzed, or true
 // otherwise.
-bool IdempotentOperationChecker::PathWasCompletelyAnalyzed(
-                                                       const CFG *C,
-                                                       const CFGBlock *CB,
-                                                       const GRCoreEngine &CE) {
-  std::deque<const CFGBlock *> WorkList;
-  llvm::SmallSet<unsigned, 8> Aborted;
-  llvm::SmallSet<unsigned, 128> Visited;
-
-  // Create a set of all aborted blocks
-  typedef GRCoreEngine::BlocksAborted::const_iterator AbortedIterator;
+bool
+IdempotentOperationChecker::pathWasCompletelyAnalyzed(const CFG *cfg,
+                                                      const CFGBlock *CB,
+                                                      const CFGStmtMap *CBM,
+                                                      const CoreEngine &CE) {
+
+  if (!CRA.get())
+    CRA.reset(new CFGReachabilityAnalysis(*cfg));
+  
+  // Test for reachability from any aborted blocks to this block
+  typedef CoreEngine::BlocksAborted::const_iterator AbortedIterator;
   for (AbortedIterator I = CE.blocks_aborted_begin(),
       E = CE.blocks_aborted_end(); I != E; ++I) {
     const BlockEdge &BE =  I->first;
 
     // The destination block on the BlockEdge is the first block that was not
-    // analyzed.
-    Aborted.insert(BE.getDst()->getBlockID());
-  }
-
-  // Save the entry block ID for early exiting
-  unsigned EntryBlockID = C->getEntry().getBlockID();
-
-  // Create initial node
-  WorkList.push_back(CB);
-
-  while (!WorkList.empty()) {
-    const CFGBlock *Head = WorkList.front();
-    WorkList.pop_front();
-    Visited.insert(Head->getBlockID());
-
-    // If we found the entry block, then there exists a path from the target
-    // node to the entry point of this function -> the path was completely
-    // analyzed.
-    if (Head->getBlockID() == EntryBlockID)
-      return true;
-
-    // If any of the aborted blocks are on the path to the beginning, then all
-    // paths to this block were not analyzed.
-    if (Aborted.count(Head->getBlockID()))
+    // analyzed. If we can reach this block from the aborted block, then this
+    // block was not completely analyzed.
+    //
+    // Also explicitly check if the current block is the destination block.
+    // While technically reachable, it means we aborted the analysis on
+    // a path that included that block.
+    const CFGBlock *destBlock = BE.getDst();
+    if (destBlock == CB || CRA->isReachable(destBlock, CB))
       return false;
-
-    // Add the predecessors to the worklist unless we have already visited them
-    for (CFGBlock::const_pred_iterator I = Head->pred_begin();
-        I != Head->pred_end(); ++I)
-      if (!Visited.count((*I)->getBlockID()))
-        WorkList.push_back(*I);
   }
+  
+  // For the items still on the worklist, see if they are in blocks that
+  // can eventually reach 'CB'.
+  class VisitWL : public WorkList::Visitor {
+    const CFGStmtMap *CBM;
+    const CFGBlock *TargetBlock;
+    CFGReachabilityAnalysis &CRA;
+  public:
+    VisitWL(const CFGStmtMap *cbm, const CFGBlock *targetBlock,
+            CFGReachabilityAnalysis &cra)
+      : CBM(cbm), TargetBlock(targetBlock), CRA(cra) {}
+    virtual bool visit(const WorkListUnit &U) {
+      ProgramPoint P = U.getNode()->getLocation();
+      const CFGBlock *B = 0;
+      if (StmtPoint *SP = dyn_cast<StmtPoint>(&P)) {
+        B = CBM->getBlock(SP->getStmt());
+      }
+      else if (BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
+        B = BE->getDst();
+      }
+      else if (BlockEntrance *BEnt = dyn_cast<BlockEntrance>(&P)) {
+        B = BEnt->getBlock();
+      }
+      else if (BlockExit *BExit = dyn_cast<BlockExit>(&P)) {
+        B = BExit->getBlock();
+      }
+      if (!B)
+        return true;
+      
+      return CRA.isReachable(B, TargetBlock);
+    }
+  };
+  VisitWL visitWL(CBM, CB, *CRA.get());
+  // Were there any items in the worklist that could potentially reach
+  // this block?
+  if (CE.getWorkList()->visitItemsInWorkList(visitWL))
+    return false;
+
+  // Verify that this block is reachable from the entry block
+  if (!CRA->isReachable(&cfg->getEntry(), CB))
+    return false;
 
   // If we get to this point, there is no connection to the entry block or an
   // aborted block. This path is unreachable and we can report the error.
@@ -614,7 +670,7 @@ bool IdempotentOperationChecker::CanVary(const Expr *Ex,
   case Stmt::OffsetOfExprClass:
   case Stmt::CompoundLiteralExprClass:
   case Stmt::AddrLabelExprClass:
-  case Stmt::TypesCompatibleExprClass:
+  case Stmt::BinaryTypeTraitExprClass:
   case Stmt::GNUNullExprClass:
   case Stmt::InitListExprClass:
   case Stmt::DesignatedInitExprClass:
@@ -636,6 +692,13 @@ bool IdempotentOperationChecker::CanVary(const Expr *Ex,
   // The next cases require recursion for subexpressions
   case Stmt::BinaryOperatorClass: {
     const BinaryOperator *B = cast<const BinaryOperator>(Ex);
+
+    // Exclude cases involving pointer arithmetic.  These are usually
+    // false positives.
+    if (B->getOpcode() == BO_Sub || B->getOpcode() == BO_Add)
+      if (B->getLHS()->getType()->getAs<PointerType>())
+        return false;
+
     return CanVary(B->getRHS(), AC)
         || CanVary(B->getLHS(), AC);
    }
@@ -653,7 +716,8 @@ bool IdempotentOperationChecker::CanVary(const Expr *Ex,
     return CanVary(cast<const ChooseExpr>(Ex)->getChosenSubExpr(
         AC->getASTContext()), AC);
   case Stmt::ConditionalOperatorClass:
-    return CanVary(cast<const ConditionalOperator>(Ex)->getCond(), AC);
+  case Stmt::BinaryConditionalOperatorClass:
+    return CanVary(cast<AbstractConditionalOperator>(Ex)->getCond(), AC);
   }
 }
 
@@ -701,3 +765,58 @@ bool IdempotentOperationChecker::containsNonLocalVarDecl(const Stmt *S) {
 
   return false;
 }
+
+bool IdempotentOperationChecker::CFGReachabilityAnalysis::isReachable(
+                                                          const CFGBlock *Src,
+                                                          const CFGBlock *Dst) {
+  const unsigned DstBlockID = Dst->getBlockID();
+
+  // If we haven't analyzed the destination node, run the analysis now
+  if (!analyzed[DstBlockID]) {
+    MapReachability(Dst);
+    analyzed[DstBlockID] = true;
+  }
+
+  // Return the cached result
+  return reachable[DstBlockID][Src->getBlockID()];
+}
+
+// Maps reachability to a common node by walking the predecessors of the
+// destination node.
+void IdempotentOperationChecker::CFGReachabilityAnalysis::MapReachability(
+                                                          const CFGBlock *Dst) {
+
+  llvm::SmallVector<const CFGBlock *, 11> worklist;
+  llvm::BitVector visited(analyzed.size());
+  
+  ReachableSet &DstReachability = reachable[Dst->getBlockID()];
+  DstReachability.resize(analyzed.size(), false);
+
+  // Start searching from the destination node, since we commonly will perform
+  // multiple queries relating to a destination node.
+  worklist.push_back(Dst);
+  bool firstRun = true;
+
+  while (!worklist.empty()) {    
+    const CFGBlock *block = worklist.back();
+    worklist.pop_back();
+    
+    if (visited[block->getBlockID()])
+      continue;
+    visited[block->getBlockID()] = true;
+    
+    // Update reachability information for this node -> Dst
+    if (!firstRun) {
+      // Don't insert Dst -> Dst unless it was a predecessor of itself
+      DstReachability[block->getBlockID()] = true;
+    }
+    else
+      firstRun = false;
+
+    // Add the predecessors to the worklist.
+    for (CFGBlock::const_pred_iterator i = block->pred_begin(), 
+         e = block->pred_end(); i != e; ++i) {
+      worklist.push_back(*i);
+    }
+  }
+}
diff --git a/lib/StaticAnalyzer/Checkers/InternalChecks.h b/lib/StaticAnalyzer/Checkers/InternalChecks.h
new file mode 100644
index 000000000000..e855386fffea
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/InternalChecks.h
@@ -0,0 +1,51 @@
+//=-- InternalChecks.h- Builtin ExprEngine Checks -------------------*- C++ -*-=
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines functions to instantiate and register the "built-in"
+//  checks in ExprEngine.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_GR_ExprEngine_INTERNAL_CHECKS
+#define LLVM_CLANG_GR_ExprEngine_INTERNAL_CHECKS
+
+namespace clang {
+
+namespace ento {
+
+class ExprEngine;
+
+// Foundational checks that handle basic semantics.
+void RegisterAdjustedReturnValueChecker(ExprEngine &Eng);
+void RegisterArrayBoundChecker(ExprEngine &Eng);
+void RegisterArrayBoundCheckerV2(ExprEngine &Eng);
+void RegisterAttrNonNullChecker(ExprEngine &Eng);
+void RegisterBuiltinFunctionChecker(ExprEngine &Eng);
+void RegisterCallAndMessageChecker(ExprEngine &Eng);
+void RegisterCastSizeChecker(ExprEngine &Eng);
+void RegisterDereferenceChecker(ExprEngine &Eng);
+void RegisterDivZeroChecker(ExprEngine &Eng);
+void RegisterNoReturnFunctionChecker(ExprEngine &Eng);
+void RegisterReturnPointerRangeChecker(ExprEngine &Eng);
+void RegisterReturnUndefChecker(ExprEngine &Eng);
+void RegisterUndefBranchChecker(ExprEngine &Eng);
+void RegisterUndefCapturedBlockVarChecker(ExprEngine &Eng);
+void RegisterUndefResultChecker(ExprEngine &Eng);
+void RegisterUndefinedArraySubscriptChecker(ExprEngine &Eng);
+void RegisterUndefinedAssignmentChecker(ExprEngine &Eng);
+void RegisterVLASizeChecker(ExprEngine &Eng);
+
+// API checks.
+void RegisterOSAtomicChecker(ExprEngine &Eng);
+
+} // end GR namespace
+
+} // end clang namespace
+
+#endif
diff --git a/lib/Checker/LLVMConventionsChecker.cpp b/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
index 2f87da142c1a..9e3adc804f67 100644
--- a/lib/Checker/LLVMConventionsChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
@@ -12,14 +12,17 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/StmtVisitor.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
 #include <string>
 #include "llvm/ADT/StringRef.h"
 
 using namespace clang;
+using namespace ento;
 
 //===----------------------------------------------------------------------===//
 // Generic type checking routines.
@@ -37,15 +40,13 @@ static bool IsLLVMStringRef(QualType T) {
 /// Check whether the declaration is semantically inside the top-level
 /// namespace named by ns.
 static bool InNamespace(const Decl *D, llvm::StringRef NS) {
-  const DeclContext *DC = D->getDeclContext();
   const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(D->getDeclContext());
   if (!ND)
     return false;
   const IdentifierInfo *II = ND->getIdentifier();
   if (!II || !II->getName().equals(NS))
     return false;
-  DC = ND->getDeclContext();
-  return isa<TranslationUnitDecl>(DC);
+  return isa<TranslationUnitDecl>(ND->getDeclContext());
 }
 
 static bool IsStdString(QualType T) {
@@ -153,7 +154,7 @@ void StringRefCheckerVisitor::VisitVarDecl(VarDecl *VD) {
   // llvm::StringRef x = call() (where call returns std::string)
   if (!IsLLVMStringRef(VD->getType()))
     return;
-  CXXExprWithTemporaries *Ex1 = dyn_cast<CXXExprWithTemporaries>(Init);
+  ExprWithCleanups *Ex1 = dyn_cast<ExprWithCleanups>(Init);
   if (!Ex1)
     return;
   CXXConstructExpr *Ex2 = dyn_cast<CXXConstructExpr>(Ex1->getSubExpr());
@@ -211,10 +212,10 @@ static bool IsPartOfAST(const CXXRecordDecl *R) {
 namespace {
 class ASTFieldVisitor {
   llvm::SmallVector<FieldDecl*, 10> FieldChain;
-  CXXRecordDecl *Root;
+  const CXXRecordDecl *Root;
   BugReporter &BR;
 public:
-  ASTFieldVisitor(CXXRecordDecl *root, BugReporter &br)
+  ASTFieldVisitor(const CXXRecordDecl *root, BugReporter &br)
     : Root(root), BR(br) {}
 
   void Visit(FieldDecl *D);
@@ -222,7 +223,7 @@ public:
 };
 } // end anonymous namespace
 
-static void CheckASTMemory(CXXRecordDecl *R, BugReporter &BR) {
+static void CheckASTMemory(const CXXRecordDecl *R, BugReporter &BR) {
   if (!IsPartOfAST(R))
     return;
 
@@ -284,29 +285,27 @@ void ASTFieldVisitor::ReportError(QualType T) {
 }
 
 //===----------------------------------------------------------------------===//
-// Entry point for all checks.
+// LLVMConventionsChecker
 //===----------------------------------------------------------------------===//
 
-static void ScanCodeDecls(DeclContext *DC, BugReporter &BR) {
-  for (DeclContext::decl_iterator I=DC->decls_begin(), E=DC->decls_end();
-       I!=E ; ++I) {
-
-    Decl *D = *I;
-
-    if (D->hasBody())
-      CheckStringRefAssignedTemporary(D, BR);
-
-    if (CXXRecordDecl *R = dyn_cast<CXXRecordDecl>(D))
-      if (R->isDefinition())
-        CheckASTMemory(R, BR);
+namespace {
+class LLVMConventionsChecker : public CheckerV2<
+                                                check::ASTDecl<CXXRecordDecl>,
+                                                check::ASTCodeBody > {
+public:
+  void checkASTDecl(const CXXRecordDecl *R, AnalysisManager& mgr,
+                    BugReporter &BR) const {
+    if (R->isDefinition())
+      CheckASTMemory(R, BR);
+  }
 
-    if (DeclContext *DC_child = dyn_cast<DeclContext>(D))
-      ScanCodeDecls(DC_child, BR);
+  void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
+                        BugReporter &BR) const {
+    CheckStringRefAssignedTemporary(D, BR);
   }
+};
 }
 
-void clang::CheckLLVMConventions(TranslationUnitDecl &TU,
-                                 BugReporter &BR) {
-  ScanCodeDecls(&TU, BR);
+void ento::registerLLVMConventionsChecker(CheckerManager &mgr) {
+  mgr.registerChecker<LLVMConventionsChecker>();
 }
-
diff --git a/lib/Checker/MacOSXAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
index bcd96e73305e..358be124b9e4 100644
--- a/lib/Checker/MacOSXAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
@@ -15,16 +15,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
+#include "ClangSACheckers.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class MacOSXAPIChecker : public CheckerVisitor<MacOSXAPIChecker> {
@@ -44,9 +46,12 @@ public:
 };
 } //end anonymous namespace
 
-void clang::RegisterMacOSXAPIChecker(GRExprEngine &Eng) {
-  if (Eng.getContext().Target.getTriple().getVendor() == llvm::Triple::Apple)
-    Eng.registerCheck(new MacOSXAPIChecker());
+static void RegisterMacOSXAPIChecker(ExprEngine &Eng) {
+  Eng.registerCheck(new MacOSXAPIChecker());
+}
+
+void ento::registerMacOSXAPIChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterMacOSXAPIChecker);
 }
 
 //===----------------------------------------------------------------------===//
@@ -73,7 +78,7 @@ static void CheckDispatchOnce(CheckerContext &C, const CallExpr *CE,
   if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
     return;
 
-  ExplodedNode *N = C.GenerateSink(state);
+  ExplodedNode *N = C.generateSink(state);
   if (!N)
     return;
 
diff --git a/lib/StaticAnalyzer/Checkers/Makefile b/lib/StaticAnalyzer/Checkers/Makefile
new file mode 100644
index 000000000000..97f46424e472
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/Makefile
@@ -0,0 +1,24 @@
+##===- clang/lib/Checker/Makefile --------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+# This implements analyses built on top of source-level CFGs. 
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangStaticAnalyzerCheckers
+
+BUILT_SOURCES = Checkers.inc
+TABLEGEN_INC_FILES_COMMON = 1
+
+include $(CLANG_LEVEL)/Makefile
+
+$(ObjDir)/Checkers.inc.tmp : Checkers.td $(PROJ_SRC_DIR)/$(CLANG_LEVEL)/include/clang/StaticAnalyzer/Checkers/CheckerBase.td $(TBLGEN) $(ObjDir)/.dir
+	$(Echo) "Building Clang SA Checkers tables with tblgen"
+	$(Verb) $(TableGen) -gen-clang-sa-checkers -I $(PROJ_SRC_DIR)/$(CLANG_LEVEL)/include -o $(call SYSPATH, $@) $<
diff --git a/lib/Checker/MallocChecker.cpp b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index c9b6d7588be1..9d3a887cdbf1 100644
--- a/lib/Checker/MallocChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -12,14 +12,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineExperimentalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/SymbolManager.h"
+#include "ExperimentalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ImmutableMap.h"
 using namespace clang;
+using namespace ento;
 
 namespace {
 
@@ -75,13 +76,13 @@ public:
       BT_BadFree(0),
       II_malloc(0), II_free(0), II_realloc(0), II_calloc(0) {}
   static void *getTag();
-  bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
-  void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
-  void EvalEndPath(GREndPathNodeBuilder &B, void *tag, GRExprEngine &Eng);
+  bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
+  void evalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
+  void evalEndPath(EndOfFunctionNodeBuilder &B, void *tag, ExprEngine &Eng);
   void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S);
-  const GRState *EvalAssume(const GRState *state, SVal Cond, bool Assumption,
+  const GRState *evalAssume(const GRState *state, SVal Cond, bool Assumption,
                             bool *respondsToCallback);
-  void VisitLocation(CheckerContext &C, const Stmt *S, SVal l);
+  void visitLocation(CheckerContext &C, const Stmt *S, SVal l, bool isLoad);
   virtual void PreVisitBind(CheckerContext &C, const Stmt *StoreE,
                             SVal location, SVal val);
 
@@ -116,14 +117,16 @@ private:
 typedef llvm::ImmutableMap<SymbolRef, RefState> RegionStateTy;
 
 namespace clang {
+namespace ento {
   template <>
   struct GRStateTrait<RegionState> 
     : public GRStatePartialTrait<RegionStateTy> {
     static void *GDMIndex() { return MallocChecker::getTag(); }
   };
 }
+}
 
-void clang::RegisterMallocChecker(GRExprEngine &Eng) {
+void ento::RegisterMallocChecker(ExprEngine &Eng) {
   Eng.registerCheck(new MallocChecker());
 }
 
@@ -132,7 +135,7 @@ void *MallocChecker::getTag() {
   return &x;
 }
 
-bool MallocChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
+bool MallocChecker::evalCallExpr(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
@@ -227,27 +230,28 @@ const GRState *MallocChecker::MallocMemAux(CheckerContext &C,
                                            SVal Size, SVal Init,
                                            const GRState *state) {
   unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-  ValueManager &ValMgr = C.getValueManager();
+  SValBuilder &svalBuilder = C.getSValBuilder();
 
   // Set the return value.
-  SVal RetVal = ValMgr.getConjuredSymbolVal(NULL, CE, CE->getType(), Count);
-  state = state->BindExpr(CE, RetVal);
+  SVal retVal = svalBuilder.getConjuredSymbolVal(NULL, CE, CE->getType(), Count);
+  state = state->BindExpr(CE, retVal);
 
   // Fill the region with the initialization value.
-  state = state->bindDefault(RetVal, Init);
+  state = state->bindDefault(retVal, Init);
 
   // Set the region's extent equal to the Size parameter.
-  const SymbolicRegion *R = cast<SymbolicRegion>(RetVal.getAsRegion());
-  DefinedOrUnknownSVal Extent = R->getExtent(ValMgr);
+  const SymbolicRegion *R = cast<SymbolicRegion>(retVal.getAsRegion());
+  DefinedOrUnknownSVal Extent = R->getExtent(svalBuilder);
   DefinedOrUnknownSVal DefinedSize = cast<DefinedOrUnknownSVal>(Size);
+  DefinedOrUnknownSVal extentMatchesSize =
+    svalBuilder.evalEQ(state, Extent, DefinedSize);
 
-  SValuator &SVator = ValMgr.getSValuator();
-  DefinedOrUnknownSVal ExtentMatchesSize =
-    SVator.EvalEQ(state, Extent, DefinedSize);
-  state = state->Assume(ExtentMatchesSize, true);
-
-  SymbolRef Sym = RetVal.getAsLocSymbol();
+  state = state->assume(extentMatchesSize, true);
+  assert(state);
+  
+  SymbolRef Sym = retVal.getAsLocSymbol();
   assert(Sym);
+
   // Set the symbol's state to Allocated.
   return state->set<RegionState>(Sym, RefState::getAllocateUnchecked(CE));
 }
@@ -288,7 +292,7 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
   // FIXME: Technically using 'Assume' here can result in a path
   //  bifurcation.  In such cases we need to return two states, not just one.
   const GRState *notNullState, *nullState;
-  llvm::tie(notNullState, nullState) = state->Assume(location);
+  llvm::tie(notNullState, nullState) = state->assume(location);
 
   // The explicit NULL case, no operation is performed.
   if (nullState && !notNullState)
@@ -351,7 +355,7 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
 
   // Check double free.
   if (RS->isReleased()) {
-    if (ExplodedNode *N = C.GenerateSink()) {
+    if (ExplodedNode *N = C.generateSink()) {
       if (!BT_DoubleFree)
         BT_DoubleFree
           = new BuiltinBug("Double free",
@@ -376,9 +380,7 @@ bool MallocChecker::SummarizeValue(llvm::raw_ostream& os, SVal V) {
   else if (loc::ConcreteInt *ConstAddr = dyn_cast<loc::ConcreteInt>(&V))
     os << "a constant address (" << ConstAddr->getValue() << ")";
   else if (loc::GotoLabel *Label = dyn_cast<loc::GotoLabel>(&V))
-    os << "the address of the label '"
-       << Label->getLabel()->getID()->getName()
-       << "'";
+    os << "the address of the label '" << Label->getLabel()->getName() << "'";
   else
     return false;
   
@@ -462,7 +464,7 @@ bool MallocChecker::SummarizeRegion(llvm::raw_ostream& os,
 
 void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
                                   SourceRange range) {
-  if (ExplodedNode *N = C.GenerateSink()) {
+  if (ExplodedNode *N = C.generateSink()) {
     if (!BT_BadFree)
       BT_BadFree = new BuiltinBug("Bad free");
     
@@ -500,21 +502,22 @@ void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
 
 void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
-  const Expr *Arg0 = CE->getArg(0);
-  DefinedOrUnknownSVal Arg0Val=cast<DefinedOrUnknownSVal>(state->getSVal(Arg0));
+  const Expr *arg0Expr = CE->getArg(0);
+  DefinedOrUnknownSVal arg0Val 
+    = cast<DefinedOrUnknownSVal>(state->getSVal(arg0Expr));
 
-  ValueManager &ValMgr = C.getValueManager();
-  SValuator &SVator = C.getSValuator();
+  SValBuilder &svalBuilder = C.getSValBuilder();
 
-  DefinedOrUnknownSVal PtrEQ = SVator.EvalEQ(state, Arg0Val, ValMgr.makeNull());
+  DefinedOrUnknownSVal PtrEQ =
+    svalBuilder.evalEQ(state, arg0Val, svalBuilder.makeNull());
 
   // If the ptr is NULL, the call is equivalent to malloc(size).
-  if (const GRState *stateEqual = state->Assume(PtrEQ, true)) {
+  if (const GRState *stateEqual = state->assume(PtrEQ, true)) {
     // Hack: set the NULL symbolic region to released to suppress false warning.
     // In the future we should add more states for allocated regions, e.g., 
     // CheckedNull, CheckedNonNull.
     
-    SymbolRef Sym = Arg0Val.getAsLocSymbol();
+    SymbolRef Sym = arg0Val.getAsLocSymbol();
     if (Sym)
       stateEqual = stateEqual->set<RegionState>(Sym, RefState::getReleased(CE));
 
@@ -523,49 +526,44 @@ void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) {
     C.addTransition(stateMalloc);
   }
 
-  if (const GRState *stateNotEqual = state->Assume(PtrEQ, false)) {
+  if (const GRState *stateNotEqual = state->assume(PtrEQ, false)) {
     const Expr *Arg1 = CE->getArg(1);
     DefinedOrUnknownSVal Arg1Val = 
       cast<DefinedOrUnknownSVal>(stateNotEqual->getSVal(Arg1));
-    DefinedOrUnknownSVal SizeZero = SVator.EvalEQ(stateNotEqual, Arg1Val,
-                                      ValMgr.makeIntValWithPtrWidth(0, false));
+    DefinedOrUnknownSVal SizeZero =
+      svalBuilder.evalEQ(stateNotEqual, Arg1Val,
+                         svalBuilder.makeIntValWithPtrWidth(0, false));
 
-    if (const GRState *stateSizeZero = stateNotEqual->Assume(SizeZero, true)) {
-      const GRState *stateFree = FreeMemAux(C, CE, stateSizeZero, 0, false);
-      if (stateFree)
+    if (const GRState *stateSizeZero = stateNotEqual->assume(SizeZero, true))
+      if (const GRState *stateFree = FreeMemAux(C, CE, stateSizeZero, 0, false))
         C.addTransition(stateFree->BindExpr(CE, UndefinedVal(), true));
-    }
 
-    if (const GRState *stateSizeNotZero=stateNotEqual->Assume(SizeZero,false)) {
-      const GRState *stateFree = FreeMemAux(C, CE, stateSizeNotZero, 0, false);
-      if (stateFree) {
+    if (const GRState *stateSizeNotZero = stateNotEqual->assume(SizeZero,false))
+      if (const GRState *stateFree = FreeMemAux(C, CE, stateSizeNotZero,
+                                                0, false)) {
         // FIXME: We should copy the content of the original buffer.
         const GRState *stateRealloc = MallocMemAux(C, CE, CE->getArg(1), 
                                                    UnknownVal(), stateFree);
         C.addTransition(stateRealloc);
       }
-    }
   }
 }
 
 void MallocChecker::CallocMem(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
-  
-  ValueManager &ValMgr = C.getValueManager();
-  SValuator &SVator = C.getSValuator();
+  SValBuilder &svalBuilder = C.getSValBuilder();
 
-  SVal Count = state->getSVal(CE->getArg(0));
-  SVal EleSize = state->getSVal(CE->getArg(1));
-  SVal TotalSize = SVator.EvalBinOp(state, BO_Mul, Count, EleSize,
-                                    ValMgr.getContext().getSizeType());
-  
-  SVal Zero = ValMgr.makeZeroVal(ValMgr.getContext().CharTy);
+  SVal count = state->getSVal(CE->getArg(0));
+  SVal elementSize = state->getSVal(CE->getArg(1));
+  SVal TotalSize = svalBuilder.evalBinOp(state, BO_Mul, count, elementSize,
+                                        svalBuilder.getContext().getSizeType());  
+  SVal zeroVal = svalBuilder.makeZeroVal(svalBuilder.getContext().CharTy);
 
-  state = MallocMemAux(C, CE, TotalSize, Zero, state);
-  C.addTransition(state);
+  C.addTransition(MallocMemAux(C, CE, TotalSize, zeroVal, state));
 }
 
-void MallocChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
+void MallocChecker::evalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper)
+{
   if (!SymReaper.hasDeadSymbols())
     return;
 
@@ -576,7 +574,7 @@ void MallocChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
   for (RegionStateTy::iterator I = RS.begin(), E = RS.end(); I != E; ++I) {
     if (SymReaper.isDead(I->first)) {
       if (I->second.isAllocated()) {
-        if (ExplodedNode *N = C.GenerateNode()) {
+        if (ExplodedNode *N = C.generateNode()) {
           if (!BT_Leak)
             BT_Leak = new BuiltinBug("Memory leak",
                      "Allocated memory never released. Potential memory leak.");
@@ -587,17 +585,14 @@ void MallocChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
       }
 
       // Remove the dead symbol from the map.
-      RS = F.Remove(RS, I->first);
+      RS = F.remove(RS, I->first);
     }
   }
-
-  state = state->set<RegionState>(RS);
-  C.GenerateNode(state);
+  C.generateNode(state->set<RegionState>(RS));
 }
 
-void MallocChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
-                                GRExprEngine &Eng) {
-  SaveAndRestore<bool> OldHasGen(B.HasGeneratedNode);
+void MallocChecker::evalEndPath(EndOfFunctionNodeBuilder &B, void *tag,
+                                ExprEngine &Eng) {
   const GRState *state = B.getState();
   RegionStateTy M = state->get<RegionState>();
 
@@ -617,14 +612,13 @@ void MallocChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
 }
 
 void MallocChecker::PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S) {
-  const Expr *RetE = S->getRetValue();
-  if (!RetE)
+  const Expr *retExpr = S->getRetValue();
+  if (!retExpr)
     return;
 
   const GRState *state = C.getState();
 
-  SymbolRef Sym = state->getSVal(RetE).getAsSymbol();
-
+  SymbolRef Sym = state->getSVal(retExpr).getAsSymbol();
   if (!Sym)
     return;
 
@@ -639,7 +633,7 @@ void MallocChecker::PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S) {
   C.addTransition(state);
 }
 
-const GRState *MallocChecker::EvalAssume(const GRState *state, SVal Cond, 
+const GRState *MallocChecker::evalAssume(const GRState *state, SVal Cond, 
                                          bool Assumption,
                                          bool * /* respondsToCallback */) {
   // If a symblic region is assumed to NULL, set its state to AllocateFailed.
@@ -656,12 +650,13 @@ const GRState *MallocChecker::EvalAssume(const GRState *state, SVal Cond,
 }
 
 // Check if the location is a freed symbolic region.
-void MallocChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l) {
+void MallocChecker::visitLocation(CheckerContext &C, const Stmt *S, SVal l,
+                                  bool isLoad) {
   SymbolRef Sym = l.getLocSymbolInBase();
   if (Sym) {
     const RefState *RS = C.getState()->get<RegionState>(Sym);
     if (RS && RS->isReleased()) {
-      if (ExplodedNode *N = C.GenerateNode()) {
+      if (ExplodedNode *N = C.generateNode()) {
         if (!BT_UseFree)
           BT_UseFree = new BuiltinBug("Use dynamically allocated memory after"
                                       " it is freed.");
@@ -697,7 +692,7 @@ void MallocChecker::PreVisitBind(CheckerContext &C,
     if (const RefState *RS = state->get<RegionState>(Sym)) {
       // If ptr is NULL, no operation is performed.
       const GRState *notNullState, *nullState;
-      llvm::tie(notNullState, nullState) = state->Assume(l);
+      llvm::tie(notNullState, nullState) = state->assume(l);
 
       // Generate a transition for 'nullState' to record the assumption
       // that the state was null.
diff --git a/lib/Checker/NSAutoreleasePoolChecker.cpp b/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
index 48f03a369ed5..9fb89d79fc79 100644
--- a/lib/Checker/NSAutoreleasePoolChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
@@ -15,14 +15,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "BasicObjCFoundationChecks.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Decl.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class NSAutoreleasePoolChecker
@@ -38,13 +40,13 @@ public:
     return &x;
   }
 
-  void PreVisitObjCMessageExpr(CheckerContext &C, const ObjCMessageExpr *ME);    
+  void preVisitObjCMessage(CheckerContext &C, ObjCMessage msg);    
 };
 
 } // end anonymous namespace
 
 
-void clang::RegisterNSAutoreleasePoolChecks(GRExprEngine &Eng) {
+static void RegisterNSAutoreleasePoolChecker(ExprEngine &Eng) {
   ASTContext &Ctx = Eng.getContext();
   if (Ctx.getLangOptions().getGCMode() != LangOptions::NonGC) {    
     Eng.registerCheck(new NSAutoreleasePoolChecker(GetNullarySelector("release",
@@ -52,11 +54,15 @@ void clang::RegisterNSAutoreleasePoolChecks(GRExprEngine &Eng) {
   }
 }
 
+void ento::registerNSAutoreleasePoolChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterNSAutoreleasePoolChecker);
+}
+
 void
-NSAutoreleasePoolChecker::PreVisitObjCMessageExpr(CheckerContext &C,
-                                                  const ObjCMessageExpr *ME) {
+NSAutoreleasePoolChecker::preVisitObjCMessage(CheckerContext &C,
+                                              ObjCMessage msg) {
   
-  const Expr *receiver = ME->getInstanceReceiver();
+  const Expr *receiver = msg.getInstanceReceiver();
   if (!receiver)
     return;
   
@@ -74,13 +80,13 @@ NSAutoreleasePoolChecker::PreVisitObjCMessageExpr(CheckerContext &C,
     return;
   
   // Sending 'release' message?
-  if (ME->getSelector() != releaseS)
+  if (msg.getSelector() != releaseS)
     return;
                      
-  SourceRange R = ME->getSourceRange();
+  SourceRange R = msg.getSourceRange();
 
   C.getBugReporter().EmitBasicReport("Use -drain instead of -release",
     "API Upgrade (Apple)",
     "Use -drain instead of -release when using NSAutoreleasePool "
-    "and garbage collection", ME->getLocStart(), &R, 1);
+    "and garbage collection", R.getBegin(), &R, 1);
 }
diff --git a/lib/Checker/NSErrorChecker.cpp b/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
index e30d54ccd79c..7a1b9787754d 100644
--- a/lib/Checker/NSErrorChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
@@ -15,23 +15,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/Checkers/DereferenceChecker.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Checkers/DereferenceChecker.h"
 #include "BasicObjCFoundationChecks.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Decl.h"
 #include "llvm/ADT/SmallVector.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class NSErrorChecker : public BugType {
   const Decl &CodeDecl;
   const bool isNSErrorWarning;
   IdentifierInfo * const II;
-  GRExprEngine &Eng;
+  ExprEngine &Eng;
 
   void CheckSignature(const ObjCMethodDecl& MD, QualType& ResultTy,
                       llvm::SmallVectorImpl<VarDecl*>& ErrorParams);
@@ -48,7 +49,7 @@ class NSErrorChecker : public BugType {
   void EmitRetTyWarning(BugReporter& BR, const Decl& CodeDecl);
 
 public:
-  NSErrorChecker(const Decl &D, bool isNSError, GRExprEngine& eng)
+  NSErrorChecker(const Decl &D, bool isNSError, ExprEngine& eng)
     : BugType(isNSError ? "NSError** null dereference"
                         : "CFErrorRef* null dereference",
               "Coding conventions (Apple)"),
@@ -62,7 +63,7 @@ public:
 
 } // end anonymous namespace
 
-void clang::RegisterNSErrorChecks(BugReporter& BR, GRExprEngine &Eng,
+void ento::RegisterNSErrorChecks(BugReporter& BR, ExprEngine &Eng,
                                   const Decl &D) {
   BR.Register(new NSErrorChecker(D, true, Eng));
   BR.Register(new NSErrorChecker(D, false, Eng));
diff --git a/lib/Checker/NoReturnFunctionChecker.cpp b/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
index 12527e076221..40040eada7b0 100644
--- a/lib/Checker/NoReturnFunctionChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/NoReturnFunctionChecker.cpp
@@ -12,11 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
@@ -28,7 +29,7 @@ public:
 
 }
 
-void clang::RegisterNoReturnFunctionChecker(GRExprEngine &Eng) {
+void ento::RegisterNoReturnFunctionChecker(ExprEngine &Eng) {
   Eng.registerCheck(new NoReturnFunctionChecker());
 }
 
@@ -75,5 +76,5 @@ void NoReturnFunctionChecker::PostVisitCallExpr(CheckerContext &C,
   }
 
   if (BuildSinks)
-    C.GenerateSink(CE);
+    C.generateSink(CE);
 }
diff --git a/lib/Checker/OSAtomicChecker.cpp b/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
index 02de0a84a4e4..e1126b617b7c 100644
--- a/lib/Checker/OSAtomicChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
@@ -11,30 +11,31 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/PathSensitive/Checker.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 #include "clang/Basic/Builtins.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
 class OSAtomicChecker : public Checker {
 public:
   static void *getTag() { static int tag = 0; return &tag; }
-  virtual bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
+  virtual bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
 
 private:
-  bool EvalOSAtomicCompareAndSwap(CheckerContext &C, const CallExpr *CE);
+  bool evalOSAtomicCompareAndSwap(CheckerContext &C, const CallExpr *CE);
 };
 
 }
 
-void clang::RegisterOSAtomicChecker(GRExprEngine &Eng) {
+void ento::RegisterOSAtomicChecker(ExprEngine &Eng) {
   Eng.registerCheck(new OSAtomicChecker());
 }
 
-bool OSAtomicChecker::EvalCallExpr(CheckerContext &C,const CallExpr *CE) {
+bool OSAtomicChecker::evalCallExpr(CheckerContext &C,const CallExpr *CE) {
   const GRState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
@@ -52,13 +53,13 @@ bool OSAtomicChecker::EvalCallExpr(CheckerContext &C,const CallExpr *CE) {
   // Check for compare and swap.
   if (FName.startswith("OSAtomicCompareAndSwap") ||
       FName.startswith("objc_atomicCompareAndSwap"))
-    return EvalOSAtomicCompareAndSwap(C, CE);
+    return evalOSAtomicCompareAndSwap(C, CE);
 
   // FIXME: Other atomics.
   return false;
 }
 
-bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C, 
+bool OSAtomicChecker::evalOSAtomicCompareAndSwap(CheckerContext &C, 
                                                  const CallExpr *CE) {
   // Not enough arguments to match OSAtomicCompareAndSwap?
   if (CE->getNumArgs() != 3)
@@ -96,7 +97,7 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
   const void *OSAtomicStoreTag = &magic_store;
 
   // Load 'theValue'.
-  GRExprEngine &Engine = C.getEngine();
+  ExprEngine &Engine = C.getEngine();
   const GRState *state = C.getState();
   ExplodedNodeSet Tmp;
   SVal location = state->getSVal(theValueExpr);
@@ -112,7 +113,7 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
       dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
     LoadTy = TR->getValueType();
   }
-  Engine.EvalLoad(Tmp, theValueExpr, C.getPredecessor(), 
+  Engine.evalLoad(Tmp, theValueExpr, C.getPredecessor(), 
                   state, location, OSAtomicLoadTag, LoadTy);
 
   if (Tmp.empty()) {
@@ -142,12 +143,12 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
     DefinedOrUnknownSVal oldValueVal =
       cast<DefinedOrUnknownSVal>(oldValueVal_untested);
 
-    SValuator &SVator = Engine.getSValuator();
+    SValBuilder &svalBuilder = Engine.getSValBuilder();
 
     // Perform the comparison.
-    DefinedOrUnknownSVal Cmp = SVator.EvalEQ(stateLoad,theValueVal,oldValueVal);
+    DefinedOrUnknownSVal Cmp = svalBuilder.evalEQ(stateLoad,theValueVal,oldValueVal);
 
-    const GRState *stateEqual = stateLoad->Assume(Cmp, true);
+    const GRState *stateEqual = stateLoad->assume(Cmp, true);
 
     // Were they equal?
     if (stateEqual) {
@@ -158,10 +159,10 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
       // Handle implicit value casts.
       if (const TypedRegion *R =
           dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
-        val = SVator.EvalCast(val,R->getValueType(), newValueExpr->getType());
+        val = svalBuilder.evalCast(val,R->getValueType(), newValueExpr->getType());
       }
 
-      Engine.EvalStore(TmpStore, NULL, theValueExpr, N, 
+      Engine.evalStore(TmpStore, NULL, theValueExpr, N, 
                        stateEqual, location, val, OSAtomicStoreTag);
 
       if (TmpStore.empty()) {
@@ -182,19 +183,19 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
         SVal Res = UnknownVal();
         QualType T = CE->getType();
         if (!T->isVoidType())
-          Res = Engine.getValueManager().makeTruthVal(true, T);
-        C.GenerateNode(stateNew->BindExpr(CE, Res), predNew);
+          Res = Engine.getSValBuilder().makeTruthVal(true, T);
+        C.generateNode(stateNew->BindExpr(CE, Res), predNew);
       }
     }
 
     // Were they not equal?
-    if (const GRState *stateNotEqual = stateLoad->Assume(Cmp, false)) {
+    if (const GRState *stateNotEqual = stateLoad->assume(Cmp, false)) {
       // Check for 'void' return type if we have a bogus function prototype.
       SVal Res = UnknownVal();
       QualType T = CE->getType();
       if (!T->isVoidType())
-        Res = Engine.getValueManager().makeTruthVal(false, CE->getType());
-      C.GenerateNode(stateNotEqual->BindExpr(CE, Res), N);
+        Res = Engine.getSValBuilder().makeTruthVal(false, CE->getType());
+      C.generateNode(stateNotEqual->BindExpr(CE, Res), N);
     }
   }
 
diff --git a/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
new file mode 100644
index 000000000000..7d440ab17e9f
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
@@ -0,0 +1,100 @@
+//== ObjCAtSyncChecker.cpp - nil mutex checker for @synchronized -*- C++ -*--=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This defines ObjCAtSyncChecker, a builtin check that checks for null pointers
+// used as mutexes for @synchronized.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Checkers/DereferenceChecker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class ObjCAtSyncChecker : public CheckerVisitor<ObjCAtSyncChecker> {
+  BuiltinBug *BT_null;
+  BuiltinBug *BT_undef;
+public:
+  ObjCAtSyncChecker() : BT_null(0), BT_undef(0) {}
+  static void *getTag() { static int tag = 0; return &tag; }
+  void PreVisitObjCAtSynchronizedStmt(CheckerContext &C,
+                                      const ObjCAtSynchronizedStmt *S);
+};
+} // end anonymous namespace
+
+static void RegisterObjCAtSyncChecker(ExprEngine &Eng) {
+  // @synchronized is an Objective-C 2 feature.
+  if (Eng.getContext().getLangOptions().ObjC2)
+    Eng.registerCheck(new ObjCAtSyncChecker());
+}
+
+void ento::registerObjCAtSyncChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterObjCAtSyncChecker);
+}
+
+void ObjCAtSyncChecker::PreVisitObjCAtSynchronizedStmt(CheckerContext &C,
+                                         const ObjCAtSynchronizedStmt *S) {
+
+  const Expr *Ex = S->getSynchExpr();
+  const GRState *state = C.getState();
+  SVal V = state->getSVal(Ex);
+
+  // Uninitialized value used for the mutex?
+  if (isa<UndefinedVal>(V)) {
+    if (ExplodedNode *N = C.generateSink()) {
+      if (!BT_undef)
+        BT_undef = new BuiltinBug("Uninitialized value used as mutex "
+                                  "for @synchronized");
+      EnhancedBugReport *report =
+        new EnhancedBugReport(*BT_undef, BT_undef->getDescription(), N);
+      report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, Ex);
+      C.EmitReport(report);
+    }
+    return;
+  }
+
+  if (V.isUnknown())
+    return;
+
+  // Check for null mutexes.
+  const GRState *notNullState, *nullState;
+  llvm::tie(notNullState, nullState) = state->assume(cast<DefinedSVal>(V));
+
+  if (nullState) {
+    if (!notNullState) {
+      // Generate an error node.  This isn't a sink since
+      // a null mutex just means no synchronization occurs.
+      if (ExplodedNode *N = C.generateNode(nullState)) {
+        if (!BT_null)
+          BT_null = new BuiltinBug("Nil value used as mutex for @synchronized() "
+                                   "(no synchronization will occur)");
+        EnhancedBugReport *report =
+          new EnhancedBugReport(*BT_null, BT_null->getDescription(), N);
+        report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
+                                  Ex);
+
+        C.EmitReport(report);
+        return;
+      }
+    }
+    // Don't add a transition for 'nullState'.  If the value is
+    // under-constrained to be null or non-null, assume it is non-null
+    // afterwards.
+  }
+
+  if (notNullState)
+    C.addTransition(notNullState);
+}
+
diff --git a/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
new file mode 100644
index 000000000000..4f247ea14670
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
@@ -0,0 +1,356 @@
+//== ObjCSelfInitChecker.cpp - Checker for 'self' initialization -*- C++ -*--=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This defines ObjCSelfInitChecker, a builtin check that checks for uses of
+// 'self' before proper initialization.
+//
+//===----------------------------------------------------------------------===//
+
+// This checks initialization methods to verify that they assign 'self' to the
+// result of an initialization call (e.g. [super init], or [self initWith..])
+// before using 'self' or any instance variable.
+//
+// To perform the required checking, values are tagged wih flags that indicate
+// 1) if the object is the one pointed to by 'self', and 2) if the object
+// is the result of an initializer (e.g. [super init]).
+//
+// Uses of an object that is true for 1) but not 2) trigger a diagnostic.
+// The uses that are currently checked are:
+//  - Using instance variables.
+//  - Returning the object.
+//
+// Note that we don't check for an invalid 'self' that is the receiver of an
+// obj-c message expression to cut down false positives where logging functions
+// get information from self (like its class) or doing "invalidation" on self
+// when the initialization fails.
+//
+// Because the object that 'self' points to gets invalidated when a call
+// receives a reference to 'self', the checker keeps track and passes the flags
+// for 1) and 2) to the new object that 'self' points to after the call.
+//
+// FIXME (rdar://7937506): In the case of:
+//   [super init];
+//   return self;
+// Have an extra PathDiagnosticPiece in the path that says "called [super init],
+// but didn't assign the result to self."
+
+//===----------------------------------------------------------------------===//
+
+// FIXME: Somehow stick the link to Apple's documentation about initializing
+// objects in the diagnostics.
+// http://developer.apple.com/library/mac/#documentation/Cocoa/Conceptual/ObjectiveC/Articles/ocAllocInit.html
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
+#include "clang/AST/ParentMap.h"
+
+using namespace clang;
+using namespace ento;
+
+static bool shouldRunOnFunctionOrMethod(const NamedDecl *ND);
+static bool isInitializationMethod(const ObjCMethodDecl *MD);
+static bool isInitMessage(const ObjCMessage &msg);
+static bool isSelfVar(SVal location, CheckerContext &C);
+
+namespace {
+enum SelfFlagEnum {
+  /// \brief No flag set.
+  SelfFlag_None = 0x0,
+  /// \brief Value came from 'self'.
+  SelfFlag_Self    = 0x1,
+  /// \brief Value came from the result of an initializer (e.g. [super init]).
+  SelfFlag_InitRes = 0x2
+};
+}
+
+namespace {
+class ObjCSelfInitChecker : public CheckerVisitor<ObjCSelfInitChecker> {
+  /// \brief A call receiving a reference to 'self' invalidates the object that
+  /// 'self' contains. This field keeps the "self flags" assigned to the 'self'
+  /// object before the call and assign them to the new object that 'self'
+  /// points to after the call.
+  SelfFlagEnum preCallSelfFlags;
+
+public:
+  static void *getTag() { static int tag = 0; return &tag; }
+  void postVisitObjCMessage(CheckerContext &C, ObjCMessage msg);
+  void PostVisitObjCIvarRefExpr(CheckerContext &C, const ObjCIvarRefExpr *E);
+  void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S);
+  void PreVisitGenericCall(CheckerContext &C, const CallExpr *CE);
+  void PostVisitGenericCall(CheckerContext &C, const CallExpr *CE);
+  virtual void visitLocation(CheckerContext &C, const Stmt *S, SVal location,
+                             bool isLoad);
+};
+} // end anonymous namespace
+
+static void RegisterObjCSelfInitChecker(ExprEngine &Eng) {
+  if (Eng.getContext().getLangOptions().ObjC1)
+    Eng.registerCheck(new ObjCSelfInitChecker());
+}
+
+void ento::registerObjCSelfInitChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterObjCSelfInitChecker);
+}
+
+namespace {
+
+class InitSelfBug : public BugType {
+  const std::string desc;
+public:
+  InitSelfBug() : BugType("missing \"self = [(super or self) init...]\"",
+                          "missing \"self = [(super or self) init...]\"") {}
+};
+
+} // end anonymous namespace
+
+typedef llvm::ImmutableMap<SymbolRef, unsigned> SelfFlag;
+namespace { struct CalledInit {}; }
+
+namespace clang {
+namespace ento {
+  template<>
+  struct GRStateTrait<SelfFlag> : public GRStatePartialTrait<SelfFlag> {
+    static void* GDMIndex() {
+      static int index = 0;
+      return &index;
+    }
+  };
+  template <>
+  struct GRStateTrait<CalledInit> : public GRStatePartialTrait<bool> {
+    static void *GDMIndex() { static int index = 0; return &index; }
+  };
+}
+}
+
+static SelfFlagEnum getSelfFlags(SVal val, const GRState *state) {
+  if (SymbolRef sym = val.getAsSymbol())
+    if (const unsigned *attachedFlags = state->get<SelfFlag>(sym))
+      return (SelfFlagEnum)*attachedFlags;
+  return SelfFlag_None;
+}
+
+static SelfFlagEnum getSelfFlags(SVal val, CheckerContext &C) {
+  return getSelfFlags(val, C.getState());
+}
+
+static void addSelfFlag(const GRState *state, SVal val,
+                        SelfFlagEnum flag, CheckerContext &C) {
+  // We tag the symbol that the SVal wraps.
+  if (SymbolRef sym = val.getAsSymbol())
+    C.addTransition(state->set<SelfFlag>(sym, getSelfFlags(val, C) | flag));
+}
+
+static bool hasSelfFlag(SVal val, SelfFlagEnum flag, CheckerContext &C) {
+  return getSelfFlags(val, C) & flag;
+}
+
+/// \brief Returns true of the value of the expression is the object that 'self'
+/// points to and is an object that did not come from the result of calling
+/// an initializer.
+static bool isInvalidSelf(const Expr *E, CheckerContext &C) {
+  SVal exprVal = C.getState()->getSVal(E);
+  if (!hasSelfFlag(exprVal, SelfFlag_Self, C))
+    return false; // value did not come from 'self'.
+  if (hasSelfFlag(exprVal, SelfFlag_InitRes, C))
+    return false; // 'self' is properly initialized.
+
+  return true;
+}
+
+static void checkForInvalidSelf(const Expr *E, CheckerContext &C,
+                                const char *errorStr) {
+  if (!E)
+    return;
+  
+  if (!C.getState()->get<CalledInit>())
+    return;
+  
+  if (!isInvalidSelf(E, C))
+    return;
+  
+  // Generate an error node.
+  ExplodedNode *N = C.generateSink();
+  if (!N)
+    return;
+
+  EnhancedBugReport *report =
+    new EnhancedBugReport(*new InitSelfBug(), errorStr, N);
+  C.EmitReport(report);
+}
+
+void ObjCSelfInitChecker::postVisitObjCMessage(CheckerContext &C,
+                                               ObjCMessage msg) {
+  // When encountering a message that does initialization (init rule),
+  // tag the return value so that we know later on that if self has this value
+  // then it is properly initialized.
+
+  // FIXME: A callback should disable checkers at the start of functions.
+  if (!shouldRunOnFunctionOrMethod(dyn_cast<NamedDecl>(
+                                     C.getCurrentAnalysisContext()->getDecl())))
+    return;
+
+  if (isInitMessage(msg)) {
+    // Tag the return value as the result of an initializer.
+    const GRState *state = C.getState();
+    
+    // FIXME this really should be context sensitive, where we record
+    // the current stack frame (for IPA).  Also, we need to clean this
+    // value out when we return from this method.
+    state = state->set<CalledInit>(true);
+    
+    SVal V = state->getSVal(msg.getOriginExpr());
+    addSelfFlag(state, V, SelfFlag_InitRes, C);
+    return;
+  }
+
+  // We don't check for an invalid 'self' in an obj-c message expression to cut
+  // down false positives where logging functions get information from self
+  // (like its class) or doing "invalidation" on self when the initialization
+  // fails.
+}
+
+void ObjCSelfInitChecker::PostVisitObjCIvarRefExpr(CheckerContext &C,
+                                                   const ObjCIvarRefExpr *E) {
+  // FIXME: A callback should disable checkers at the start of functions.
+  if (!shouldRunOnFunctionOrMethod(dyn_cast<NamedDecl>(
+                                     C.getCurrentAnalysisContext()->getDecl())))
+    return;
+
+  checkForInvalidSelf(E->getBase(), C,
+    "Instance variable used while 'self' is not set to the result of "
+                                                 "'[(super or self) init...]'");
+}
+
+void ObjCSelfInitChecker::PreVisitReturnStmt(CheckerContext &C,
+                                             const ReturnStmt *S) {
+  // FIXME: A callback should disable checkers at the start of functions.
+  if (!shouldRunOnFunctionOrMethod(dyn_cast<NamedDecl>(
+                                     C.getCurrentAnalysisContext()->getDecl())))
+    return;
+
+  checkForInvalidSelf(S->getRetValue(), C,
+    "Returning 'self' while it is not set to the result of "
+                                                 "'[(super or self) init...]'");
+}
+
+// When a call receives a reference to 'self', [Pre/Post]VisitGenericCall pass
+// the SelfFlags from the object 'self' point to before the call, to the new
+// object after the call. This is to avoid invalidation of 'self' by logging
+// functions.
+// Another common pattern in classes with multiple initializers is to put the
+// subclass's common initialization bits into a static function that receives
+// the value of 'self', e.g:
+// @code
+//   if (!(self = [super init]))
+//     return nil;
+//   if (!(self = _commonInit(self)))
+//     return nil;
+// @endcode
+// Until we can use inter-procedural analysis, in such a call, transfer the
+// SelfFlags to the result of the call.
+
+void ObjCSelfInitChecker::PreVisitGenericCall(CheckerContext &C,
+                                              const CallExpr *CE) {
+  const GRState *state = C.getState();
+  for (CallExpr::const_arg_iterator
+         I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) {
+    SVal argV = state->getSVal(*I);
+    if (isSelfVar(argV, C)) {
+      preCallSelfFlags = getSelfFlags(state->getSVal(cast<Loc>(argV)), C);
+      return;
+    } else if (hasSelfFlag(argV, SelfFlag_Self, C)) {
+      preCallSelfFlags = getSelfFlags(argV, C);
+      return;
+    }
+  }
+}
+
+void ObjCSelfInitChecker::PostVisitGenericCall(CheckerContext &C,
+                                               const CallExpr *CE) {
+  const GRState *state = C.getState();
+  for (CallExpr::const_arg_iterator
+         I = CE->arg_begin(), E = CE->arg_end(); I != E; ++I) {
+    SVal argV = state->getSVal(*I);
+    if (isSelfVar(argV, C)) {
+      addSelfFlag(state, state->getSVal(cast<Loc>(argV)), preCallSelfFlags, C);
+      return;
+    } else if (hasSelfFlag(argV, SelfFlag_Self, C)) {
+      addSelfFlag(state, state->getSVal(CE), preCallSelfFlags, C);
+      return;
+    }
+  }
+}
+
+void ObjCSelfInitChecker::visitLocation(CheckerContext &C, const Stmt *S,
+                                        SVal location, bool isLoad) {
+  // Tag the result of a load from 'self' so that we can easily know that the
+  // value is the object that 'self' points to.
+  const GRState *state = C.getState();
+  if (isSelfVar(location, C))
+    addSelfFlag(state, state->getSVal(cast<Loc>(location)), SelfFlag_Self, C);
+}
+
+// FIXME: A callback should disable checkers at the start of functions.
+static bool shouldRunOnFunctionOrMethod(const NamedDecl *ND) {
+  if (!ND)
+    return false;
+
+  const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(ND);
+  if (!MD)
+    return false;
+  if (!isInitializationMethod(MD))
+    return false;
+
+  // self = [super init] applies only to NSObject subclasses.
+  // For instance, NSProxy doesn't implement -init.
+  ASTContext& Ctx = MD->getASTContext();
+  IdentifierInfo* NSObjectII = &Ctx.Idents.get("NSObject");
+  ObjCInterfaceDecl* ID = MD->getClassInterface()->getSuperClass();
+  for ( ; ID ; ID = ID->getSuperClass()) {
+    IdentifierInfo *II = ID->getIdentifier();
+
+    if (II == NSObjectII)
+      break;
+  }
+  if (!ID)
+    return false;
+
+  return true;
+}
+
+/// \brief Returns true if the location is 'self'.
+static bool isSelfVar(SVal location, CheckerContext &C) {
+  AnalysisContext *analCtx = C.getCurrentAnalysisContext(); 
+  if (!analCtx->getSelfDecl())
+    return false;
+  if (!isa<loc::MemRegionVal>(location))
+    return false;
+
+  loc::MemRegionVal MRV = cast<loc::MemRegionVal>(location);
+  if (const DeclRegion *DR = dyn_cast<DeclRegion>(MRV.getRegion()))
+    return (DR->getDecl() == analCtx->getSelfDecl());
+
+  return false;
+}
+
+static bool isInitializationMethod(const ObjCMethodDecl *MD) {
+  // Init methods with prefix like '-(id)_init' are private and the requirements
+  // are less strict so we don't check those.
+  return MD->isInstanceMethod() &&
+      cocoa::deriveNamingConvention(MD->getSelector(),
+                                    /*ignorePrefix=*/false) == cocoa::InitRule;
+}
+
+static bool isInitMessage(const ObjCMessage &msg) {
+  return cocoa::deriveNamingConvention(msg.getSelector()) == cocoa::InitRule;
+}
diff --git a/lib/Checker/ObjCUnusedIVarsChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
index 2523cff9d8d0..6e92498769f7 100644
--- a/lib/Checker/ObjCUnusedIVarsChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ObjCUnusedIVarsChecker.cpp
@@ -13,9 +13,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerV2.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/DeclObjC.h"
@@ -23,6 +24,7 @@
 #include "clang/Basic/SourceManager.h"
 
 using namespace clang;
+using namespace ento;
 
 enum IVarState { Unused, Used };
 typedef llvm::DenseMap<const ObjCIvarDecl*,IVarState> IvarUsageMap;
@@ -97,7 +99,7 @@ static void Scan(IvarUsageMap &M, const DeclContext *C, const FileID FID,
     }
 }
 
-void clang::CheckObjCUnusedIvar(const ObjCImplementationDecl *D,
+static void checkObjCUnusedIvar(const ObjCImplementationDecl *D,
                                 BugReporter &BR) {
 
   const ObjCInterfaceDecl* ID = D->getClassInterface();
@@ -113,9 +115,11 @@ void clang::CheckObjCUnusedIvar(const ObjCImplementationDecl *D,
     // (a) aren't private
     // (b) explicitly marked unused
     // (c) are iboutlets
+    // (d) are unnamed bitfields
     if (ID->getAccessControl() != ObjCIvarDecl::Private ||
         ID->getAttr<UnusedAttr>() || ID->getAttr<IBOutletAttr>() ||
-        ID->getAttr<IBOutletCollectionAttr>())
+        ID->getAttr<IBOutletCollectionAttr>() ||
+        ID->isUnnamedBitfield())
       continue;
 
     M[ID] = Unused;
@@ -159,3 +163,22 @@ void clang::CheckObjCUnusedIvar(const ObjCImplementationDecl *D,
                          os.str(), I->first->getLocation());
     }
 }
+
+//===----------------------------------------------------------------------===//
+// ObjCUnusedIvarsChecker
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ObjCUnusedIvarsChecker : public CheckerV2<
+                                      check::ASTDecl<ObjCImplementationDecl> > {
+public:
+  void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager& mgr,
+                    BugReporter &BR) const {
+    checkObjCUnusedIvar(D, BR);
+  }
+};
+}
+
+void ento::registerObjCUnusedIvarsChecker(CheckerManager &mgr) {
+  mgr.registerChecker<ObjCUnusedIvarsChecker>();
+}
diff --git a/lib/Checker/PointerArithChecker.cpp b/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
index cbac423b467c..741e48bfcbe0 100644
--- a/lib/Checker/PointerArithChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
@@ -12,11 +12,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class PointerArithChecker 
@@ -53,7 +55,7 @@ void PointerArithChecker::PreVisitBinaryOperator(CheckerContext &C,
   if (isa<VarRegion>(LR) || isa<CodeTextRegion>(LR) || 
       isa<CompoundLiteralRegion>(LR)) {
 
-    if (ExplodedNode *N = C.GenerateNode()) {
+    if (ExplodedNode *N = C.generateNode()) {
       if (!BT)
         BT = new BuiltinBug("Dangerous pointer arithmetic",
                             "Pointer arithmetic done on non-array variables "
@@ -66,6 +68,10 @@ void PointerArithChecker::PreVisitBinaryOperator(CheckerContext &C,
   }
 }
 
-void clang::RegisterPointerArithChecker(GRExprEngine &Eng) {
+static void RegisterPointerArithChecker(ExprEngine &Eng) {
   Eng.registerCheck(new PointerArithChecker());
 }
+
+void ento::registerPointerArithChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterPointerArithChecker);
+}
diff --git a/lib/Checker/PointerSubChecker.cpp b/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index d64b6ae3fe3f..f28fe9a5379e 100644
--- a/lib/Checker/PointerSubChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -13,11 +13,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class PointerSubChecker 
@@ -62,7 +64,7 @@ void PointerSubChecker::PreVisitBinaryOperator(CheckerContext &C,
   if (isa<SymbolicRegion>(BaseLR) || isa<SymbolicRegion>(BaseRR))
     return;
 
-  if (ExplodedNode *N = C.GenerateNode()) {
+  if (ExplodedNode *N = C.generateNode()) {
     if (!BT)
       BT = new BuiltinBug("Pointer subtraction", 
                           "Subtraction of two pointers that do not point to "
@@ -73,6 +75,10 @@ void PointerSubChecker::PreVisitBinaryOperator(CheckerContext &C,
   }
 }
 
-void clang::RegisterPointerSubChecker(GRExprEngine &Eng) {
+static void RegisterPointerSubChecker(ExprEngine &Eng) {
   Eng.registerCheck(new PointerSubChecker());
 }
+
+void ento::registerPointerSubChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterPointerSubChecker);
+}
diff --git a/lib/Checker/PthreadLockChecker.cpp b/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
index 74e266c3edfc..34c095f42e08 100644
--- a/lib/Checker/PthreadLockChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
@@ -8,17 +8,19 @@
 //===----------------------------------------------------------------------===//
 //
 // This defines PthreadLockChecker, a simple lock -> unlock checker.  Eventually
-// this shouldn't be registered with GRExprEngineInternalChecks.
+// this shouldn't be registered with ExprEngineInternalChecks.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "GRExprEngineExperimentalChecks.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
 #include "llvm/ADT/ImmutableSet.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class PthreadLockChecker
@@ -44,16 +46,22 @@ public:
 // GDM Entry for tracking lock state.
 namespace { class LockSet {}; }
 namespace clang {
+namespace ento {
 template <> struct GRStateTrait<LockSet> :
   public GRStatePartialTrait<llvm::ImmutableSet<const MemRegion*> > {
     static void* GDMIndex() { return PthreadLockChecker::getTag(); }
 };
+} // end GR namespace
 } // end clang namespace
 
-void clang::RegisterPthreadLockChecker(GRExprEngine &Eng) {
+static void RegisterPthreadLockChecker(ExprEngine &Eng) {
   Eng.registerCheck(new PthreadLockChecker());
 }
 
+void ento::registerPthreadLockChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterPthreadLockChecker);
+}
+
 
 void PthreadLockChecker::PostVisitCallExpr(CheckerContext &C,
                                            const CallExpr *CE) {
@@ -65,7 +73,10 @@ void PthreadLockChecker::PostVisitCallExpr(CheckerContext &C,
   if (!R)
     return;
   
-  llvm::StringRef FName = R->getDecl()->getName();
+  IdentifierInfo *II = R->getDecl()->getIdentifier();
+  if (!II)   // if no identifier, not a simple C function
+    return;
+  llvm::StringRef FName = II->getName();
   
   if (FName == "pthread_mutex_lock") {
     if (CE->getNumArgs() != 1)
@@ -103,20 +114,20 @@ void PthreadLockChecker::AcquireLock(CheckerContext &C, const CallExpr *CE,
   if (isTryLock) {
       // Bifurcate the state, and allow a mode where the lock acquisition fails.
     const GRState *lockFail;
-    llvm::tie(lockFail, lockSucc) = state->Assume(retVal);    
+    llvm::tie(lockFail, lockSucc) = state->assume(retVal);    
     assert(lockFail && lockSucc);
-    C.addTransition(C.GenerateNode(CE, lockFail));
+    C.addTransition(C.generateNode(CE, lockFail));
   }
   else {
       // Assume that the return value was 0.
-    lockSucc = state->Assume(retVal, false);
+    lockSucc = state->assume(retVal, false);
     assert(lockSucc);
   }
   
     // Record that the lock was acquired.  
   lockSucc = lockSucc->add<LockSet>(lockR);
   
-  C.addTransition(lockSucc != state ? C.GenerateNode(CE, lockSucc) :
+  C.addTransition(lockSucc != state ? C.generateNode(CE, lockSucc) :
                   C.getPredecessor());
 }
 
@@ -137,5 +148,5 @@ void PthreadLockChecker::ReleaseLock(CheckerContext &C, const CallExpr *CE,
   if (state == unlockState)
     return;
   
-  C.addTransition(C.GenerateNode(CE, unlockState));  
+  C.addTransition(C.generateNode(CE, unlockState));  
 }
diff --git a/lib/Checker/ReturnPointerRangeChecker.cpp b/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
index a9eb5ce1a700..838a00f18785 100644
--- a/lib/Checker/ReturnPointerRangeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
@@ -12,12 +12,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class ReturnPointerRangeChecker : 
@@ -30,7 +31,7 @@ public:
 };
 }
 
-void clang::RegisterReturnPointerRangeChecker(GRExprEngine &Eng) {
+void ento::RegisterReturnPointerRangeChecker(ExprEngine &Eng) {
   Eng.registerCheck(new ReturnPointerRangeChecker());
 }
 
@@ -48,19 +49,16 @@ void ReturnPointerRangeChecker::PreVisitReturnStmt(CheckerContext &C,
  
   SVal V = state->getSVal(RetE);
   const MemRegion *R = V.getAsRegion();
-  if (!R)
-    return;
-
-  R = R->StripCasts();
-  if (!R)
-    return;
 
   const ElementRegion *ER = dyn_cast_or_null<ElementRegion>(R);
   if (!ER)
     return;
 
-  DefinedOrUnknownSVal &Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
-
+  DefinedOrUnknownSVal Idx = cast<DefinedOrUnknownSVal>(ER->getIndex());
+  // Zero index is always in bound, this also passes ElementRegions created for
+  // pointer casts.
+  if (Idx.isZeroConstant())
+    return;
   // FIXME: All of this out-of-bounds checking should eventually be refactored
   // into a common place.
 
@@ -68,10 +66,10 @@ void ReturnPointerRangeChecker::PreVisitReturnStmt(CheckerContext &C,
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(),
                                            ER->getValueType());
 
-  const GRState *StInBound = state->AssumeInBound(Idx, NumElements, true);
-  const GRState *StOutBound = state->AssumeInBound(Idx, NumElements, false);
+  const GRState *StInBound = state->assumeInBound(Idx, NumElements, true);
+  const GRState *StOutBound = state->assumeInBound(Idx, NumElements, false);
   if (StOutBound && !StInBound) {
-    ExplodedNode *N = C.GenerateSink(StOutBound);
+    ExplodedNode *N = C.generateSink(StOutBound);
 
     if (!N)
       return;
diff --git a/lib/Checker/ReturnUndefChecker.cpp b/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
index 73d1890f336f..555eaf41282b 100644
--- a/lib/Checker/ReturnUndefChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
@@ -13,12 +13,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class ReturnUndefChecker : 
@@ -31,7 +32,7 @@ public:
 };
 }
 
-void clang::RegisterReturnUndefChecker(GRExprEngine &Eng) {
+void ento::RegisterReturnUndefChecker(ExprEngine &Eng) {
   Eng.registerCheck(new ReturnUndefChecker());
 }
 
@@ -49,7 +50,7 @@ void ReturnUndefChecker::PreVisitReturnStmt(CheckerContext &C,
   if (!C.getState()->getSVal(RetE).isUndef())
     return;
   
-  ExplodedNode *N = C.GenerateSink();
+  ExplodedNode *N = C.generateSink();
 
   if (!N)
     return;
diff --git a/lib/Checker/StackAddrLeakChecker.cpp b/lib/StaticAnalyzer/Checkers/StackAddrLeakChecker.cpp
index c67a81dced0a..363f4042db6b 100644
--- a/lib/Checker/StackAddrLeakChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/StackAddrLeakChecker.cpp
@@ -12,13 +12,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallString.h"
 using namespace clang;
+using namespace ento;
 
 namespace {
 class StackAddrLeakChecker : public CheckerVisitor<StackAddrLeakChecker> {
@@ -32,7 +34,7 @@ public:
     return &x;
   }
   void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *RS);
-  void EvalEndPath(GREndPathNodeBuilder &B, void *tag, GRExprEngine &Eng);
+  void evalEndPath(EndOfFunctionNodeBuilder &B, void *tag, ExprEngine &Eng);
 private:
   void EmitStackError(CheckerContext &C, const MemRegion *R, const Expr *RetE);
   SourceRange GenName(llvm::raw_ostream &os, const MemRegion *R,
@@ -40,10 +42,14 @@ private:
 };
 }
 
-void clang::RegisterStackAddrLeakChecker(GRExprEngine &Eng) {
+static void RegisterStackAddrLeakChecker(ExprEngine &Eng) {
   Eng.registerCheck(new StackAddrLeakChecker());
 }
 
+void ento::registerStackAddrLeakChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterStackAddrLeakChecker);
+}
+
 SourceRange StackAddrLeakChecker::GenName(llvm::raw_ostream &os,
                                           const MemRegion *R,
                                           SourceManager &SM) {
@@ -89,7 +95,7 @@ SourceRange StackAddrLeakChecker::GenName(llvm::raw_ostream &os,
 
 void StackAddrLeakChecker::EmitStackError(CheckerContext &C, const MemRegion *R,
                                           const Expr *RetE) {
-  ExplodedNode *N = C.GenerateSink();
+  ExplodedNode *N = C.generateSink();
 
   if (!N)
     return;
@@ -129,9 +135,9 @@ void StackAddrLeakChecker::PreVisitReturnStmt(CheckerContext &C,
   }
 }
 
-void StackAddrLeakChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
-                                       GRExprEngine &Eng) {
-  SaveAndRestore<bool> OldHasGen(B.HasGeneratedNode);
+void StackAddrLeakChecker::evalEndPath(EndOfFunctionNodeBuilder &B, void *tag,
+                                       ExprEngine &Eng) {
+
   const GRState *state = B.getState();
 
   // Iterate over all bindings to global variables and see if it contains
diff --git a/lib/Checker/StreamChecker.cpp b/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index 8553875a24ff..2655be28a462 100644
--- a/lib/Checker/StreamChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -11,15 +11,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineExperimentalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/SymbolManager.h"
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/ImmutableMap.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
@@ -72,9 +74,9 @@ public:
     return &x;
   }
 
-  virtual bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
-  void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
-  void EvalEndPath(GREndPathNodeBuilder &B, void *tag, GRExprEngine &Eng);
+  virtual bool evalCallExpr(CheckerContext &C, const CallExpr *CE);
+  void evalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
+  void evalEndPath(EndOfFunctionNodeBuilder &B, void *tag, ExprEngine &Eng);
   void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S);
 
 private:
@@ -104,18 +106,24 @@ private:
 } // end anonymous namespace
 
 namespace clang {
+namespace ento {
   template <>
   struct GRStateTrait<StreamState> 
     : public GRStatePartialTrait<llvm::ImmutableMap<SymbolRef, StreamState> > {
     static void *GDMIndex() { return StreamChecker::getTag(); }
   };
 }
+}
 
-void clang::RegisterStreamChecker(GRExprEngine &Eng) {
+static void RegisterStreamChecker(ExprEngine &Eng) {
   Eng.registerCheck(new StreamChecker());
 }
 
-bool StreamChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
+void ento::registerStreamChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterStreamChecker);
+}
+
+bool StreamChecker::evalCallExpr(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
   const Expr *Callee = CE->getCallee();
   SVal L = state->getSVal(Callee);
@@ -224,16 +232,16 @@ void StreamChecker::Tmpfile(CheckerContext &C, const CallExpr *CE) {
 void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
   unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-  ValueManager &ValMgr = C.getValueManager();
-  DefinedSVal RetVal = cast<DefinedSVal>(ValMgr.getConjuredSymbolVal(0, CE, 
-                                                                     Count));
+  SValBuilder &svalBuilder = C.getSValBuilder();
+  DefinedSVal RetVal =
+    cast<DefinedSVal>(svalBuilder.getConjuredSymbolVal(0, CE, Count));
   state = state->BindExpr(CE, RetVal);
   
   ConstraintManager &CM = C.getConstraintManager();
   // Bifurcate the state into two: one with a valid FILE* pointer, the other
   // with a NULL.
   const GRState *stateNotNull, *stateNull;
-  llvm::tie(stateNotNull, stateNull) = CM.AssumeDual(state, RetVal);
+  llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, RetVal);
   
   if (SymbolRef Sym = RetVal.getAsSymbol()) {
     // if RetVal is not NULL, set the symbol's state to Opened.
@@ -271,29 +279,24 @@ void StreamChecker::Fseek(CheckerContext &C, const CallExpr *CE) {
     return;
   // Check the legality of the 'whence' argument of 'fseek'.
   SVal Whence = state->getSVal(CE->getArg(2));
-  bool WhenceIsLegal = true;
   const nonloc::ConcreteInt *CI = dyn_cast<nonloc::ConcreteInt>(&Whence);
+
   if (!CI)
-    WhenceIsLegal = false;
+    return;
 
   int64_t x = CI->getValue().getSExtValue();
-  if (!(x == 0 || x == 1 || x == 2))
-    WhenceIsLegal = false;
-
-  if (!WhenceIsLegal) {
-    if (ExplodedNode *N = C.GenerateSink(state)) {
-      if (!BT_illegalwhence)
-        BT_illegalwhence = new BuiltinBug("Illegal whence argument",
-                                     "The whence argument to fseek() should be "
-                                          "SEEK_SET, SEEK_END, or SEEK_CUR.");
-      BugReport *R = new BugReport(*BT_illegalwhence, 
-                                   BT_illegalwhence->getDescription(), N);
-      C.EmitReport(R);
-    }
+  if (x >= 0 && x <= 2)
     return;
-  }
 
-  C.addTransition(state);
+  if (ExplodedNode *N = C.generateNode(state)) {
+    if (!BT_illegalwhence)
+      BT_illegalwhence = new BuiltinBug("Illegal whence argument",
+					"The whence argument to fseek() should be "
+					"SEEK_SET, SEEK_END, or SEEK_CUR.");
+    BugReport *R = new BugReport(*BT_illegalwhence, 
+				 BT_illegalwhence->getDescription(), N);
+    C.EmitReport(R);
+  }
 }
 
 void StreamChecker::Ftell(CheckerContext &C, const CallExpr *CE) {
@@ -352,10 +355,10 @@ const GRState *StreamChecker::CheckNullStream(SVal SV, const GRState *state,
 
   ConstraintManager &CM = C.getConstraintManager();
   const GRState *stateNotNull, *stateNull;
-  llvm::tie(stateNotNull, stateNull) = CM.AssumeDual(state, *DV);
+  llvm::tie(stateNotNull, stateNull) = CM.assumeDual(state, *DV);
 
   if (!stateNotNull && stateNull) {
-    if (ExplodedNode *N = C.GenerateSink(stateNull)) {
+    if (ExplodedNode *N = C.generateSink(stateNull)) {
       if (!BT_nullfp)
         BT_nullfp = new BuiltinBug("NULL stream pointer",
                                      "Stream pointer might be NULL.");
@@ -383,7 +386,7 @@ const GRState *StreamChecker::CheckDoubleClose(const CallExpr *CE,
   // Check: Double close a File Descriptor could cause undefined behaviour.
   // Conforming to man-pages
   if (SS->isClosed()) {
-    ExplodedNode *N = C.GenerateSink();
+    ExplodedNode *N = C.generateSink();
     if (N) {
       if (!BT_doubleclose)
         BT_doubleclose = new BuiltinBug("Double fclose",
@@ -400,7 +403,7 @@ const GRState *StreamChecker::CheckDoubleClose(const CallExpr *CE,
   return state->set<StreamState>(Sym, StreamState::getClosed(CE));
 }
 
-void StreamChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
+void StreamChecker::evalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
          E = SymReaper.dead_end(); I != E; ++I) {
     SymbolRef Sym = *I;
@@ -410,7 +413,7 @@ void StreamChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
       return;
 
     if (SS->isOpened()) {
-      ExplodedNode *N = C.GenerateSink();
+      ExplodedNode *N = C.generateSink();
       if (N) {
         if (!BT_ResourceLeak)
           BT_ResourceLeak = new BuiltinBug("Resource Leak", 
@@ -423,9 +426,8 @@ void StreamChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
   }
 }
 
-void StreamChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
-                                GRExprEngine &Eng) {
-  SaveAndRestore<bool> OldHasGen(B.HasGeneratedNode);
+void StreamChecker::evalEndPath(EndOfFunctionNodeBuilder &B, void *tag,
+                                ExprEngine &Eng) {
   const GRState *state = B.getState();
   typedef llvm::ImmutableMap<SymbolRef, StreamState> SymMap;
   SymMap M = state->get<StreamState>();
diff --git a/lib/Checker/UndefBranchChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
index 1ff0641be5df..14ae9edc7b7e 100644
--- a/lib/Checker/UndefBranchChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
@@ -12,11 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/Checker.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
@@ -49,13 +50,13 @@ class UndefBranchChecker : public Checker {
 public:
   UndefBranchChecker() : BT(0) {}
   static void *getTag();
-  void VisitBranchCondition(GRBranchNodeBuilder &Builder, GRExprEngine &Eng,
+  void VisitBranchCondition(BranchNodeBuilder &Builder, ExprEngine &Eng,
                             const Stmt *Condition, void *tag);
 };
 
 }
 
-void clang::RegisterUndefBranchChecker(GRExprEngine &Eng) {
+void ento::RegisterUndefBranchChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UndefBranchChecker());
 }
 
@@ -64,8 +65,8 @@ void *UndefBranchChecker::getTag() {
   return &x;
 }
 
-void UndefBranchChecker::VisitBranchCondition(GRBranchNodeBuilder &Builder, 
-                                              GRExprEngine &Eng,
+void UndefBranchChecker::VisitBranchCondition(BranchNodeBuilder &Builder, 
+                                              ExprEngine &Eng,
                                               const Stmt *Condition, void *tag){
   const GRState *state = Builder.getState();
   SVal X = state->getSVal(Condition);
diff --git a/lib/Checker/UndefCapturedBlockVarChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
index b1010c9c4892..6d3c966e6db1 100644
--- a/lib/Checker/UndefCapturedBlockVarChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
@@ -11,13 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/BugReporter/BugType.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class UndefCapturedBlockVarChecker
@@ -31,7 +32,7 @@ public:
 };
 } // end anonymous namespace
 
-void clang::RegisterUndefCapturedBlockVarChecker(GRExprEngine &Eng) {
+void ento::RegisterUndefCapturedBlockVarChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UndefCapturedBlockVarChecker());
 }
 
@@ -55,7 +56,7 @@ static const BlockDeclRefExpr *FindBlockDeclRefExpr(const Stmt *S,
 void
 UndefCapturedBlockVarChecker::PostVisitBlockExpr(CheckerContext &C,
                                                  const BlockExpr *BE) {
-  if (!BE->hasBlockDeclRefExprs())
+  if (!BE->getBlockDecl()->hasCaptures())
     return;
 
   const GRState *state = C.getState();
@@ -76,19 +77,19 @@ UndefCapturedBlockVarChecker::PostVisitBlockExpr(CheckerContext &C,
 
     // Get the VarRegion associated with VD in the local stack frame.
     const LocationContext *LC = C.getPredecessor()->getLocationContext();
-    VR = C.getValueManager().getRegionManager().getVarRegion(VD, LC);
+    VR = C.getSValBuilder().getRegionManager().getVarRegion(VD, LC);
 
     if (state->getSVal(VR).isUndef())
-      if (ExplodedNode *N = C.GenerateSink()) {
+      if (ExplodedNode *N = C.generateSink()) {
         if (!BT)
-          BT = new BuiltinBug("Captured block variable is uninitialized");
+          BT = new BuiltinBug("uninitialized variable captured by block");
 
         // Generate a bug report.
         llvm::SmallString<128> buf;
         llvm::raw_svector_ostream os(buf);
 
-        os << "Variable '" << VD->getName() << "' is captured by block with "
-              "a garbage value";
+        os << "Variable '" << VD->getName() 
+           << "' is uninitialized when captured by block";
 
         EnhancedBugReport *R = new EnhancedBugReport(*BT, os.str(), N);
         if (const Expr *Ex = FindBlockDeclRefExpr(BE->getBody(), VD))
diff --git a/lib/Checker/UndefResultChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
index 8b07aed10e30..64a3567bc342 100644
--- a/lib/Checker/UndefResultChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
@@ -7,17 +7,18 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines UndefResultChecker, a builtin check in GRExprEngine that 
+// This defines UndefResultChecker, a builtin check in ExprEngine that 
 // performs checks for undefined results of non-assignment binary operators.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class UndefResultChecker 
@@ -32,7 +33,7 @@ public:
 };
 } // end anonymous namespace
 
-void clang::RegisterUndefResultChecker(GRExprEngine &Eng) {
+void ento::RegisterUndefResultChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UndefResultChecker());
 }
 
@@ -41,7 +42,7 @@ void UndefResultChecker::PostVisitBinaryOperator(CheckerContext &C,
   const GRState *state = C.getState();
   if (state->getSVal(B).isUndef()) {
     // Generate an error node.
-    ExplodedNode *N = C.GenerateSink();
+    ExplodedNode *N = C.generateSink();
     if (!N)
       return;
     
diff --git a/lib/Checker/UndefinedArraySubscriptChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
index 148629e0093f..ff0344802d90 100644
--- a/lib/Checker/UndefinedArraySubscriptChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
@@ -7,16 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines UndefinedArraySubscriptChecker, a builtin check in GRExprEngine
+// This defines UndefinedArraySubscriptChecker, a builtin check in ExprEngine
 // that performs checks for undefined array subscripts.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class UndefinedArraySubscriptChecker
@@ -33,7 +34,7 @@ public:
 };
 } // end anonymous namespace
 
-void clang::RegisterUndefinedArraySubscriptChecker(GRExprEngine &Eng) {
+void ento::RegisterUndefinedArraySubscriptChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UndefinedArraySubscriptChecker());
 }
 
@@ -41,7 +42,7 @@ void
 UndefinedArraySubscriptChecker::PreVisitArraySubscriptExpr(CheckerContext &C, 
                                                 const ArraySubscriptExpr *A) {
   if (C.getState()->getSVal(A->getIdx()).isUndef()) {
-    if (ExplodedNode *N = C.GenerateSink()) {
+    if (ExplodedNode *N = C.generateSink()) {
       if (!BT)
         BT = new BuiltinBug("Array subscript is undefined");
 
diff --git a/lib/Checker/UndefinedAssignmentChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
index ccc97489e631..e53cbba41cbc 100644
--- a/lib/Checker/UndefinedAssignmentChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
@@ -7,16 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines UndefinedAssginmentChecker, a builtin check in GRExprEngine that
+// This defines UndefinedAssginmentChecker, a builtin check in ExprEngine that
 // checks for assigning undefined values.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "InternalChecks.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class UndefinedAssignmentChecker
@@ -30,7 +31,7 @@ public:
 };
 }
 
-void clang::RegisterUndefinedAssignmentChecker(GRExprEngine &Eng){
+void ento::RegisterUndefinedAssignmentChecker(ExprEngine &Eng){
   Eng.registerCheck(new UndefinedAssignmentChecker());
 }
 
@@ -46,7 +47,7 @@ void UndefinedAssignmentChecker::PreVisitBind(CheckerContext &C,
   if (!val.isUndef())
     return;
 
-  ExplodedNode *N = C.GenerateSink();
+  ExplodedNode *N = C.generateSink();
 
   if (!N)
     return;
diff --git a/lib/Checker/UnixAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index de7346d6273a..a53ebb5ae19c 100644
--- a/lib/Checker/UnixAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -12,15 +12,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
+#include "ClangSACheckers.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
 #include <fcntl.h>
 
 using namespace clang;
+using namespace ento;
 using llvm::Optional;
 
 namespace {
@@ -28,6 +30,7 @@ class UnixAPIChecker : public CheckerVisitor<UnixAPIChecker> {
   enum SubChecks {
     OpenFn = 0,
     PthreadOnceFn = 1,
+    MallocZero = 2,
     NumChecks
   };
 
@@ -44,10 +47,14 @@ public:
 };
 } //end anonymous namespace
 
-void clang::RegisterUnixAPIChecker(GRExprEngine &Eng) {
+static void RegisterUnixAPIChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UnixAPIChecker());
 }
 
+void ento::registerUnixAPIChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterUnixAPIChecker);
+}
+
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
@@ -72,7 +79,7 @@ static void CheckOpen(CheckerContext &C, UnixAPIChecker &UC,
     else {
       // FIXME: We need a more general way of getting the O_CREAT value.
       // We could possibly grovel through the preprocessor state, but
-      // that would require passing the Preprocessor object to the GRExprEngine.
+      // that would require passing the Preprocessor object to the ExprEngine.
       return;
     }
   }
@@ -98,18 +105,18 @@ static void CheckOpen(CheckerContext &C, UnixAPIChecker &UC,
   }
   NonLoc oflags = cast<NonLoc>(V);
   NonLoc ocreateFlag =
-    cast<NonLoc>(C.getValueManager().makeIntVal(UC.Val_O_CREAT.getValue(),
+    cast<NonLoc>(C.getSValBuilder().makeIntVal(UC.Val_O_CREAT.getValue(),
                                                 oflagsEx->getType()));
-  SVal maskedFlagsUC = C.getSValuator().EvalBinOpNN(state, BO_And,
-                                                    oflags, ocreateFlag,
-                                                    oflagsEx->getType());
+  SVal maskedFlagsUC = C.getSValBuilder().evalBinOpNN(state, BO_And,
+                                                      oflags, ocreateFlag,
+                                                      oflagsEx->getType());
   if (maskedFlagsUC.isUnknownOrUndef())
     return;
   DefinedSVal maskedFlags = cast<DefinedSVal>(maskedFlagsUC);
 
   // Check if maskedFlags is non-zero.
   const GRState *trueState, *falseState;
-  llvm::tie(trueState, falseState) = state->Assume(maskedFlags);
+  llvm::tie(trueState, falseState) = state->assume(maskedFlags);
 
   // Only emit an error if the value of 'maskedFlags' is properly
   // constrained;
@@ -117,7 +124,7 @@ static void CheckOpen(CheckerContext &C, UnixAPIChecker &UC,
     return;
 
   if (CE->getNumArgs() < 3) {
-    ExplodedNode *N = C.GenerateSink(trueState);
+    ExplodedNode *N = C.generateSink(trueState);
     if (!N)
       return;
 
@@ -152,7 +159,7 @@ static void CheckPthreadOnce(CheckerContext &C, UnixAPIChecker &,
   if (!R || !isa<StackSpaceRegion>(R->getMemorySpace()))
     return;
 
-  ExplodedNode *N = C.GenerateSink(state);
+  ExplodedNode *N = C.generateSink(state);
   if (!N)
     return;
 
@@ -174,6 +181,56 @@ static void CheckPthreadOnce(CheckerContext &C, UnixAPIChecker &,
 }
 
 //===----------------------------------------------------------------------===//
+// "malloc" with allocation size 0
+//===----------------------------------------------------------------------===//
+
+// FIXME: Eventually this should be rolled into the MallocChecker, but this
+// check is more basic and is valuable for widespread use.
+static void CheckMallocZero(CheckerContext &C, UnixAPIChecker &UC,
+                            const CallExpr *CE, BugType *&BT) {
+
+  // Sanity check that malloc takes one argument.
+  if (CE->getNumArgs() != 1)
+    return;
+
+  // Check if the allocation size is 0.
+  const GRState *state = C.getState();
+  SVal argVal = state->getSVal(CE->getArg(0));
+
+  if (argVal.isUnknownOrUndef())
+    return;
+  
+  const GRState *trueState, *falseState;
+  llvm::tie(trueState, falseState) = state->assume(cast<DefinedSVal>(argVal));
+  
+  // Is the value perfectly constrained to zero?
+  if (falseState && !trueState) {
+    ExplodedNode *N = C.generateSink(falseState);
+    if (!N)
+      return;
+    
+    // FIXME: Add reference to CERT advisory, and/or C99 standard in bug
+    // output.
+
+    LazyInitialize(BT, "Undefined allocation of 0 bytes");
+    
+    EnhancedBugReport *report =
+      new EnhancedBugReport(*BT, "Call to 'malloc' has an allocation size"
+                                 " of 0 bytes", N);
+    report->addRange(CE->getArg(0)->getSourceRange());
+    report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue,
+                              CE->getArg(0));
+    C.EmitReport(report);
+    return;
+  }
+  // Assume the the value is non-zero going forward.
+  assert(trueState);
+  if (trueState != state) {
+    C.addTransition(trueState);
+  }
+}
+  
+//===----------------------------------------------------------------------===//
 // Central dispatch function.
 //===----------------------------------------------------------------------===//
 
@@ -213,9 +270,12 @@ void UnixAPIChecker::PreVisitCallExpr(CheckerContext &C, const CallExpr *CE) {
 
   const SubCheck &SC =
     llvm::StringSwitch<SubCheck>(FI->getName())
-      .Case("open", SubCheck(CheckOpen, this, BTypes[OpenFn]))
-      .Case("pthread_once", SubCheck(CheckPthreadOnce, this,
-                                     BTypes[PthreadOnceFn]))
+      .Case("open",
+            SubCheck(CheckOpen, this, BTypes[OpenFn]))
+      .Case("pthread_once",
+            SubCheck(CheckPthreadOnce, this, BTypes[PthreadOnceFn]))
+      .Case("malloc",
+            SubCheck(CheckMallocZero, this, BTypes[MallocZero]))
       .Default(SubCheck());
 
   SC.run(C, CE);
diff --git a/lib/Checker/UnreachableCodeChecker.cpp b/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
index 7a56c7f46b48..3038e29c0efc 100644
--- a/lib/Checker/UnreachableCodeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UnreachableCodeChecker.cpp
@@ -13,15 +13,16 @@
 // A similar flow-sensitive only check exists in Analysis/ReachableCode.cpp
 //===----------------------------------------------------------------------===//
 
+#include "ClangSACheckers.h"
 #include "clang/AST/ParentMap.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/ExplodedGraph.h"
-#include "clang/Checker/PathSensitive/SVals.h"
-#include "clang/Checker/PathSensitive/CheckerHelpers.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "GRExprEngineExperimentalChecks.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 // The number of CFGBlock pointers we want to reserve memory for. This is used
@@ -29,14 +30,15 @@
 #define DEFAULT_CFGBLOCKS 256
 
 using namespace clang;
+using namespace ento;
 
 namespace {
-class UnreachableCodeChecker : public CheckerVisitor<UnreachableCodeChecker> {
+class UnreachableCodeChecker : public Checker {
 public:
   static void *getTag();
   void VisitEndAnalysis(ExplodedGraph &G,
                         BugReporter &B,
-                        GRExprEngine &Eng);
+                        ExprEngine &Eng);
 private:
   static inline const Stmt *getUnreachableStmt(const CFGBlock *CB);
   void FindUnreachableEntryPoints(const CFGBlock *CB);
@@ -53,13 +55,17 @@ void *UnreachableCodeChecker::getTag() {
   return &x;
 }
 
-void clang::RegisterUnreachableCodeChecker(GRExprEngine &Eng) {
+static void RegisterUnreachableCodeChecker(ExprEngine &Eng) {
   Eng.registerCheck(new UnreachableCodeChecker());
 }
 
+void ento::registerUnreachableCodeChecker(CheckerManager &mgr) {
+  mgr.addCheckerRegisterFunction(RegisterUnreachableCodeChecker);
+}
+
 void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
                                               BugReporter &B,
-                                              GRExprEngine &Eng) {
+                                              ExprEngine &Eng) {
   // Bail out if we didn't cover all paths
   if (Eng.hasWorkRemaining())
     return;
@@ -91,7 +97,7 @@ void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
   ASTContext &Ctx = B.getContext();
 
   // Find CFGBlocks that were not covered by any node
-  for (CFG::const_iterator I = C->begin(); I != C->end(); ++I) {
+  for (CFG::const_iterator I = C->begin(), E = C->end(); I != E; ++I) {
     const CFGBlock *CB = *I;
     // Check if the block is unreachable
     if (reachable.count(CB->getBlockID()))
@@ -102,7 +108,8 @@ void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
       continue;
 
     // Find the entry points for this block
-    FindUnreachableEntryPoints(CB);
+    if (!visited.count(CB->getBlockID()))
+      FindUnreachableEntryPoints(CB);
 
     // This block may have been pruned; check if we still want to report it
     if (reachable.count(CB->getBlockID()))
@@ -116,10 +123,12 @@ void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
     // FIXME: This should be extended to include other unreachable markers,
     // such as llvm_unreachable.
     if (!CB->empty()) {
-      const Stmt *First = CB->front();
-      if (const CallExpr *CE = dyn_cast<CallExpr>(First)) {
-        if (CE->isBuiltinCall(Ctx) == Builtin::BI__builtin_unreachable)
-          continue;
+      CFGElement First = CB->front();
+      if (CFGStmt S = First.getAs<CFGStmt>()) {
+        if (const CallExpr *CE = dyn_cast<CallExpr>(S.getStmt())) {
+          if (CE->isBuiltinCall(Ctx) == Builtin::BI__builtin_unreachable)
+            continue;
+        }
       }
     }
 
@@ -147,35 +156,28 @@ void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
 
 // Recursively finds the entry point(s) for this dead CFGBlock.
 void UnreachableCodeChecker::FindUnreachableEntryPoints(const CFGBlock *CB) {
-  bool allPredecessorsReachable = true;
-
   visited.insert(CB->getBlockID());
 
-  for (CFGBlock::const_pred_iterator I = CB->pred_begin(); I != CB->pred_end();
-      ++I) {
-    // Recurse over all unreachable blocks
+  for (CFGBlock::const_pred_iterator I = CB->pred_begin(), E = CB->pred_end();
+      I != E; ++I) {
     if (!reachable.count((*I)->getBlockID())) {
-      // At least one predeccessor was unreachable
-      allPredecessorsReachable = false;
-
-      // Only visit the block once
+      // If we find an unreachable predecessor, mark this block as reachable so
+      // we don't report this block
+      reachable.insert(CB->getBlockID());
       if (!visited.count((*I)->getBlockID()))
+        // If we haven't previously visited the unreachable predecessor, recurse
         FindUnreachableEntryPoints(*I);
     }
   }
-
-  // If at least one predecessor is unreachable, mark this block as reachable
-  // so we don't report this block.
-  if (!allPredecessorsReachable) {
-    reachable.insert(CB->getBlockID());
-  }
 }
 
 // Find the Stmt* in a CFGBlock for reporting a warning
 const Stmt *UnreachableCodeChecker::getUnreachableStmt(const CFGBlock *CB) {
-  if (CB->size() > 0)
-    return CB->front().getStmt();
-  else if (const Stmt *S = CB->getTerminator())
+  for (CFGBlock::const_iterator I = CB->begin(), E = CB->end(); I != E; ++I) {
+    if (CFGStmt S = I->getAs<CFGStmt>())
+      return S;
+  }
+  if (const Stmt *S = CB->getTerminator())
     return S;
   else
     return 0;
diff --git a/lib/Checker/VLASizeChecker.cpp b/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
index 0800b8be1881..ba46e177f822 100644
--- a/lib/Checker/VLASizeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
@@ -7,20 +7,21 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This defines VLASizeChecker, a builtin check in GRExprEngine that 
+// This defines VLASizeChecker, a builtin check in ExprEngine that 
 // performs checks for declaration of VLA of undefined or zero size.
 // In addition, VLASizeChecker is responsible for defining the extent
 // of the MemRegion that represents a VLA.
 //
 //===----------------------------------------------------------------------===//
 
-#include "GRExprEngineInternalChecks.h"
+#include "InternalChecks.h"
 #include "clang/AST/CharUnits.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 class VLASizeChecker : public CheckerVisitor<VLASizeChecker> {
@@ -34,7 +35,7 @@ public:
 };
 } // end anonymous namespace
 
-void clang::RegisterVLASizeChecker(GRExprEngine &Eng) {
+void ento::RegisterVLASizeChecker(ExprEngine &Eng) {
   Eng.registerCheck(new VLASizeChecker());
 }
 
@@ -58,7 +59,7 @@ void VLASizeChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
 
   if (sizeV.isUndef()) {
     // Generate an error node.
-    ExplodedNode *N = C.GenerateSink();
+    ExplodedNode *N = C.generateSink();
     if (!N)
       return;
     
@@ -83,10 +84,10 @@ void VLASizeChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
   DefinedSVal sizeD = cast<DefinedSVal>(sizeV);
 
   const GRState *stateNotZero, *stateZero;
-  llvm::tie(stateNotZero, stateZero) = state->Assume(sizeD);
+  llvm::tie(stateNotZero, stateZero) = state->assume(sizeD);
 
   if (stateZero && !stateNotZero) {
-    ExplodedNode* N = C.GenerateSink(stateZero);
+    ExplodedNode* N = C.generateSink(stateZero);
     if (!BT_zero)
       BT_zero = new BuiltinBug("Declared variable-length array (VLA) has zero "
                                "size");
@@ -107,25 +108,27 @@ void VLASizeChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
   // then matching that with the array region's extent symbol.
 
   // Convert the array length to size_t.
-  ValueManager &ValMgr = C.getValueManager();
-  SValuator &SV = ValMgr.getSValuator();
+  SValBuilder &svalBuilder = C.getSValBuilder();
   QualType SizeTy = Ctx.getSizeType();
-  NonLoc ArrayLength = cast<NonLoc>(SV.EvalCast(sizeD, SizeTy, SE->getType()));
+  NonLoc ArrayLength = cast<NonLoc>(svalBuilder.evalCast(sizeD, SizeTy, 
+                                                         SE->getType()));
 
   // Get the element size.
   CharUnits EleSize = Ctx.getTypeSizeInChars(VLA->getElementType());
-  SVal EleSizeVal = ValMgr.makeIntVal(EleSize.getQuantity(), SizeTy);
+  SVal EleSizeVal = svalBuilder.makeIntVal(EleSize.getQuantity(), SizeTy);
 
   // Multiply the array length by the element size.
-  SVal ArraySizeVal = SV.EvalBinOpNN(state, BO_Mul, ArrayLength,
-                                     cast<NonLoc>(EleSizeVal), SizeTy);
+  SVal ArraySizeVal = svalBuilder.evalBinOpNN(state, BO_Mul, ArrayLength,
+                                              cast<NonLoc>(EleSizeVal), SizeTy);
 
-  // Finally, Assume that the array's extent matches the given size.
+  // Finally, assume that the array's extent matches the given size.
   const LocationContext *LC = C.getPredecessor()->getLocationContext();
-  DefinedOrUnknownSVal Extent = state->getRegion(VD, LC)->getExtent(ValMgr);
+  DefinedOrUnknownSVal Extent =
+    state->getRegion(VD, LC)->getExtent(svalBuilder);
   DefinedOrUnknownSVal ArraySize = cast<DefinedOrUnknownSVal>(ArraySizeVal);
-  DefinedOrUnknownSVal SizeIsKnown = SV.EvalEQ(state, Extent, ArraySize);
-  state = state->Assume(SizeIsKnown, true);
+  DefinedOrUnknownSVal sizeIsKnown =
+    svalBuilder.evalEQ(state, Extent, ArraySize);
+  state = state->assume(sizeIsKnown, true);
 
   // Assume should not fail at this point.
   assert(state);
diff --git a/lib/Checker/AggExprVisitor.cpp b/lib/StaticAnalyzer/Core/AggExprVisitor.cpp
index 6d472f46b45a..e80cf9b4a331 100644
--- a/lib/Checker/AggExprVisitor.cpp
+++ b/lib/StaticAnalyzer/Core/AggExprVisitor.cpp
@@ -12,32 +12,34 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/AST/StmtVisitor.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 /// AggExprVisitor is designed after AggExprEmitter of the CodeGen module.  It
 /// is used for evaluating exprs of C++ object type. Evaluating such exprs
 /// requires a destination pointer pointing to the object being evaluated
 /// into. Passing such a pointer around would pollute the Visit* interface of
-/// GRExprEngine. AggExprVisitor encapsulates code that goes through various
+/// ExprEngine. AggExprVisitor encapsulates code that goes through various
 /// cast and construct exprs (and others), and at the final point, dispatches
-/// back to the GRExprEngine to let the real evaluation logic happen.
+/// back to the ExprEngine to let the real evaluation logic happen.
 class AggExprVisitor : public StmtVisitor<AggExprVisitor> {
-  SVal DestPtr;
+  const MemRegion *Dest;
   ExplodedNode *Pred;
   ExplodedNodeSet &DstSet;
-  GRExprEngine &Eng;
+  ExprEngine &Eng;
 
 public:
-  AggExprVisitor(SVal dest, ExplodedNode *N, ExplodedNodeSet &dst, 
-                 GRExprEngine &eng)
-    : DestPtr(dest), Pred(N), DstSet(dst), Eng(eng) {}
+  AggExprVisitor(const MemRegion *dest, ExplodedNode *N, ExplodedNodeSet &dst, 
+                 ExprEngine &eng)
+    : Dest(dest), Pred(N), DstSet(dst), Eng(eng) {}
 
   void VisitCastExpr(CastExpr *E);
   void VisitCXXConstructExpr(CXXConstructExpr *E);
+  void VisitCXXMemberCallExpr(CXXMemberCallExpr *E);
 };
 }
 
@@ -47,16 +49,21 @@ void AggExprVisitor::VisitCastExpr(CastExpr *E) {
     assert(0 && "Unhandled cast kind");
   case CK_NoOp:
   case CK_ConstructorConversion:
+  case CK_UserDefinedConversion:
     Visit(E->getSubExpr());
     break;
   }
 }
 
 void AggExprVisitor::VisitCXXConstructExpr(CXXConstructExpr *E) {
-  Eng.VisitCXXConstructExpr(E, DestPtr, Pred, DstSet);
+  Eng.VisitCXXConstructExpr(E, Dest, Pred, DstSet);
 }
 
-void GRExprEngine::VisitAggExpr(const Expr *E, SVal Dest, ExplodedNode *Pred,
-                                ExplodedNodeSet &Dst) {
+void AggExprVisitor::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
+  Eng.VisitCXXMemberCallExpr(E, Pred, DstSet);
+}
+
+void ExprEngine::VisitAggExpr(const Expr *E, const MemRegion *Dest, 
+                                ExplodedNode *Pred, ExplodedNodeSet &Dst) {
   AggExprVisitor(Dest, Pred, Dst, *this).Visit(const_cast<Expr *>(E));
 }
diff --git a/lib/Checker/AnalysisManager.cpp b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
index 339cdab80bdb..5f4f83c0a8dc 100644
--- a/lib/Checker/AnalysisManager.cpp
+++ b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
@@ -7,13 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
 #include "clang/Index/Entity.h"
 #include "clang/Index/Indexer.h"
 
 using namespace clang;
+using namespace ento;
 
-const AnalysisContext *
+AnalysisContext *
 AnalysisManager::getAnalysisContextInAnotherTU(const Decl *D) {
   idx::Entity Ent = idx::Entity::get(const_cast<Decl *>(D), 
                                      Idxer->getProgram());
diff --git a/lib/Checker/BasicConstraintManager.cpp b/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
index eee5c5946674..3050ca3ce3cc 100644
--- a/lib/Checker/BasicConstraintManager.cpp
+++ b/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
@@ -13,12 +13,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "SimpleConstraintManager.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/GRTransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 
 namespace { class ConstNotEq {}; }
@@ -31,6 +32,7 @@ static int ConstEqIndex = 0;
 static int ConstNotEqIndex = 0;
 
 namespace clang {
+namespace ento {
 template<>
 struct GRStateTrait<ConstNotEq> : public GRStatePartialTrait<ConstNotEqTy> {
   static inline void* GDMIndex() { return &ConstNotEqIndex; }
@@ -41,6 +43,7 @@ struct GRStateTrait<ConstEq> : public GRStatePartialTrait<ConstEqTy> {
   static inline void* GDMIndex() { return &ConstEqIndex; }
 };
 }
+}
 
 namespace {
 // BasicConstraintManager only tracks equality and inequality constraints of
@@ -49,31 +52,31 @@ class BasicConstraintManager
   : public SimpleConstraintManager {
   GRState::IntSetTy::Factory ISetFactory;
 public:
-  BasicConstraintManager(GRStateManager &statemgr, GRSubEngine &subengine)
+  BasicConstraintManager(GRStateManager &statemgr, SubEngine &subengine)
     : SimpleConstraintManager(subengine), 
       ISetFactory(statemgr.getAllocator()) {}
 
-  const GRState* AssumeSymNE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymNE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymEQ(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymEQ(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymLT(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymLT(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymGT(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymGT(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymGE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymGE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymLE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymLE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& V,
                              const llvm::APSInt& Adjustment);
 
@@ -87,7 +90,7 @@ public:
   bool isEqual(const GRState* state, SymbolRef sym, const llvm::APSInt& V)
       const;
 
-  const GRState* RemoveDeadBindings(const GRState* state, SymbolReaper& SymReaper);
+  const GRState* removeDeadBindings(const GRState* state, SymbolReaper& SymReaper);
 
   void print(const GRState* state, llvm::raw_ostream& Out,
              const char* nl, const char *sep);
@@ -95,14 +98,14 @@ public:
 
 } // end anonymous namespace
 
-ConstraintManager* clang::CreateBasicConstraintManager(GRStateManager& statemgr,
-                                                       GRSubEngine &subengine) {
+ConstraintManager* ento::CreateBasicConstraintManager(GRStateManager& statemgr,
+                                                       SubEngine &subengine) {
   return new BasicConstraintManager(statemgr, subengine);
 }
 
 
 const GRState*
-BasicConstraintManager::AssumeSymNE(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymNE(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // First, determine if sym == X, where X+Adjustment != V.
@@ -122,7 +125,7 @@ BasicConstraintManager::AssumeSymNE(const GRState *state, SymbolRef sym,
 }
 
 const GRState*
-BasicConstraintManager::AssumeSymEQ(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymEQ(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // First, determine if sym == X, where X+Adjustment != V.
@@ -143,7 +146,7 @@ BasicConstraintManager::AssumeSymEQ(const GRState *state, SymbolRef sym,
 
 // The logic for these will be handled in another ConstraintManager.
 const GRState*
-BasicConstraintManager::AssumeSymLT(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymLT(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Is 'V' the smallest possible value?
@@ -153,11 +156,11 @@ BasicConstraintManager::AssumeSymLT(const GRState *state, SymbolRef sym,
   }
 
   // FIXME: For now have assuming x < y be the same as assuming sym != V;
-  return AssumeSymNE(state, sym, V, Adjustment);
+  return assumeSymNE(state, sym, V, Adjustment);
 }
 
 const GRState*
-BasicConstraintManager::AssumeSymGT(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymGT(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Is 'V' the largest possible value?
@@ -167,11 +170,11 @@ BasicConstraintManager::AssumeSymGT(const GRState *state, SymbolRef sym,
   }
 
   // FIXME: For now have assuming x > y be the same as assuming sym != V;
-  return AssumeSymNE(state, sym, V, Adjustment);
+  return assumeSymNE(state, sym, V, Adjustment);
 }
 
 const GRState*
-BasicConstraintManager::AssumeSymGE(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymGE(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Reject a path if the value of sym is a constant X and !(X+Adj >= V).
@@ -199,7 +202,7 @@ BasicConstraintManager::AssumeSymGE(const GRState *state, SymbolRef sym,
 }
 
 const GRState*
-BasicConstraintManager::AssumeSymLE(const GRState *state, SymbolRef sym,
+BasicConstraintManager::assumeSymLE(const GRState *state, SymbolRef sym,
                                     const llvm::APSInt &V,
                                     const llvm::APSInt &Adjustment) {
   // Reject a path if the value of sym is a constant X and !(X+Adj <= V).
@@ -237,10 +240,10 @@ const GRState* BasicConstraintManager::AddNE(const GRState* state, SymbolRef sym
 
   // First, retrieve the NE-set associated with the given symbol.
   ConstNotEqTy::data_type* T = state->get<ConstNotEq>(sym);
-  GRState::IntSetTy S = T ? *T : ISetFactory.GetEmptySet();
+  GRState::IntSetTy S = T ? *T : ISetFactory.getEmptySet();
 
   // Now add V to the NE set.
-  S = ISetFactory.Add(S, &state->getBasicVals().getValue(V));
+  S = ISetFactory.add(S, &state->getBasicVals().getValue(V));
 
   // Create a new state with the old binding replaced.
   return state->set<ConstNotEq>(sym, S);
@@ -273,7 +276,7 @@ bool BasicConstraintManager::isEqual(const GRState* state, SymbolRef sym,
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
 const GRState*
-BasicConstraintManager::RemoveDeadBindings(const GRState* state,
+BasicConstraintManager::removeDeadBindings(const GRState* state,
                                            SymbolReaper& SymReaper) {
 
   ConstEqTy CE = state->get<ConstEq>();
@@ -281,7 +284,8 @@ BasicConstraintManager::RemoveDeadBindings(const GRState* state,
 
   for (ConstEqTy::iterator I = CE.begin(), E = CE.end(); I!=E; ++I) {
     SymbolRef sym = I.getKey();
-    if (SymReaper.maybeDead(sym)) CE = CEFactory.Remove(CE, sym);
+    if (SymReaper.maybeDead(sym))
+      CE = CEFactory.remove(CE, sym);
   }
   state = state->set<ConstEq>(CE);
 
@@ -290,7 +294,8 @@ BasicConstraintManager::RemoveDeadBindings(const GRState* state,
 
   for (ConstNotEqTy::iterator I = CNE.begin(), E = CNE.end(); I != E; ++I) {
     SymbolRef sym = I.getKey();
-    if (SymReaper.maybeDead(sym)) CNE = CNEFactory.Remove(CNE, sym);
+    if (SymReaper.maybeDead(sym))
+      CNE = CNEFactory.remove(CNE, sym);
   }
 
   return state->set<ConstNotEq>(CNE);
diff --git a/lib/Checker/BasicStore.cpp b/lib/StaticAnalyzer/Core/BasicStore.cpp
index f82e1b20be9b..98365e7f4e62 100644
--- a/lib/Checker/BasicStore.cpp
+++ b/lib/StaticAnalyzer/Core/BasicStore.cpp
@@ -11,13 +11,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/AnalysisContext.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 #include "llvm/ADT/ImmutableMap.h"
 
 using namespace clang;
+using namespace ento;
 
 typedef llvm::ImmutableMap<const MemRegion*,SVal> BindingsTy;
 
@@ -46,47 +48,48 @@ public:
 
   SVal Retrieve(Store store, Loc loc, QualType T = QualType());
 
-  Store InvalidateRegion(Store store, const MemRegion *R, const Expr *E,
-                         unsigned Count, InvalidatedSymbols *IS);
+  StoreRef invalidateRegion(Store store, const MemRegion *R, const Expr *E,
+                            unsigned Count, InvalidatedSymbols *IS);
 
-  Store InvalidateRegions(Store store, const MemRegion * const *Begin,
-                          const MemRegion * const *End, const Expr *E,
-                          unsigned Count, InvalidatedSymbols *IS,
-                          bool invalidateGlobals, InvalidatedRegions *Regions);
+  StoreRef invalidateRegions(Store store, const MemRegion * const *Begin,
+                             const MemRegion * const *End, const Expr *E,
+                             unsigned Count, InvalidatedSymbols *IS,
+                             bool invalidateGlobals,
+                             InvalidatedRegions *Regions);
 
-  Store scanForIvars(Stmt *B, const Decl* SelfDecl,
-                     const MemRegion *SelfRegion, Store St);
+  StoreRef scanForIvars(Stmt *B, const Decl* SelfDecl,
+                        const MemRegion *SelfRegion, Store St);
 
-  Store Bind(Store St, Loc loc, SVal V);
-  Store Remove(Store St, Loc loc);
-  Store getInitialStore(const LocationContext *InitLoc);
+  StoreRef Bind(Store St, Loc loc, SVal V);
+  StoreRef Remove(Store St, Loc loc);
+  StoreRef getInitialStore(const LocationContext *InitLoc);
 
-  Store BindCompoundLiteral(Store store, const CompoundLiteralExpr*,
+  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr*,
                             const LocationContext*, SVal val) {
-    return store;
+    return StoreRef(store, *this);
   }
 
-  /// ArrayToPointer - Used by GRExprEngine::VistCast to handle implicit
+  /// ArrayToPointer - Used by ExprEngine::VistCast to handle implicit
   ///  conversions between arrays and pointers.
   SVal ArrayToPointer(Loc Array) { return Array; }
 
-  /// RemoveDeadBindings - Scans a BasicStore of 'state' for dead values.
+  /// removeDeadBindings - Scans a BasicStore of 'state' for dead values.
   ///  It updatees the GRState object in place with the values removed.
-  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
-                           SymbolReaper& SymReaper,
+  StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
+                              SymbolReaper& SymReaper,
                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
 
   void iterBindings(Store store, BindingsHandler& f);
 
-  Store BindDecl(Store store, const VarRegion *VR, SVal InitVal) {
+  StoreRef BindDecl(Store store, const VarRegion *VR, SVal InitVal) {
     return BindDeclInternal(store, VR, &InitVal);
   }
 
-  Store BindDeclWithNoInit(Store store, const VarRegion *VR) {
+  StoreRef BindDeclWithNoInit(Store store, const VarRegion *VR) {
     return BindDeclInternal(store, VR, 0);
   }
 
-  Store BindDeclInternal(Store store, const VarRegion *VR, SVal *InitVal);
+  StoreRef BindDeclInternal(Store store, const VarRegion *VR, SVal *InitVal);
 
   static inline BindingsTy GetBindings(Store store) {
     return BindingsTy(static_cast<const BindingsTy::TreeTy*>(store));
@@ -102,7 +105,7 @@ private:
 } // end anonymous namespace
 
 
-StoreManager* clang::CreateBasicStoreManager(GRStateManager& StMgr) {
+StoreManager* ento::CreateBasicStoreManager(GRStateManager& StMgr) {
   return new BasicStoreManager(StMgr);
 }
 
@@ -142,17 +145,17 @@ SVal BasicStoreManager::LazyRetrieve(Store store, const TypedRegion *R) {
   // Globals and parameters start with symbolic values.
   // Local variables initially are undefined.
 
-  // Non-static globals may have had their values reset by InvalidateRegions.
+  // Non-static globals may have had their values reset by invalidateRegions.
   const MemSpaceRegion *MS = VR->getMemorySpace();
   if (isa<NonStaticGlobalSpaceRegion>(MS)) {
     BindingsTy B = GetBindings(store);
     // FIXME: Copy-and-pasted from RegionStore.cpp.
     if (BindingsTy::data_type *Val = B.lookup(MS)) {
       if (SymbolRef parentSym = Val->getAsSymbol())
-        return ValMgr.getDerivedRegionValueSymbolVal(parentSym, R);
+        return svalBuilder.getDerivedRegionValueSymbolVal(parentSym, R);
 
       if (Val->isZeroConstant())
-        return ValMgr.makeZeroVal(T);
+        return svalBuilder.makeZeroVal(T);
 
       if (Val->isUnknownOrUndef())
         return *Val;
@@ -163,7 +166,7 @@ SVal BasicStoreManager::LazyRetrieve(Store store, const TypedRegion *R) {
 
   if (VR->hasGlobalsOrParametersStorage() ||
       isa<UnknownSpaceRegion>(VR->getMemorySpace()))
-    return ValMgr.getRegionValueSymbolVal(R);
+    return svalBuilder.getRegionValueSymbolVal(R);
 
   return UndefinedVal();
 }
@@ -179,7 +182,8 @@ SVal BasicStoreManager::Retrieve(Store store, Loc loc, QualType T) {
     case loc::MemRegionKind: {
       const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
 
-      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R)))
+      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) ||
+          isa<CXXThisRegion>(R)))
         return UnknownVal();
 
       BindingsTy B = GetBindings(store);
@@ -193,11 +197,11 @@ SVal BasicStoreManager::Retrieve(Store store, Loc loc, QualType T) {
       return V.isUnknownOrUndef() ? V : CastRetrievedVal(V, TR, T);
     }
 
+    case loc::ObjCPropRefKind:
     case loc::ConcreteIntKind:
-      // Some clients may call GetSVal with such an option simply because
-      // they are doing a quick scan through their Locs (potentially to
-      // invalidate their bindings).  Just return Undefined.
-      return UndefinedVal();
+      // Support direct accesses to memory.  It's up to individual checkers
+      // to flag an error.
+      return UnknownVal();
 
     default:
       assert (false && "Invalid Loc.");
@@ -207,9 +211,9 @@ SVal BasicStoreManager::Retrieve(Store store, Loc loc, QualType T) {
   return UnknownVal();
 }
 
-Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
+StoreRef BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
   if (isa<loc::ConcreteInt>(loc))
-    return store;
+    return StoreRef(store, *this);
 
   const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
 
@@ -217,7 +221,7 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
   //  that is used to derive other symbols.
   if (isa<NonStaticGlobalSpaceRegion>(R)) {
     BindingsTy B = GetBindings(store);
-    return VBFactory.Add(B, R, V).getRoot();
+    return StoreRef(VBFactory.add(B, R, V).getRoot(), *this);
   }
 
   // Special case: handle store of pointer values (Loc) to pointers via
@@ -232,15 +236,15 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
         R = ER->getSuperRegion();
     }
 
-  if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R)))
-    return store;
+  if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) || isa<CXXThisRegion>(R)))
+    return StoreRef(store, *this);
 
   const TypedRegion *TyR = cast<TypedRegion>(R);
 
   // Do not bind to arrays.  We need to explicitly check for this so that
   // we do not encounter any weirdness of trying to load/store from arrays.
   if (TyR->isBoundable() && TyR->getValueType()->isArrayType())
-    return store;
+    return StoreRef(store, *this);
 
   if (nonloc::LocAsInteger *X = dyn_cast<nonloc::LocAsInteger>(&V)) {
     // Only convert 'V' to a location iff the underlying region type
@@ -249,35 +253,36 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
     // a pointer.  We may wish to flag a type error here if the types
     // are incompatible.  This may also cause lots of breakage
     // elsewhere. Food for thought.
-    if (TyR->isBoundable() && Loc::IsLocType(TyR->getValueType()))
+    if (TyR->isBoundable() && Loc::isLocType(TyR->getValueType()))
       V = X->getLoc();
   }
 
   BindingsTy B = GetBindings(store);
-  return V.isUnknown()
-    ? VBFactory.Remove(B, R).getRoot()
-    : VBFactory.Add(B, R, V).getRoot();
+  return StoreRef(V.isUnknown()
+                    ? VBFactory.remove(B, R).getRoot()
+                    : VBFactory.add(B, R, V).getRoot(), *this);
 }
 
-Store BasicStoreManager::Remove(Store store, Loc loc) {
+StoreRef BasicStoreManager::Remove(Store store, Loc loc) {
   switch (loc.getSubKind()) {
     case loc::MemRegionKind: {
       const MemRegion* R = cast<loc::MemRegionVal>(loc).getRegion();
 
-      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R)))
-        return store;
+      if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R) ||
+          isa<CXXThisRegion>(R)))
+        return StoreRef(store, *this);
 
-      return VBFactory.Remove(GetBindings(store), R).getRoot();
+      return StoreRef(VBFactory.remove(GetBindings(store), R).getRoot(), *this);
     }
     default:
       assert ("Remove for given Loc type not yet implemented.");
-      return store;
+      return StoreRef(store, *this);
   }
 }
 
-Store BasicStoreManager::RemoveDeadBindings(Store store,
-                                            const StackFrameContext *LCtx,
-                                            SymbolReaper& SymReaper,
+StoreRef BasicStoreManager::removeDeadBindings(Store store,
+                                               const StackFrameContext *LCtx,
+                                               SymbolReaper& SymReaper,
                            llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
 {
   BindingsTy B = GetBindings(store);
@@ -292,7 +297,8 @@ Store BasicStoreManager::RemoveDeadBindings(Store store,
         continue;
     }
     else if (isa<ObjCIvarRegion>(I.getKey()) ||
-             isa<NonStaticGlobalSpaceRegion>(I.getKey()))
+             isa<NonStaticGlobalSpaceRegion>(I.getKey()) ||
+             isa<CXXThisRegion>(I.getKey()))
       RegionRoots.push_back(I.getKey());
     else
       continue;
@@ -316,7 +322,7 @@ Store BasicStoreManager::RemoveDeadBindings(Store store,
         break;
       }
       else if (isa<VarRegion>(MR) || isa<ObjCIvarRegion>(MR) ||
-               isa<NonStaticGlobalSpaceRegion>(MR)) {
+               isa<NonStaticGlobalSpaceRegion>(MR) || isa<CXXThisRegion>(MR)) {
         if (Marked.count(MR))
           break;
 
@@ -342,11 +348,12 @@ Store BasicStoreManager::RemoveDeadBindings(Store store,
   }
 
   // Remove dead variable bindings.
+  StoreRef newStore(store, *this);
   for (BindingsTy::iterator I=B.begin(), E=B.end(); I!=E ; ++I) {
     const MemRegion* R = I.getKey();
 
     if (!Marked.count(R)) {
-      store = Remove(store, ValMgr.makeLoc(R));
+      newStore = Remove(newStore.getStore(), svalBuilder.makeLoc(R));
       SVal X = I.getData();
 
       for (symbol_iterator SI=X.symbol_begin(), SE=X.symbol_end(); SI!=SE; ++SI)
@@ -354,11 +361,15 @@ Store BasicStoreManager::RemoveDeadBindings(Store store,
     }
   }
 
-  return store;
+  return newStore;
 }
 
-Store BasicStoreManager::scanForIvars(Stmt *B, const Decl* SelfDecl,
-                                      const MemRegion *SelfRegion, Store St) {
+StoreRef BasicStoreManager::scanForIvars(Stmt *B, const Decl* SelfDecl,
+                                         const MemRegion *SelfRegion,
+                                         Store St) {
+  
+  StoreRef newStore(St, *this);
+
   for (Stmt::child_iterator CI=B->child_begin(), CE=B->child_end();
        CI != CE; ++CI) {
 
@@ -373,25 +384,25 @@ Store BasicStoreManager::scanForIvars(Stmt *B, const Decl* SelfDecl,
         if (DR->getDecl() == SelfDecl) {
           const ObjCIvarRegion *IVR = MRMgr.getObjCIvarRegion(IV->getDecl(),
                                                          SelfRegion);
-          SVal X = ValMgr.getRegionValueSymbolVal(IVR);
-          St = Bind(St, ValMgr.makeLoc(IVR), X);
+          SVal X = svalBuilder.getRegionValueSymbolVal(IVR);
+          newStore = Bind(newStore.getStore(), svalBuilder.makeLoc(IVR), X);
         }
       }
     }
     else
-      St = scanForIvars(*CI, SelfDecl, SelfRegion, St);
+      newStore = scanForIvars(*CI, SelfDecl, SelfRegion, newStore.getStore());
   }
 
-  return St;
+  return newStore;
 }
 
-Store BasicStoreManager::getInitialStore(const LocationContext *InitLoc) {
+StoreRef BasicStoreManager::getInitialStore(const LocationContext *InitLoc) {
   // The LiveVariables information already has a compilation of all VarDecls
   // used in the function.  Iterate through this set, and "symbolicate"
   // any VarDecl whose value originally comes from outside the function.
   typedef LiveVariables::AnalysisDataTy LVDataTy;
   LVDataTy& D = InitLoc->getLiveVariables()->getAnalysisData();
-  Store St = VBFactory.GetEmptyMap().getRoot();
+  StoreRef St(VBFactory.getEmptyMap().getRoot(), *this);
 
   for (LVDataTy::decl_iterator I=D.begin_decl(), E=D.end_decl(); I != E; ++I) {
     const NamedDecl* ND = I->first;
@@ -405,30 +416,41 @@ Store BasicStoreManager::getInitialStore(const LocationContext *InitLoc) {
           // SelfRegion?  (i.e., it implements MD->getClassInterface()).
           const VarRegion *VR = MRMgr.getVarRegion(PD, InitLoc);
           const MemRegion *SelfRegion =
-            ValMgr.getRegionValueSymbolVal(VR).getAsRegion();
+            svalBuilder.getRegionValueSymbolVal(VR).getAsRegion();
           assert(SelfRegion);
-          St = Bind(St, ValMgr.makeLoc(VR), loc::MemRegionVal(SelfRegion));
+          St = Bind(St.getStore(), svalBuilder.makeLoc(VR),
+                    loc::MemRegionVal(SelfRegion));
           // Scan the method for ivar references.  While this requires an
           // entire AST scan, the cost should not be high in practice.
-          St = scanForIvars(MD->getBody(), PD, SelfRegion, St);
+          St = scanForIvars(MD->getBody(), PD, SelfRegion, St.getStore());
         }
       }
     }
   }
 
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(InitLoc->getDecl())) {
+    // For C++ methods add symbolic region for 'this' in initial stack frame.
+    QualType ThisT = MD->getThisType(StateMgr.getContext());
+    MemRegionManager &RegMgr = svalBuilder.getRegionManager();
+    const CXXThisRegion *ThisR = RegMgr.getCXXThisRegion(ThisT, InitLoc);
+    SVal ThisV = svalBuilder.getRegionValueSymbolVal(ThisR);
+    St = Bind(St.getStore(), svalBuilder.makeLoc(ThisR), ThisV);
+  }
+
   return St;
 }
 
-Store BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
-                                          SVal* InitVal) {
+StoreRef BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
+                                             SVal* InitVal) {
 
   BasicValueFactory& BasicVals = StateMgr.getBasicVals();
   const VarDecl *VD = VR->getDecl();
+  StoreRef newStore(store, *this);
 
   // BasicStore does not model arrays and structs.
   if (VD->getType()->isArrayType() || VD->getType()->isStructureOrClassType())
-    return store;
-
+    return newStore;
+  
   if (VD->hasGlobalStorage()) {
     // Handle variables with global storage: extern, static, PrivateExtern.
 
@@ -445,32 +467,33 @@ Store BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
       // C99: 6.7.8 Initialization
       //  If an object that has static storage duration is not initialized
       //  explicitly, then:
-      //   —if it has pointer type, it is initialized to a null pointer;
-      //   —if it has arithmetic type, it is initialized to (positive or
+      //   -if it has pointer type, it is initialized to a null pointer;
+      //   -if it has arithmetic type, it is initialized to (positive or
       //     unsigned) zero;
       if (!InitVal) {
         QualType T = VD->getType();
-        if (Loc::IsLocType(T))
-          store = Bind(store, loc::MemRegionVal(VR),
-                       loc::ConcreteInt(BasicVals.getValue(0, T)));
+        if (Loc::isLocType(T))
+          newStore = Bind(store, loc::MemRegionVal(VR),
+                          loc::ConcreteInt(BasicVals.getValue(0, T)));
         else if (T->isIntegerType() && T->isScalarType())
-          store = Bind(store, loc::MemRegionVal(VR),
-                       nonloc::ConcreteInt(BasicVals.getValue(0, T)));
+          newStore = Bind(store, loc::MemRegionVal(VR),
+                          nonloc::ConcreteInt(BasicVals.getValue(0, T)));
       } else {
-        store = Bind(store, loc::MemRegionVal(VR), *InitVal);
+          newStore = Bind(store, loc::MemRegionVal(VR), *InitVal);
       }
     }
   } else {
     // Process local scalar variables.
     QualType T = VD->getType();
     // BasicStore only supports scalars.
-    if (T->isScalarType() && ValMgr.getSymbolManager().canSymbolicate(T)) {
+    if ((T->isScalarType() || T->isReferenceType()) &&
+        svalBuilder.getSymbolManager().canSymbolicate(T)) {
       SVal V = InitVal ? *InitVal : UndefinedVal();
-      store = Bind(store, loc::MemRegionVal(VR), V);
+      newStore = Bind(store, loc::MemRegionVal(VR), V);
     }
   }
 
-  return store;
+  return newStore;
 }
 
 void BasicStoreManager::print(Store store, llvm::raw_ostream& Out,
@@ -508,19 +531,21 @@ StoreManager::BindingsHandler::~BindingsHandler() {}
 //===----------------------------------------------------------------------===//
 
 
-Store BasicStoreManager::InvalidateRegions(Store store,
-                                           const MemRegion * const *I,
-                                           const MemRegion * const *End,
-                                           const Expr *E, unsigned Count,
-                                           InvalidatedSymbols *IS,
-                                           bool invalidateGlobals,
-                                           InvalidatedRegions *Regions) {
+StoreRef BasicStoreManager::invalidateRegions(Store store,
+                                              const MemRegion * const *I,
+                                              const MemRegion * const *End,
+                                              const Expr *E, unsigned Count,
+                                              InvalidatedSymbols *IS,
+                                              bool invalidateGlobals,
+                                              InvalidatedRegions *Regions) {
+  StoreRef newStore(store, *this);
+  
   if (invalidateGlobals) {
     BindingsTy B = GetBindings(store);
     for (BindingsTy::iterator I=B.begin(), End=B.end(); I != End; ++I) {
       const MemRegion *R = I.getKey();
       if (isa<NonStaticGlobalSpaceRegion>(R->getMemorySpace()))
-        store = InvalidateRegion(store, R, E, Count, IS);
+        newStore = invalidateRegion(newStore.getStore(), R, E, Count, IS);
     }
   }
 
@@ -531,7 +556,7 @@ Store BasicStoreManager::InvalidateRegions(Store store,
       if (isa<NonStaticGlobalSpaceRegion>(R->getMemorySpace()))
         continue;
     }
-    store = InvalidateRegion(store, *I, E, Count, IS);
+    newStore = invalidateRegion(newStore.getStore(), *I, E, Count, IS);
     if (Regions)
       Regions->push_back(R);
   }
@@ -542,28 +567,28 @@ Store BasicStoreManager::InvalidateRegions(Store store,
     // use to derive the bindings for all non-static globals.
     const GlobalsSpaceRegion *GS = MRMgr.getGlobalsRegion();
     SVal V =
-      ValMgr.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, E,
+      svalBuilder.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, E,
                                   /* symbol type, doesn't matter */ Ctx.IntTy,
                                   Count);
 
-    store = Bind(store, loc::MemRegionVal(GS), V);
+    newStore = Bind(newStore.getStore(), loc::MemRegionVal(GS), V);
     if (Regions)
       Regions->push_back(GS);
   }
 
-  return store;
+  return newStore;
 }
 
 
-Store BasicStoreManager::InvalidateRegion(Store store,
-                                          const MemRegion *R,
-                                          const Expr *E,
-                                          unsigned Count,
-                                          InvalidatedSymbols *IS) {
+StoreRef BasicStoreManager::invalidateRegion(Store store,
+                                             const MemRegion *R,
+                                             const Expr *E,
+                                             unsigned Count,
+                                             InvalidatedSymbols *IS) {
   R = R->StripCasts();
 
   if (!(isa<VarRegion>(R) || isa<ObjCIvarRegion>(R)))
-      return store;
+      return StoreRef(store, *this);
 
   if (IS) {
     BindingsTy B = GetBindings(store);
@@ -574,7 +599,6 @@ Store BasicStoreManager::InvalidateRegion(Store store,
   }
 
   QualType T = cast<TypedRegion>(R)->getValueType();
-  SVal V = ValMgr.getConjuredSymbolVal(R, E, T, Count);
+  SVal V = svalBuilder.getConjuredSymbolVal(R, E, T, Count);
   return Bind(store, loc::MemRegionVal(R), V);
 }
-
diff --git a/lib/Checker/BasicValueFactory.cpp b/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
index 4c9b109c8823..6315d83d894d 100644
--- a/lib/Checker/BasicValueFactory.cpp
+++ b/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
@@ -8,14 +8,15 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines BasicValueFactory, a class that manages the lifetime
-//  of APSInt objects and symbolic constraints used by GRExprEngine
+//  of APSInt objects and symbolic constraints used by ExprEngine
 //  and related classes.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/BasicValueFactory.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h"
 
 using namespace clang;
+using namespace ento;
 
 void CompoundValData::Profile(llvm::FoldingSetNodeID& ID, QualType T,
                               llvm::ImmutableList<SVal> L) {
@@ -98,7 +99,7 @@ const llvm::APSInt& BasicValueFactory::getValue(uint64_t X, unsigned BitWidth,
 const llvm::APSInt& BasicValueFactory::getValue(uint64_t X, QualType T) {
 
   unsigned bits = Ctx.getTypeSize(T);
-  llvm::APSInt V(bits, T->isUnsignedIntegerType() || Loc::IsLocType(T));
+  llvm::APSInt V(bits, T->isUnsignedIntegerType() || Loc::isLocType(T));
   V = X;
   return getValue(V);
 }
@@ -142,7 +143,7 @@ BasicValueFactory::getLazyCompoundValData(const void *store,
 }
 
 const llvm::APSInt*
-BasicValueFactory::EvaluateAPSInt(BinaryOperator::Opcode Op,
+BasicValueFactory::evalAPSInt(BinaryOperator::Opcode Op,
                              const llvm::APSInt& V1, const llvm::APSInt& V2) {
 
   switch (Op) {
@@ -285,5 +286,3 @@ BasicValueFactory::getPersistentSValPair(const SVal& V1, const SVal& V2) {
 const SVal* BasicValueFactory::getPersistentSVal(SVal X) {
   return &getPersistentSValWithData(X, 0).first;
 }
-
-
diff --git a/lib/Checker/GRBlockCounter.cpp b/lib/StaticAnalyzer/Core/BlockCounter.cpp
index cd26060ef0aa..ed52b6b012b7 100644
--- a/lib/Checker/GRBlockCounter.cpp
+++ b/lib/StaticAnalyzer/Core/BlockCounter.cpp
@@ -1,4 +1,4 @@
-//==- GRBlockCounter.h - ADT for counting block visits -------------*- C++ -*-//
+//==- BlockCounter.h - ADT for counting block visits -------------*- C++ -*-//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,16 +7,17 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines GRBlockCounter, an abstract data type used to count
+//  This file defines BlockCounter, an abstract data type used to count
 //  the number of times a given block has been visited along a path
-//  analyzed by GRCoreEngine.
+//  analyzed by CoreEngine.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/GRBlockCounter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/BlockCounter.h"
 #include "llvm/ADT/ImmutableMap.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
 
@@ -55,31 +56,31 @@ static inline CountMap::Factory& GetFactory(void* F) {
   return *static_cast<CountMap::Factory*>(F);
 }
 
-unsigned GRBlockCounter::getNumVisited(const StackFrameContext *CallSite, 
+unsigned BlockCounter::getNumVisited(const StackFrameContext *CallSite, 
                                        unsigned BlockID) const {
   CountMap M = GetMap(Data);
   CountMap::data_type* T = M.lookup(CountKey(CallSite, BlockID));
   return T ? *T : 0;
 }
 
-GRBlockCounter::Factory::Factory(llvm::BumpPtrAllocator& Alloc) {
+BlockCounter::Factory::Factory(llvm::BumpPtrAllocator& Alloc) {
   F = new CountMap::Factory(Alloc);
 }
 
-GRBlockCounter::Factory::~Factory() {
+BlockCounter::Factory::~Factory() {
   delete static_cast<CountMap::Factory*>(F);
 }
 
-GRBlockCounter
-GRBlockCounter::Factory::IncrementCount(GRBlockCounter BC, 
+BlockCounter
+BlockCounter::Factory::IncrementCount(BlockCounter BC, 
                                         const StackFrameContext *CallSite,
                                         unsigned BlockID) {
-  return GRBlockCounter(GetFactory(F).Add(GetMap(BC.Data), 
+  return BlockCounter(GetFactory(F).add(GetMap(BC.Data), 
                                           CountKey(CallSite, BlockID),
                              BC.getNumVisited(CallSite, BlockID)+1).getRoot());
 }
 
-GRBlockCounter
-GRBlockCounter::Factory::GetEmptyCounter() {
-  return GRBlockCounter(GetFactory(F).GetEmptyMap().getRoot());
+BlockCounter
+BlockCounter::Factory::GetEmptyCounter() {
+  return BlockCounter(GetFactory(F).getEmptyMap().getRoot());
 }
diff --git a/lib/Checker/BugReporter.cpp b/lib/StaticAnalyzer/Core/BugReporter.cpp
index bffbd52b7d88..9a84045ebd97 100644
--- a/lib/Checker/BugReporter.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -12,9 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Analysis/CFG.h"
 #include "clang/AST/Expr.h"
@@ -22,7 +22,7 @@
 #include "clang/AST/StmtObjC.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Analysis/ProgramPoint.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
@@ -30,6 +30,7 @@
 #include <queue>
 
 using namespace clang;
+using namespace ento;
 
 BugReporterVisitor::~BugReporterVisitor() {}
 BugReporterContext::~BugReporterContext() {
@@ -51,7 +52,7 @@ void BugReporterContext::addVisitor(BugReporterVisitor* visitor) {
   }
 
   CallbacksSet.InsertNode(visitor, InsertPos);
-  Callbacks = F.Add(visitor, Callbacks);
+  Callbacks = F.add(visitor, Callbacks);
 }
 
 //===----------------------------------------------------------------------===//
@@ -92,6 +93,7 @@ static const Stmt* GetNextStmt(const ExplodedNode* N) {
       // not actual statement points.
       switch (S->getStmtClass()) {
         case Stmt::ChooseExprClass:
+        case Stmt::BinaryConditionalOperatorClass: continue;
         case Stmt::ConditionalOperatorClass: continue;
         case Stmt::BinaryOperatorClass: {
           BinaryOperatorKind Op = cast<BinaryOperator>(S)->getOpcode();
@@ -168,9 +170,9 @@ public:
   PathDiagnosticLocation ExecutionContinues(llvm::raw_string_ostream& os,
                                             const ExplodedNode* N);
 
-  Decl const &getCodeDecl() { return R->getEndNode()->getCodeDecl(); }
+  Decl const &getCodeDecl() { return R->getErrorNode()->getCodeDecl(); }
 
-  ParentMap& getParentMap() { return R->getEndNode()->getParentMap(); }
+  ParentMap& getParentMap() { return R->getErrorNode()->getParentMap(); }
 
   const Stmt *getParent(const Stmt *S) {
     return getParentMap().getParent(S);
@@ -278,10 +280,11 @@ PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
           return PathDiagnosticLocation(Parent, SMgr);
         else
           return PathDiagnosticLocation(S, SMgr);
+      case Stmt::BinaryConditionalOperatorClass:
       case Stmt::ConditionalOperatorClass:
         // For '?', if we are referring to condition, just have the edge point
         // to the entire '?' expression.
-        if (cast<ConditionalOperator>(Parent)->getCond() == S)
+        if (cast<AbstractConditionalOperator>(Parent)->getCond() == S)
           return PathDiagnosticLocation(Parent, SMgr);
         else
           return PathDiagnosticLocation(S, SMgr);
@@ -445,7 +448,7 @@ public:
     // Create the diagnostic.
     FullSourceLoc L(S->getLocStart(), BR.getSourceManager());
 
-    if (Loc::IsLocType(VD->getType())) {
+    if (Loc::isLocType(VD->getType())) {
       std::string msg = "'" + std::string(VD->getNameAsString()) +
       "' now aliases '" + MostRecent->getNameAsString() + "'";
 
@@ -634,6 +637,7 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
         }
 
           // Determine control-flow for ternary '?'.
+        case Stmt::BinaryConditionalOperatorClass:
         case Stmt::ConditionalOperatorClass: {
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
@@ -809,7 +813,7 @@ static bool IsControlFlowExpr(const Stmt *S) {
 
   E = E->IgnoreParenCasts();
 
-  if (isa<ConditionalOperator>(E))
+  if (isa<AbstractConditionalOperator>(E))
     return true;
 
   if (const BinaryOperator *B = dyn_cast<BinaryOperator>(E))
@@ -858,8 +862,9 @@ class EdgeBuilder {
             S = cast<ParenExpr>(S)->IgnoreParens();
             firstCharOnly = true;
             continue;
+          case Stmt::BinaryConditionalOperatorClass:
           case Stmt::ConditionalOperatorClass:
-            S = cast<ConditionalOperator>(S)->getCond();
+            S = cast<AbstractConditionalOperator>(S)->getCond();
             firstCharOnly = true;
             continue;
           case Stmt::ChooseExprClass:
@@ -1165,15 +1170,15 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
       }
 
       if (const BlockEntrance *BE = dyn_cast<BlockEntrance>(&P)) {
-        if (const Stmt* S = BE->getFirstStmt()) {
-         if (IsControlFlowExpr(S)) {
-           // Add the proper context for '&&', '||', and '?'.
-           EB.addContext(S);
-         }
-         else
-           EB.addExtendedContext(PDB.getEnclosingStmtLocation(S).asStmt());
+        if (CFGStmt S = BE->getFirstElement().getAs<CFGStmt>()) {
+          if (IsControlFlowExpr(S)) {
+            // Add the proper context for '&&', '||', and '?'.
+            EB.addContext(S);
+          }
+          else
+            EB.addExtendedContext(PDB.getEnclosingStmtLocation(S).asStmt());
         }
-
+        
         break;
       }
     } while (0);
@@ -1216,13 +1221,13 @@ BugReport::~BugReport() {}
 RangedBugReport::~RangedBugReport() {}
 
 const Stmt* BugReport::getStmt() const {
-  ProgramPoint ProgP = EndNode->getLocation();
+  ProgramPoint ProgP = ErrorNode->getLocation();
   const Stmt *S = NULL;
 
   if (BlockEntrance* BE = dyn_cast<BlockEntrance>(&ProgP)) {
     CFGBlock &Exit = ProgP.getLocationContext()->getCFG()->getExit();
     if (BE->getBlock() == &Exit)
-      S = GetPreviousStmt(EndNode);
+      S = GetPreviousStmt(ErrorNode);
   }
   if (!S)
     S = GetStmt(ProgP);
@@ -1239,8 +1244,8 @@ BugReport::getEndPath(BugReporterContext& BRC,
   if (!S)
     return NULL;
 
-  const SourceRange *Beg, *End;
-  getRanges(Beg, End);
+  BugReport::ranges_iterator Beg, End;
+  llvm::tie(Beg, End) = getRanges();
   PathDiagnosticLocation L(S, BRC.getSourceManager());
 
   // Only add the statement itself as a range if we didn't specify any
@@ -1254,20 +1259,20 @@ BugReport::getEndPath(BugReporterContext& BRC,
   return P;
 }
 
-void BugReport::getRanges(const SourceRange*& beg, const SourceRange*& end) {
+std::pair<BugReport::ranges_iterator, BugReport::ranges_iterator>
+BugReport::getRanges() const {
   if (const Expr* E = dyn_cast_or_null<Expr>(getStmt())) {
     R = E->getSourceRange();
     assert(R.isValid());
-    beg = &R;
-    end = beg+1;
+    return std::make_pair(&R, &R+1);
   }
   else
-    beg = end = 0;
+    return std::make_pair(ranges_iterator(), ranges_iterator());
 }
 
 SourceLocation BugReport::getLocation() const {
-  if (EndNode)
-    if (const Stmt* S = GetCurrentOrPreviousStmt(EndNode)) {
+  if (ErrorNode)
+    if (const Stmt* S = GetCurrentOrPreviousStmt(ErrorNode)) {
       // For member expressions, return the location of the '.' or '->'.
       if (const MemberExpr *ME = dyn_cast<MemberExpr>(S))
         return ME->getMemberLoc();
@@ -1333,7 +1338,7 @@ void BugReporter::FlushReports() {
   }
 
   // Remove all references to the BugType objects.
-  BugTypes = F.GetEmptySet();
+  BugTypes = F.getEmptySet();
 }
 
 //===----------------------------------------------------------------------===//
@@ -1343,8 +1348,7 @@ void BugReporter::FlushReports() {
 static std::pair<std::pair<ExplodedGraph*, NodeBackMap*>,
                  std::pair<ExplodedNode*, unsigned> >
 MakeReportGraph(const ExplodedGraph* G,
-                const ExplodedNode** NStart,
-                const ExplodedNode** NEnd) {
+                llvm::SmallVectorImpl<const ExplodedNode*> &nodes) {
 
   // Create the trimmed graph.  It will contain the shortest paths from the
   // error nodes to the root.  In the new graph we should only have one
@@ -1354,7 +1358,8 @@ MakeReportGraph(const ExplodedGraph* G,
   InterExplodedGraphMap* NMap;
 
   llvm::DenseMap<const void*, const void*> InverseMap;
-  llvm::tie(GTrim, NMap) = G->Trim(NStart, NEnd, &InverseMap);
+  llvm::tie(GTrim, NMap) = G->Trim(nodes.data(), nodes.data() + nodes.size(),
+                                   &InverseMap);
 
   // Create owning pointers for GTrim and NMap just to ensure that they are
   // released when this function exists.
@@ -1369,12 +1374,13 @@ MakeReportGraph(const ExplodedGraph* G,
   typedef llvm::DenseMap<const ExplodedNode*, unsigned> IndexMapTy;
   IndexMapTy IndexMap;
 
-  for (const ExplodedNode** I = NStart; I != NEnd; ++I)
-    if (const ExplodedNode *N = NMap->getMappedNode(*I)) {
-      unsigned NodeIndex = (I - NStart) / sizeof(*I);
+  for (unsigned nodeIndex = 0 ; nodeIndex < nodes.size(); ++nodeIndex) {
+    const ExplodedNode *originalNode = nodes[nodeIndex];
+    if (const ExplodedNode *N = NMap->getMappedNode(originalNode)) {
       WS.push(N);
-      IndexMap[*I] = NodeIndex;
+      IndexMap[originalNode] = nodeIndex;
     }
+  }
 
   assert(!WS.empty() && "No error node found in the trimmed graph.");
 
@@ -1567,30 +1573,24 @@ static void CompactPathDiagnostic(PathDiagnostic &PD, const SourceManager& SM) {
 }
 
 void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
-                                           BugReportEquivClass& EQ) {
+                        llvm::SmallVectorImpl<BugReport *> &bugReports) {
 
-  std::vector<const ExplodedNode*> Nodes;
-
-  for (BugReportEquivClass::iterator I=EQ.begin(), E=EQ.end(); I!=E; ++I) {
-    const ExplodedNode* N = I->getEndNode();
-    if (N) Nodes.push_back(N);
+  assert(!bugReports.empty());
+  llvm::SmallVector<const ExplodedNode *, 10> errorNodes;
+  for (llvm::SmallVectorImpl<BugReport*>::iterator I = bugReports.begin(),
+    E = bugReports.end(); I != E; ++I) {
+      errorNodes.push_back((*I)->getErrorNode());
   }
 
-  if (Nodes.empty())
-    return;
-
   // Construct a new graph that contains only a single path from the error
   // node to a root.
   const std::pair<std::pair<ExplodedGraph*, NodeBackMap*>,
   std::pair<ExplodedNode*, unsigned> >&
-  GPair = MakeReportGraph(&getGraph(), &Nodes[0], &Nodes[0] + Nodes.size());
+    GPair = MakeReportGraph(&getGraph(), errorNodes);
 
   // Find the BugReport with the original location.
-  BugReport *R = 0;
-  unsigned i = 0;
-  for (BugReportEquivClass::iterator I=EQ.begin(), E=EQ.end(); I!=E; ++I, ++i)
-    if (i == GPair.second.second) { R = *I; break; }
-
+  assert(GPair.second.second < bugReports.size());
+  BugReport *R = bugReports[GPair.second.second];
   assert(R && "No original report found for sliced graph.");
 
   llvm::OwningPtr<ExplodedGraph> ReportGraph(GPair.first.first);
@@ -1620,7 +1620,7 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
 }
 
 void BugReporter::Register(BugType *BT) {
-  BugTypes = F.Add(BugTypes, BT);
+  BugTypes = F.add(BugTypes, BT);
 }
 
 void BugReporter::EmitReport(BugReport* R) {
@@ -1657,37 +1657,56 @@ struct FRIEC_WLItem {
 };  
 }
 
-static BugReport *FindReportInEquivalenceClass(BugReportEquivClass& EQ) {
+static BugReport *
+FindReportInEquivalenceClass(BugReportEquivClass& EQ,
+                             llvm::SmallVectorImpl<BugReport*> &bugReports) {
+
   BugReportEquivClass::iterator I = EQ.begin(), E = EQ.end();
   assert(I != E);
   BugReport *R = *I;
   BugType& BT = R->getBugType();
-  
-  if (!BT.isSuppressOnSink())
+
+  // If we don't need to suppress any of the nodes because they are
+  // post-dominated by a sink, simply add all the nodes in the equivalence class
+  // to 'Nodes'.  Any of the reports will serve as a "representative" report.
+  if (!BT.isSuppressOnSink()) {
+    for (BugReportEquivClass::iterator I=EQ.begin(), E=EQ.end(); I!=E; ++I) {
+      const ExplodedNode* N = I->getErrorNode();
+      if (N) {
+        R = *I;
+        bugReports.push_back(R);
+      }
+    }
     return R;
-  
+  }
+
   // For bug reports that should be suppressed when all paths are post-dominated
   // by a sink node, iterate through the reports in the equivalence class
   // until we find one that isn't post-dominated (if one exists).  We use a
   // DFS traversal of the ExplodedGraph to find a non-sink node.  We could write
   // this as a recursive function, but we don't want to risk blowing out the
   // stack for very long paths.
+  BugReport *exampleReport = 0;
+
   for (; I != E; ++I) {
     R = *I;
-    const ExplodedNode *N = R->getEndNode();
+    const ExplodedNode *errorNode = R->getErrorNode();
 
-    if (!N)
+    if (!errorNode)
       continue;
-
-    if (N->isSink()) {
+    if (errorNode->isSink()) {
       assert(false &&
            "BugType::isSuppressSink() should not be 'true' for sink end nodes");
-      return R;
+      return 0;
     }
-    
-    if (N->succ_empty())
-      return R;
-    
+    // No successors?  By definition this nodes isn't post-dominated by a sink.
+    if (errorNode->succ_empty()) {
+      bugReports.push_back(R);
+      if (!exampleReport)
+        exampleReport = R;
+      continue;
+    }
+
     // At this point we know that 'N' is not a sink and it has at least one
     // successor.  Use a DFS worklist to find a non-sink end-of-path node.    
     typedef FRIEC_WLItem WLItem;
@@ -1695,8 +1714,8 @@ static BugReport *FindReportInEquivalenceClass(BugReportEquivClass& EQ) {
     llvm::DenseMap<const ExplodedNode *, unsigned> Visited;
     
     DFSWorkList WL;
-    WL.push_back(N);
-    Visited[N] = 1;
+    WL.push_back(errorNode);
+    Visited[errorNode] = 1;
     
     while (!WL.empty()) {
       WLItem &WI = WL.back();
@@ -1706,15 +1725,17 @@ static BugReport *FindReportInEquivalenceClass(BugReportEquivClass& EQ) {
         const ExplodedNode *Succ = *WI.I;        
         // End-of-path node?
         if (Succ->succ_empty()) {
-          // If we found an end-of-path node that is not a sink, then return
-          // this report.
-          if (!Succ->isSink())
-            return R;
-         
+          // If we found an end-of-path node that is not a sink.
+          if (!Succ->isSink()) {
+            bugReports.push_back(R);
+            if (!exampleReport)
+              exampleReport = R;
+            WL.clear();
+            break;
+          }
           // Found a sink?  Continue on to the next successor.
           continue;
         }
-        
         // Mark the successor as visited.  If it hasn't been explored,
         // enqueue it to the DFS worklist.
         unsigned &mark = Visited[Succ];
@@ -1724,17 +1745,18 @@ static BugReport *FindReportInEquivalenceClass(BugReportEquivClass& EQ) {
           break;
         }
       }
-      
-      if (&WL.back() == &WI)
+
+      // The worklist may have been cleared at this point.  First
+      // check if it is empty before checking the last item.
+      if (!WL.empty() && &WL.back() == &WI)
         WL.pop_back();
     }
   }
-  
-  // If we reach here, the end nodes for all reports in the equivalence
-  // class are post-dominated by a sink node.
-  return NULL;
-}
 
+  // ExampleReport will be NULL if all the nodes in the equivalence class
+  // were post-dominated by sinks.
+  return exampleReport;
+}
 
 //===----------------------------------------------------------------------===//
 // DiagnosticCache.  This is a hack to cache analyzer diagnostics.  It
@@ -1780,42 +1802,45 @@ static bool IsCachedDiagnostic(BugReport *R, PathDiagnostic *PD) {
 }
 
 void BugReporter::FlushReport(BugReportEquivClass& EQ) {
-  BugReport *R = FindReportInEquivalenceClass(EQ);
-
-  if (!R)
+  llvm::SmallVector<BugReport*, 10> bugReports;
+  BugReport *exampleReport = FindReportInEquivalenceClass(EQ, bugReports);
+  if (!exampleReport)
     return;
   
   PathDiagnosticClient* PD = getPathDiagnosticClient();
 
   // FIXME: Make sure we use the 'R' for the path that was actually used.
   // Probably doesn't make a difference in practice.
-  BugType& BT = R->getBugType();
+  BugType& BT = exampleReport->getBugType();
 
   llvm::OwningPtr<PathDiagnostic>
-    D(new PathDiagnostic(R->getBugType().getName(),
+    D(new PathDiagnostic(exampleReport->getBugType().getName(),
                          !PD || PD->useVerboseDescription()
-                         ? R->getDescription() : R->getShortDescription(),
+                         ? exampleReport->getDescription() 
+                         : exampleReport->getShortDescription(),
                          BT.getCategory()));
 
-  GeneratePathDiagnostic(*D.get(), EQ);
+  if (!bugReports.empty())
+    GeneratePathDiagnostic(*D.get(), bugReports);
 
-  if (IsCachedDiagnostic(R, D.get()))
+  if (IsCachedDiagnostic(exampleReport, D.get()))
     return;
   
   // Get the meta data.
-  std::pair<const char**, const char**> Meta = R->getExtraDescriptiveText();
+  std::pair<const char**, const char**> Meta =
+    exampleReport->getExtraDescriptiveText();
   for (const char** s = Meta.first; s != Meta.second; ++s)
     D->addMeta(*s);
 
   // Emit a summary diagnostic to the regular Diagnostics engine.
-  const SourceRange *Beg = 0, *End = 0;
-  R->getRanges(Beg, End);
-  Diagnostic& Diag = getDiagnostic();
-  FullSourceLoc L(R->getLocation(), getSourceManager());
+  BugReport::ranges_iterator Beg, End;
+  llvm::tie(Beg, End) = exampleReport->getRanges();
+  Diagnostic &Diag = getDiagnostic();
+  FullSourceLoc L(exampleReport->getLocation(), getSourceManager());
   
   // Search the description for '%', as that will be interpretted as a
   // format character by FormatDiagnostics.
-  llvm::StringRef desc = R->getShortDescription();
+  llvm::StringRef desc = exampleReport->getShortDescription();
   unsigned ErrorDiag;
   {
     llvm::SmallString<512> TmpStr;
@@ -1830,12 +1855,10 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
     ErrorDiag = Diag.getCustomDiagID(Diagnostic::Warning, TmpStr);
   }        
 
-  switch (End-Beg) {
-    default: assert(0 && "Don't handle this many ranges yet!");
-    case 0: Diag.Report(L, ErrorDiag); break;
-    case 1: Diag.Report(L, ErrorDiag) << Beg[0]; break;
-    case 2: Diag.Report(L, ErrorDiag) << Beg[0] << Beg[1]; break;
-    case 3: Diag.Report(L, ErrorDiag) << Beg[0] << Beg[1] << Beg[2]; break;
+  {
+    DiagnosticBuilder diagBuilder = Diag.Report(L, ErrorDiag);
+    for (BugReport::ranges_iterator I = Beg; I != End; ++I)
+      diagBuilder << *I;
   }
 
   // Emit a full diagnostic for the path if we have a PathDiagnosticClient.
@@ -1844,7 +1867,7 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
 
   if (D->empty()) {
     PathDiagnosticPiece* piece =
-      new PathDiagnosticEventPiece(L, R->getDescription());
+      new PathDiagnosticEventPiece(L, exampleReport->getDescription());
 
     for ( ; Beg != End; ++Beg) piece->addRange(*Beg);
     D->push_back(piece);
diff --git a/lib/Checker/BugReporterVisitors.cpp b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 91cf349107ca..8e31adead188 100644
--- a/lib/Checker/BugReporterVisitors.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -14,18 +14,19 @@
 
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprObjC.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/PathSensitive/ExplodedGraph.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 
 using namespace clang;
+using namespace ento;
 
 //===----------------------------------------------------------------------===//
 // Utility functions.
 //===----------------------------------------------------------------------===//
 
-const Stmt *clang::bugreporter::GetDerefExpr(const ExplodedNode *N) {
+const Stmt *bugreporter::GetDerefExpr(const ExplodedNode *N) {
   // Pattern match for a few useful cases (do something smarter later):
   //   a[0], p->f, *p
   const Stmt *S = N->getLocationAs<PostStmt>()->getStmt();
@@ -46,16 +47,14 @@ const Stmt *clang::bugreporter::GetDerefExpr(const ExplodedNode *N) {
   return NULL;
 }
 
-const Stmt*
-clang::bugreporter::GetDenomExpr(const ExplodedNode *N) {
+const Stmt *bugreporter::GetDenomExpr(const ExplodedNode *N) {
   const Stmt *S = N->getLocationAs<PreStmt>()->getStmt();
   if (const BinaryOperator *BE = dyn_cast<BinaryOperator>(S))
     return BE->getRHS();
   return NULL;
 }
 
-const Stmt*
-clang::bugreporter::GetCalleeExpr(const ExplodedNode *N) {
+const Stmt *bugreporter::GetCalleeExpr(const ExplodedNode *N) {
   // Callee is checked as a PreVisit to the CallExpr.
   const Stmt *S = N->getLocationAs<PreStmt>()->getStmt();
   if (const CallExpr *CE = dyn_cast<CallExpr>(S))
@@ -63,8 +62,7 @@ clang::bugreporter::GetCalleeExpr(const ExplodedNode *N) {
   return NULL;
 }
 
-const Stmt*
-clang::bugreporter::GetRetValExpr(const ExplodedNode *N) {
+const Stmt *bugreporter::GetRetValExpr(const ExplodedNode *N) {
   const Stmt *S = N->getLocationAs<PostStmt>()->getStmt();
   if (const ReturnStmt *RS = dyn_cast<ReturnStmt>(S))
     return RS->getRetValue();
@@ -252,14 +250,14 @@ public:
 
     // Check if in the previous state it was feasible for this constraint
     // to *not* be true.
-    if (PrevN->getState()->Assume(Constraint, !Assumption)) {
+    if (PrevN->getState()->assume(Constraint, !Assumption)) {
 
       isSatisfied = true;
 
       // As a sanity check, make sure that the negation of the constraint
       // was infeasible in the current state.  If it is feasible, we somehow
       // missed the transition point.
-      if (N->getState()->Assume(Constraint, !Assumption))
+      if (N->getState()->assume(Constraint, !Assumption))
         return NULL;
 
       // We found the transition point for the constraint.  We now need to
@@ -306,9 +304,9 @@ static void registerTrackConstraint(BugReporterContext& BRC,
   BRC.addVisitor(new TrackConstraintBRVisitor(Constraint, Assumption));
 }
 
-void clang::bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
-                                                       const void *data,
-                                                       const ExplodedNode* N) {
+void bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
+                                                const void *data,
+                                                const ExplodedNode* N) {
 
   const Stmt *S = static_cast<const Stmt*>(data);
 
@@ -318,13 +316,14 @@ void clang::bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
   GRStateManager &StateMgr = BRC.getStateManager();
   const GRState *state = N->getState();
 
+  // Walk through lvalue-to-rvalue conversions.  
   if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S)) {
     if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
       const VarRegion *R =
-      StateMgr.getRegionManager().getVarRegion(VD, N->getLocationContext());
+        StateMgr.getRegionManager().getVarRegion(VD, N->getLocationContext());
 
       // What did we load?
-      SVal V = state->getSVal(S);
+      SVal V = state->getSVal(loc::MemRegionVal(R));
 
       if (isa<loc::ConcreteInt>(V) || isa<nonloc::ConcreteInt>(V)
           || V.isUndef()) {
@@ -353,9 +352,9 @@ void clang::bugreporter::registerTrackNullOrUndefValue(BugReporterContext& BRC,
   }
 }
 
-void clang::bugreporter::registerFindLastStore(BugReporterContext& BRC,
-                                               const void *data,
-                                               const ExplodedNode* N) {
+void bugreporter::registerFindLastStore(BugReporterContext& BRC,
+                                        const void *data,
+                                        const ExplodedNode* N) {
 
   const MemRegion *R = static_cast<const MemRegion*>(data);
 
@@ -400,7 +399,7 @@ public:
     const DefinedOrUnknownSVal *DV = dyn_cast<DefinedOrUnknownSVal>(&V);
     if (!DV)
       return 0;
-    state = state->Assume(*DV, true);
+    state = state->assume(*DV, true);
     if (state)
       return 0;
 
@@ -416,12 +415,12 @@ public:
 };
 } // end anonymous namespace
 
-void clang::bugreporter::registerNilReceiverVisitor(BugReporterContext &BRC) {
+void bugreporter::registerNilReceiverVisitor(BugReporterContext &BRC) {
   BRC.addVisitor(new NilReceiverVisitor());
 }
 
 // Registers every VarDecl inside a Stmt with a last store vistor.
-void clang::bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
+void bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
                                                    const void *stmt,
                                                    const ExplodedNode *N) {
   const Stmt *S = static_cast<const Stmt *>(stmt);
diff --git a/lib/Checker/CFRefCount.cpp b/lib/StaticAnalyzer/Core/CFRefCount.cpp
index 6fa48b2923fe..b3721d7659fe 100644
--- a/lib/Checker/CFRefCount.cpp
+++ b/lib/StaticAnalyzer/Core/CFRefCount.cpp
@@ -16,15 +16,15 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Checker/BugReporter/BugType.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/DomainSpecific/CocoaConventions.h"
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRExprEngineBuilders.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/GRTransferFuncs.h"
-#include "clang/Checker/PathSensitive/SymbolManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerVisitor.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngineBuilders.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableList.h"
@@ -34,19 +34,21 @@
 #include <stdarg.h>
 
 using namespace clang;
+using namespace ento;
 using llvm::StringRef;
 using llvm::StrInStrNoCase;
 
 namespace {
 class InstanceReceiver {
-  const ObjCMessageExpr *ME;
+  ObjCMessage Msg;
   const LocationContext *LC;
 public:
-  InstanceReceiver(const ObjCMessageExpr *me = 0,
-                   const LocationContext *lc = 0) : ME(me), LC(lc) {}
+  InstanceReceiver() : LC(0) { }
+  InstanceReceiver(const ObjCMessage &msg,
+                   const LocationContext *lc = 0) : Msg(msg), LC(lc) {}
 
   bool isValid() const {
-    return ME && ME->isInstanceMessage();
+    return Msg.isValid() && Msg.isInstanceMessage();
   }
   operator bool() const {
     return isValid();
@@ -56,7 +58,7 @@ public:
     assert(isValid());
     // We have an expression for the receiver?  Fetch the value
     // of that expression.
-    if (const Expr *Ex = ME->getInstanceReceiver())
+    if (const Expr *Ex = Msg.getInstanceReceiver())
       return state->getSValAsScalarOrLoc(Ex);
 
     // Otherwise we are sending a message to super.  In this case the
@@ -69,11 +71,11 @@ public:
 
   SourceRange getSourceRange() const {
     assert(isValid());
-    if (const Expr *Ex = ME->getInstanceReceiver())
+    if (const Expr *Ex = Msg.getInstanceReceiver())
       return Ex->getSourceRange();
 
     // Otherwise we are sending a message to super.
-    SourceLocation L = ME->getSuperLoc();
+    SourceLocation L = Msg.getSuperLoc();
     assert(L.isValid());
     return SourceRange(L, L);
   }
@@ -90,17 +92,17 @@ ResolveToInterfaceMethodDecl(const ObjCMethodDecl *MD) {
 }
 
 namespace {
-class GenericNodeBuilder {
-  GRStmtNodeBuilder *SNB;
+class GenericNodeBuilderRefCount {
+  StmtNodeBuilder *SNB;
   const Stmt *S;
   const void *tag;
-  GREndPathNodeBuilder *ENB;
+  EndOfFunctionNodeBuilder *ENB;
 public:
-  GenericNodeBuilder(GRStmtNodeBuilder &snb, const Stmt *s,
+  GenericNodeBuilderRefCount(StmtNodeBuilder &snb, const Stmt *s,
                      const void *t)
   : SNB(&snb), S(s), tag(t), ENB(0) {}
 
-  GenericNodeBuilder(GREndPathNodeBuilder &enb)
+  GenericNodeBuilderRefCount(EndOfFunctionNodeBuilder &enb)
   : SNB(0), S(0), tag(0), ENB(&enb) {}
 
   ExplodedNode *MakeNode(const GRState *state, ExplodedNode *Pred) {
@@ -396,6 +398,7 @@ void RefVal::print(llvm::raw_ostream& Out) const {
 typedef llvm::ImmutableMap<SymbolRef, RefVal> RefBindings;
 
 namespace clang {
+namespace ento {
   template<>
   struct GRStateTrait<RefBindings> : public GRStatePartialTrait<RefBindings> {
     static void* GDMIndex() {
@@ -404,6 +407,7 @@ namespace clang {
     }
   };
 }
+}
 
 //===----------------------------------------------------------------------===//
 // Summaries
@@ -447,6 +451,10 @@ public:
 
     return DefaultArgEffect;
   }
+  
+  void addArg(ArgEffects::Factory &af, unsigned idx, ArgEffect e) {
+    Args = af.add(Args, idx, e);
+  }
 
   /// setDefaultArgEffect - Set the default argument effect.
   void setDefaultArgEffect(ArgEffect E) {
@@ -463,6 +471,10 @@ public:
   ///  terminate the path.
   bool isEndPath() const { return EndPath; }
 
+  
+  /// Sets the effect on the receiver of the message.
+  void setReceiverEffect(ArgEffect e) { Receiver = e; }
+  
   /// getReceiverEffect - Returns the effect on the receiver of the call.
   ///  This is only meaningful if the summary applies to an ObjCMessageExpr*.
   ArgEffect getReceiverEffect() const { return Receiver; }
@@ -762,7 +774,7 @@ private:
   }
 
   void addPanicSummary(const char* Cls, ...) {
-    RetainSummary* Summ = getPersistentSummary(AF.GetEmptyMap(),
+    RetainSummary* Summ = getPersistentSummary(AF.getEmptyMap(),
                                                RetEffect::MakeNoRet(),
                                                DoNothing,  DoNothing, true);
     va_list argp;
@@ -776,12 +788,12 @@ public:
   RetainSummaryManager(ASTContext& ctx, bool gcenabled)
    : Ctx(ctx),
      CFDictionaryCreateII(&ctx.Idents.get("CFDictionaryCreate")),
-     GCEnabled(gcenabled), AF(BPAlloc), ScratchArgs(AF.GetEmptyMap()),
+     GCEnabled(gcenabled), AF(BPAlloc), ScratchArgs(AF.getEmptyMap()),
      ObjCAllocRetE(gcenabled ? RetEffect::MakeGCNotOwned()
                              : RetEffect::MakeOwned(RetEffect::ObjC, true)),
      ObjCInitRetE(gcenabled ? RetEffect::MakeGCNotOwned()
                             : RetEffect::MakeOwnedWhenTrackedReceiver()),
-     DefaultSummary(AF.GetEmptyMap() /* per-argument effects (none) */,
+     DefaultSummary(AF.getEmptyMap() /* per-argument effects (none) */,
                     RetEffect::MakeNoRet() /* return effect */,
                     MayEscape, /* default argument effect */
                     DoNothing /* receiver effect */),
@@ -795,14 +807,14 @@ public:
 
   RetainSummary* getSummary(const FunctionDecl* FD);
 
-  RetainSummary *getInstanceMethodSummary(const ObjCMessageExpr *ME,
+  RetainSummary *getInstanceMethodSummary(const ObjCMessage &msg,
                                           const GRState *state,
                                           const LocationContext *LC);
 
-  RetainSummary* getInstanceMethodSummary(const ObjCMessageExpr* ME,
+  RetainSummary* getInstanceMethodSummary(const ObjCMessage &msg,
                                           const ObjCInterfaceDecl* ID) {
-    return getInstanceMethodSummary(ME->getSelector(), 0,
-                            ID, ME->getMethodDecl(), ME->getType());
+    return getInstanceMethodSummary(msg.getSelector(), 0,
+                            ID, msg.getMethodDecl(), msg.getType(Ctx));
   }
 
   RetainSummary* getInstanceMethodSummary(Selector S, IdentifierInfo *ClsName,
@@ -815,23 +827,15 @@ public:
                                        const ObjCMethodDecl *MD,
                                        QualType RetTy);
 
-  RetainSummary *getClassMethodSummary(const ObjCMessageExpr *ME) {
-    ObjCInterfaceDecl *Class = 0;
-    switch (ME->getReceiverKind()) {
-    case ObjCMessageExpr::Class:
-    case ObjCMessageExpr::SuperClass:
-      Class = ME->getReceiverInterface();
-      break;
-
-    case ObjCMessageExpr::Instance:
-    case ObjCMessageExpr::SuperInstance:
-      break;
-    }
+  RetainSummary *getClassMethodSummary(const ObjCMessage &msg) {
+    const ObjCInterfaceDecl *Class = 0;
+    if (!msg.isInstanceMessage())
+      Class = msg.getReceiverInterface();
 
-    return getClassMethodSummary(ME->getSelector(),
+    return getClassMethodSummary(msg.getSelector(),
                                  Class? Class->getIdentifier() : 0,
                                  Class,
-                                 ME->getMethodDecl(), ME->getType());
+                                 msg.getMethodDecl(), msg.getType(Ctx));
   }
 
   /// getMethodSummary - This version of getMethodSummary is used to query
@@ -881,7 +885,7 @@ RetainSummaryManager::~RetainSummaryManager() {}
 
 ArgEffects RetainSummaryManager::getArgEffects() {
   ArgEffects AE = ScratchArgs;
-  ScratchArgs = AF.GetEmptyMap();
+  ScratchArgs = AF.getEmptyMap();
   return AE;
 }
 
@@ -967,13 +971,13 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
       // FIXES: <rdar://problem/6326900>
       // This should be addressed using a API table.  This strcmp is also
       // a little gross, but there is no need to super optimize here.
-      ScratchArgs = AF.Add(ScratchArgs, 1, DecRef);
+      ScratchArgs = AF.add(ScratchArgs, 1, DecRef);
       S = getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, DoNothing);
     } else if (FName == "IOServiceAddNotification" ||
                FName == "IOServiceAddMatchingNotification") {
       // Part of <rdar://problem/6961230>. (IOKit)
       // This should be addressed using a API table.
-      ScratchArgs = AF.Add(ScratchArgs, 2, DecRef);
+      ScratchArgs = AF.add(ScratchArgs, 2, DecRef);
       S = getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, DoNothing);
     } else if (FName == "CVPixelBufferCreateWithBytes") {
       // FIXES: <rdar://problem/7283567>
@@ -982,14 +986,14 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
       // a callback and doing full IPA to make sure this is done correctly.
       // FIXME: This function has an out parameter that returns an
       // allocated object.
-      ScratchArgs = AF.Add(ScratchArgs, 7, StopTracking);
+      ScratchArgs = AF.add(ScratchArgs, 7, StopTracking);
       S = getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, DoNothing);
     } else if (FName == "CGBitmapContextCreateWithData") {
       // FIXES: <rdar://problem/7358899>
       // Eventually this can be improved by recognizing that 'releaseInfo'
       // passed to CGBitmapContextCreateWithData is released via
       // a callback and doing full IPA to make sure this is done correctly.
-      ScratchArgs = AF.Add(ScratchArgs, 8, StopTracking);
+      ScratchArgs = AF.add(ScratchArgs, 8, StopTracking);
       S = getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true),
                                DoNothing, DoNothing);
     } else if (FName == "CVPixelBufferCreateWithPlanarBytes") {
@@ -998,7 +1002,7 @@ RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
       // buffer passed to CVPixelBufferCreateWithPlanarBytes is released
       // via a callback and doing full IPA to make sure this is done
       // correctly.
-      ScratchArgs = AF.Add(ScratchArgs, 12, StopTracking);
+      ScratchArgs = AF.add(ScratchArgs, 12, StopTracking);
       S = getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, DoNothing);
     }
 
@@ -1129,19 +1133,19 @@ RetainSummaryManager::getUnarySummary(const FunctionType* FT,
 
   switch (func) {
     case cfretain: {
-      ScratchArgs = AF.Add(ScratchArgs, 0, IncRef);
+      ScratchArgs = AF.add(ScratchArgs, 0, IncRef);
       return getPersistentSummary(RetEffect::MakeAlias(0),
                                   DoNothing, DoNothing);
     }
 
     case cfrelease: {
-      ScratchArgs = AF.Add(ScratchArgs, 0, DecRef);
+      ScratchArgs = AF.add(ScratchArgs, 0, DecRef);
       return getPersistentSummary(RetEffect::MakeNoRet(),
                                   DoNothing, DoNothing);
     }
 
     case cfmakecollectable: {
-      ScratchArgs = AF.Add(ScratchArgs, 0, MakeCollectable);
+      ScratchArgs = AF.add(ScratchArgs, 0, MakeCollectable);
       return getPersistentSummary(RetEffect::MakeAlias(0),DoNothing, DoNothing);
     }
 
@@ -1156,8 +1160,8 @@ RetainSummaryManager::getCFSummaryCreateRule(const FunctionDecl* FD) {
   assert (ScratchArgs.isEmpty());
 
   if (FD->getIdentifier() == CFDictionaryCreateII) {
-    ScratchArgs = AF.Add(ScratchArgs, 1, DoNothingByRef);
-    ScratchArgs = AF.Add(ScratchArgs, 2, DoNothingByRef);
+    ScratchArgs = AF.add(ScratchArgs, 1, DoNothingByRef);
+    ScratchArgs = AF.add(ScratchArgs, 2, DoNothingByRef);
   }
 
   return getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true));
@@ -1191,6 +1195,20 @@ RetainSummaryManager::updateSummaryFromAnnotations(RetainSummary &Summ,
   if (!FD)
     return;
 
+  // Effects on the parameters.
+  unsigned parm_idx = 0;
+  for (FunctionDecl::param_const_iterator pi = FD->param_begin(), 
+       pe = FD->param_end(); pi != pe; ++pi) {
+    const ParmVarDecl *pd = *pi;
+    if (pd->getAttr<NSConsumedAttr>()) {
+      if (!GCEnabled)
+        Summ.addArg(AF, parm_idx, DecRef);      
+    }
+    else if(pd->getAttr<CFConsumedAttr>()) {
+      Summ.addArg(AF, parm_idx, DecRef);      
+    }   
+  }
+  
   QualType RetTy = FD->getResultType();
 
   // Determine if there is a special return effect for this method.
@@ -1223,6 +1241,26 @@ RetainSummaryManager::updateSummaryFromAnnotations(RetainSummary &Summ,
 
   bool isTrackedLoc = false;
 
+  // Effects on the receiver.
+  if (MD->getAttr<NSConsumesSelfAttr>()) {
+    if (!GCEnabled)
+      Summ.setReceiverEffect(DecRefMsg);      
+  }
+  
+  // Effects on the parameters.
+  unsigned parm_idx = 0;
+  for (ObjCMethodDecl::param_iterator pi=MD->param_begin(), pe=MD->param_end();
+       pi != pe; ++pi, ++parm_idx) {
+    const ParmVarDecl *pd = *pi;
+    if (pd->getAttr<NSConsumedAttr>()) {
+      if (!GCEnabled)
+        Summ.addArg(AF, parm_idx, DecRef);      
+    }
+    else if(pd->getAttr<CFConsumedAttr>()) {
+      Summ.addArg(AF, parm_idx, DecRef);      
+    }   
+  }
+  
   // Determine if there is a special return effect for this method.
   if (cocoa::isCocoaObjectRef(MD->getResultType())) {
     if (MD->getAttr<NSReturnsRetainedAttr>()) {
@@ -1263,7 +1301,7 @@ RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
       if (ParmVarDecl *PD = *I) {
         QualType Ty = Ctx.getCanonicalType(PD->getType());
         if (Ty.getLocalUnqualifiedType() == Ctx.VoidPtrTy)
-          ScratchArgs = AF.Add(ScratchArgs, i, StopTracking);
+          ScratchArgs = AF.add(ScratchArgs, i, StopTracking);
       }
   }
 
@@ -1283,7 +1321,7 @@ RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
 
   // Look for methods that return an owned object.
   if (cocoa::isCocoaObjectRef(RetTy)) {
-    // EXPERIMENTAL: Assume the Cocoa conventions for all objects returned
+    // EXPERIMENTAL: assume the Cocoa conventions for all objects returned
     //  by instance methods.
     RetEffect E = cocoa::followsFundamentalRule(S)
                   ? ObjCAllocRetE : RetEffect::MakeNotOwned(RetEffect::ObjC);
@@ -1307,13 +1345,13 @@ RetainSummaryManager::getCommonMethodSummary(const ObjCMethodDecl* MD,
 }
 
 RetainSummary*
-RetainSummaryManager::getInstanceMethodSummary(const ObjCMessageExpr *ME,
+RetainSummaryManager::getInstanceMethodSummary(const ObjCMessage &msg,
                                                const GRState *state,
                                                const LocationContext *LC) {
 
   // We need the type-information of the tracked receiver object
   // Retrieve it from the state.
-  const Expr *Receiver = ME->getInstanceReceiver();
+  const Expr *Receiver = msg.getInstanceReceiver();
   const ObjCInterfaceDecl* ID = 0;
 
   // FIXME: Is this really working as expected?  There are cases where
@@ -1341,30 +1379,12 @@ RetainSummaryManager::getInstanceMethodSummary(const ObjCMessageExpr *ME,
     }
   } else {
     // FIXME: Hack for 'super'.
-    ID = ME->getReceiverInterface();
+    ID = msg.getReceiverInterface();
   }
 
   // FIXME: The receiver could be a reference to a class, meaning that
   //  we should use the class method.
-  RetainSummary *Summ = getInstanceMethodSummary(ME, ID);
-
-  // Special-case: are we sending a mesage to "self"?
-  //  This is a hack.  When we have full-IP this should be removed.
-  if (isa<ObjCMethodDecl>(LC->getDecl()) && Receiver) {
-    if (const loc::MemRegionVal *L = dyn_cast<loc::MemRegionVal>(&receiverV)) {
-      // Get the region associated with 'self'.
-      if (const ImplicitParamDecl *SelfDecl = LC->getSelfDecl()) {
-        SVal SelfVal = state->getSVal(state->getRegion(SelfDecl, LC));
-        if (L->StripCasts() == SelfVal.getAsRegion()) {
-          // Update the summary to make the default argument effect
-          // 'StopTracking'.
-          Summ = copySummary(Summ);
-          Summ->setDefaultArgEffect(StopTracking);
-        }
-      }
-    }
-  }
-
+  RetainSummary *Summ = getInstanceMethodSummary(msg, ID);
   return Summ ? Summ : getDefaultSummary();
 }
 
@@ -1421,27 +1441,16 @@ void RetainSummaryManager::InitializeClassMethodSummaries() {
   assert(ScratchArgs.isEmpty());
   RetainSummary* Summ = getPersistentSummary(ObjCAllocRetE);
 
-  // Create the summaries for "alloc", "new", and "allocWithZone:" for
-  // NSObject and its derivatives.
-  addNSObjectClsMethSummary(GetNullarySelector("alloc", Ctx), Summ);
-  addNSObjectClsMethSummary(GetNullarySelector("new", Ctx), Summ);
-  addNSObjectClsMethSummary(GetUnarySelector("allocWithZone", Ctx), Summ);
-
   // Create the [NSAssertionHandler currentHander] summary.
   addClassMethSummary("NSAssertionHandler", "currentHandler",
                 getPersistentSummary(RetEffect::MakeNotOwned(RetEffect::ObjC)));
 
   // Create the [NSAutoreleasePool addObject:] summary.
-  ScratchArgs = AF.Add(ScratchArgs, 0, Autorelease);
+  ScratchArgs = AF.add(ScratchArgs, 0, Autorelease);
   addClassMethSummary("NSAutoreleasePool", "addObject",
                       getPersistentSummary(RetEffect::MakeNoRet(),
                                            DoNothing, Autorelease));
 
-  // Create a summary for [NSCursor dragCopyCursor].
-  addClassMethSummary("NSCursor", "dragCopyCursor",
-                      getPersistentSummary(RetEffect::MakeNoRet(), DoNothing,
-                                           DoNothing));
-
   // Create the summaries for [NSObject performSelector...].  We treat
   // these as 'stop tracking' for the arguments because they are often
   // used for delegates that can release the object.  When we have better
@@ -1463,15 +1472,6 @@ void RetainSummaryManager::InitializeClassMethodSummaries() {
                     "withObject", "waitUntilDone", "modes", NULL);
   addClsMethSummary(NSObjectII, Summ, "performSelectorInBackground",
                     "withObject", NULL);
-
-  // Specially handle NSData.
-  RetainSummary *dataWithBytesNoCopySumm =
-    getPersistentSummary(RetEffect::MakeNotOwned(RetEffect::ObjC), DoNothing,
-                         DoNothing);
-  addClsMethSummary("NSData", dataWithBytesNoCopySumm,
-                    "dataWithBytesNoCopy", "length", NULL);
-  addClsMethSummary("NSData", dataWithBytesNoCopySumm,
-                    "dataWithBytesNoCopy", "length", "freeWhenDone", NULL);
 }
 
 void RetainSummaryManager::InitializeMethodSummaries() {
@@ -1493,12 +1493,6 @@ void RetainSummaryManager::InitializeMethodSummaries() {
   RetainSummary *CFAllocSumm =
     getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true));
 
-  // Create the "copy" selector.
-  addNSObjectMethSummary(GetNullarySelector("copy", Ctx), AllocSumm);
-
-  // Create the "mutableCopy" selector.
-  addNSObjectMethSummary(GetNullarySelector("mutableCopy", Ctx), AllocSumm);
-
   // Create the "retain" selector.
   RetEffect E = RetEffect::MakeReceiverAlias();
   RetainSummary *Summ = getPersistentSummary(E, IncRefMsg);
@@ -1602,6 +1596,7 @@ namespace { class AutoreleasePoolContents {}; }
 namespace { class AutoreleaseStack {}; }
 
 namespace clang {
+namespace ento {
 template<> struct GRStateTrait<AutoreleaseStack>
   : public GRStatePartialTrait<ARStack> {
   static inline void* GDMIndex() { return &AutoRBIndex; }
@@ -1611,6 +1606,7 @@ template<> struct GRStateTrait<AutoreleasePoolContents>
   : public GRStatePartialTrait<ARPoolContents> {
   static inline void* GDMIndex() { return &AutoRCIndex; }
 };
+} // end GR namespace
 } // end clang namespace
 
 static SymbolRef GetCurrentAutoreleasePool(const GRState* state) {
@@ -1627,10 +1623,10 @@ static const GRState * SendAutorelease(const GRState *state,
 
   if (cnts) {
     const unsigned *cnt = (*cnts).lookup(sym);
-    newCnts = F.Add(*cnts, sym, cnt ? *cnt  + 1 : 1);
+    newCnts = F.add(*cnts, sym, cnt ? *cnt  + 1 : 1);
   }
   else
-    newCnts = F.Add(F.GetEmptyMap(), sym, 1);
+    newCnts = F.add(F.getEmptyMap(), sym, 1);
 
   return state->set<AutoreleasePoolContents>(pool, newCnts);
 }
@@ -1641,7 +1637,7 @@ static const GRState * SendAutorelease(const GRState *state,
 
 namespace {
 
-class CFRefCount : public GRTransferFuncs {
+class CFRefCount : public TransferFuncs {
 public:
   class BindingsPrinter : public GRState::Printer {
   public:
@@ -1669,7 +1665,7 @@ private:
                     RefVal::Kind& hasErr);
 
   void ProcessNonLeakError(ExplodedNodeSet& Dst,
-                           GRStmtNodeBuilder& Builder,
+                           StmtNodeBuilder& Builder,
                            const Expr* NodeExpr, SourceRange ErrorRange,
                            ExplodedNode* Pred,
                            const GRState* St,
@@ -1680,8 +1676,8 @@ private:
 
   ExplodedNode* ProcessLeaks(const GRState * state,
                                       llvm::SmallVectorImpl<SymbolRef> &Leaked,
-                                      GenericNodeBuilder &Builder,
-                                      GRExprEngine &Eng,
+                                      GenericNodeBuilderRefCount &Builder,
+                                      ExprEngine &Eng,
                                       ExplodedNode *Pred = 0);
 
 public:
@@ -1694,7 +1690,7 @@ public:
 
   virtual ~CFRefCount() {}
 
-  void RegisterChecks(GRExprEngine &Eng);
+  void RegisterChecks(ExprEngine &Eng);
 
   virtual void RegisterPrinters(std::vector<GRState::Printer*>& Printers) {
     Printers.push_back(new BindingsPrinter());
@@ -1710,59 +1706,59 @@ public:
 
   // Calls.
 
-  void EvalSummary(ExplodedNodeSet& Dst,
-                   GRExprEngine& Eng,
-                   GRStmtNodeBuilder& Builder,
+  void evalSummary(ExplodedNodeSet& Dst,
+                   ExprEngine& Eng,
+                   StmtNodeBuilder& Builder,
                    const Expr* Ex,
+                   const CallOrObjCMessage &callOrMsg,
                    InstanceReceiver Receiver,
                    const RetainSummary& Summ,
                    const MemRegion *Callee,
-                   ConstExprIterator arg_beg, ConstExprIterator arg_end,
                    ExplodedNode* Pred, const GRState *state);
 
-  virtual void EvalCall(ExplodedNodeSet& Dst,
-                        GRExprEngine& Eng,
-                        GRStmtNodeBuilder& Builder,
+  virtual void evalCall(ExplodedNodeSet& Dst,
+                        ExprEngine& Eng,
+                        StmtNodeBuilder& Builder,
                         const CallExpr* CE, SVal L,
                         ExplodedNode* Pred);
 
 
-  virtual void EvalObjCMessageExpr(ExplodedNodeSet& Dst,
-                                   GRExprEngine& Engine,
-                                   GRStmtNodeBuilder& Builder,
-                                   const ObjCMessageExpr* ME,
-                                   ExplodedNode* Pred,
-                                   const GRState *state);
+  virtual void evalObjCMessage(ExplodedNodeSet& Dst,
+                               ExprEngine& Engine,
+                               StmtNodeBuilder& Builder,
+                               ObjCMessage msg,
+                               ExplodedNode* Pred,
+                               const GRState *state);
   // Stores.
-  virtual void EvalBind(GRStmtNodeBuilderRef& B, SVal location, SVal val);
+  virtual void evalBind(StmtNodeBuilderRef& B, SVal location, SVal val);
 
   // End-of-path.
 
-  virtual void EvalEndPath(GRExprEngine& Engine,
-                           GREndPathNodeBuilder& Builder);
+  virtual void evalEndPath(ExprEngine& Engine,
+                           EndOfFunctionNodeBuilder& Builder);
 
-  virtual void EvalDeadSymbols(ExplodedNodeSet& Dst,
-                               GRExprEngine& Engine,
-                               GRStmtNodeBuilder& Builder,
+  virtual void evalDeadSymbols(ExplodedNodeSet& Dst,
+                               ExprEngine& Engine,
+                               StmtNodeBuilder& Builder,
                                ExplodedNode* Pred,
                                const GRState* state,
                                SymbolReaper& SymReaper);
 
   std::pair<ExplodedNode*, const GRState *>
-  HandleAutoreleaseCounts(const GRState * state, GenericNodeBuilder Bd,
-                          ExplodedNode* Pred, GRExprEngine &Eng,
+  HandleAutoreleaseCounts(const GRState * state, GenericNodeBuilderRefCount Bd,
+                          ExplodedNode* Pred, ExprEngine &Eng,
                           SymbolRef Sym, RefVal V, bool &stop);
   // Return statements.
 
-  virtual void EvalReturn(ExplodedNodeSet& Dst,
-                          GRExprEngine& Engine,
-                          GRStmtNodeBuilder& Builder,
+  virtual void evalReturn(ExplodedNodeSet& Dst,
+                          ExprEngine& Engine,
+                          StmtNodeBuilder& Builder,
                           const ReturnStmt* S,
                           ExplodedNode* Pred);
 
   // Assumptions.
 
-  virtual const GRState *EvalAssume(const GRState* state, SVal condition,
+  virtual const GRState *evalAssume(const GRState* state, SVal condition,
                                     bool assumption);
 };
 
@@ -1941,15 +1937,15 @@ namespace {
 
     virtual ~CFRefReport() {}
 
-    CFRefBug& getBugType() {
+    CFRefBug& getBugType() const {
       return (CFRefBug&) RangedBugReport::getBugType();
     }
 
-    virtual void getRanges(const SourceRange*& beg, const SourceRange*& end) {
+    virtual std::pair<ranges_iterator, ranges_iterator> getRanges() const {
       if (!getBugType().isLeak())
-        RangedBugReport::getRanges(beg, end);
+        return RangedBugReport::getRanges();
       else
-        beg = end = 0;
+        return std::make_pair(ranges_iterator(), ranges_iterator());
     }
 
     SymbolRef getSymbol() const { return Sym; }
@@ -1970,7 +1966,7 @@ namespace {
   public:
     CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
                     ExplodedNode *n, SymbolRef sym,
-                    GRExprEngine& Eng);
+                    ExprEngine& Eng);
 
     PathDiagnosticPiece* getEndPath(BugReporterContext& BRC,
                                     const ExplodedNode* N);
@@ -2061,9 +2057,10 @@ PathDiagnosticPiece* CFRefReport::VisitNode(const ExplodedNode* N,
       else
         os << "function call";
     }
-    else {
-      assert (isa<ObjCMessageExpr>(S));
+    else if (isa<ObjCMessageExpr>(S)) {
       os << "Method";
+    } else {
+      os << "Property";
     }
 
     if (CurrV.getObjKind() == RetEffect::CF) {
@@ -2420,15 +2417,15 @@ CFRefLeakReport::getEndPath(BugReporterContext& BRC,
           "collector";
   }
   else
-    os << " is no longer referenced after this point and has a retain count of"
-          " +" << RV->getCount() << " (object leaked)";
+    os << " is not referenced later in this execution path and has a retain "
+          "count of +" << RV->getCount() << " (object leaked)";
 
   return new PathDiagnosticEventPiece(L, os.str());
 }
 
 CFRefLeakReport::CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
                                  ExplodedNode *n,
-                                 SymbolRef sym, GRExprEngine& Eng)
+                                 SymbolRef sym, ExprEngine& Eng)
 : CFRefReport(D, tf, n, sym) {
 
   // Most bug reports are cached at the location where they occured.
@@ -2442,7 +2439,7 @@ CFRefLeakReport::CFRefLeakReport(CFRefBug& D, const CFRefCount &tf,
   const ExplodedNode* AllocNode = 0;
 
   llvm::tie(AllocNode, AllocBinding) =  // Set AllocBinding.
-    GetAllocationSite(Eng.getStateManager(), getEndNode(), getSymbol());
+    GetAllocationSite(Eng.getStateManager(), getErrorNode(), getSymbol());
 
   // Get the SourceLocation for the allocation site.
   ProgramPoint P = AllocNode->getLocation();
@@ -2494,20 +2491,18 @@ static QualType GetReturnType(const Expr* RetE, ASTContext& Ctx) {
   return RetTy;
 }
 
-void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
-                             GRExprEngine& Eng,
-                             GRStmtNodeBuilder& Builder,
+void CFRefCount::evalSummary(ExplodedNodeSet& Dst,
+                             ExprEngine& Eng,
+                             StmtNodeBuilder& Builder,
                              const Expr* Ex,
+                             const CallOrObjCMessage &callOrMsg,
                              InstanceReceiver Receiver,
                              const RetainSummary& Summ,
                              const MemRegion *Callee,
-                             ConstExprIterator arg_beg, 
-                             ConstExprIterator arg_end,
                              ExplodedNode* Pred, const GRState *state) {
 
   // Evaluate the effect of the arguments.
   RefVal::Kind hasErr = (RefVal::Kind) 0;
-  unsigned idx = 0;
   SourceRange ErrorRange;
   SymbolRef ErrorSym = 0;
 
@@ -2519,8 +2514,20 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
   //  done an invalidation pass.
   llvm::DenseSet<SymbolRef> WhitelistedSymbols;
 
-  for (ConstExprIterator I = arg_beg; I != arg_end; ++I, ++idx) {
-    SVal V = state->getSValAsScalarOrLoc(*I);
+  // Invalidate all instance variables of the receiver of a message.
+  // FIXME: We should be able to do better with inter-procedural analysis.
+  if (Receiver) {
+    SVal V = Receiver.getSValAsScalarOrLoc(state);
+    if (SymbolRef Sym = V.getAsLocSymbol()) {
+      if (state->get<RefBindings>(Sym))
+        WhitelistedSymbols.insert(Sym);
+    }
+    if (const MemRegion *region = V.getAsRegion())
+      RegionsToInvalidate.push_back(region);
+  }
+  
+  for (unsigned idx = 0, e = callOrMsg.getNumArgs(); idx != e; ++idx) {
+    SVal V = callOrMsg.getArgSValAsScalarOrLoc(idx);
     SymbolRef Sym = V.getAsLocSymbol();
 
     if (Sym)
@@ -2528,7 +2535,7 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
         WhitelistedSymbols.insert(Sym);
         state = Update(state, Sym, *T, Summ.getArg(idx), hasErr);
         if (hasErr) {
-          ErrorRange = (*I)->getSourceRange();
+          ErrorRange = callOrMsg.getArgSourceRange(idx);
           ErrorSym = Sym;
           break;
         }
@@ -2606,7 +2613,7 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
 
   // NOTE: Even if RegionsToInvalidate is empty, we must still invalidate
   //  global variables.
-  state = state->InvalidateRegions(RegionsToInvalidate.data(),
+  state = state->invalidateRegions(RegionsToInvalidate.data(),
                                    RegionsToInvalidate.data() +
                                    RegionsToInvalidate.size(),
                                    Ex, Count, &IS,
@@ -2671,23 +2678,11 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
       // FIXME: We eventually should handle structs and other compound types
       // that are returned by value.
 
-      QualType T = Ex->getType();
-
-      // For CallExpr, use the result type to know if it returns a reference.
-      if (const CallExpr *CE = dyn_cast<CallExpr>(Ex)) {
-        const Expr *Callee = CE->getCallee();
-        if (const FunctionDecl *FD = state->getSVal(Callee).getAsFunctionDecl())
-          T = FD->getResultType();
-      }
-      else if (const ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(Ex)) {
-        if (const ObjCMethodDecl *MD = ME->getMethodDecl())
-          T = MD->getResultType();
-      }
-
-      if (Loc::IsLocType(T) || (T->isIntegerType() && T->isScalarType())) {
+      QualType T = callOrMsg.getResultType(Eng.getContext());
+      if (Loc::isLocType(T) || (T->isIntegerType() && T->isScalarType())) {
         unsigned Count = Builder.getCurrentBlockCount();
-        ValueManager &ValMgr = Eng.getValueManager();
-        SVal X = ValMgr.getConjuredSymbolVal(NULL, Ex, T, Count);
+        SValBuilder &svalBuilder = Eng.getSValBuilder();
+        SVal X = svalBuilder.getConjuredSymbolVal(NULL, Ex, T, Count);
         state = state->BindExpr(Ex, X, false);
       }
 
@@ -2696,9 +2691,8 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
 
     case RetEffect::Alias: {
       unsigned idx = RE.getIndex();
-      assert (arg_end >= arg_beg);
-      assert (idx < (unsigned) (arg_end - arg_beg));
-      SVal V = state->getSValAsScalarOrLoc(*(arg_beg+idx));
+      assert (idx < callOrMsg.getNumArgs());
+      SVal V = callOrMsg.getArgSValAsScalarOrLoc(idx);
       state = state->BindExpr(Ex, V, false);
       break;
     }
@@ -2713,19 +2707,19 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
     case RetEffect::OwnedAllocatedSymbol:
     case RetEffect::OwnedSymbol: {
       unsigned Count = Builder.getCurrentBlockCount();
-      ValueManager &ValMgr = Eng.getValueManager();
-      SymbolRef Sym = ValMgr.getConjuredSymbol(Ex, Count);
-      QualType RetT = GetReturnType(Ex, ValMgr.getContext());
+      SValBuilder &svalBuilder = Eng.getSValBuilder();
+      SymbolRef Sym = svalBuilder.getConjuredSymbol(Ex, Count);
+      QualType RetT = GetReturnType(Ex, svalBuilder.getContext());
       state = state->set<RefBindings>(Sym, RefVal::makeOwned(RE.getObjKind(),
                                                             RetT));
-      state = state->BindExpr(Ex, ValMgr.makeLoc(Sym), false);
+      state = state->BindExpr(Ex, svalBuilder.makeLoc(Sym), false);
 
       // FIXME: Add a flag to the checker where allocations are assumed to
       // *not fail.
 #if 0
       if (RE.getKind() == RetEffect::OwnedAllocatedSymbol) {
         bool isFeasible;
-        state = state.Assume(loc::SymbolVal(Sym), true, isFeasible);
+        state = state.assume(loc::SymbolVal(Sym), true, isFeasible);
         assert(isFeasible && "Cannot assume fresh symbol is non-null.");
       }
 #endif
@@ -2736,12 +2730,12 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
     case RetEffect::GCNotOwnedSymbol:
     case RetEffect::NotOwnedSymbol: {
       unsigned Count = Builder.getCurrentBlockCount();
-      ValueManager &ValMgr = Eng.getValueManager();
-      SymbolRef Sym = ValMgr.getConjuredSymbol(Ex, Count);
-      QualType RetT = GetReturnType(Ex, ValMgr.getContext());
+      SValBuilder &svalBuilder = Eng.getSValBuilder();
+      SymbolRef Sym = svalBuilder.getConjuredSymbol(Ex, Count);
+      QualType RetT = GetReturnType(Ex, svalBuilder.getContext());
       state = state->set<RefBindings>(Sym, RefVal::makeNotOwned(RE.getObjKind(),
                                                                RetT));
-      state = state->BindExpr(Ex, ValMgr.makeLoc(Sym), false);
+      state = state->BindExpr(Ex, svalBuilder.makeLoc(Sym), false);
       break;
     }
   }
@@ -2756,9 +2750,9 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
 }
 
 
-void CFRefCount::EvalCall(ExplodedNodeSet& Dst,
-                          GRExprEngine& Eng,
-                          GRStmtNodeBuilder& Builder,
+void CFRefCount::evalCall(ExplodedNodeSet& Dst,
+                          ExprEngine& Eng,
+                          StmtNodeBuilder& Builder,
                           const CallExpr* CE, SVal L,
                           ExplodedNode* Pred) {
 
@@ -2777,25 +2771,28 @@ void CFRefCount::EvalCall(ExplodedNodeSet& Dst,
   }
 
   assert(Summ);
-  EvalSummary(Dst, Eng, Builder, CE, 0, *Summ, L.getAsRegion(),
-              CE->arg_begin(), CE->arg_end(), Pred, Builder.GetState(Pred));
+  evalSummary(Dst, Eng, Builder, CE,
+              CallOrObjCMessage(CE, Builder.GetState(Pred)),
+              InstanceReceiver(), *Summ,L.getAsRegion(),
+              Pred, Builder.GetState(Pred));
 }
 
-void CFRefCount::EvalObjCMessageExpr(ExplodedNodeSet& Dst,
-                                     GRExprEngine& Eng,
-                                     GRStmtNodeBuilder& Builder,
-                                     const ObjCMessageExpr* ME,
-                                     ExplodedNode* Pred,
-                                     const GRState *state) {
+void CFRefCount::evalObjCMessage(ExplodedNodeSet& Dst,
+                                 ExprEngine& Eng,
+                                 StmtNodeBuilder& Builder,
+                                 ObjCMessage msg,
+                                 ExplodedNode* Pred,
+                                 const GRState *state) {
   RetainSummary *Summ =
-    ME->isInstanceMessage()
-      ? Summaries.getInstanceMethodSummary(ME, state,Pred->getLocationContext())
-      : Summaries.getClassMethodSummary(ME);
+    msg.isInstanceMessage()
+      ? Summaries.getInstanceMethodSummary(msg, state,Pred->getLocationContext())
+      : Summaries.getClassMethodSummary(msg);
 
   assert(Summ && "RetainSummary is null");
-  EvalSummary(Dst, Eng, Builder, ME,
-              InstanceReceiver(ME, Pred->getLocationContext()), *Summ, NULL,
-              ME->arg_begin(), ME->arg_end(), Pred, state);
+  evalSummary(Dst, Eng, Builder, msg.getOriginExpr(),
+              CallOrObjCMessage(msg, Builder.GetState(Pred)),
+              InstanceReceiver(msg, Pred->getLocationContext()), *Summ, NULL,
+              Pred, state);
 }
 
 namespace {
@@ -2813,7 +2810,7 @@ public:
 } // end anonymous namespace
 
 
-void CFRefCount::EvalBind(GRStmtNodeBuilderRef& B, SVal location, SVal val) {
+void CFRefCount::evalBind(StmtNodeBuilderRef& B, SVal location, SVal val) {
   // Are we storing to something that causes the value to "escape"?
   bool escapes = false;
 
@@ -2852,9 +2849,9 @@ void CFRefCount::EvalBind(GRStmtNodeBuilderRef& B, SVal location, SVal val) {
 
  // Return statements.
 
-void CFRefCount::EvalReturn(ExplodedNodeSet& Dst,
-                            GRExprEngine& Eng,
-                            GRStmtNodeBuilder& Builder,
+void CFRefCount::evalReturn(ExplodedNodeSet& Dst,
+                            ExprEngine& Eng,
+                            StmtNodeBuilder& Builder,
                             const ReturnStmt* S,
                             ExplodedNode* Pred) {
 
@@ -2912,7 +2909,7 @@ void CFRefCount::EvalReturn(ExplodedNodeSet& Dst,
 
   // Update the autorelease counts.
   static unsigned autoreleasetag = 0;
-  GenericNodeBuilder Bd(Builder, S, &autoreleasetag);
+  GenericNodeBuilderRefCount Bd(Builder, S, &autoreleasetag);
   bool stop = false;
   llvm::tie(Pred, state) = HandleAutoreleaseCounts(state , Bd, Pred, Eng, Sym,
                                                    X, stop);
@@ -2994,14 +2991,14 @@ void CFRefCount::EvalReturn(ExplodedNodeSet& Dst,
 
 // Assumptions.
 
-const GRState* CFRefCount::EvalAssume(const GRState *state,
+const GRState* CFRefCount::evalAssume(const GRState *state,
                                       SVal Cond, bool Assumption) {
 
-  // FIXME: We may add to the interface of EvalAssume the list of symbols
+  // FIXME: We may add to the interface of evalAssume the list of symbols
   //  whose assumptions have changed.  For now we just iterate through the
   //  bindings and check if any of the tracked symbols are NULL.  This isn't
   //  too bad since the number of symbols we will track in practice are
-  //  probably small and EvalAssume is only called at branches and a few
+  //  probably small and evalAssume is only called at branches and a few
   //  other places.
   RefBindings B = state->get<RefBindings>();
 
@@ -3016,7 +3013,7 @@ const GRState* CFRefCount::EvalAssume(const GRState *state,
     // If this is the case, stop tracking the symbol.
     if (state->getSymVal(I.getKey())) {
       changed = true;
-      B = RefBFactory.Remove(B, I.getKey());
+      B = RefBFactory.remove(B, I.getKey());
     }
   }
 
@@ -3160,9 +3157,10 @@ const GRState * CFRefCount::Update(const GRState * state, SymbolRef sym,
 //===----------------------------------------------------------------------===//
 
 std::pair<ExplodedNode*, const GRState *>
-CFRefCount::HandleAutoreleaseCounts(const GRState * state, GenericNodeBuilder Bd,
+CFRefCount::HandleAutoreleaseCounts(const GRState * state, 
+                                    GenericNodeBuilderRefCount Bd,
                                     ExplodedNode* Pred,
-                                    GRExprEngine &Eng,
+                                    ExprEngine &Eng,
                                     SymbolRef Sym, RefVal V, bool &stop) {
 
   unsigned ACnt = V.getAutoreleaseCount();
@@ -3245,8 +3243,8 @@ CFRefCount::HandleSymbolDeath(const GRState * state, SymbolRef sid, RefVal V,
 ExplodedNode*
 CFRefCount::ProcessLeaks(const GRState * state,
                          llvm::SmallVectorImpl<SymbolRef> &Leaked,
-                         GenericNodeBuilder &Builder,
-                         GRExprEngine& Eng,
+                         GenericNodeBuilderRefCount &Builder,
+                         ExprEngine& Eng,
                          ExplodedNode *Pred) {
 
   if (Leaked.empty())
@@ -3270,11 +3268,11 @@ CFRefCount::ProcessLeaks(const GRState * state,
   return N;
 }
 
-void CFRefCount::EvalEndPath(GRExprEngine& Eng,
-                             GREndPathNodeBuilder& Builder) {
+void CFRefCount::evalEndPath(ExprEngine& Eng,
+                             EndOfFunctionNodeBuilder& Builder) {
 
   const GRState *state = Builder.getState();
-  GenericNodeBuilder Bd(Builder);
+  GenericNodeBuilderRefCount Bd(Builder);
   RefBindings B = state->get<RefBindings>();
   ExplodedNode *Pred = 0;
 
@@ -3297,9 +3295,9 @@ void CFRefCount::EvalEndPath(GRExprEngine& Eng,
   ProcessLeaks(state, Leaked, Bd, Eng, Pred);
 }
 
-void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
-                                 GRExprEngine& Eng,
-                                 GRStmtNodeBuilder& Builder,
+void CFRefCount::evalDeadSymbols(ExplodedNodeSet& Dst,
+                                 ExprEngine& Eng,
+                                 StmtNodeBuilder& Builder,
                                  ExplodedNode* Pred,
                                  const GRState* state,
                                  SymbolReaper& SymReaper) {
@@ -3313,7 +3311,7 @@ void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
     if (const RefVal* T = B.lookup(Sym)){
       // Use the symbol as the tag.
       // FIXME: This might not be as unique as we would like.
-      GenericNodeBuilder Bd(Builder, S, Sym);
+      GenericNodeBuilderRefCount Bd(Builder, S, Sym);
       bool stop = false;
       llvm::tie(Pred, state) = HandleAutoreleaseCounts(state, Bd, Pred, Eng,
                                                        Sym, *T, stop);
@@ -3333,7 +3331,7 @@ void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
 
   static unsigned LeakPPTag = 0;
   {
-    GenericNodeBuilder Bd(Builder, S, &LeakPPTag);
+    GenericNodeBuilderRefCount Bd(Builder, S, &LeakPPTag);
     Pred = ProcessLeaks(state, Leaked, Bd, Eng, Pred);
   }
 
@@ -3345,14 +3343,14 @@ void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
   RefBindings::Factory& F = state->get_context<RefBindings>();
 
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
-       E = SymReaper.dead_end(); I!=E; ++I) B = F.Remove(B, *I);
+       E = SymReaper.dead_end(); I!=E; ++I) B = F.remove(B, *I);
 
   state = state->set<RefBindings>(B);
   Builder.MakeNode(Dst, S, Pred, state);
 }
 
 void CFRefCount::ProcessNonLeakError(ExplodedNodeSet& Dst,
-                                     GRStmtNodeBuilder& Builder,
+                                     StmtNodeBuilder& Builder,
                                      const Expr* NodeExpr, 
                                      SourceRange ErrorRange,
                                      ExplodedNode* Pred,
@@ -3413,7 +3411,7 @@ void RetainReleaseChecker::PostVisitBlockExpr(CheckerContext &C,
 
   // Scan the BlockDecRefExprs for any object the retain/release checker
   // may be tracking.
-  if (!BE->hasBlockDeclRefExprs())
+  if (!BE->getBlockDecl()->hasCaptures())
     return;
 
   const GRState *state = C.getState();
@@ -3431,7 +3429,7 @@ void RetainReleaseChecker::PostVisitBlockExpr(CheckerContext &C,
   // and in implicit increment/decrement of a retain count.
   llvm::SmallVector<const MemRegion*, 10> Regions;
   const LocationContext *LC = C.getPredecessor()->getLocationContext();
-  MemRegionManager &MemMgr = C.getValueManager().getRegionManager();
+  MemRegionManager &MemMgr = C.getSValBuilder().getRegionManager();
 
   for ( ; I != E; ++I) {
     const VarRegion *VR = *I;
@@ -3451,7 +3449,7 @@ void RetainReleaseChecker::PostVisitBlockExpr(CheckerContext &C,
 // Transfer function creation for external clients.
 //===----------------------------------------------------------------------===//
 
-void CFRefCount::RegisterChecks(GRExprEngine& Eng) {
+void CFRefCount::RegisterChecks(ExprEngine& Eng) {
   BugReporter &BR = Eng.getBugReporter();
 
   useAfterRelease = new UseAfterRelease(this);
@@ -3509,13 +3507,13 @@ void CFRefCount::RegisterChecks(GRExprEngine& Eng) {
   // Save the reference to the BugReporter.
   this->BR = &BR;
 
-  // Register the RetainReleaseChecker with the GRExprEngine object.
+  // Register the RetainReleaseChecker with the ExprEngine object.
   // Functionality in CFRefCount will be migrated to RetainReleaseChecker
   // over time.
   Eng.registerCheck(new RetainReleaseChecker(this));
 }
 
-GRTransferFuncs* clang::MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
+TransferFuncs* ento::MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
                                          const LangOptions& lopts) {
   return new CFRefCount(Ctx, GCEnabled, lopts);
 }
diff --git a/lib/StaticAnalyzer/Core/CMakeLists.txt b/lib/StaticAnalyzer/Core/CMakeLists.txt
new file mode 100644
index 000000000000..14c636cf761a
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/CMakeLists.txt
@@ -0,0 +1,41 @@
+set(LLVM_LINK_COMPONENTS support)
+
+set(LLVM_USED_LIBS clangBasic clangLex clangAST clangFrontend clangRewrite)
+
+add_clang_library(clangStaticAnalyzerCore
+  AggExprVisitor.cpp
+  AnalysisManager.cpp
+  BasicConstraintManager.cpp
+  BasicStore.cpp
+  BasicValueFactory.cpp
+  BugReporter.cpp
+  BugReporterVisitors.cpp
+  CFRefCount.cpp
+  Checker.cpp
+  CheckerHelpers.cpp
+  CheckerManager.cpp
+  Environment.cpp
+  ExplodedGraph.cpp
+  FlatStore.cpp
+  BlockCounter.cpp
+  CXXExprEngine.cpp
+  CoreEngine.cpp
+  GRState.cpp
+  HTMLDiagnostics.cpp
+  MemRegion.cpp
+  ObjCMessage.cpp
+  PathDiagnostic.cpp
+  PlistDiagnostics.cpp
+  RangeConstraintManager.cpp
+  RegionStore.cpp
+  SimpleConstraintManager.cpp
+  SimpleSValBuilder.cpp
+  Store.cpp
+  SValBuilder.cpp
+  SVals.cpp
+  SymbolManager.cpp
+  TextPathDiagnostics.cpp
+  )
+
+add_dependencies(clangStaticAnalyzerCore ClangAttrClasses ClangAttrList ClangDeclNodes
+                 ClangStmtNodes)
diff --git a/lib/StaticAnalyzer/Core/CXXExprEngine.cpp b/lib/StaticAnalyzer/Core/CXXExprEngine.cpp
new file mode 100644
index 000000000000..56dfe8cb04a9
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/CXXExprEngine.cpp
@@ -0,0 +1,322 @@
+//===- GRCXXExprEngine.cpp - C++ expr evaluation engine ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the C++ expression evaluation engine.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/AST/DeclCXX.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class CallExprWLItem {
+public:
+  CallExpr::const_arg_iterator I;
+  ExplodedNode *N;
+
+  CallExprWLItem(const CallExpr::const_arg_iterator &i, ExplodedNode *n)
+    : I(i), N(n) {}
+};
+}
+
+void ExprEngine::evalArguments(ConstExprIterator AI, ConstExprIterator AE,
+                                 const FunctionProtoType *FnType, 
+                                 ExplodedNode *Pred, ExplodedNodeSet &Dst,
+                                 bool FstArgAsLValue) {
+
+
+  llvm::SmallVector<CallExprWLItem, 20> WorkList;
+  WorkList.reserve(AE - AI);
+  WorkList.push_back(CallExprWLItem(AI, Pred));
+
+  while (!WorkList.empty()) {
+    CallExprWLItem Item = WorkList.back();
+    WorkList.pop_back();
+
+    if (Item.I == AE) {
+      Dst.insert(Item.N);
+      continue;
+    }
+
+    // Evaluate the argument.
+    ExplodedNodeSet Tmp;
+    if (FstArgAsLValue) {
+      FstArgAsLValue = false;
+    }
+
+    Visit(*Item.I, Item.N, Tmp);
+    ++(Item.I);
+    for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI != NE; ++NI)
+      WorkList.push_back(CallExprWLItem(Item.I, *NI));
+  }
+}
+
+const CXXThisRegion *ExprEngine::getCXXThisRegion(const CXXRecordDecl *D,
+                                                 const StackFrameContext *SFC) {
+  const Type *T = D->getTypeForDecl();
+  QualType PT = getContext().getPointerType(QualType(T, 0));
+  return svalBuilder.getRegionManager().getCXXThisRegion(PT, SFC);
+}
+
+const CXXThisRegion *ExprEngine::getCXXThisRegion(const CXXMethodDecl *decl,
+                                            const StackFrameContext *frameCtx) {
+  return svalBuilder.getRegionManager().
+                    getCXXThisRegion(decl->getThisType(getContext()), frameCtx);
+}
+
+void ExprEngine::CreateCXXTemporaryObject(const Expr *Ex, ExplodedNode *Pred,
+                                            ExplodedNodeSet &Dst) {
+  ExplodedNodeSet Tmp;
+  Visit(Ex, Pred, Tmp);
+  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E; ++I) {
+    const GRState *state = GetState(*I);
+
+    // Bind the temporary object to the value of the expression. Then bind
+    // the expression to the location of the object.
+    SVal V = state->getSVal(Ex);
+
+    const MemRegion *R =
+      svalBuilder.getRegionManager().getCXXTempObjectRegion(Ex,
+                                                   Pred->getLocationContext());
+
+    state = state->bindLoc(loc::MemRegionVal(R), V);
+    MakeNode(Dst, Ex, Pred, state->BindExpr(Ex, loc::MemRegionVal(R)));
+  }
+}
+
+void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E, 
+                                         const MemRegion *Dest,
+                                         ExplodedNode *Pred,
+                                         ExplodedNodeSet &Dst) {
+  if (!Dest)
+    Dest = svalBuilder.getRegionManager().getCXXTempObjectRegion(E,
+                                                    Pred->getLocationContext());
+
+  if (E->isElidable()) {
+    VisitAggExpr(E->getArg(0), Dest, Pred, Dst);
+    return;
+  }
+
+  const CXXConstructorDecl *CD = E->getConstructor();
+  assert(CD);
+
+  if (!(CD->isThisDeclarationADefinition() && AMgr.shouldInlineCall()))
+    // FIXME: invalidate the object.
+    return;
+
+  
+  // Evaluate other arguments.
+  ExplodedNodeSet argsEvaluated;
+  const FunctionProtoType *FnType = CD->getType()->getAs<FunctionProtoType>();
+  evalArguments(E->arg_begin(), E->arg_end(), FnType, Pred, argsEvaluated);
+  // The callee stack frame context used to create the 'this' parameter region.
+  const StackFrameContext *SFC = AMgr.getStackFrame(CD, 
+                                                    Pred->getLocationContext(),
+                                                    E, Builder->getBlock(),
+                                                    Builder->getIndex());
+
+  const CXXThisRegion *ThisR =getCXXThisRegion(E->getConstructor()->getParent(),
+                                               SFC);
+
+  CallEnter Loc(E, SFC, Pred->getLocationContext());
+  for (ExplodedNodeSet::iterator NI = argsEvaluated.begin(),
+                                 NE = argsEvaluated.end(); NI != NE; ++NI) {
+    const GRState *state = GetState(*NI);
+    // Setup 'this' region, so that the ctor is evaluated on the object pointed
+    // by 'Dest'.
+    state = state->bindLoc(loc::MemRegionVal(ThisR), loc::MemRegionVal(Dest));
+    ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
+    if (N)
+      Dst.Add(N);
+  }
+}
+
+void ExprEngine::VisitCXXDestructor(const CXXDestructorDecl *DD,
+                                      const MemRegion *Dest,
+                                      const Stmt *S,
+                                      ExplodedNode *Pred, 
+                                      ExplodedNodeSet &Dst) {
+  if (!(DD->isThisDeclarationADefinition() && AMgr.shouldInlineCall()))
+    return;
+  // Create the context for 'this' region.
+  const StackFrameContext *SFC = AMgr.getStackFrame(DD,
+                                                    Pred->getLocationContext(),
+                                                    S, Builder->getBlock(),
+                                                    Builder->getIndex());
+
+  const CXXThisRegion *ThisR = getCXXThisRegion(DD->getParent(), SFC);
+
+  CallEnter PP(S, SFC, Pred->getLocationContext());
+
+  const GRState *state = Pred->getState();
+  state = state->bindLoc(loc::MemRegionVal(ThisR), loc::MemRegionVal(Dest));
+  ExplodedNode *N = Builder->generateNode(PP, state, Pred);
+  if (N)
+    Dst.Add(N);
+}
+
+void ExprEngine::VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE, 
+                                          ExplodedNode *Pred, 
+                                          ExplodedNodeSet &Dst) {
+  // Get the method type.
+  const FunctionProtoType *FnType = 
+                       MCE->getCallee()->getType()->getAs<FunctionProtoType>();
+  assert(FnType && "Method type not available");
+
+  // Evaluate explicit arguments with a worklist.
+  ExplodedNodeSet argsEvaluated;
+  evalArguments(MCE->arg_begin(), MCE->arg_end(), FnType, Pred, argsEvaluated);
+ 
+  // Evaluate the implicit object argument.
+  ExplodedNodeSet AllargsEvaluated;
+  const MemberExpr *ME = dyn_cast<MemberExpr>(MCE->getCallee()->IgnoreParens());
+  if (!ME)
+    return;
+  Expr *ObjArgExpr = ME->getBase();
+  for (ExplodedNodeSet::iterator I = argsEvaluated.begin(), 
+                                 E = argsEvaluated.end(); I != E; ++I) {
+      Visit(ObjArgExpr, *I, AllargsEvaluated);
+  }
+
+  // Now evaluate the call itself.
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
+  assert(MD && "not a CXXMethodDecl?");
+  evalMethodCall(MCE, MD, ObjArgExpr, Pred, AllargsEvaluated, Dst);
+}
+
+void ExprEngine::VisitCXXOperatorCallExpr(const CXXOperatorCallExpr *C,
+                                            ExplodedNode *Pred,
+                                            ExplodedNodeSet &Dst) {
+  const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(C->getCalleeDecl());
+  if (!MD) {
+    // If the operator doesn't represent a method call treat as regural call.
+    VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst);
+    return;
+  }
+
+  // Determine the type of function we're calling (if available).
+  const FunctionProtoType *Proto = NULL;
+  QualType FnType = C->getCallee()->IgnoreParens()->getType();
+  if (const PointerType *FnTypePtr = FnType->getAs<PointerType>())
+    Proto = FnTypePtr->getPointeeType()->getAs<FunctionProtoType>();
+
+  // Evaluate arguments treating the first one (object method is called on)
+  // as alvalue.
+  ExplodedNodeSet argsEvaluated;
+  evalArguments(C->arg_begin(), C->arg_end(), Proto, Pred, argsEvaluated, true);
+
+  // Now evaluate the call itself.
+  evalMethodCall(C, MD, C->getArg(0), Pred, argsEvaluated, Dst);
+}
+
+void ExprEngine::evalMethodCall(const CallExpr *MCE, const CXXMethodDecl *MD,
+                                  const Expr *ThisExpr, ExplodedNode *Pred,
+                                  ExplodedNodeSet &Src, ExplodedNodeSet &Dst) {
+  // Allow checkers to pre-visit the member call.
+  ExplodedNodeSet PreVisitChecks;
+  CheckerVisit(MCE, PreVisitChecks, Src, PreVisitStmtCallback);
+
+  if (!(MD->isThisDeclarationADefinition() && AMgr.shouldInlineCall())) {
+    // FIXME: conservative method call evaluation.
+    CheckerVisit(MCE, Dst, PreVisitChecks, PostVisitStmtCallback);
+    return;
+  }
+
+  const StackFrameContext *SFC = AMgr.getStackFrame(MD, 
+                                                    Pred->getLocationContext(),
+                                                    MCE,
+                                                    Builder->getBlock(), 
+                                                    Builder->getIndex());
+  const CXXThisRegion *ThisR = getCXXThisRegion(MD, SFC);
+  CallEnter Loc(MCE, SFC, Pred->getLocationContext());
+  for (ExplodedNodeSet::iterator I = PreVisitChecks.begin(),
+         E = PreVisitChecks.end(); I != E; ++I) {
+    // Set up 'this' region.
+    const GRState *state = GetState(*I);
+    state = state->bindLoc(loc::MemRegionVal(ThisR), state->getSVal(ThisExpr));
+    Dst.Add(Builder->generateNode(Loc, state, *I));
+  }
+}
+
+void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
+                                   ExplodedNodeSet &Dst) {
+  if (CNE->isArray()) {
+    // FIXME: allocating an array has not been handled.
+    return;
+  }
+
+  unsigned Count = Builder->getCurrentBlockCount();
+  DefinedOrUnknownSVal symVal =
+    svalBuilder.getConjuredSymbolVal(NULL, CNE, CNE->getType(), Count);
+  const MemRegion *NewReg = cast<loc::MemRegionVal>(symVal).getRegion();
+
+  QualType ObjTy = CNE->getType()->getAs<PointerType>()->getPointeeType();
+
+  const ElementRegion *EleReg = 
+                         getStoreManager().GetElementZeroRegion(NewReg, ObjTy);
+
+  // Evaluate constructor arguments.
+  const FunctionProtoType *FnType = NULL;
+  const CXXConstructorDecl *CD = CNE->getConstructor();
+  if (CD)
+    FnType = CD->getType()->getAs<FunctionProtoType>();
+  ExplodedNodeSet argsEvaluated;
+  evalArguments(CNE->constructor_arg_begin(), CNE->constructor_arg_end(),
+                FnType, Pred, argsEvaluated);
+
+  // Initialize the object region and bind the 'new' expression.
+  for (ExplodedNodeSet::iterator I = argsEvaluated.begin(), 
+                                 E = argsEvaluated.end(); I != E; ++I) {
+    const GRState *state = GetState(*I);
+
+    if (ObjTy->isRecordType()) {
+      state = state->invalidateRegion(EleReg, CNE, Count);
+    } else {
+      if (CNE->hasInitializer()) {
+        SVal V = state->getSVal(*CNE->constructor_arg_begin());
+        state = state->bindLoc(loc::MemRegionVal(EleReg), V);
+      } else {
+        // Explicitly set to undefined, because currently we retrieve symbolic
+        // value from symbolic region.
+        state = state->bindLoc(loc::MemRegionVal(EleReg), UndefinedVal());
+      }
+    }
+    state = state->BindExpr(CNE, loc::MemRegionVal(EleReg));
+    MakeNode(Dst, CNE, *I, state);
+  }
+}
+
+void ExprEngine::VisitCXXDeleteExpr(const CXXDeleteExpr *CDE, 
+                                      ExplodedNode *Pred,ExplodedNodeSet &Dst) {
+  // Should do more checking.
+  ExplodedNodeSet Argevaluated;
+  Visit(CDE->getArgument(), Pred, Argevaluated);
+  for (ExplodedNodeSet::iterator I = Argevaluated.begin(), 
+                                 E = Argevaluated.end(); I != E; ++I) {
+    const GRState *state = GetState(*I);
+    MakeNode(Dst, CDE, *I, state);
+  }
+}
+
+void ExprEngine::VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred,
+                                    ExplodedNodeSet &Dst) {
+  // Get the this object region from StoreManager.
+  const MemRegion *R =
+    svalBuilder.getRegionManager().getCXXThisRegion(
+                                  getContext().getCanonicalType(TE->getType()),
+                                               Pred->getLocationContext());
+
+  const GRState *state = GetState(Pred);
+  SVal V = state->getSVal(loc::MemRegionVal(R));
+  MakeNode(Dst, TE, Pred, state->BindExpr(TE, V));
+}
diff --git a/lib/Checker/Checker.cpp b/lib/StaticAnalyzer/Core/Checker.cpp
index 36323b9efb61..a014eec76bd4 100644
--- a/lib/Checker/Checker.cpp
+++ b/lib/StaticAnalyzer/Core/Checker.cpp
@@ -12,8 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 using namespace clang;
+using namespace ento;
 
 Checker::~Checker() {}
 
@@ -23,11 +24,10 @@ CheckerContext::~CheckerContext() {
   // without actually generated a new node.  We also shouldn't autotransition
   // if we are building sinks or we generated a node and decided to not
   // add it as a transition.
-  if (Dst.size() == size && !B.BuildSinks && !B.HasGeneratedNode) {
+  if (Dst.size() == size && !B.BuildSinks && !B.hasGeneratedNode) {
     if (ST && ST != B.GetState(Pred)) {
       static int autoTransitionTag = 0;
-      B.Tag = &autoTransitionTag;
-      addTransition(ST);
+      addTransition(ST, &autoTransitionTag);
     }
     else
       Dst.Add(Pred);
diff --git a/lib/Checker/CheckerHelpers.cpp b/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
index ece69435cac7..28df6959aff5 100644
--- a/lib/Checker/CheckerHelpers.cpp
+++ b/lib/StaticAnalyzer/Core/CheckerHelpers.cpp
@@ -11,11 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/CheckerHelpers.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h"
 #include "clang/AST/Expr.h"
 
 // Recursively find any substatements containing macros
-bool clang::containsMacro(const Stmt *S) {
+bool clang::ento::containsMacro(const Stmt *S) {
   if (S->getLocStart().isMacroID())
     return true;
 
@@ -32,7 +32,7 @@ bool clang::containsMacro(const Stmt *S) {
 }
 
 // Recursively find any substatements containing enum constants
-bool clang::containsEnum(const Stmt *S) {
+bool clang::ento::containsEnum(const Stmt *S) {
   const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
 
   if (DR && isa<EnumConstantDecl>(DR->getDecl()))
@@ -48,7 +48,7 @@ bool clang::containsEnum(const Stmt *S) {
 }
 
 // Recursively find any substatements containing static vars
-bool clang::containsStaticLocal(const Stmt *S) {
+bool clang::ento::containsStaticLocal(const Stmt *S) {
   const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
 
   if (DR)
@@ -66,7 +66,7 @@ bool clang::containsStaticLocal(const Stmt *S) {
 }
 
 // Recursively find any substatements containing __builtin_offsetof
-bool clang::containsBuiltinOffsetOf(const Stmt *S) {
+bool clang::ento::containsBuiltinOffsetOf(const Stmt *S) {
   if (isa<OffsetOfExpr>(S))
     return true;
 
diff --git a/lib/StaticAnalyzer/Core/CheckerManager.cpp b/lib/StaticAnalyzer/Core/CheckerManager.cpp
new file mode 100644
index 000000000000..1989b822ae40
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -0,0 +1,85 @@
+//===--- CheckerManager.cpp - Static Analyzer Checker Manager -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the Static Analyzer Checker Manager.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
+#include "clang/AST/DeclBase.h"
+
+using namespace clang;
+using namespace ento;
+
+void CheckerManager::runCheckersOnASTDecl(const Decl *D, AnalysisManager& mgr,
+                                          BugReporter &BR) {
+  assert(D);
+
+  unsigned DeclKind = D->getKind();
+  CachedDeclCheckers *checkers = 0;
+  CachedDeclCheckersMapTy::iterator CCI = CachedDeclCheckersMap.find(DeclKind);
+  if (CCI != CachedDeclCheckersMap.end()) {
+    checkers = &(CCI->second);
+  } else {
+    // Find the checkers that should run for this Decl and cache them.
+    checkers = &CachedDeclCheckersMap[DeclKind];
+    for (unsigned i = 0, e = DeclCheckers.size(); i != e; ++i) {
+      DeclCheckerInfo &info = DeclCheckers[i];
+      if (info.IsForDeclFn(D))
+        checkers->push_back(std::make_pair(info.Checker, info.CheckFn));
+    }
+  }
+
+  assert(checkers);
+  for (CachedDeclCheckers::iterator
+         I = checkers->begin(), E = checkers->end(); I != E; ++I) {
+    CheckerRef checker = I->first;
+    CheckDeclFunc fn = I->second;
+    fn(checker, D, mgr, BR);
+  }
+}
+
+void CheckerManager::runCheckersOnASTBody(const Decl *D, AnalysisManager& mgr,
+                                          BugReporter &BR) {
+  assert(D && D->hasBody());
+
+  for (unsigned i = 0, e = BodyCheckers.size(); i != e; ++i) {
+    CheckerRef checker = BodyCheckers[i].first;
+    CheckDeclFunc fn = BodyCheckers[i].second;
+    fn(checker, D, mgr, BR);
+  }
+}
+
+void CheckerManager::_registerForDecl(CheckerRef checker, CheckDeclFunc checkfn,
+                                      HandlesDeclFunc isForDeclFn) {
+  DeclCheckerInfo info = { checker, checkfn, isForDeclFn };
+  DeclCheckers.push_back(info);
+}
+
+void CheckerManager::_registerForBody(CheckerRef checker,
+                                      CheckDeclFunc checkfn) {
+  BodyCheckers.push_back(std::make_pair(checker, checkfn));
+}
+
+void CheckerManager::registerCheckersToEngine(ExprEngine &eng) {
+  for (unsigned i = 0, e = Funcs.size(); i != e; ++i)
+    Funcs[i](eng);
+}
+
+CheckerManager::~CheckerManager() {
+  for (unsigned i = 0, e = Checkers.size(); i != e; ++i) {
+    CheckerRef checker = Checkers[i].first;
+    Dtor dtor = Checkers[i].second;
+    dtor(checker);
+  }
+}
+
+// Anchor for the vtable.
+CheckerProvider::~CheckerProvider() { }
diff --git a/lib/Checker/GRCoreEngine.cpp b/lib/StaticAnalyzer/Core/CoreEngine.cpp
index 5125f366de2a..070042a641f6 100644
--- a/lib/Checker/GRCoreEngine.cpp
+++ b/lib/StaticAnalyzer/Core/CoreEngine.cpp
@@ -1,4 +1,4 @@
-//==- GRCoreEngine.cpp - Path-Sensitive Dataflow Engine ------------*- C++ -*-//
+//==- CoreEngine.cpp - Path-Sensitive Dataflow Engine ------------*- C++ -*-//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,9 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/AnalysisManager.h"
-#include "clang/Checker/PathSensitive/GRCoreEngine.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CoreEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/Index/TranslationUnit.h"
 #include "clang/AST/Expr.h"
 #include "llvm/Support/Casting.h"
@@ -25,86 +25,107 @@
 using llvm::cast;
 using llvm::isa;
 using namespace clang;
+using namespace ento;
 
 // This should be removed in the future.
 namespace clang {
-GRTransferFuncs* MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
+namespace ento {
+TransferFuncs* MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
                                   const LangOptions& lopts);
 }
+}
 
 //===----------------------------------------------------------------------===//
 // Worklist classes for exploration of reachable states.
 //===----------------------------------------------------------------------===//
 
+WorkList::Visitor::~Visitor() {}
+
 namespace {
-class DFS : public GRWorkList {
-  llvm::SmallVector<GRWorkListUnit,20> Stack;
+class DFS : public WorkList {
+  llvm::SmallVector<WorkListUnit,20> Stack;
 public:
   virtual bool hasWork() const {
     return !Stack.empty();
   }
 
-  virtual void Enqueue(const GRWorkListUnit& U) {
+  virtual void enqueue(const WorkListUnit& U) {
     Stack.push_back(U);
   }
 
-  virtual GRWorkListUnit Dequeue() {
+  virtual WorkListUnit dequeue() {
     assert (!Stack.empty());
-    const GRWorkListUnit& U = Stack.back();
+    const WorkListUnit& U = Stack.back();
     Stack.pop_back(); // This technically "invalidates" U, but we are fine.
     return U;
   }
+  
+  virtual bool visitItemsInWorkList(Visitor &V) {
+    for (llvm::SmallVectorImpl<WorkListUnit>::iterator
+         I = Stack.begin(), E = Stack.end(); I != E; ++I) {
+      if (V.visit(*I))
+        return true;
+    }
+    return false;
+  }
 };
 
-class BFS : public GRWorkList {
-  std::queue<GRWorkListUnit> Queue;
+class BFS : public WorkList {
+  std::deque<WorkListUnit> Queue;
 public:
   virtual bool hasWork() const {
     return !Queue.empty();
   }
 
-  virtual void Enqueue(const GRWorkListUnit& U) {
-    Queue.push(U);
+  virtual void enqueue(const WorkListUnit& U) {
+    Queue.push_front(U);
   }
 
-  virtual GRWorkListUnit Dequeue() {
-    // Don't use const reference.  The subsequent pop_back() might make it
-    // unsafe.
-    GRWorkListUnit U = Queue.front();
-    Queue.pop();
+  virtual WorkListUnit dequeue() {
+    WorkListUnit U = Queue.front();
+    Queue.pop_front();
     return U;
   }
+  
+  virtual bool visitItemsInWorkList(Visitor &V) {
+    for (std::deque<WorkListUnit>::iterator
+         I = Queue.begin(), E = Queue.end(); I != E; ++I) {
+      if (V.visit(*I))
+        return true;
+    }
+    return false;
+  }
 };
 
 } // end anonymous namespace
 
-// Place the dstor for GRWorkList here because it contains virtual member
+// Place the dstor for WorkList here because it contains virtual member
 // functions, and we the code for the dstor generated in one compilation unit.
-GRWorkList::~GRWorkList() {}
+WorkList::~WorkList() {}
 
-GRWorkList *GRWorkList::MakeDFS() { return new DFS(); }
-GRWorkList *GRWorkList::MakeBFS() { return new BFS(); }
+WorkList *WorkList::makeDFS() { return new DFS(); }
+WorkList *WorkList::makeBFS() { return new BFS(); }
 
 namespace {
-  class BFSBlockDFSContents : public GRWorkList {
-    std::queue<GRWorkListUnit> Queue;
-    llvm::SmallVector<GRWorkListUnit,20> Stack;
+  class BFSBlockDFSContents : public WorkList {
+    std::deque<WorkListUnit> Queue;
+    llvm::SmallVector<WorkListUnit,20> Stack;
   public:
     virtual bool hasWork() const {
       return !Queue.empty() || !Stack.empty();
     }
 
-    virtual void Enqueue(const GRWorkListUnit& U) {
+    virtual void enqueue(const WorkListUnit& U) {
       if (isa<BlockEntrance>(U.getNode()->getLocation()))
-        Queue.push(U);
+        Queue.push_front(U);
       else
         Stack.push_back(U);
     }
 
-    virtual GRWorkListUnit Dequeue() {
+    virtual WorkListUnit dequeue() {
       // Process all basic blocks to completion.
       if (!Stack.empty()) {
-        const GRWorkListUnit& U = Stack.back();
+        const WorkListUnit& U = Stack.back();
         Stack.pop_back(); // This technically "invalidates" U, but we are fine.
         return U;
       }
@@ -112,14 +133,28 @@ namespace {
       assert(!Queue.empty());
       // Don't use const reference.  The subsequent pop_back() might make it
       // unsafe.
-      GRWorkListUnit U = Queue.front();
-      Queue.pop();
+      WorkListUnit U = Queue.front();
+      Queue.pop_front();
       return U;
     }
+    virtual bool visitItemsInWorkList(Visitor &V) {
+      for (llvm::SmallVectorImpl<WorkListUnit>::iterator
+           I = Stack.begin(), E = Stack.end(); I != E; ++I) {
+        if (V.visit(*I))
+          return true;
+      }
+      for (std::deque<WorkListUnit>::iterator
+           I = Queue.begin(), E = Queue.end(); I != E; ++I) {
+        if (V.visit(*I))
+          return true;
+      }
+      return false;
+    }
+
   };
 } // end anonymous namespace
 
-GRWorkList* GRWorkList::MakeBFSBlockDFSContents() {
+WorkList* WorkList::makeBFSBlockDFSContents() {
   return new BFSBlockDFSContents();
 }
 
@@ -128,7 +163,7 @@ GRWorkList* GRWorkList::MakeBFSBlockDFSContents() {
 //===----------------------------------------------------------------------===//
 
 /// ExecuteWorkList - Run the worklist algorithm for a maximum number of steps.
-bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
+bool CoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
                                    const GRState *InitState) {
 
   if (G->num_roots() == 0) { // Initialize the analysis by constructing
@@ -154,14 +189,22 @@ bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
 
     if (!InitState)
       // Generate the root.
-      GenerateNode(StartLoc, getInitialState(L), 0);
+      generateNode(StartLoc, SubEng.getInitialState(L), 0);
     else
-      GenerateNode(StartLoc, InitState, 0);
+      generateNode(StartLoc, InitState, 0);
   }
 
-  while (Steps && WList->hasWork()) {
-    --Steps;
-    const GRWorkListUnit& WU = WList->Dequeue();
+  // Check if we have a steps limit
+  bool UnlimitedSteps = Steps == 0;
+
+  while (WList->hasWork()) {
+    if (!UnlimitedSteps) {
+      if (Steps == 0)
+        break;
+      --Steps;
+    }
+
+    const WorkListUnit& WU = WList->dequeue();
 
     // Set the current block counter.
     WList->setBlockCounter(WU.getBlockCounter());
@@ -193,18 +236,18 @@ bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
         break;
 
       default:
-        assert(isa<PostStmt>(Node->getLocation()));
-        HandlePostStmt(cast<PostStmt>(Node->getLocation()), WU.getBlock(),
-                       WU.getIndex(), Node);
+        assert(isa<PostStmt>(Node->getLocation()) || 
+               isa<PostInitializer>(Node->getLocation()));
+        HandlePostStmt(WU.getBlock(), WU.getIndex(), Node);
         break;
     }
   }
 
-  SubEngine.ProcessEndWorklist(hasWorkRemaining());
+  SubEng.processEndWorklist(hasWorkRemaining());
   return WList->hasWork();
 }
 
-void GRCoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L, 
+void CoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L, 
                                                    unsigned Steps,
                                                    const GRState *InitState, 
                                                    ExplodedNodeSet &Dst) {
@@ -215,19 +258,19 @@ void GRCoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L,
   }
 }
 
-void GRCoreEngine::HandleCallEnter(const CallEnter &L, const CFGBlock *Block,
+void CoreEngine::HandleCallEnter(const CallEnter &L, const CFGBlock *Block,
                                    unsigned Index, ExplodedNode *Pred) {
-  GRCallEnterNodeBuilder Builder(*this, Pred, L.getCallExpr(), 
+  CallEnterNodeBuilder Builder(*this, Pred, L.getCallExpr(), 
                                  L.getCalleeContext(), Block, Index);
-  ProcessCallEnter(Builder);
+  SubEng.processCallEnter(Builder);
 }
 
-void GRCoreEngine::HandleCallExit(const CallExit &L, ExplodedNode *Pred) {
-  GRCallExitNodeBuilder Builder(*this, Pred);
-  ProcessCallExit(Builder);
+void CoreEngine::HandleCallExit(const CallExit &L, ExplodedNode *Pred) {
+  CallExitNodeBuilder Builder(*this, Pred);
+  SubEng.processCallExit(Builder);
 }
 
-void GRCoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
+void CoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
 
   const CFGBlock* Blk = L.getDst();
 
@@ -238,28 +281,44 @@ void GRCoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
             && "EXIT block cannot contain Stmts.");
 
     // Process the final state transition.
-    GREndPathNodeBuilder Builder(Blk, Pred, this);
-    ProcessEndPath(Builder);
+    EndOfFunctionNodeBuilder Builder(Blk, Pred, this);
+    SubEng.processEndOfFunction(Builder);
 
     // This path is done. Don't enqueue any more nodes.
     return;
   }
 
-  // FIXME: Should we allow ProcessBlockEntrance to also manipulate state?
+  // Call into the subengine to process entering the CFGBlock.
+  ExplodedNodeSet dstNodes;
+  BlockEntrance BE(Blk, Pred->getLocationContext());
+  GenericNodeBuilder<BlockEntrance> nodeBuilder(*this, Pred, BE);
+  SubEng.processCFGBlockEntrance(dstNodes, nodeBuilder);
 
-  if (ProcessBlockEntrance(Blk, Pred, WList->getBlockCounter()))
-    GenerateNode(BlockEntrance(Blk, Pred->getLocationContext()),
-                 Pred->State, Pred);
+  if (dstNodes.empty()) {
+    if (!nodeBuilder.hasGeneratedNode) {
+      // Auto-generate a node and enqueue it to the worklist.
+      generateNode(BE, Pred->State, Pred);    
+    }
+  }
   else {
-    blocksAborted.push_back(std::make_pair(L, Pred));
+    for (ExplodedNodeSet::iterator I = dstNodes.begin(), E = dstNodes.end();
+         I != E; ++I) {
+      WList->enqueue(*I);
+    }
+  }
+
+  for (llvm::SmallVectorImpl<ExplodedNode*>::const_iterator
+       I = nodeBuilder.sinks().begin(), E = nodeBuilder.sinks().end();
+       I != E; ++I) {
+    blocksAborted.push_back(std::make_pair(L, *I));
   }
 }
 
-void GRCoreEngine::HandleBlockEntrance(const BlockEntrance& L,
+void CoreEngine::HandleBlockEntrance(const BlockEntrance& L,
                                        ExplodedNode* Pred) {
 
   // Increment the block counter.
-  GRBlockCounter Counter = WList->getBlockCounter();
+  BlockCounter Counter = WList->getBlockCounter();
   Counter = BCounterFactory.IncrementCount(Counter, 
                              Pred->getLocationContext()->getCurrentStackFrame(),
                                            L.getBlock()->getBlockID());
@@ -267,15 +326,15 @@ void GRCoreEngine::HandleBlockEntrance(const BlockEntrance& L,
 
   // Process the entrance of the block.
   if (CFGElement E = L.getFirstElement()) {
-    GRStmtNodeBuilder Builder(L.getBlock(), 0, Pred, this,
-                              SubEngine.getStateManager());
-    ProcessStmt(E, Builder);
+    StmtNodeBuilder Builder(L.getBlock(), 0, Pred, this,
+                              SubEng.getStateManager());
+    SubEng.processCFGElement(E, Builder);
   }
   else
     HandleBlockExit(L.getBlock(), Pred);
 }
 
-void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
+void CoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
 
   if (const Stmt* Term = B->getTerminator()) {
     switch (Term->getStmtClass()) {
@@ -287,8 +346,10 @@ void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
         HandleBranch(cast<BinaryOperator>(Term)->getLHS(), Term, B, Pred);
         return;
 
+      case Stmt::BinaryConditionalOperatorClass:
       case Stmt::ConditionalOperatorClass:
-        HandleBranch(cast<ConditionalOperator>(Term)->getCond(), Term, B, Pred);
+        HandleBranch(cast<AbstractConditionalOperator>(Term)->getCond(),
+                     Term, B, Pred);
         return;
 
         // FIXME: Use constant-folding in CFG construction to simplify this
@@ -319,11 +380,11 @@ void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
         // Only 1 successor: the indirect goto dispatch block.
         assert (B->succ_size() == 1);
 
-        GRIndirectGotoNodeBuilder
+        IndirectGotoNodeBuilder
            builder(Pred, B, cast<IndirectGotoStmt>(Term)->getTarget(),
                    *(B->succ_begin()), this);
 
-        ProcessIndirectGoto(builder);
+        SubEng.processIndirectGoto(builder);
         return;
       }
 
@@ -343,10 +404,10 @@ void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
       }
 
       case Stmt::SwitchStmtClass: {
-        GRSwitchNodeBuilder builder(Pred, B, cast<SwitchStmt>(Term)->getCond(),
+        SwitchNodeBuilder builder(Pred, B, cast<SwitchStmt>(Term)->getCond(),
                                     this);
 
-        ProcessSwitch(builder);
+        SubEng.processSwitch(builder);
         return;
       }
 
@@ -359,38 +420,35 @@ void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
   assert (B->succ_size() == 1 &&
           "Blocks with no terminator should have at most 1 successor.");
 
-  GenerateNode(BlockEdge(B, *(B->succ_begin()), Pred->getLocationContext()),
+  generateNode(BlockEdge(B, *(B->succ_begin()), Pred->getLocationContext()),
                Pred->State, Pred);
 }
 
-void GRCoreEngine::HandleBranch(const Stmt* Cond, const Stmt* Term, 
+void CoreEngine::HandleBranch(const Stmt* Cond, const Stmt* Term, 
                                 const CFGBlock * B, ExplodedNode* Pred) {
-  assert (B->succ_size() == 2);
-
-  GRBranchNodeBuilder Builder(B, *(B->succ_begin()), *(B->succ_begin()+1),
-                              Pred, this);
-
-  ProcessBranch(Cond, Term, Builder);
+  assert(B->succ_size() == 2);
+  BranchNodeBuilder Builder(B, *(B->succ_begin()), *(B->succ_begin()+1),
+                            Pred, this);
+  SubEng.processBranch(Cond, Term, Builder);
 }
 
-void GRCoreEngine::HandlePostStmt(const PostStmt& L, const CFGBlock* B,
-                                  unsigned StmtIdx, ExplodedNode* Pred) {
-
+void CoreEngine::HandlePostStmt(const CFGBlock* B, unsigned StmtIdx, 
+                                  ExplodedNode* Pred) {
   assert (!B->empty());
 
   if (StmtIdx == B->size())
     HandleBlockExit(B, Pred);
   else {
-    GRStmtNodeBuilder Builder(B, StmtIdx, Pred, this,
-                              SubEngine.getStateManager());
-    ProcessStmt((*B)[StmtIdx], Builder);
+    StmtNodeBuilder Builder(B, StmtIdx, Pred, this,
+                              SubEng.getStateManager());
+    SubEng.processCFGElement((*B)[StmtIdx], Builder);
   }
 }
 
-/// GenerateNode - Utility method to generate nodes, hook up successors,
+/// generateNode - Utility method to generate nodes, hook up successors,
 ///  and add nodes to the worklist.
-void GRCoreEngine::GenerateNode(const ProgramPoint& Loc,
-                                const GRState* State, ExplodedNode* Pred) {
+void CoreEngine::generateNode(const ProgramPoint& Loc,
+                              const GRState* State, ExplodedNode* Pred) {
 
   bool IsNew;
   ExplodedNode* Node = G->getNode(Loc, State, &IsNew);
@@ -403,33 +461,60 @@ void GRCoreEngine::GenerateNode(const ProgramPoint& Loc,
   }
 
   // Only add 'Node' to the worklist if it was freshly generated.
-  if (IsNew) WList->Enqueue(Node);
+  if (IsNew) WList->enqueue(Node);
+}
+
+ExplodedNode *
+GenericNodeBuilderImpl::generateNodeImpl(const GRState *state,
+                                         ExplodedNode *pred,
+                                         ProgramPoint programPoint,
+                                         bool asSink) {
+  
+  hasGeneratedNode = true;
+  bool isNew;
+  ExplodedNode *node = engine.getGraph().getNode(programPoint, state, &isNew);
+  if (pred)
+    node->addPredecessor(pred, engine.getGraph());
+  if (isNew) {
+    if (asSink) {
+      node->markAsSink();
+      sinksGenerated.push_back(node);
+    }
+    return node;
+  }
+  return 0;
 }
 
-GRStmtNodeBuilder::GRStmtNodeBuilder(const CFGBlock* b, unsigned idx,
-                                     ExplodedNode* N, GRCoreEngine* e,
+StmtNodeBuilder::StmtNodeBuilder(const CFGBlock* b, unsigned idx,
+                                     ExplodedNode* N, CoreEngine* e,
                                      GRStateManager &mgr)
-  : Eng(*e), B(*b), Idx(idx), Pred(N), Mgr(mgr), Auditor(0),
-    PurgingDeadSymbols(false), BuildSinks(false), HasGeneratedNode(false),
+  : Eng(*e), B(*b), Idx(idx), Pred(N), Mgr(mgr),
+    PurgingDeadSymbols(false), BuildSinks(false), hasGeneratedNode(false),
     PointKind(ProgramPoint::PostStmtKind), Tag(0) {
   Deferred.insert(N);
   CleanedState = Pred->getState();
 }
 
-GRStmtNodeBuilder::~GRStmtNodeBuilder() {
+StmtNodeBuilder::~StmtNodeBuilder() {
   for (DeferredTy::iterator I=Deferred.begin(), E=Deferred.end(); I!=E; ++I)
     if (!(*I)->isSink())
       GenerateAutoTransition(*I);
 }
 
-void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
+void StmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   assert (!N->isSink());
 
   // Check if this node entered a callee.
   if (isa<CallEnter>(N->getLocation())) {
     // Still use the index of the CallExpr. It's needed to create the callee
     // StackFrameContext.
-    Eng.WList->Enqueue(N, &B, Idx);
+    Eng.WList->enqueue(N, &B, Idx);
+    return;
+  }
+
+  // Do not create extra nodes. Move to the next CFG element.
+  if (isa<PostInitializer>(N->getLocation())) {
+    Eng.WList->enqueue(N, &B, Idx+1);
     return;
   }
 
@@ -438,7 +523,7 @@ void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   if (Loc == N->getLocation()) {
     // Note: 'N' should be a fresh node because otherwise it shouldn't be
     // a member of Deferred.
-    Eng.WList->Enqueue(N, &B, Idx+1);
+    Eng.WList->enqueue(N, &B, Idx+1);
     return;
   }
 
@@ -447,31 +532,20 @@ void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   Succ->addPredecessor(N, *Eng.G);
 
   if (IsNew)
-    Eng.WList->Enqueue(Succ, &B, Idx+1);
+    Eng.WList->enqueue(Succ, &B, Idx+1);
 }
 
-ExplodedNode* GRStmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
+ExplodedNode* StmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
                                           ExplodedNode* Pred, const GRState* St,
                                           ProgramPoint::Kind K) {
-  const GRState* PredState = GetState(Pred);
-
-  // If the state hasn't changed, don't generate a new node.
-  if (!BuildSinks && St == PredState && Auditor == 0) {
-    Dst.Add(Pred);
-    return NULL;
-  }
 
   ExplodedNode* N = generateNode(S, St, Pred, K);
 
   if (N) {
     if (BuildSinks)
       N->markAsSink();
-    else {
-      if (Auditor && Auditor->Audit(N, Mgr))
-        N->markAsSink();
-      
+    else
       Dst.Add(N);
-    }
   }
   
   return N;
@@ -502,7 +576,7 @@ static ProgramPoint GetProgramPoint(const Stmt *S, ProgramPoint::Kind K,
 }
 
 ExplodedNode*
-GRStmtNodeBuilder::generateNodeInternal(const Stmt* S, const GRState* state,
+StmtNodeBuilder::generateNodeInternal(const Stmt* S, const GRState* state,
                                         ExplodedNode* Pred,
                                         ProgramPoint::Kind K,
                                         const void *tag) {
@@ -512,7 +586,7 @@ GRStmtNodeBuilder::generateNodeInternal(const Stmt* S, const GRState* state,
 }
 
 ExplodedNode*
-GRStmtNodeBuilder::generateNodeInternal(const ProgramPoint &Loc,
+StmtNodeBuilder::generateNodeInternal(const ProgramPoint &Loc,
                                         const GRState* State,
                                         ExplodedNode* Pred) {
   bool IsNew;
@@ -528,7 +602,7 @@ GRStmtNodeBuilder::generateNodeInternal(const ProgramPoint &Loc,
   return NULL;
 }
 
-ExplodedNode* GRBranchNodeBuilder::generateNode(const GRState* State,
+ExplodedNode* BranchNodeBuilder::generateNode(const GRState* State,
                                                 bool branch) {
 
   // If the branch has been marked infeasible we should not generate a node.
@@ -556,17 +630,17 @@ ExplodedNode* GRBranchNodeBuilder::generateNode(const GRState* State,
   return NULL;
 }
 
-GRBranchNodeBuilder::~GRBranchNodeBuilder() {
+BranchNodeBuilder::~BranchNodeBuilder() {
   if (!GeneratedTrue) generateNode(Pred->State, true);
   if (!GeneratedFalse) generateNode(Pred->State, false);
 
   for (DeferredTy::iterator I=Deferred.begin(), E=Deferred.end(); I!=E; ++I)
-    if (!(*I)->isSink()) Eng.WList->Enqueue(*I);
+    if (!(*I)->isSink()) Eng.WList->enqueue(*I);
 }
 
 
 ExplodedNode*
-GRIndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
+IndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
                                         bool isSink) {
   bool IsNew;
 
@@ -580,7 +654,7 @@ GRIndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
     if (isSink)
       Succ->markAsSink();
     else
-      Eng.WList->Enqueue(Succ);
+      Eng.WList->enqueue(Succ);
 
     return Succ;
   }
@@ -590,7 +664,7 @@ GRIndirectGotoNodeBuilder::generateNode(const iterator& I, const GRState* St,
 
 
 ExplodedNode*
-GRSwitchNodeBuilder::generateCaseStmtNode(const iterator& I, const GRState* St){
+SwitchNodeBuilder::generateCaseStmtNode(const iterator& I, const GRState* St){
 
   bool IsNew;
 
@@ -599,7 +673,7 @@ GRSwitchNodeBuilder::generateCaseStmtNode(const iterator& I, const GRState* St){
   Succ->addPredecessor(Pred, *Eng.G);
 
   if (IsNew) {
-    Eng.WList->Enqueue(Succ);
+    Eng.WList->enqueue(Succ);
     return Succ;
   }
 
@@ -608,7 +682,7 @@ GRSwitchNodeBuilder::generateCaseStmtNode(const iterator& I, const GRState* St){
 
 
 ExplodedNode*
-GRSwitchNodeBuilder::generateDefaultCaseNode(const GRState* St, bool isSink) {
+SwitchNodeBuilder::generateDefaultCaseNode(const GRState* St, bool isSink) {
 
   // Get the block for the default case.
   assert (Src->succ_rbegin() != Src->succ_rend());
@@ -624,7 +698,7 @@ GRSwitchNodeBuilder::generateDefaultCaseNode(const GRState* St, bool isSink) {
     if (isSink)
       Succ->markAsSink();
     else
-      Eng.WList->Enqueue(Succ);
+      Eng.WList->enqueue(Succ);
 
     return Succ;
   }
@@ -632,9 +706,9 @@ GRSwitchNodeBuilder::generateDefaultCaseNode(const GRState* St, bool isSink) {
   return NULL;
 }
 
-GREndPathNodeBuilder::~GREndPathNodeBuilder() {
+EndOfFunctionNodeBuilder::~EndOfFunctionNodeBuilder() {
   // Auto-generate an EOP node if one has not been generated.
-  if (!HasGeneratedNode) {
+  if (!hasGeneratedNode) {
     // If we are in an inlined call, generate CallExit node.
     if (Pred->getLocationContext()->getParent())
       GenerateCallExitNode(Pred->State);
@@ -644,9 +718,9 @@ GREndPathNodeBuilder::~GREndPathNodeBuilder() {
 }
 
 ExplodedNode*
-GREndPathNodeBuilder::generateNode(const GRState* State, const void *tag,
+EndOfFunctionNodeBuilder::generateNode(const GRState* State, const void *tag,
                                    ExplodedNode* P) {
-  HasGeneratedNode = true;
+  hasGeneratedNode = true;
   bool IsNew;
 
   ExplodedNode* Node = Eng.G->getNode(BlockEntrance(&B,
@@ -662,8 +736,8 @@ GREndPathNodeBuilder::generateNode(const GRState* State, const void *tag,
   return NULL;
 }
 
-void GREndPathNodeBuilder::GenerateCallExitNode(const GRState *state) {
-  HasGeneratedNode = true;
+void EndOfFunctionNodeBuilder::GenerateCallExitNode(const GRState *state) {
+  hasGeneratedNode = true;
   // Create a CallExit node and enqueue it.
   const StackFrameContext *LocCtx
                          = cast<StackFrameContext>(Pred->getLocationContext());
@@ -677,19 +751,18 @@ void GREndPathNodeBuilder::GenerateCallExitNode(const GRState *state) {
   Node->addPredecessor(Pred, *Eng.G);
 
   if (isNew)
-    Eng.WList->Enqueue(Node);
+    Eng.WList->enqueue(Node);
 }
                                                 
 
-void GRCallEnterNodeBuilder::GenerateNode(const GRState *state,
-                                          const LocationContext *LocCtx) {
+void CallEnterNodeBuilder::generateNode(const GRState *state) {
   // Check if the callee is in the same translation unit.
   if (CalleeCtx->getTranslationUnit() != 
       Pred->getLocationContext()->getTranslationUnit()) {
     // Create a new engine. We must be careful that the new engine should not
     // reference data structures owned by the old engine.
 
-    AnalysisManager &OldMgr = Eng.SubEngine.getAnalysisManager();
+    AnalysisManager &OldMgr = Eng.SubEng.getAnalysisManager();
     
     // Get the callee's translation unit.
     idx::TranslationUnit *TU = CalleeCtx->getTranslationUnit();
@@ -702,25 +775,29 @@ void GRCallEnterNodeBuilder::GenerateNode(const GRState *state,
                          OldMgr.getPathDiagnosticClient(),
                          OldMgr.getStoreManagerCreator(),
                          OldMgr.getConstraintManagerCreator(),
+                         OldMgr.getCheckerManager(),
                          OldMgr.getIndexer(),
-                         OldMgr.getMaxNodes(), OldMgr.getMaxLoop(),
+                         OldMgr.getMaxNodes(), OldMgr.getMaxVisit(),
                          OldMgr.shouldVisualizeGraphviz(),
                          OldMgr.shouldVisualizeUbigraph(),
                          OldMgr.shouldPurgeDead(),
                          OldMgr.shouldEagerlyAssume(),
                          OldMgr.shouldTrimGraph(),
                          OldMgr.shouldInlineCall(),
-                     OldMgr.getAnalysisContextManager().getUseUnoptimizedCFG());
-    llvm::OwningPtr<GRTransferFuncs> TF(MakeCFRefCountTF(AMgr.getASTContext(),
+                     OldMgr.getAnalysisContextManager().getUseUnoptimizedCFG(),
+                     OldMgr.getAnalysisContextManager().getAddImplicitDtors(),
+                     OldMgr.getAnalysisContextManager().getAddInitializers(),
+                     OldMgr.shouldEagerlyTrimExplodedGraph());
+    llvm::OwningPtr<TransferFuncs> TF(MakeCFRefCountTF(AMgr.getASTContext(),
                                                          /* GCEnabled */ false,
                                                         AMgr.getLangOptions()));
     // Create the new engine.
-    GRExprEngine NewEng(AMgr, TF.take());
+    ExprEngine NewEng(AMgr, TF.take());
 
     // Create the new LocationContext.
     AnalysisContext *NewAnaCtx = AMgr.getAnalysisContext(CalleeCtx->getDecl(), 
                                                CalleeCtx->getTranslationUnit());
-    const StackFrameContext *OldLocCtx = cast<StackFrameContext>(LocCtx);
+    const StackFrameContext *OldLocCtx = CalleeCtx;
     const StackFrameContext *NewLocCtx = AMgr.getStackFrame(NewAnaCtx, 
                                                OldLocCtx->getParent(),
                                                OldLocCtx->getCallSite(),
@@ -737,7 +814,7 @@ void GRCallEnterNodeBuilder::GenerateNode(const GRState *state,
   }
 
   // Get the callee entry block.
-  const CFGBlock *Entry = &(LocCtx->getCFG()->getEntry());
+  const CFGBlock *Entry = &(CalleeCtx->getCFG()->getEntry());
   assert(Entry->empty());
   assert(Entry->succ_size() == 1);
 
@@ -745,26 +822,27 @@ void GRCallEnterNodeBuilder::GenerateNode(const GRState *state,
   const CFGBlock *SuccB = *(Entry->succ_begin());
 
   // Construct an edge representing the starting location in the callee.
-  BlockEdge Loc(Entry, SuccB, LocCtx);
+  BlockEdge Loc(Entry, SuccB, CalleeCtx);
 
   bool isNew;
   ExplodedNode *Node = Eng.G->getNode(Loc, state, &isNew);
   Node->addPredecessor(const_cast<ExplodedNode*>(Pred), *Eng.G);
 
   if (isNew)
-    Eng.WList->Enqueue(Node);
+    Eng.WList->enqueue(Node);
 }
 
-void GRCallExitNodeBuilder::GenerateNode(const GRState *state) {
+void CallExitNodeBuilder::generateNode(const GRState *state) {
   // Get the callee's location context.
   const StackFrameContext *LocCtx 
                          = cast<StackFrameContext>(Pred->getLocationContext());
-
+  // When exiting an implicit automatic obj dtor call, the callsite is the Stmt
+  // that triggers the dtor.
   PostStmt Loc(LocCtx->getCallSite(), LocCtx->getParent());
   bool isNew;
   ExplodedNode *Node = Eng.G->getNode(Loc, state, &isNew);
   Node->addPredecessor(const_cast<ExplodedNode*>(Pred), *Eng.G);
   if (isNew)
-    Eng.WList->Enqueue(Node, LocCtx->getCallSiteBlock(),
+    Eng.WList->enqueue(Node, LocCtx->getCallSiteBlock(),
                        LocCtx->getIndex() + 1);
 }
diff --git a/lib/Checker/Environment.cpp b/lib/StaticAnalyzer/Core/Environment.cpp
index 02291f43500c..ecaff295b390 100644
--- a/lib/Checker/Environment.cpp
+++ b/lib/StaticAnalyzer/Core/Environment.cpp
@@ -13,36 +13,39 @@
 
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 
 using namespace clang;
+using namespace ento;
 
-SVal Environment::GetSVal(const Stmt *E, ValueManager& ValMgr) const {
+SVal Environment::lookupExpr(const Stmt* E) const {
+  const SVal* X = ExprBindings.lookup(E);
+  if (X) {
+    SVal V = *X;
+    return V;
+  }
+  return UnknownVal();
+}
 
+SVal Environment::getSVal(const Stmt *E, SValBuilder& svalBuilder) const {
   for (;;) {
-
     switch (E->getStmtClass()) {
-
       case Stmt::AddrLabelExprClass:
-        return ValMgr.makeLoc(cast<AddrLabelExpr>(E));
-
-        // ParenExprs are no-ops.
-
+        return svalBuilder.makeLoc(cast<AddrLabelExpr>(E));
       case Stmt::ParenExprClass:
+        // ParenExprs are no-ops.
         E = cast<ParenExpr>(E)->getSubExpr();
         continue;
-
       case Stmt::CharacterLiteralClass: {
         const CharacterLiteral* C = cast<CharacterLiteral>(E);
-        return ValMgr.makeIntVal(C->getValue(), C->getType());
+        return svalBuilder.makeIntVal(C->getValue(), C->getType());
       }
-
       case Stmt::CXXBoolLiteralExprClass: {
         const SVal *X = ExprBindings.lookup(E);
         if (X) 
           return *X;
         else 
-          return ValMgr.makeIntVal(cast<CXXBoolLiteralExpr>(E));
+          return svalBuilder.makeBoolVal(cast<CXXBoolLiteralExpr>(E));
       }
       case Stmt::IntegerLiteralClass: {
         // In C++, this expression may have been bound to a temporary object.
@@ -50,34 +53,36 @@ SVal Environment::GetSVal(const Stmt *E, ValueManager& ValMgr) const {
         if (X)
           return *X;
         else
-          return ValMgr.makeIntVal(cast<IntegerLiteral>(E));
+          return svalBuilder.makeIntVal(cast<IntegerLiteral>(E));
       }
-
-      // Casts where the source and target type are the same
-      // are no-ops.  We blast through these to get the descendant
-      // subexpression that has a value.
-
       case Stmt::ImplicitCastExprClass:
+      case Stmt::CXXFunctionalCastExprClass:
       case Stmt::CStyleCastExprClass: {
+        // We blast through no-op casts to get the descendant
+        // subexpression that has a value.
         const CastExpr* C = cast<CastExpr>(E);
         QualType CT = C->getType();
-
         if (CT->isVoidType())
           return UnknownVal();
-
+        if (C->getCastKind() == CK_NoOp) {
+          E = C->getSubExpr();
+          continue;
+        }
         break;
       }
-
-        // Handle all other Stmt* using a lookup.
-
+      case Stmt::ExprWithCleanupsClass:
+        E = cast<ExprWithCleanups>(E)->getSubExpr();
+        continue;
+      case Stmt::CXXBindTemporaryExprClass:
+        E = cast<CXXBindTemporaryExpr>(E)->getSubExpr();
+        continue;
+      // Handle all other Stmt* using a lookup.
       default:
         break;
     };
-
     break;
   }
-
-  return LookupExpr(E);
+  return lookupExpr(E);
 }
 
 Environment EnvironmentManager::bindExpr(Environment Env, const Stmt *S,
@@ -86,12 +91,12 @@ Environment EnvironmentManager::bindExpr(Environment Env, const Stmt *S,
 
   if (V.isUnknown()) {
     if (Invalidate)
-      return Environment(F.Remove(Env.ExprBindings, S));
+      return Environment(F.remove(Env.ExprBindings, S));
     else
       return Env;
   }
 
-  return Environment(F.Add(Env.ExprBindings, S, V));
+  return Environment(F.add(Env.ExprBindings, S, V));
 }
 
 static inline const Stmt *MakeLocation(const Stmt *S) {
@@ -101,7 +106,8 @@ static inline const Stmt *MakeLocation(const Stmt *S) {
 Environment EnvironmentManager::bindExprAndLocation(Environment Env,
                                                     const Stmt *S,
                                                     SVal location, SVal V) {
-  return Environment(F.Add(F.Add(Env.ExprBindings, MakeLocation(S), V), S, V));
+  return Environment(F.add(F.add(Env.ExprBindings, MakeLocation(S), location),
+                           S, V));
 }
 
 namespace {
@@ -132,7 +138,7 @@ static inline bool IsLocation(const Stmt *S) {
   return (bool) (((uintptr_t) S) & 0x1);
 }
 
-// RemoveDeadBindings:
+// removeDeadBindings:
 //  - Remove subexpression bindings.
 //  - Remove dead block expression bindings.
 //  - Keep live block expression bindings:
@@ -140,7 +146,7 @@ static inline bool IsLocation(const Stmt *S) {
 //     see ScanReachableSymbols.
 //   - Mark the region in DRoots if the binding is a loc::MemRegionVal.
 Environment
-EnvironmentManager::RemoveDeadBindings(Environment Env,
+EnvironmentManager::removeDeadBindings(Environment Env,
                                        SymbolReaper &SymReaper,
                                        const GRState *ST,
                               llvm::SmallVectorImpl<const MemRegion*> &DRoots) {
@@ -174,7 +180,7 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
 
     // Block-level expressions in callers are assumed always live.
     if (isBlockExprInCallers(BlkExpr, SymReaper.getLocationContext())) {
-      NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, BlkExpr, X);
+      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
 
       if (isa<loc::MemRegionVal>(X)) {
         const MemRegion* R = cast<loc::MemRegionVal>(X).getRegion();
@@ -193,7 +199,7 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
 
     if (SymReaper.isLive(BlkExpr)) {
       // Copy the binding to the new map.
-      NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, BlkExpr, X);
+      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
 
       // If the block expr's value is a memory region, then mark that region.
       if (isa<loc::MemRegionVal>(X)) {
@@ -212,7 +218,7 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
     // beginning of itself, but we need its UndefinedVal to determine its
     // SVal.
     if (X.isUndef() && cast<UndefinedVal>(X).getData())
-      NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, BlkExpr, X);
+      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, BlkExpr, X);
   }
   
   // Go through he deferred locations and add them to the new environment if
@@ -221,7 +227,7 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
       I = deferredLocations.begin(), E = deferredLocations.end(); I != E; ++I) {
     const Stmt *S = (Stmt*) (((uintptr_t) I->first) & (uintptr_t) ~0x1);
     if (NewEnv.ExprBindings.lookup(S))
-      NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, I->first, I->second);
+      NewEnv.ExprBindings = F.add(NewEnv.ExprBindings, I->first, I->second);
   }
 
   return NewEnv;
diff --git a/lib/Checker/ExplodedGraph.cpp b/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
index 20429b951992..2a8364d4117f 100644
--- a/lib/Checker/ExplodedGraph.cpp
+++ b/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/ExplodedGraph.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 #include "clang/AST/Stmt.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DenseMap.h"
@@ -21,6 +21,7 @@
 #include <vector>
 
 using namespace clang;
+using namespace ento;
 
 //===----------------------------------------------------------------------===//
 // Node auditing.
@@ -40,6 +41,96 @@ void ExplodedNode::SetAuditor(ExplodedNode::Auditor* A) {
 }
 
 //===----------------------------------------------------------------------===//
+// Cleanup.
+//===----------------------------------------------------------------------===//
+
+typedef std::vector<ExplodedNode*> NodeList;
+static inline NodeList*& getNodeList(void *&p) { return (NodeList*&) p; }
+
+ExplodedGraph::~ExplodedGraph() {
+  if (reclaimNodes) {
+    delete getNodeList(recentlyAllocatedNodes);
+    delete getNodeList(freeNodes);
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Node reclamation.
+//===----------------------------------------------------------------------===//
+
+void ExplodedGraph::reclaimRecentlyAllocatedNodes() {
+  if (!recentlyAllocatedNodes)
+    return;
+  NodeList &nl = *getNodeList(recentlyAllocatedNodes);
+ 
+  // Reclaimn all nodes that match *all* the following criteria:
+  //
+  // (1) 1 predecessor (that has one successor)
+  // (2) 1 successor (that has one predecessor)
+  // (3) The ProgramPoint is for a PostStmt.
+  // (4) There is no 'tag' for the ProgramPoint.
+  // (5) The 'store' is the same as the predecessor.
+  // (6) The 'GDM' is the same as the predecessor.
+  // (7) The LocationContext is the same as the predecessor.
+  // (8) The PostStmt is for a non-CFGElement expression.
+  
+  for (NodeList::iterator i = nl.begin(), e = nl.end() ; i != e; ++i) {
+    ExplodedNode *node = *i;
+    
+    // Conditions 1 and 2.
+    if (node->pred_size() != 1 || node->succ_size() != 1)
+      continue;
+
+    ExplodedNode *pred = *(node->pred_begin());
+    if (pred->succ_size() != 1)
+      continue;
+
+    ExplodedNode *succ = *(node->succ_begin());
+    if (succ->pred_size() != 1)
+      continue;
+
+    // Condition 3.
+    ProgramPoint progPoint = node->getLocation();
+    if (!isa<PostStmt>(progPoint))
+      continue;
+
+    // Condition 4.
+    PostStmt ps = cast<PostStmt>(progPoint);
+    if (ps.getTag() || isa<PostStmtCustom>(ps))
+      continue;
+
+    if (isa<BinaryOperator>(ps.getStmt()))
+      continue;
+
+    // Conditions 5, 6, and 7.
+    const GRState *state = node->getState();
+    const GRState *pred_state = pred->getState();    
+    if (state->store != pred_state->store || state->GDM != pred_state->GDM ||
+        progPoint.getLocationContext() != pred->getLocationContext())
+      continue;
+
+    // Condition 8.
+    if (node->getCFG().isBlkExpr(ps.getStmt()))
+      continue;
+    
+    // If we reach here, we can remove the node.  This means:
+    // (a) changing the predecessors successor to the successor of this node
+    // (b) changing the successors predecessor to the predecessor of this node
+    // (c) Putting 'node' onto freeNodes.
+    pred->replaceSuccessor(succ);
+    succ->replacePredecessor(pred);
+    if (!freeNodes)
+      freeNodes = new NodeList();
+    getNodeList(freeNodes)->push_back(node);
+    Nodes.RemoveNode(node);
+    --NumNodes;
+    node->~ExplodedNode();
+  }
+  
+  nl.clear();
+}
+
+//===----------------------------------------------------------------------===//
 // ExplodedNode.
 //===----------------------------------------------------------------------===//
 
@@ -56,6 +147,12 @@ void ExplodedNode::addPredecessor(ExplodedNode* V, ExplodedGraph &G) {
 #endif
 }
 
+void ExplodedNode::NodeGroup::replaceNode(ExplodedNode *node) {
+  assert(getKind() == Size1);
+  P = reinterpret_cast<uintptr_t>(node);
+  assert(getKind() == Size1);
+}
+
 void ExplodedNode::NodeGroup::addNode(ExplodedNode* N, ExplodedGraph &G) {
   assert((reinterpret_cast<uintptr_t>(N) & Mask) == 0x0);
   assert(!getFlag());
@@ -128,10 +225,24 @@ ExplodedNode *ExplodedGraph::getNode(const ProgramPoint& L,
   NodeTy* V = Nodes.FindNodeOrInsertPos(profile, InsertPos);
 
   if (!V) {
-    // Allocate a new node.
-    V = (NodeTy*) getAllocator().Allocate<NodeTy>();
+    if (freeNodes && !getNodeList(freeNodes)->empty()) {
+      NodeList *nl = getNodeList(freeNodes);
+      V = nl->back();
+      nl->pop_back();
+    }
+    else {
+      // Allocate a new node.
+      V = (NodeTy*) getAllocator().Allocate<NodeTy>();
+    }
+
     new (V) NodeTy(L, State);
 
+    if (reclaimNodes) {
+      if (!recentlyAllocatedNodes)
+        recentlyAllocatedNodes = new NodeList();
+      getNodeList(recentlyAllocatedNodes)->push_back(V);
+    }
+
     // Insert the node into the node set and return it.
     Nodes.InsertNode(V, InsertPos);
 
diff --git a/lib/Checker/FlatStore.cpp b/lib/StaticAnalyzer/Core/FlatStore.cpp
index 21fa422166f0..99a5eadaca20 100644
--- a/lib/Checker/FlatStore.cpp
+++ b/lib/StaticAnalyzer/Core/FlatStore.cpp
@@ -7,11 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 #include "llvm/ADT/ImmutableIntervalMap.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace clang;
+using namespace ento;
 using llvm::Interval;
 
 // The actual store type.
@@ -30,13 +31,13 @@ public:
       BVFactory(mgr.getAllocator()) {}
 
   SVal Retrieve(Store store, Loc L, QualType T);
-  Store Bind(Store store, Loc L, SVal val);
-  Store Remove(Store St, Loc L);
-  Store BindCompoundLiteral(Store store, const CompoundLiteralExpr* cl,
+  StoreRef Bind(Store store, Loc L, SVal val);
+  StoreRef Remove(Store St, Loc L);
+  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr* cl,
                             const LocationContext *LC, SVal v);
 
-  Store getInitialStore(const LocationContext *InitLoc) {
-    return RBFactory.GetEmptyMap().getRoot();
+  StoreRef getInitialStore(const LocationContext *InitLoc) {
+    return StoreRef(RBFactory.getEmptyMap().getRoot(), *this);
   }
 
   SubRegionMap *getSubRegionMap(Store store) {
@@ -44,22 +45,23 @@ public:
   }
 
   SVal ArrayToPointer(Loc Array);
-  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
-                           SymbolReaper& SymReaper,
-                         llvm::SmallVectorImpl<const MemRegion*>& RegionRoots){
-    return store;
+  StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
+                              SymbolReaper& SymReaper,
+                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots){
+    return StoreRef(store, *this);
   }
 
-  Store BindDecl(Store store, const VarRegion *VR, SVal initVal);
+  StoreRef BindDecl(Store store, const VarRegion *VR, SVal initVal);
 
-  Store BindDeclWithNoInit(Store store, const VarRegion *VR);
+  StoreRef BindDeclWithNoInit(Store store, const VarRegion *VR);
 
   typedef llvm::DenseSet<SymbolRef> InvalidatedSymbols;
   
-  Store InvalidateRegions(Store store, const MemRegion * const *I,
-                          const MemRegion * const *E, const Expr *Ex,
-                          unsigned Count, InvalidatedSymbols *IS,
-                          bool invalidateGlobals, InvalidatedRegions *Regions);
+  StoreRef invalidateRegions(Store store, const MemRegion * const *I,
+                             const MemRegion * const *E, const Expr *Ex,
+                             unsigned Count, InvalidatedSymbols *IS,
+                             bool invalidateGlobals,
+                             InvalidatedRegions *Regions);
 
   void print(Store store, llvm::raw_ostream& Out, const char* nl, 
              const char *sep);
@@ -83,7 +85,7 @@ private:
 };
 } // end anonymous namespace
 
-StoreManager *clang::CreateFlatStoreManager(GRStateManager &StMgr) {
+StoreManager *ento::CreateFlatStoreManager(GRStateManager &StMgr) {
   return new FlatStoreManager(StMgr);
 }
 
@@ -97,7 +99,7 @@ SVal FlatStoreManager::Retrieve(Store store, Loc L, QualType T) {
   RegionBindings B = getRegionBindings(store);
   const BindingVal *BV = B.lookup(RI.R);
   if (BV) {
-    const SVal *V = BVFactory.Lookup(*BV, RI.I);
+    const SVal *V = BVFactory.lookup(*BV, RI.I);
     if (V)
       return *V;
     else
@@ -111,60 +113,60 @@ SVal FlatStoreManager::RetrieveRegionWithNoBinding(const MemRegion *R,
   if (R->hasStackNonParametersStorage())
     return UndefinedVal();
   else
-    return ValMgr.getRegionValueSymbolVal(cast<TypedRegion>(R));
+    return svalBuilder.getRegionValueSymbolVal(cast<TypedRegion>(R));
 }
 
-Store FlatStoreManager::Bind(Store store, Loc L, SVal val) {
+StoreRef FlatStoreManager::Bind(Store store, Loc L, SVal val) {
   const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
   RegionBindings B = getRegionBindings(store);
   const BindingVal *V = B.lookup(R);
 
-  BindingVal BV = BVFactory.GetEmptyMap();
+  BindingVal BV = BVFactory.getEmptyMap();
   if (V)
     BV = *V;
 
   RegionInterval RI = RegionToInterval(R);
   // FIXME: FlatStore should handle regions with unknown intervals.
   if (!RI.R)
-    return B.getRoot();
-  BV = BVFactory.Add(BV, RI.I, val);
-  B = RBFactory.Add(B, RI.R, BV);
-  return B.getRoot();
+    return StoreRef(B.getRoot(), *this);
+  BV = BVFactory.add(BV, RI.I, val);
+  B = RBFactory.add(B, RI.R, BV);
+  return StoreRef(B.getRoot(), *this);
 }
 
-Store FlatStoreManager::Remove(Store store, Loc L) {
-  return store;
+StoreRef FlatStoreManager::Remove(Store store, Loc L) {
+  return StoreRef(store, *this);
 }
 
-Store FlatStoreManager::BindCompoundLiteral(Store store,
+StoreRef FlatStoreManager::BindCompoundLiteral(Store store,
                                             const CompoundLiteralExpr* cl,
                                             const LocationContext *LC,
                                             SVal v) {
-  return store;
+  return StoreRef(store, *this);
 }
 
 SVal FlatStoreManager::ArrayToPointer(Loc Array) {
   return Array;
 }
 
-Store FlatStoreManager::BindDecl(Store store, const VarRegion *VR, 
-                                 SVal initVal) {
-  return Bind(store, ValMgr.makeLoc(VR), initVal);
+StoreRef FlatStoreManager::BindDecl(Store store, const VarRegion *VR, 
+                                    SVal initVal) {
+  return Bind(store, svalBuilder.makeLoc(VR), initVal);
 }
 
-Store FlatStoreManager::BindDeclWithNoInit(Store store, const VarRegion *VR) {
-  return store;
+StoreRef FlatStoreManager::BindDeclWithNoInit(Store store, const VarRegion *VR){
+  return StoreRef(store, *this);
 }
 
-Store FlatStoreManager::InvalidateRegions(Store store,
-                                          const MemRegion * const *I,
-                                          const MemRegion * const *E,
-                                          const Expr *Ex, unsigned Count,
-                                          InvalidatedSymbols *IS,
-                                          bool invalidateGlobals,
-                                          InvalidatedRegions *Regions) {
+StoreRef FlatStoreManager::invalidateRegions(Store store,
+                                             const MemRegion * const *I,
+                                             const MemRegion * const *E,
+                                             const Expr *Ex, unsigned Count,
+                                             InvalidatedSymbols *IS,
+                                             bool invalidateGlobals,
+                                             InvalidatedRegions *Regions) {
   assert(false && "Not implemented");
-  return store;
+  return StoreRef(store, *this);
 }
 
 void FlatStoreManager::print(Store store, llvm::raw_ostream& Out, 
diff --git a/lib/Checker/GRState.cpp b/lib/StaticAnalyzer/Core/GRState.cpp
index d38ae21fce94..7b216775b8e6 100644
--- a/lib/Checker/GRState.cpp
+++ b/lib/StaticAnalyzer/Core/GRState.cpp
@@ -12,18 +12,44 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/CFG.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRSubEngine.h"
-#include "clang/Checker/PathSensitive/GRTransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SubEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 // Give the vtable for ConstraintManager somewhere to live.
 // FIXME: Move this elsewhere.
 ConstraintManager::~ConstraintManager() {}
 
+GRState::GRState(GRStateManager *mgr, const Environment& env,
+                 StoreRef st, GenericDataMap gdm)
+  : stateMgr(mgr),
+    Env(env),
+    store(st.getStore()),
+    GDM(gdm),
+    refCount(0) {
+  stateMgr->getStoreManager().incrementReferenceCount(store);
+}
+
+GRState::GRState(const GRState& RHS)
+    : llvm::FoldingSetNode(),
+      stateMgr(RHS.stateMgr),
+      Env(RHS.Env),
+      store(RHS.store),
+      GDM(RHS.GDM),
+      refCount(0) {
+  stateMgr->getStoreManager().incrementReferenceCount(store);
+}
+
+GRState::~GRState() {
+  if (store)
+    stateMgr->getStoreManager().decrementReferenceCount(store);
+}
+
 GRStateManager::~GRStateManager() {
   for (std::vector<GRState::Printer*>::iterator I=Printers.begin(),
         E=Printers.end(); I!=E; ++I)
@@ -35,7 +61,7 @@ GRStateManager::~GRStateManager() {
 }
 
 const GRState*
-GRStateManager::RemoveDeadBindings(const GRState* state,
+GRStateManager::removeDeadBindings(const GRState* state,
                                    const StackFrameContext *LCtx,
                                    SymbolReaper& SymReaper) {
 
@@ -48,14 +74,14 @@ GRStateManager::RemoveDeadBindings(const GRState* state,
   llvm::SmallVector<const MemRegion*, 10> RegionRoots;
   GRState NewState = *state;
 
-  NewState.Env = EnvMgr.RemoveDeadBindings(NewState.Env, SymReaper,
+  NewState.Env = EnvMgr.removeDeadBindings(NewState.Env, SymReaper,
                                            state, RegionRoots);
 
   // Clean up the store.
-  NewState.St = StoreMgr->RemoveDeadBindings(NewState.St, LCtx, 
-                                             SymReaper, RegionRoots);
+  NewState.setStore(StoreMgr->removeDeadBindings(NewState.getStore(), LCtx,
+                                                 SymReaper, RegionRoots));
   state = getPersistentState(NewState);
-  return ConstraintMgr->RemoveDeadBindings(state, SymReaper);
+  return ConstraintMgr->removeDeadBindings(state, SymReaper);
 }
 
 const GRState *GRStateManager::MarshalState(const GRState *state,
@@ -64,7 +90,7 @@ const GRState *GRStateManager::MarshalState(const GRState *state,
   GRState State(this,
                 EnvMgr.getInitialEnvironment(),
                 StoreMgr->getInitialStore(InitLoc),
-                GDMFactory.GetEmptyMap());
+                GDMFactory.getEmptyMap());
 
   return getPersistentState(State);
 }
@@ -72,84 +98,84 @@ const GRState *GRStateManager::MarshalState(const GRState *state,
 const GRState *GRState::bindCompoundLiteral(const CompoundLiteralExpr* CL,
                                             const LocationContext *LC,
                                             SVal V) const {
-  Store new_store = 
-    getStateManager().StoreMgr->BindCompoundLiteral(St, CL, LC, V);
-  return makeWithStore(new_store);
+  const StoreRef &newStore = 
+    getStateManager().StoreMgr->BindCompoundLiteral(getStore(), CL, LC, V);
+  return makeWithStore(newStore);
 }
 
 const GRState *GRState::bindDecl(const VarRegion* VR, SVal IVal) const {
-  Store new_store = getStateManager().StoreMgr->BindDecl(St, VR, IVal);
-  return makeWithStore(new_store);
+  const StoreRef &newStore =
+    getStateManager().StoreMgr->BindDecl(getStore(), VR, IVal);
+  return makeWithStore(newStore);
 }
 
 const GRState *GRState::bindDeclWithNoInit(const VarRegion* VR) const {
-  Store new_store = getStateManager().StoreMgr->BindDeclWithNoInit(St, VR);
-  return makeWithStore(new_store);
+  const StoreRef &newStore =
+    getStateManager().StoreMgr->BindDeclWithNoInit(getStore(), VR);
+  return makeWithStore(newStore);
 }
 
 const GRState *GRState::bindLoc(Loc LV, SVal V) const {
   GRStateManager &Mgr = getStateManager();
-  Store new_store = Mgr.StoreMgr->Bind(St, LV, V);
-  const GRState *new_state = makeWithStore(new_store);
-
+  const GRState *newState = makeWithStore(Mgr.StoreMgr->Bind(getStore(), 
+                                                             LV, V));
   const MemRegion *MR = LV.getAsRegion();
-  if (MR)
-    return Mgr.getOwningEngine().ProcessRegionChange(new_state, MR);
+  if (MR && Mgr.getOwningEngine())
+    return Mgr.getOwningEngine()->processRegionChange(newState, MR);
 
-  return new_state;
+  return newState;
 }
 
 const GRState *GRState::bindDefault(SVal loc, SVal V) const {
   GRStateManager &Mgr = getStateManager();
   const MemRegion *R = cast<loc::MemRegionVal>(loc).getRegion();
-  Store new_store = Mgr.StoreMgr->BindDefault(St, R, V);
-  const GRState *new_state = makeWithStore(new_store);
-  return Mgr.getOwningEngine().ProcessRegionChange(new_state, R);
+  const StoreRef &newStore = Mgr.StoreMgr->BindDefault(getStore(), R, V);
+  const GRState *new_state = makeWithStore(newStore);
+  return Mgr.getOwningEngine() ? 
+           Mgr.getOwningEngine()->processRegionChange(new_state, R) : 
+           new_state;
 }
 
-const GRState *GRState::InvalidateRegions(const MemRegion * const *Begin,
+const GRState *GRState::invalidateRegions(const MemRegion * const *Begin,
                                           const MemRegion * const *End,
                                           const Expr *E, unsigned Count,
                                           StoreManager::InvalidatedSymbols *IS,
                                           bool invalidateGlobals) const {
   GRStateManager &Mgr = getStateManager();
-  GRSubEngine &Eng = Mgr.getOwningEngine();
-
-  if (Eng.WantsRegionChangeUpdate(this)) {
+  SubEngine* Eng = Mgr.getOwningEngine();
+ 
+  if (Eng && Eng->wantsRegionChangeUpdate(this)) {
     StoreManager::InvalidatedRegions Regions;
-
-    Store new_store = Mgr.StoreMgr->InvalidateRegions(St, Begin, End,
-                                                      E, Count, IS,
-                                                      invalidateGlobals,
-                                                      &Regions);
-    const GRState *new_state = makeWithStore(new_store);
-
-    return Eng.ProcessRegionChanges(new_state,
-                                    &Regions.front(),
-                                    &Regions.back()+1);
+    const StoreRef &newStore
+      = Mgr.StoreMgr->invalidateRegions(getStore(), Begin, End, E, Count, IS,
+                                        invalidateGlobals, &Regions);
+    const GRState *newState = makeWithStore(newStore);
+    return Eng->processRegionChanges(newState,
+                                     &Regions.front(),
+                                     &Regions.back()+1);
   }
 
-  Store new_store = Mgr.StoreMgr->InvalidateRegions(St, Begin, End,
-                                                    E, Count, IS,
-                                                    invalidateGlobals,
-                                                    NULL);
-  return makeWithStore(new_store);
+  const StoreRef &newStore =
+    Mgr.StoreMgr->invalidateRegions(getStore(), Begin, End, E, Count, IS,
+                                    invalidateGlobals, NULL);
+  return makeWithStore(newStore);
 }
 
 const GRState *GRState::unbindLoc(Loc LV) const {
-  assert(!isa<loc::MemRegionVal>(LV) && "Use InvalidateRegion instead.");
+  assert(!isa<loc::MemRegionVal>(LV) && "Use invalidateRegion instead.");
 
   Store OldStore = getStore();
-  Store NewStore = getStateManager().StoreMgr->Remove(OldStore, LV);
+  const StoreRef &newStore = getStateManager().StoreMgr->Remove(OldStore, LV);
 
-  if (NewStore == OldStore)
+  if (newStore.getStore() == OldStore)
     return this;
 
-  return makeWithStore(NewStore);
+  return makeWithStore(newStore);
 }
 
-const GRState *GRState::EnterStackFrame(const StackFrameContext *frame) const {
-  Store new_store = getStateManager().StoreMgr->EnterStackFrame(this, frame);
+const GRState *GRState::enterStackFrame(const StackFrameContext *frame) const {
+  const StoreRef &new_store =
+    getStateManager().StoreMgr->enterStackFrame(this, frame);
   return makeWithStore(new_store);
 }
 
@@ -162,16 +188,16 @@ SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
 
   if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
     QualType T = TR->getValueType();
-    if (Loc::IsLocType(T) || T->isIntegerType())
+    if (Loc::isLocType(T) || T->isIntegerType())
       return getSVal(R);
   }
 
   return UnknownVal();
 }
 
-SVal GRState::getSimplifiedSVal(Loc location, QualType T) const {
-  SVal V = getSVal(cast<Loc>(location), T);
-  
+SVal GRState::getSVal(Loc location, QualType T) const {
+  SVal V = getRawSVal(cast<Loc>(location), T);
+
   // If 'V' is a symbolic value that is *perfectly* constrained to
   // be a constant value, use that value instead to lessen the burden
   // on later analysis stages (so we have less symbolic values to reason
@@ -230,7 +256,7 @@ const GRState *GRState::bindExprAndLocation(const Stmt *S, SVal location,
   return getStateManager().getPersistentState(NewSt);
 }
 
-const GRState *GRState::AssumeInBound(DefinedOrUnknownSVal Idx,
+const GRState *GRState::assumeInBound(DefinedOrUnknownSVal Idx,
                                       DefinedOrUnknownSVal UpperBound,
                                       bool Assumption) const {
   if (Idx.isUnknown() || UpperBound.isUnknown())
@@ -238,51 +264,65 @@ const GRState *GRState::AssumeInBound(DefinedOrUnknownSVal Idx,
 
   // Build an expression for 0 <= Idx < UpperBound.
   // This is the same as Idx + MIN < UpperBound + MIN, if overflow is allowed.
-  // FIXME: This should probably be part of SValuator.
+  // FIXME: This should probably be part of SValBuilder.
   GRStateManager &SM = getStateManager();
-  ValueManager &VM = SM.getValueManager();
-  SValuator &SV = VM.getSValuator();
-  ASTContext &Ctx = VM.getContext();
+  SValBuilder &svalBuilder = SM.getSValBuilder();
+  ASTContext &Ctx = svalBuilder.getContext();
 
   // Get the offset: the minimum value of the array index type.
-  BasicValueFactory &BVF = VM.getBasicValueFactory();
-  // FIXME: This should be using ValueManager::ArrayIndexTy...somehow.
-  QualType IndexTy = Ctx.IntTy;
-  nonloc::ConcreteInt Min = BVF.getMinValue(IndexTy);
+  BasicValueFactory &BVF = svalBuilder.getBasicValueFactory();
+  // FIXME: This should be using ValueManager::ArrayindexTy...somehow.
+  QualType indexTy = Ctx.IntTy;
+  nonloc::ConcreteInt Min(BVF.getMinValue(indexTy));
 
   // Adjust the index.
-  SVal NewIdx = SV.EvalBinOpNN(this, BO_Add,
-                               cast<NonLoc>(Idx), Min, IndexTy);
-  if (NewIdx.isUnknownOrUndef())
+  SVal newIdx = svalBuilder.evalBinOpNN(this, BO_Add,
+                                        cast<NonLoc>(Idx), Min, indexTy);
+  if (newIdx.isUnknownOrUndef())
     return this;
 
   // Adjust the upper bound.
-  SVal NewBound = SV.EvalBinOpNN(this, BO_Add,
-                                 cast<NonLoc>(UpperBound), Min, IndexTy);
-  if (NewBound.isUnknownOrUndef())
+  SVal newBound =
+    svalBuilder.evalBinOpNN(this, BO_Add, cast<NonLoc>(UpperBound),
+                            Min, indexTy);
+
+  if (newBound.isUnknownOrUndef())
     return this;
 
   // Build the actual comparison.
-  SVal InBound = SV.EvalBinOpNN(this, BO_LT,
-                                cast<NonLoc>(NewIdx), cast<NonLoc>(NewBound),
+  SVal inBound = svalBuilder.evalBinOpNN(this, BO_LT,
+                                cast<NonLoc>(newIdx), cast<NonLoc>(newBound),
                                 Ctx.IntTy);
-  if (InBound.isUnknownOrUndef())
+  if (inBound.isUnknownOrUndef())
     return this;
 
   // Finally, let the constraint manager take care of it.
   ConstraintManager &CM = SM.getConstraintManager();
-  return CM.Assume(this, cast<DefinedSVal>(InBound), Assumption);
+  return CM.assume(this, cast<DefinedSVal>(inBound), Assumption);
 }
 
 const GRState* GRStateManager::getInitialState(const LocationContext *InitLoc) {
   GRState State(this,
                 EnvMgr.getInitialEnvironment(),
                 StoreMgr->getInitialStore(InitLoc),
-                GDMFactory.GetEmptyMap());
+                GDMFactory.getEmptyMap());
 
   return getPersistentState(State);
 }
 
+void GRStateManager::recycleUnusedStates() {
+  for (std::vector<GRState*>::iterator i = recentlyAllocatedStates.begin(),
+       e = recentlyAllocatedStates.end(); i != e; ++i) {
+    GRState *state = *i;
+    if (state->referencedByExplodedNode())
+      continue;
+    StateSet.RemoveNode(state);
+    freeStates.push_back(state);
+    state->~GRState();
+  }
+  recentlyAllocatedStates.clear();
+}
+
 const GRState* GRStateManager::getPersistentState(GRState& State) {
 
   llvm::FoldingSetNodeID ID;
@@ -292,18 +332,35 @@ const GRState* GRStateManager::getPersistentState(GRState& State) {
   if (GRState* I = StateSet.FindNodeOrInsertPos(ID, InsertPos))
     return I;
 
-  GRState* I = (GRState*) Alloc.Allocate<GRState>();
-  new (I) GRState(State);
-  StateSet.InsertNode(I, InsertPos);
-  return I;
+  GRState *newState = 0;
+  if (!freeStates.empty()) {
+    newState = freeStates.back();
+    freeStates.pop_back();    
+  }
+  else {
+    newState = (GRState*) Alloc.Allocate<GRState>();
+  }
+  new (newState) GRState(State);
+  StateSet.InsertNode(newState, InsertPos);
+  recentlyAllocatedStates.push_back(newState);
+  return newState;
 }
 
-const GRState* GRState::makeWithStore(Store store) const {
+const GRState* GRState::makeWithStore(const StoreRef &store) const {
   GRState NewSt = *this;
-  NewSt.St = store;
+  NewSt.setStore(store);
   return getStateManager().getPersistentState(NewSt);
 }
 
+void GRState::setStore(const StoreRef &newStore) {
+  Store newStoreStore = newStore.getStore();
+  if (newStoreStore)
+    stateMgr->getStoreManager().incrementReferenceCount(newStoreStore);
+  if (store)
+    stateMgr->getStoreManager().decrementReferenceCount(store);
+  store = newStoreStore;
+}
+
 //===----------------------------------------------------------------------===//
 //  State pretty-printing.
 //===----------------------------------------------------------------------===//
@@ -420,7 +477,7 @@ GRStateManager::FindGDMContext(void* K,
 
 const GRState* GRStateManager::addGDM(const GRState* St, void* Key, void* Data){
   GRState::GenericDataMap M1 = St->getGDM();
-  GRState::GenericDataMap M2 = GDMFactory.Add(M1, Key, Data);
+  GRState::GenericDataMap M2 = GDMFactory.add(M1, Key, Data);
 
   if (M1 == M2)
     return St;
@@ -432,7 +489,7 @@ const GRState* GRStateManager::addGDM(const GRState* St, void* Key, void* Data){
 
 const GRState *GRStateManager::removeGDM(const GRState *state, void *Key) {
   GRState::GenericDataMap OldM = state->getGDM();
-  GRState::GenericDataMap NewM = GDMFactory.Remove(OldM, Key);
+  GRState::GenericDataMap NewM = GDMFactory.remove(OldM, Key);
 
   if (NewM == OldM)
     return state;
diff --git a/lib/Checker/HTMLDiagnostics.cpp b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index ff9867fb5f16..1ebc28c044fd 100644
--- a/lib/Checker/HTMLDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathDiagnosticClients.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/Basic/SourceManager.h"
@@ -21,11 +21,13 @@
 #include "clang/Rewrite/HTMLRewrite.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
+#include "llvm/Support/Path.h"
 
 using namespace clang;
+using namespace ento;
 
 //===----------------------------------------------------------------------===//
 // Boilerplate.
@@ -77,8 +79,8 @@ HTMLDiagnostics::HTMLDiagnostics(const std::string& prefix,
 }
 
 PathDiagnosticClient*
-clang::CreateHTMLDiagnosticClient(const std::string& prefix,
-                                  const Preprocessor &PP) {
+ento::createHTMLDiagnosticClient(const std::string& prefix,
+                                 const Preprocessor &PP) {
   return new HTMLDiagnostics(prefix, PP);
 }
 
@@ -120,7 +122,9 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
     std::string ErrorMsg;
     Directory.createDirectoryOnDisk(true, &ErrorMsg);
 
-    if (!Directory.isDirectory()) {
+    bool IsDirectory;
+    if (llvm::sys::fs::is_directory(Directory.str(), IsDirectory) ||
+        !IsDirectory) {
       llvm::errs() << "warning: could not create directory '"
                    << Directory.str() << "'\n"
                    << "reason: " << ErrorMsg << '\n';
@@ -199,7 +203,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
 
   std::string DirName = "";
 
-  if (!llvm::sys::Path(Entry->getName()).isAbsolute()) {
+  if (llvm::sys::path::is_relative(Entry->getName())) {
     llvm::sys::Path P = llvm::sys::Path::GetCurrentDirectory();
     DirName = P.str() + "/";
   }
@@ -300,7 +304,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
   }
 
   if (FilesMade)
-    FilesMade->push_back(H.getLast());
+    FilesMade->push_back(llvm::sys::path::filename(H.str()));
 
   // Emit the HTML to disk.
   for (RewriteBuffer::iterator I = Buf->begin(), E = Buf->end(); I!=E; ++I)
diff --git a/lib/StaticAnalyzer/Core/Makefile b/lib/StaticAnalyzer/Core/Makefile
new file mode 100644
index 000000000000..4aebc163dddc
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/Makefile
@@ -0,0 +1,17 @@
+##===- clang/lib/StaticAnalyzer/Core/Makefile --------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+# This implements analyses built on top of source-level CFGs. 
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangStaticAnalyzerCore
+
+include $(CLANG_LEVEL)/Makefile
diff --git a/lib/Checker/MemRegion.cpp b/lib/StaticAnalyzer/Core/MemRegion.cpp
index 3f706e145a82..d9e884a4b208 100644
--- a/lib/Checker/MemRegion.cpp
+++ b/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -13,8 +13,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/MemRegion.h"
-#include "clang/Checker/PathSensitive/ValueManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/Support/BumpVector.h"
 #include "clang/AST/CharUnits.h"
@@ -22,6 +22,7 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 //===----------------------------------------------------------------------===//
 // MemRegion Construction.
@@ -176,27 +177,27 @@ const StackFrameContext *VarRegion::getStackFrame() const {
 // Region extents.
 //===----------------------------------------------------------------------===//
 
-DefinedOrUnknownSVal DeclRegion::getExtent(ValueManager& ValMgr) const {
-  ASTContext& Ctx = ValMgr.getContext();
-  QualType T = getDesugaredValueType();
+DefinedOrUnknownSVal DeclRegion::getExtent(SValBuilder &svalBuilder) const {
+  ASTContext& Ctx = svalBuilder.getContext();
+  QualType T = getDesugaredValueType(Ctx);
 
   if (isa<VariableArrayType>(T))
-    return nonloc::SymbolVal(ValMgr.getSymbolManager().getExtentSymbol(this));
+    return nonloc::SymbolVal(svalBuilder.getSymbolManager().getExtentSymbol(this));
   if (isa<IncompleteArrayType>(T))
     return UnknownVal();
 
-  CharUnits Size = Ctx.getTypeSizeInChars(T);
-  QualType SizeTy = Ctx.getSizeType();
-  return ValMgr.makeIntVal(Size.getQuantity(), SizeTy);
+  CharUnits size = Ctx.getTypeSizeInChars(T);
+  QualType sizeTy = svalBuilder.getArrayIndexType();
+  return svalBuilder.makeIntVal(size.getQuantity(), sizeTy);
 }
 
-DefinedOrUnknownSVal FieldRegion::getExtent(ValueManager& ValMgr) const {
-  DefinedOrUnknownSVal Extent = DeclRegion::getExtent(ValMgr);
+DefinedOrUnknownSVal FieldRegion::getExtent(SValBuilder &svalBuilder) const {
+  DefinedOrUnknownSVal Extent = DeclRegion::getExtent(svalBuilder);
 
   // A zero-length array at the end of a struct often stands for dynamically-
   // allocated extra memory.
   if (Extent.isZeroConstant()) {
-    QualType T = getDesugaredValueType();
+    QualType T = getDesugaredValueType(svalBuilder.getContext());
 
     if (isa<ConstantArrayType>(T))
       return UnknownVal();
@@ -205,17 +206,21 @@ DefinedOrUnknownSVal FieldRegion::getExtent(ValueManager& ValMgr) const {
   return Extent;
 }
 
-DefinedOrUnknownSVal AllocaRegion::getExtent(ValueManager& ValMgr) const {
-  return nonloc::SymbolVal(ValMgr.getSymbolManager().getExtentSymbol(this));
+DefinedOrUnknownSVal AllocaRegion::getExtent(SValBuilder &svalBuilder) const {
+  return nonloc::SymbolVal(svalBuilder.getSymbolManager().getExtentSymbol(this));
 }
 
-DefinedOrUnknownSVal SymbolicRegion::getExtent(ValueManager& ValMgr) const {
-  return nonloc::SymbolVal(ValMgr.getSymbolManager().getExtentSymbol(this));
+DefinedOrUnknownSVal SymbolicRegion::getExtent(SValBuilder &svalBuilder) const {
+  return nonloc::SymbolVal(svalBuilder.getSymbolManager().getExtentSymbol(this));
 }
 
-DefinedOrUnknownSVal StringRegion::getExtent(ValueManager& ValMgr) const {
-  QualType SizeTy = ValMgr.getContext().getSizeType();
-  return ValMgr.makeIntVal(getStringLiteral()->getByteLength()+1, SizeTy);
+DefinedOrUnknownSVal StringRegion::getExtent(SValBuilder &svalBuilder) const {
+  return svalBuilder.makeIntVal(getStringLiteral()->getByteLength()+1,
+                                svalBuilder.getArrayIndexType());
+}
+
+QualType CXXBaseObjectRegion::getValueType() const {
+  return QualType(decl->getTypeForDecl(), 0);
 }
 
 //===----------------------------------------------------------------------===//
@@ -356,17 +361,28 @@ void BlockDataRegion::Profile(llvm::FoldingSetNodeID& ID) const {
   BlockDataRegion::ProfileRegion(ID, BC, LC, getSuperRegion());
 }
 
-void CXXObjectRegion::ProfileRegion(llvm::FoldingSetNodeID &ID,
-                                    Expr const *Ex,
-                                    const MemRegion *sReg) {
+void CXXTempObjectRegion::ProfileRegion(llvm::FoldingSetNodeID &ID,
+                                        Expr const *Ex,
+                                        const MemRegion *sReg) {
   ID.AddPointer(Ex);
   ID.AddPointer(sReg);
 }
 
-void CXXObjectRegion::Profile(llvm::FoldingSetNodeID &ID) const {
+void CXXTempObjectRegion::Profile(llvm::FoldingSetNodeID &ID) const {
   ProfileRegion(ID, Ex, getSuperRegion());
 }
 
+void CXXBaseObjectRegion::ProfileRegion(llvm::FoldingSetNodeID &ID,
+                                        const CXXRecordDecl *decl,
+                                        const MemRegion *sReg) {
+  ID.AddPointer(decl);
+  ID.AddPointer(sReg);
+}
+
+void CXXBaseObjectRegion::Profile(llvm::FoldingSetNodeID &ID) const {
+  ProfileRegion(ID, decl, superRegion);
+}
+
 //===----------------------------------------------------------------------===//
 // Region pretty-printing.
 //===----------------------------------------------------------------------===//
@@ -407,6 +423,14 @@ void CompoundLiteralRegion::dumpToStream(llvm::raw_ostream& os) const {
   os << "{ " << (void*) CL <<  " }";
 }
 
+void CXXTempObjectRegion::dumpToStream(llvm::raw_ostream &os) const {
+  os << "temp_object";
+}
+
+void CXXBaseObjectRegion::dumpToStream(llvm::raw_ostream &os) const {
+  os << "base " << decl->getName();
+}
+
 void CXXThisRegion::dumpToStream(llvm::raw_ostream &os) const {
   os << "this";
 }
@@ -445,7 +469,7 @@ void RegionRawOffset::dump() const {
 }
 
 void RegionRawOffset::dumpToStream(llvm::raw_ostream& os) const {
-  os << "raw_offset{" << getRegion() << ',' << getByteOffset() << '}';
+  os << "raw_offset{" << getRegion() << ',' << getOffset().getQuantity() << '}';
 }
 
 void StaticGlobalSpaceRegion::dumpToStream(llvm::raw_ostream &os) const {
@@ -624,7 +648,7 @@ MemRegionManager::getCompoundLiteralRegion(const CompoundLiteralExpr* CL,
 }
 
 const ElementRegion*
-MemRegionManager::getElementRegion(QualType elementType, SVal Idx,
+MemRegionManager::getElementRegion(QualType elementType, NonLoc Idx,
                                    const MemRegion* superRegion,
                                    ASTContext& Ctx){
 
@@ -675,12 +699,18 @@ MemRegionManager::getObjCIvarRegion(const ObjCIvarDecl* d,
   return getSubRegion<ObjCIvarRegion>(d, superRegion);
 }
 
-const CXXObjectRegion*
-MemRegionManager::getCXXObjectRegion(Expr const *E,
-                                     LocationContext const *LC) {
+const CXXTempObjectRegion*
+MemRegionManager::getCXXTempObjectRegion(Expr const *E,
+                                         LocationContext const *LC) {
   const StackFrameContext *SFC = LC->getCurrentStackFrame();
   assert(SFC);
-  return getSubRegion<CXXObjectRegion>(E, getStackLocalsRegion(SFC));
+  return getSubRegion<CXXTempObjectRegion>(E, getStackLocalsRegion(SFC));
+}
+
+const CXXBaseObjectRegion *
+MemRegionManager::getCXXBaseObjectRegion(const CXXRecordDecl *decl,
+                                         const MemRegion *superRegion) {
+  return getSubRegion<CXXBaseObjectRegion>(decl, superRegion);
 }
 
 const CXXThisRegion*
@@ -740,6 +770,7 @@ const MemRegion *MemRegion::getBaseRegion() const {
       case MemRegion::ElementRegionKind:
       case MemRegion::FieldRegionKind:
       case MemRegion::ObjCIvarRegionKind:
+      case MemRegion::CXXBaseObjectRegionKind:
         R = cast<SubRegion>(R)->getSuperRegion();
         continue;
       default:
@@ -824,7 +855,7 @@ RegionRawOffset ElementRegion::getAsArrayOffset() const {
   }
 
   assert(superR && "super region cannot be NULL");
-  return RegionRawOffset(superR, offset.getQuantity());
+  return RegionRawOffset(superR, offset);
 }
 
 RegionOffset MemRegion::getAsOffset() const {
@@ -841,7 +872,7 @@ RegionOffset MemRegion::getAsOffset() const {
     case CXXThisRegionKind:
     case StringRegionKind:
     case VarRegionKind:
-    case CXXObjectRegionKind:
+    case CXXTempObjectRegionKind:
       goto Finish;
     case ElementRegionKind: {
       const ElementRegion *ER = cast<ElementRegion>(R);
diff --git a/lib/StaticAnalyzer/Core/ObjCMessage.cpp b/lib/StaticAnalyzer/Core/ObjCMessage.cpp
new file mode 100644
index 000000000000..2e370d62e440
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/ObjCMessage.cpp
@@ -0,0 +1,99 @@
+//===- ObjCMessage.cpp - Wrapper for ObjC messages and dot syntax -*- C++ -*--//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines ObjCMessage which serves as a common wrapper for ObjC
+// message expressions or implicit messages for loading/storing ObjC properties.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/ObjCMessage.h"
+
+using namespace clang;
+using namespace ento;
+
+QualType ObjCMessage::getType(ASTContext &ctx) const {
+  assert(isValid() && "This ObjCMessage is uninitialized!");
+  if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+    return msgE->getType();
+  const ObjCPropertyRefExpr *propE = cast<ObjCPropertyRefExpr>(MsgOrPropE);
+  if (isPropertySetter())
+    return ctx.VoidTy;
+  return propE->getType();
+}
+
+Selector ObjCMessage::getSelector() const {
+  assert(isValid() && "This ObjCMessage is uninitialized!");
+  if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+    return msgE->getSelector();
+  const ObjCPropertyRefExpr *propE = cast<ObjCPropertyRefExpr>(MsgOrPropE);
+  if (isPropertySetter())
+    return propE->getSetterSelector();
+  return propE->getGetterSelector();
+}
+
+const ObjCMethodDecl *ObjCMessage::getMethodDecl() const {
+  assert(isValid() && "This ObjCMessage is uninitialized!");
+  if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+    return msgE->getMethodDecl();
+  const ObjCPropertyRefExpr *propE = cast<ObjCPropertyRefExpr>(MsgOrPropE);
+  if (propE->isImplicitProperty())
+    return isPropertySetter() ? propE->getImplicitPropertySetter()
+                              : propE->getImplicitPropertyGetter();
+  return 0;
+}
+
+const ObjCInterfaceDecl *ObjCMessage::getReceiverInterface() const {
+  assert(isValid() && "This ObjCMessage is uninitialized!");
+  if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+    return msgE->getReceiverInterface();
+  const ObjCPropertyRefExpr *propE = cast<ObjCPropertyRefExpr>(MsgOrPropE);
+  if (propE->isClassReceiver())
+    return propE->getClassReceiver();
+  QualType recT;
+  if (const Expr *recE = getInstanceReceiver())
+    recT = recE->getType();
+  else {
+    assert(propE->isSuperReceiver());
+    recT = propE->getSuperReceiverType();
+  }
+  if (const ObjCObjectPointerType *Ptr = recT->getAs<ObjCObjectPointerType>())
+    return Ptr->getInterfaceDecl();
+  return 0;
+}
+
+const Expr *ObjCMessage::getArgExpr(unsigned i) const {
+  assert(isValid() && "This ObjCMessage is uninitialized!");
+  assert(i < getNumArgs() && "Invalid index for argument");
+  if (const ObjCMessageExpr *msgE = dyn_cast<ObjCMessageExpr>(MsgOrPropE))
+    return msgE->getArg(i);
+  assert(isPropertySetter());
+  if (const BinaryOperator *bop = dyn_cast<BinaryOperator>(OriginE))
+    if (bop->isAssignmentOp())
+      return bop->getRHS();
+  return 0;
+}
+
+QualType CallOrObjCMessage::getResultType(ASTContext &ctx) const {
+  if (CallE) {
+    const Expr *Callee = CallE->getCallee();
+    if (const FunctionDecl *FD = State->getSVal(Callee).getAsFunctionDecl())
+      return FD->getResultType();
+    return CallE->getType();
+  }
+  return Msg.getResultType(ctx);
+}
+
+SVal CallOrObjCMessage::getArgSValAsScalarOrLoc(unsigned i) const {
+  assert(i < getNumArgs());
+  if (CallE) return State->getSValAsScalarOrLoc(CallE->getArg(i));
+  QualType argT = Msg.getArgType(i);
+  if (Loc::isLocType(argT) || argT->isIntegerType())
+    return Msg.getArgSVal(i, State);
+  return UnknownVal();
+}
diff --git a/lib/Checker/PathDiagnostic.cpp b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
index cf05a7df67f3..872bbfe9e160 100644
--- a/lib/Checker/PathDiagnostic.cpp
+++ b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
@@ -20,6 +20,7 @@
 #include "llvm/Support/Casting.h"
 
 using namespace clang;
+using namespace ento;
 using llvm::dyn_cast;
 using llvm::isa;
 
@@ -84,6 +85,8 @@ PathDiagnostic::PathDiagnostic(llvm::StringRef bugtype, llvm::StringRef desc,
 
 void PathDiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
                                             const DiagnosticInfo &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticClient::HandleDiagnostic(DiagLevel, Info);
 
   // Create a PathDiagnostic with a single piece.
 
@@ -104,7 +107,9 @@ void PathDiagnosticClient::HandleDiagnostic(Diagnostic::Level DiagLevel,
   Info.FormatDiagnostic(StrC);
 
   PathDiagnosticPiece *P =
-    new PathDiagnosticEventPiece(Info.getLocation(), StrC.str());
+    new PathDiagnosticEventPiece(FullSourceLoc(Info.getLocation(),
+                                               Info.getSourceManager()),
+                                 StrC.str());
 
   for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i)
     P->addRange(Info.getRange(i).getAsRange());
@@ -168,6 +173,7 @@ PathDiagnosticRange PathDiagnosticLocation::asRange() const {
         case Stmt::ChooseExprClass:
         case Stmt::IndirectGotoStmtClass:
         case Stmt::SwitchStmtClass:
+        case Stmt::BinaryConditionalOperatorClass:
         case Stmt::ConditionalOperatorClass:
         case Stmt::ObjCForCollectionStmtClass: {
           SourceLocation L = S->getLocStart();
diff --git a/lib/Checker/PlistDiagnostics.cpp b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
index 13accbbff8c7..fbbbd46ac149 100644
--- a/lib/Checker/PlistDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathDiagnosticClients.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Lex/Preprocessor.h"
@@ -21,6 +21,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 using namespace clang;
+using namespace ento;
 using llvm::cast;
 
 typedef llvm::DenseMap<FileID, unsigned> FIDMap;
@@ -97,8 +98,8 @@ PlistDiagnostics::PlistDiagnostics(const std::string& output,
   : OutputFile(output), LangOpts(LO), SubPD(subPD), flushed(false) {}
 
 PathDiagnosticClient*
-clang::CreatePlistDiagnosticClient(const std::string& s, const Preprocessor &PP,
-                                   PathDiagnosticClient *subPD) {
+ento::createPlistDiagnosticClient(const std::string& s, const Preprocessor &PP,
+                                  PathDiagnosticClient *subPD) {
   return new PlistDiagnostics(s, PP.getLangOptions(), subPD);
 }
 
diff --git a/lib/Checker/RangeConstraintManager.cpp b/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 697694e72a36..389fff5ed593 100644
--- a/lib/Checker/RangeConstraintManager.cpp
+++ b/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -13,16 +13,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "SimpleConstraintManager.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/GRTransferFuncs.h"
-#include "clang/Checker/ManagerRegistry.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableSet.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace { class ConstraintRange {}; }
 static int ConstraintRangeIndex = 0;
@@ -91,7 +91,7 @@ public:
 
   /// Construct a new RangeSet representing '{ [from, to] }'.
   RangeSet(Factory &F, const llvm::APSInt &from, const llvm::APSInt &to)
-    : ranges(F.Add(F.GetEmptySet(), Range(from, to))) {}
+    : ranges(F.add(F.getEmptySet(), Range(from, to))) {}
 
   /// Profile - Generates a hash profile of this RangeSet for use
   ///  by FoldingSet.
@@ -129,17 +129,17 @@ private:
 
       if (i->Includes(Lower)) {
         if (i->Includes(Upper)) {
-          newRanges = F.Add(newRanges, Range(BV.getValue(Lower),
+          newRanges = F.add(newRanges, Range(BV.getValue(Lower),
                                              BV.getValue(Upper)));
           break;
         } else
-          newRanges = F.Add(newRanges, Range(BV.getValue(Lower), i->To()));
+          newRanges = F.add(newRanges, Range(BV.getValue(Lower), i->To()));
       } else {
         if (i->Includes(Upper)) {
-          newRanges = F.Add(newRanges, Range(i->From(), BV.getValue(Upper)));
+          newRanges = F.add(newRanges, Range(i->From(), BV.getValue(Upper)));
           break;
         } else
-          newRanges = F.Add(newRanges, *i);
+          newRanges = F.add(newRanges, *i);
       }
     }
   }
@@ -155,7 +155,7 @@ public:
   RangeSet Intersect(BasicValueFactory &BV, Factory &F,
                      const llvm::APSInt &Lower,
                      const llvm::APSInt &Upper) const {
-    PrimRangeSet newRanges = F.GetEmptySet();
+    PrimRangeSet newRanges = F.getEmptySet();
 
     PrimRangeSet::iterator i = begin(), e = end();
     if (Lower <= Upper)
@@ -194,41 +194,43 @@ public:
 typedef llvm::ImmutableMap<SymbolRef,RangeSet> ConstraintRangeTy;
 
 namespace clang {
+namespace ento {
 template<>
 struct GRStateTrait<ConstraintRange>
   : public GRStatePartialTrait<ConstraintRangeTy> {
   static inline void* GDMIndex() { return &ConstraintRangeIndex; }
 };
 }
+}
 
 namespace {
 class RangeConstraintManager : public SimpleConstraintManager{
   RangeSet GetRange(const GRState *state, SymbolRef sym);
 public:
-  RangeConstraintManager(GRSubEngine &subengine)
+  RangeConstraintManager(SubEngine &subengine)
     : SimpleConstraintManager(subengine) {}
 
-  const GRState* AssumeSymNE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymNE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymEQ(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymEQ(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymLT(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymLT(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymGT(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymGT(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymGE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymGE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
-  const GRState* AssumeSymLE(const GRState* state, SymbolRef sym,
+  const GRState *assumeSymLE(const GRState* state, SymbolRef sym,
                              const llvm::APSInt& Int,
                              const llvm::APSInt& Adjustment);
 
@@ -240,7 +242,7 @@ public:
     return i ? *i == V : false;
   }
 
-  const GRState* RemoveDeadBindings(const GRState* St, SymbolReaper& SymReaper);
+  const GRState* removeDeadBindings(const GRState* St, SymbolReaper& SymReaper);
 
   void print(const GRState* St, llvm::raw_ostream& Out,
              const char* nl, const char *sep);
@@ -251,8 +253,8 @@ private:
 
 } // end anonymous namespace
 
-ConstraintManager* clang::CreateRangeConstraintManager(GRStateManager&,
-                                                       GRSubEngine &subeng) {
+ConstraintManager* ento::CreateRangeConstraintManager(GRStateManager&,
+                                                    SubEngine &subeng) {
   return new RangeConstraintManager(subeng);
 }
 
@@ -265,7 +267,7 @@ const llvm::APSInt* RangeConstraintManager::getSymVal(const GRState* St,
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
 const GRState*
-RangeConstraintManager::RemoveDeadBindings(const GRState* state,
+RangeConstraintManager::removeDeadBindings(const GRState* state,
                                            SymbolReaper& SymReaper) {
 
   ConstraintRangeTy CR = state->get<ConstraintRange>();
@@ -274,7 +276,7 @@ RangeConstraintManager::RemoveDeadBindings(const GRState* state,
   for (ConstraintRangeTy::iterator I = CR.begin(), E = CR.end(); I != E; ++I) {
     SymbolRef sym = I.getKey();
     if (SymReaper.maybeDead(sym))
-      CR = CRFactory.Remove(CR, sym);
+      CR = CRFactory.remove(CR, sym);
   }
 
   return state->set<ConstraintRange>(CR);
@@ -293,7 +295,7 @@ RangeConstraintManager::GetRange(const GRState *state, SymbolRef sym) {
 }
 
 //===------------------------------------------------------------------------===
-// AssumeSymX methods: public interface for RangeConstraintManager.
+// assumeSymX methods: public interface for RangeConstraintManager.
 //===------------------------------------------------------------------------===/
 
 // The syntax for ranges below is mathematical, using [x, y] for closed ranges
@@ -305,7 +307,7 @@ RangeConstraintManager::GetRange(const GRState *state, SymbolRef sym) {
 // UINT_MAX, 0, 1, and 2.
 
 const GRState*
-RangeConstraintManager::AssumeSymNE(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymNE(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -322,7 +324,7 @@ RangeConstraintManager::AssumeSymNE(const GRState* state, SymbolRef sym,
 }
 
 const GRState*
-RangeConstraintManager::AssumeSymEQ(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymEQ(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   // [Int-Adjustment, Int-Adjustment]
@@ -333,7 +335,7 @@ RangeConstraintManager::AssumeSymEQ(const GRState* state, SymbolRef sym,
 }
 
 const GRState*
-RangeConstraintManager::AssumeSymLT(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymLT(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -354,7 +356,7 @@ RangeConstraintManager::AssumeSymLT(const GRState* state, SymbolRef sym,
 }
 
 const GRState*
-RangeConstraintManager::AssumeSymGT(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymGT(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -375,7 +377,7 @@ RangeConstraintManager::AssumeSymGT(const GRState* state, SymbolRef sym,
 }
 
 const GRState*
-RangeConstraintManager::AssumeSymGE(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymGE(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
@@ -397,7 +399,7 @@ RangeConstraintManager::AssumeSymGE(const GRState* state, SymbolRef sym,
 }
 
 const GRState*
-RangeConstraintManager::AssumeSymLE(const GRState* state, SymbolRef sym,
+RangeConstraintManager::assumeSymLE(const GRState* state, SymbolRef sym,
                                     const llvm::APSInt& Int,
                                     const llvm::APSInt& Adjustment) {
   BasicValueFactory &BV = state->getBasicVals();
diff --git a/lib/Checker/RegionStore.cpp b/lib/StaticAnalyzer/Core/RegionStore.cpp
index 1a3eded7cb06..19e0e1257215 100644
--- a/lib/Checker/RegionStore.cpp
+++ b/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -20,15 +20,16 @@
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/GRStateTrait.h"
-#include "clang/Checker/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRStateTrait.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "llvm/ADT/ImmutableList.h"
 #include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 using llvm::Optional;
 
 //===----------------------------------------------------------------------===//
@@ -85,7 +86,7 @@ BindingKey BindingKey::Make(const MemRegion *R, Kind k) {
     // FIXME: There are some ElementRegions for which we cannot compute
     // raw offsets yet, including regions with symbolic offsets. These will be
     // ignored by the store.
-    return BindingKey(O.getRegion(), O.getByteOffset(), k);
+    return BindingKey(O.getRegion(), O.getOffset().getQuantity(), k);
   }
 
   return BindingKey(R, 0, k);
@@ -148,11 +149,11 @@ public:
     Map::iterator I = M.find(Parent);
 
     if (I == M.end()) {
-      M.insert(std::make_pair(Parent, F.Add(F.GetEmptySet(), SubRegion)));
+      M.insert(std::make_pair(Parent, F.add(F.getEmptySet(), SubRegion)));
       return true;
     }
 
-    I->second = F.Add(I->second, SubRegion);
+    I->second = F.add(I->second, SubRegion);
     return false;
   }
 
@@ -214,86 +215,98 @@ public:
   /// setImplicitDefaultValue - Set the default binding for the provided
   ///  MemRegion to the value implicitly defined for compound literals when
   ///  the value is not specified.
-  Store setImplicitDefaultValue(Store store, const MemRegion *R, QualType T);
+  StoreRef setImplicitDefaultValue(Store store, const MemRegion *R, QualType T);
 
   /// ArrayToPointer - Emulates the "decay" of an array to a pointer
   ///  type.  'Array' represents the lvalue of the array being decayed
   ///  to a pointer, and the returned SVal represents the decayed
   ///  version of that lvalue (i.e., a pointer to the first element of
-  ///  the array).  This is called by GRExprEngine when evaluating
+  ///  the array).  This is called by ExprEngine when evaluating
   ///  casts from arrays to pointers.
   SVal ArrayToPointer(Loc Array);
 
-  SVal EvalBinOp(BinaryOperator::Opcode Op,Loc L, NonLoc R, QualType resultTy);
+  /// For DerivedToBase casts, create a CXXBaseObjectRegion and return it.
+  virtual SVal evalDerivedToBase(SVal derived, QualType basePtrType);
 
-  Store getInitialStore(const LocationContext *InitLoc) {
-    return RBFactory.GetEmptyMap().getRoot();
+  StoreRef getInitialStore(const LocationContext *InitLoc) {
+    return StoreRef(RBFactory.getEmptyMap().getRootWithoutRetain(), *this);
   }
 
   //===-------------------------------------------------------------------===//
   // Binding values to regions.
   //===-------------------------------------------------------------------===//
 
-  Store InvalidateRegions(Store store,
-                          const MemRegion * const *Begin,
-                          const MemRegion * const *End,
-                          const Expr *E, unsigned Count,
-                          InvalidatedSymbols *IS,
-                          bool invalidateGlobals,
-                          InvalidatedRegions *Regions);
+  StoreRef invalidateRegions(Store store,
+                             const MemRegion * const *Begin,
+                             const MemRegion * const *End,
+                             const Expr *E, unsigned Count,
+                             InvalidatedSymbols *IS,
+                             bool invalidateGlobals,
+                             InvalidatedRegions *Regions);
 
 public:   // Made public for helper classes.
 
   void RemoveSubRegionBindings(RegionBindings &B, const MemRegion *R,
                                RegionStoreSubRegionMap &M);
 
-  RegionBindings Add(RegionBindings B, BindingKey K, SVal V);
+  RegionBindings addBinding(RegionBindings B, BindingKey K, SVal V);
 
-  RegionBindings Add(RegionBindings B, const MemRegion *R,
+  RegionBindings addBinding(RegionBindings B, const MemRegion *R,
                      BindingKey::Kind k, SVal V);
 
-  const SVal *Lookup(RegionBindings B, BindingKey K);
-  const SVal *Lookup(RegionBindings B, const MemRegion *R, BindingKey::Kind k);
+  const SVal *lookup(RegionBindings B, BindingKey K);
+  const SVal *lookup(RegionBindings B, const MemRegion *R, BindingKey::Kind k);
 
-  RegionBindings Remove(RegionBindings B, BindingKey K);
-  RegionBindings Remove(RegionBindings B, const MemRegion *R,
+  RegionBindings removeBinding(RegionBindings B, BindingKey K);
+  RegionBindings removeBinding(RegionBindings B, const MemRegion *R,
                         BindingKey::Kind k);
 
-  RegionBindings Remove(RegionBindings B, const MemRegion *R) {
-    return Remove(Remove(B, R, BindingKey::Direct), R, BindingKey::Default);
+  RegionBindings removeBinding(RegionBindings B, const MemRegion *R) {
+    return removeBinding(removeBinding(B, R, BindingKey::Direct), R,
+                        BindingKey::Default);
   }
 
 public: // Part of public interface to class.
 
-  Store Bind(Store store, Loc LV, SVal V);
+  StoreRef Bind(Store store, Loc LV, SVal V);
 
   // BindDefault is only used to initialize a region with a default value.
-  Store BindDefault(Store store, const MemRegion *R, SVal V) {
+  StoreRef BindDefault(Store store, const MemRegion *R, SVal V) {
     RegionBindings B = GetRegionBindings(store);
-    assert(!Lookup(B, R, BindingKey::Default));
-    assert(!Lookup(B, R, BindingKey::Direct));
-    return Add(B, R, BindingKey::Default, V).getRoot();
+    assert(!lookup(B, R, BindingKey::Default));
+    assert(!lookup(B, R, BindingKey::Direct));
+    return StoreRef(addBinding(B, R, BindingKey::Default, V).getRootWithoutRetain(), *this);
   }
 
-  Store BindCompoundLiteral(Store store, const CompoundLiteralExpr* CL,
-                            const LocationContext *LC, SVal V);
+  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr* CL,
+                               const LocationContext *LC, SVal V);
 
-  Store BindDecl(Store store, const VarRegion *VR, SVal InitVal);
+  StoreRef BindDecl(Store store, const VarRegion *VR, SVal InitVal);
 
-  Store BindDeclWithNoInit(Store store, const VarRegion *) {
-    return store;
+  StoreRef BindDeclWithNoInit(Store store, const VarRegion *) {
+    return StoreRef(store, *this);
   }
 
   /// BindStruct - Bind a compound value to a structure.
-  Store BindStruct(Store store, const TypedRegion* R, SVal V);
+  StoreRef BindStruct(Store store, const TypedRegion* R, SVal V);
 
-  Store BindArray(Store store, const TypedRegion* R, SVal V);
+  StoreRef BindArray(Store store, const TypedRegion* R, SVal V);
 
   /// KillStruct - Set the entire struct to unknown.
-  Store KillStruct(Store store, const TypedRegion* R, SVal DefaultVal);
+  StoreRef KillStruct(Store store, const TypedRegion* R, SVal DefaultVal);
 
-  Store Remove(Store store, Loc LV);
+  StoreRef Remove(Store store, Loc LV);
 
+  void incrementReferenceCount(Store store) {
+    GetRegionBindings(store).manualRetain();    
+  }
+  
+  /// If the StoreManager supports it, decrement the reference count of
+  /// the specified Store object.  If the reference count hits 0, the memory
+  /// associated with the object is recycled.
+  void decrementReferenceCount(Store store) {
+    GetRegionBindings(store).manualRelease();
+  }
 
   //===------------------------------------------------------------------===//
   // Loading values from regions.
@@ -344,20 +357,20 @@ public: // Part of public interface to class.
   std::pair<Store, const MemRegion*>
   GetLazyBinding(RegionBindings B, const MemRegion *R);
 
-  Store CopyLazyBindings(nonloc::LazyCompoundVal V, Store store,
-                         const TypedRegion *R);
+  StoreRef CopyLazyBindings(nonloc::LazyCompoundVal V, Store store,
+                            const TypedRegion *R);
 
   //===------------------------------------------------------------------===//
   // State pruning.
   //===------------------------------------------------------------------===//
 
-  /// RemoveDeadBindings - Scans the RegionStore of 'state' for dead values.
+  /// removeDeadBindings - Scans the RegionStore of 'state' for dead values.
   ///  It returns a new Store with these values removed.
-  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
+  StoreRef removeDeadBindings(Store store, const StackFrameContext *LCtx,
                            SymbolReaper& SymReaper,
                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
 
-  Store EnterStackFrame(const GRState *state, const StackFrameContext *frame);
+  StoreRef enterStackFrame(const GRState *state, const StackFrameContext *frame);
 
   //===------------------------------------------------------------------===//
   // Region "extents".
@@ -399,12 +412,12 @@ public: // Part of public interface to class.
 // RegionStore creation.
 //===----------------------------------------------------------------------===//
 
-StoreManager *clang::CreateRegionStoreManager(GRStateManager& StMgr) {
+StoreManager *ento::CreateRegionStoreManager(GRStateManager& StMgr) {
   RegionStoreFeatures F = maximal_features_tag();
   return new RegionStoreManager(StMgr, F);
 }
 
-StoreManager *clang::CreateFieldsOnlyRegionStoreManager(GRStateManager &StMgr) {
+StoreManager *ento::CreateFieldsOnlyRegionStoreManager(GRStateManager &StMgr) {
   RegionStoreFeatures F = minimal_features_tag();
   F.enableFields(true);
   return new RegionStoreManager(StMgr, F);
@@ -453,15 +466,18 @@ protected:
 
   RegionStoreManager &RM;
   ASTContext &Ctx;
-  ValueManager &ValMgr;
+  SValBuilder &svalBuilder;
 
   RegionBindings B;
+  
+  const bool includeGlobals;
 
 public:
   ClusterAnalysis(RegionStoreManager &rm, GRStateManager &StateMgr,
-                  RegionBindings b)
-    : RM(rm), Ctx(StateMgr.getContext()), ValMgr(StateMgr.getValueManager()),
-      B(b) {}
+                  RegionBindings b, const bool includeGlobals)
+    : RM(rm), Ctx(StateMgr.getContext()),
+      svalBuilder(StateMgr.getSValBuilder()),
+      B(b), includeGlobals(includeGlobals) {}
 
   RegionBindings getRegionBindings() const { return B; }
 
@@ -487,7 +503,7 @@ public:
     return *CRef;
   }
 
-  void GenerateClusters(bool includeGlobals = false) {
+  void GenerateClusters() {
       // Scan the entire set of bindings and make the region clusters.
     for (RegionBindings::iterator RI = B.begin(), RE = B.end(); RI != RE; ++RI){
       RegionCluster &C = AddToCluster(RI.getKey());
@@ -558,24 +574,25 @@ void RegionStoreManager::RemoveSubRegionBindings(RegionBindings &B,
          I != E; ++I)
       RemoveSubRegionBindings(B, *I, M);
 
-  B = Remove(B, R);
+  B = removeBinding(B, R);
 }
 
 namespace {
-class InvalidateRegionsWorker : public ClusterAnalysis<InvalidateRegionsWorker>
+class invalidateRegionsWorker : public ClusterAnalysis<invalidateRegionsWorker>
 {
   const Expr *Ex;
   unsigned Count;
   StoreManager::InvalidatedSymbols *IS;
   StoreManager::InvalidatedRegions *Regions;
 public:
-  InvalidateRegionsWorker(RegionStoreManager &rm,
+  invalidateRegionsWorker(RegionStoreManager &rm,
                           GRStateManager &stateMgr,
                           RegionBindings b,
                           const Expr *ex, unsigned count,
                           StoreManager::InvalidatedSymbols *is,
-                          StoreManager::InvalidatedRegions *r)
-    : ClusterAnalysis<InvalidateRegionsWorker>(rm, stateMgr, b),
+                          StoreManager::InvalidatedRegions *r,
+                          bool includeGlobals)
+    : ClusterAnalysis<invalidateRegionsWorker>(rm, stateMgr, b, includeGlobals),
       Ex(ex), Count(count), IS(is), Regions(r) {}
 
   void VisitCluster(const MemRegion *baseR, BindingKey *I, BindingKey *E);
@@ -586,7 +603,7 @@ private:
 };
 }
 
-void InvalidateRegionsWorker::VisitBinding(SVal V) {
+void invalidateRegionsWorker::VisitBinding(SVal V) {
   // A symbol?  Mark it touched by the invalidation.
   if (IS)
     if (SymbolRef Sym = V.getAsSymbol())
@@ -614,19 +631,19 @@ void InvalidateRegionsWorker::VisitBinding(SVal V) {
   }
 }
 
-void InvalidateRegionsWorker::VisitCluster(const MemRegion *baseR,
+void invalidateRegionsWorker::VisitCluster(const MemRegion *baseR,
                                            BindingKey *I, BindingKey *E) {
   for ( ; I != E; ++I) {
     // Get the old binding.  Is it a region?  If so, add it to the worklist.
     const BindingKey &K = *I;
-    if (const SVal *V = RM.Lookup(B, K))
+    if (const SVal *V = RM.lookup(B, K))
       VisitBinding(*V);
 
-    B = RM.Remove(B, K);
+    B = RM.removeBinding(B, K);
   }
 }
 
-void InvalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
+void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (IS) {
     // Symbolic region?  Mark that symbol touched by the invalidation.
     if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(baseR))
@@ -654,9 +671,9 @@ void InvalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (isa<AllocaRegion>(baseR) || isa<SymbolicRegion>(baseR)) {
     // Invalidate the region by setting its default value to
     // conjured symbol. The type of the symbol is irrelavant.
-    DefinedOrUnknownSVal V = ValMgr.getConjuredSymbolVal(baseR, Ex, Ctx.IntTy,
-                                                         Count);
-    B = RM.Add(B, baseR, BindingKey::Default, V);
+    DefinedOrUnknownSVal V =
+      svalBuilder.getConjuredSymbolVal(baseR, Ex, Ctx.IntTy, Count);
+    B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
   }
 
@@ -670,38 +687,48 @@ void InvalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (T->isStructureType()) {
     // Invalidate the region by setting its default value to
     // conjured symbol. The type of the symbol is irrelavant.
-    DefinedOrUnknownSVal V = ValMgr.getConjuredSymbolVal(baseR, Ex, Ctx.IntTy,
+    DefinedOrUnknownSVal V = svalBuilder.getConjuredSymbolVal(baseR, Ex, Ctx.IntTy,
                                                          Count);
-    B = RM.Add(B, baseR, BindingKey::Default, V);
+    B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
   }
 
   if (const ArrayType *AT = Ctx.getAsArrayType(T)) {
       // Set the default value of the array to conjured symbol.
     DefinedOrUnknownSVal V =
-    ValMgr.getConjuredSymbolVal(baseR, Ex, AT->getElementType(), Count);
-    B = RM.Add(B, baseR, BindingKey::Default, V);
+    svalBuilder.getConjuredSymbolVal(baseR, Ex, AT->getElementType(), Count);
+    B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
   }
+  
+  if (includeGlobals && 
+      isa<NonStaticGlobalSpaceRegion>(baseR->getMemorySpace())) {
+    // If the region is a global and we are invalidating all globals,
+    // just erase the entry.  This causes all globals to be lazily
+    // symbolicated from the same base symbol.
+    B = RM.removeBinding(B, baseR);
+    return;
+  }
+  
 
-  DefinedOrUnknownSVal V = ValMgr.getConjuredSymbolVal(baseR, Ex, T, Count);
+  DefinedOrUnknownSVal V = svalBuilder.getConjuredSymbolVal(baseR, Ex, T, Count);
   assert(SymbolManager::canSymbolicate(T) || V.isUnknown());
-  B = RM.Add(B, baseR, BindingKey::Direct, V);
+  B = RM.addBinding(B, baseR, BindingKey::Direct, V);
 }
 
-Store RegionStoreManager::InvalidateRegions(Store store,
-                                            const MemRegion * const *I,
-                                            const MemRegion * const *E,
-                                            const Expr *Ex, unsigned Count,
-                                            InvalidatedSymbols *IS,
-                                            bool invalidateGlobals,
-                                            InvalidatedRegions *Regions) {
-  InvalidateRegionsWorker W(*this, StateMgr,
+StoreRef RegionStoreManager::invalidateRegions(Store store,
+                                               const MemRegion * const *I,
+                                               const MemRegion * const *E,
+                                               const Expr *Ex, unsigned Count,
+                                               InvalidatedSymbols *IS,
+                                               bool invalidateGlobals,
+                                               InvalidatedRegions *Regions) {
+  invalidateRegionsWorker W(*this, StateMgr,
                             RegionStoreManager::GetRegionBindings(store),
-                            Ex, Count, IS, Regions);
+                            Ex, Count, IS, Regions, invalidateGlobals);
 
   // Scan the bindings and generate the clusters.
-  W.GenerateClusters(invalidateGlobals);
+  W.GenerateClusters();
 
   // Add I .. E to the worklist.
   for ( ; I != E; ++I)
@@ -717,10 +744,10 @@ Store RegionStoreManager::InvalidateRegions(Store store,
     // use to derive the bindings for all non-static globals.
     const GlobalsSpaceRegion *GS = MRMgr.getGlobalsRegion();
     SVal V =
-      ValMgr.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, Ex,
+      svalBuilder.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, Ex,
                                   /* symbol type, doesn't matter */ Ctx.IntTy,
                                   Count);
-    B = Add(B, BindingKey::Make(GS, BindingKey::Default), V);
+    B = addBinding(B, BindingKey::Make(GS, BindingKey::Default), V);
 
     // Even if there are no bindings in the global scope, we still need to
     // record that we touched it.
@@ -728,7 +755,7 @@ Store RegionStoreManager::InvalidateRegions(Store store,
       Regions->push_back(GS);
   }
 
-  return B.getRoot();
+  return StoreRef(B.getRootWithoutRetain(), *this);
 }
 
 //===----------------------------------------------------------------------===//
@@ -738,19 +765,26 @@ Store RegionStoreManager::InvalidateRegions(Store store,
 DefinedOrUnknownSVal RegionStoreManager::getSizeInElements(const GRState *state,
                                                            const MemRegion *R,
                                                            QualType EleTy) {
-  SVal Size = cast<SubRegion>(R)->getExtent(ValMgr);
-  SValuator &SVator = ValMgr.getSValuator();
-  const llvm::APSInt *SizeInt = SVator.getKnownValue(state, Size);
+  SVal Size = cast<SubRegion>(R)->getExtent(svalBuilder);
+  const llvm::APSInt *SizeInt = svalBuilder.getKnownValue(state, Size);
   if (!SizeInt)
     return UnknownVal();
 
   CharUnits RegionSize = CharUnits::fromQuantity(SizeInt->getSExtValue());
+
+  if (Ctx.getAsVariableArrayType(EleTy)) {
+    // FIXME: We need to track extra state to properly record the size
+    // of VLAs.  Returning UnknownVal here, however, is a stop-gap so that
+    // we don't have a divide-by-zero below.
+    return UnknownVal();
+  }
+
   CharUnits EleSize = Ctx.getTypeSizeInChars(EleTy);
 
   // If a variable is reinterpreted as a type that doesn't fit into a larger
   // type evenly, round it down.
   // This is a signed value, since it's used in arithmetic with signed indices.
-  return ValMgr.makeIntVal(RegionSize / EleSize, false);
+  return svalBuilder.makeIntVal(RegionSize / EleSize, false);
 }
 
 //===----------------------------------------------------------------------===//
@@ -761,7 +795,7 @@ DefinedOrUnknownSVal RegionStoreManager::getSizeInElements(const GRState *state,
 ///  type.  'Array' represents the lvalue of the array being decayed
 ///  to a pointer, and the returned SVal represents the decayed
 ///  version of that lvalue (i.e., a pointer to the first element of
-///  the array).  This is called by GRExprEngine when evaluating casts
+///  the array).  This is called by ExprEngine when evaluating casts
 ///  from arrays to pointers.
 SVal RegionStoreManager::ArrayToPointer(Loc Array) {
   if (!isa<loc::MemRegionVal>(Array))
@@ -774,127 +808,31 @@ SVal RegionStoreManager::ArrayToPointer(Loc Array) {
     return UnknownVal();
 
   // Strip off typedefs from the ArrayRegion's ValueType.
-  QualType T = ArrayR->getValueType().getDesugaredType();
-  ArrayType *AT = cast<ArrayType>(T);
+  QualType T = ArrayR->getValueType().getDesugaredType(Ctx);
+  const ArrayType *AT = cast<ArrayType>(T);
   T = AT->getElementType();
 
-  SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
+  NonLoc ZeroIdx = svalBuilder.makeZeroArrayIndex();
   return loc::MemRegionVal(MRMgr.getElementRegion(T, ZeroIdx, ArrayR, Ctx));
 }
 
-//===----------------------------------------------------------------------===//
-// Pointer arithmetic.
-//===----------------------------------------------------------------------===//
-
-SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
-                                   QualType resultTy) {
-  // Assume the base location is MemRegionVal.
-  if (!isa<loc::MemRegionVal>(L))
-    return UnknownVal();
-
-  // Special case for zero RHS.
-  if (R.isZeroConstant()) {
-    switch (Op) {
-    default:
-      // Handle it normally.
-      break;
-    case BO_Add:
-    case BO_Sub:
-      // FIXME: does this need to be casted to match resultTy?
-      return L;
-    }
-  }
-
-  const MemRegion* MR = cast<loc::MemRegionVal>(L).getRegion();
-  const ElementRegion *ER = 0;
-
-  switch (MR->getKind()) {
-    case MemRegion::SymbolicRegionKind: {
-      const SymbolicRegion *SR = cast<SymbolicRegion>(MR);
-      SymbolRef Sym = SR->getSymbol();
-      QualType T = Sym->getType(Ctx);
-      QualType EleTy;
-
-      if (const PointerType *PT = T->getAs<PointerType>())
-        EleTy = PT->getPointeeType();
-      else
-        EleTy = T->getAs<ObjCObjectPointerType>()->getPointeeType();
-
-      SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, SR, Ctx);
-      break;
-    }
-    case MemRegion::AllocaRegionKind: {
-      const AllocaRegion *AR = cast<AllocaRegion>(MR);
-      QualType EleTy = Ctx.CharTy; // Create an ElementRegion of bytes.
-      SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, AR, Ctx);
-      break;
-    }
-
-    case MemRegion::ElementRegionKind: {
-      ER = cast<ElementRegion>(MR);
-      break;
-    }
-
-    // Not yet handled.
-    case MemRegion::VarRegionKind:
-    case MemRegion::StringRegionKind: {
-
-    }
-    // Fall-through.
-    case MemRegion::CompoundLiteralRegionKind:
-    case MemRegion::FieldRegionKind:
-    case MemRegion::ObjCIvarRegionKind:
-    case MemRegion::CXXObjectRegionKind:
-      return UnknownVal();
+SVal RegionStoreManager::evalDerivedToBase(SVal derived, QualType baseType) {
+  const CXXRecordDecl *baseDecl;
+  if (baseType->isPointerType())
+    baseDecl = baseType->getCXXRecordDeclForPointerType();
+  else
+    baseDecl = baseType->getAsCXXRecordDecl();
 
-    case MemRegion::FunctionTextRegionKind:
-    case MemRegion::BlockTextRegionKind:
-    case MemRegion::BlockDataRegionKind:
-      // Technically this can happen if people do funny things with casts.
-      return UnknownVal();
+  assert(baseDecl && "not a CXXRecordDecl?");
 
-    case MemRegion::CXXThisRegionKind:
-      assert(0 &&
-             "Cannot perform pointer arithmetic on implicit argument 'this'");
-    case MemRegion::GenericMemSpaceRegionKind:
-    case MemRegion::StackLocalsSpaceRegionKind:
-    case MemRegion::StackArgumentsSpaceRegionKind:
-    case MemRegion::HeapSpaceRegionKind:
-    case MemRegion::NonStaticGlobalSpaceRegionKind:
-    case MemRegion::StaticGlobalSpaceRegionKind:
-    case MemRegion::UnknownSpaceRegionKind:
-      assert(0 && "Cannot perform pointer arithmetic on a MemSpace");
-      return UnknownVal();
-  }
+  loc::MemRegionVal *derivedRegVal = dyn_cast<loc::MemRegionVal>(&derived);
+  if (!derivedRegVal)
+    return derived;
 
-  SVal Idx = ER->getIndex();
-  nonloc::ConcreteInt* Base = dyn_cast<nonloc::ConcreteInt>(&Idx);
-
-  // For now, only support:
-  //  (a) concrete integer indices that can easily be resolved
-  //  (b) 0 + symbolic index
-  if (Base) {
-    if (nonloc::ConcreteInt *Offset = dyn_cast<nonloc::ConcreteInt>(&R)) {
-      // FIXME: Should use SValuator here.
-      SVal NewIdx =
-        Base->evalBinOp(ValMgr, Op,
-                cast<nonloc::ConcreteInt>(ValMgr.convertToArrayIndex(*Offset)));
-      const MemRegion* NewER =
-        MRMgr.getElementRegion(ER->getElementType(), NewIdx,
-                               ER->getSuperRegion(), Ctx);
-      return ValMgr.makeLoc(NewER);
-    }
-    if (0 == Base->getValue()) {
-      const MemRegion* NewER =
-        MRMgr.getElementRegion(ER->getElementType(), R,
-                               ER->getSuperRegion(), Ctx);
-      return ValMgr.makeLoc(NewER);
-    }
-  }
+  const MemRegion *baseReg = 
+    MRMgr.getCXXBaseObjectRegion(baseDecl, derivedRegVal->getRegion()); 
 
-  return UnknownVal();
+  return loc::MemRegionVal(baseReg);
 }
 
 //===----------------------------------------------------------------------===//
@@ -904,7 +842,7 @@ SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
 Optional<SVal> RegionStoreManager::getDirectBinding(RegionBindings B,
                                                     const MemRegion *R) {
 
-  if (const SVal *V = Lookup(B, R, BindingKey::Direct))
+  if (const SVal *V = lookup(B, R, BindingKey::Direct))
     return *V;
 
   return Optional<SVal>();
@@ -917,7 +855,7 @@ Optional<SVal> RegionStoreManager::getDefaultBinding(RegionBindings B,
       if (TR->getValueType()->isUnionType())
         return UnknownVal();
 
-  if (const SVal *V = Lookup(B, R, BindingKey::Default))
+  if (const SVal *V = lookup(B, R, BindingKey::Default))
     return *V;
 
   return Optional<SVal>();
@@ -927,10 +865,18 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   assert(!isa<UnknownVal>(L) && "location unknown");
   assert(!isa<UndefinedVal>(L) && "location undefined");
 
-  // FIXME: Is this even possible?  Shouldn't this be treated as a null
-  //  dereference at a higher level?
-  if (isa<loc::ConcreteInt>(L))
-    return UndefinedVal();
+  // For access to concrete addresses, return UnknownVal.  Checks
+  // for null dereferences (and similar errors) are done by checkers, not
+  // the Store.
+  // FIXME: We can consider lazily symbolicating such memory, but we really
+  // should defer this when we can reason easily about symbolicating arrays
+  // of bytes.
+  if (isa<loc::ConcreteInt>(L)) {
+    return UnknownVal();
+  }
+  if (!isa<loc::MemRegionVal>(L)) {
+    return UnknownVal();
+  }
 
   const MemRegion *MR = cast<loc::MemRegionVal>(L).getRegion();
 
@@ -1008,7 +954,7 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   }
 
   RegionBindings B = GetRegionBindings(store);
-  const SVal *V = Lookup(B, R, BindingKey::Direct);
+  const SVal *V = lookup(B, R, BindingKey::Direct);
 
   // Check if the region has a binding.
   if (V)
@@ -1026,7 +972,7 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   }
 
   // All other values are symbolic.
-  return ValMgr.getRegionValueSymbolVal(R);
+  return svalBuilder.getRegionValueSymbolVal(R);
 }
 
 std::pair<Store, const MemRegion *>
@@ -1052,6 +998,17 @@ RegionStoreManager::GetLazyBinding(RegionBindings B, const MemRegion *R) {
       return std::make_pair(X.first,
                             MRMgr.getFieldRegionWithSuper(FR, X.second));
   }
+  // C++ base object region is another kind of region that we should blast
+  // through to look for lazy compound value. It is like a field region.
+  else if (const CXXBaseObjectRegion *baseReg = 
+                            dyn_cast<CXXBaseObjectRegion>(R)) {
+    const std::pair<Store, const MemRegion *> &X =
+      GetLazyBinding(B, baseReg->getSuperRegion());
+    
+    if (X.second)
+      return std::make_pair(X.first,
+                     MRMgr.getCXXBaseObjectRegionWithSuper(baseReg, X.second));
+  }
   // The NULL MemRegion indicates an non-existent lazy binding. A NULL Store is
   // possible for a valid lazy binding.
   return std::make_pair((Store) 0, (const MemRegion *) 0);
@@ -1084,7 +1041,7 @@ SVal RegionStoreManager::RetrieveElement(Store store,
       // the only time such an access would be made is if a string literal was
       // used to initialize a larger array.
       char c = (i >= byteLength) ? '\0' : Str->getString()[i];
-      return ValMgr.makeIntVal(c, T);
+      return svalBuilder.makeIntVal(c, T);
     }
   }
   
@@ -1107,7 +1064,7 @@ SVal RegionStoreManager::RetrieveElement(Store store,
         if (Ctx.getTypeSizeInChars(baseT) >= Ctx.getTypeSizeInChars(elemT)) {
           if (const Optional<SVal> &V = getDirectBinding(B, superR)) {
             if (SymbolRef parentSym = V->getAsSymbol())
-              return ValMgr.getDerivedRegionValueSymbolVal(parentSym, R);
+              return svalBuilder.getDerivedRegionValueSymbolVal(parentSym, R);
 
             if (V->isUnknownOrUndef())
               return *V;
@@ -1142,10 +1099,10 @@ RegionStoreManager::RetrieveDerivedDefaultValue(RegionBindings B,
 
   if (const Optional<SVal> &D = getDefaultBinding(B, superR)) {
     if (SymbolRef parentSym = D->getAsSymbol())
-      return ValMgr.getDerivedRegionValueSymbolVal(parentSym, R);
+      return svalBuilder.getDerivedRegionValueSymbolVal(parentSym, R);
 
     if (D->isZeroConstant())
-      return ValMgr.makeZeroVal(Ty);
+      return svalBuilder.makeZeroVal(Ty);
 
     if (D->isUnknownOrUndef())
       return *D;
@@ -1167,16 +1124,16 @@ SVal RegionStoreManager::RetrieveFieldOrElementCommon(Store store,
   RegionBindings B = GetRegionBindings(store);
 
   while (superR) {
-    if (const Optional<SVal> &D = RetrieveDerivedDefaultValue(B, superR, R, Ty))
+    if (const Optional<SVal> &D =
+        RetrieveDerivedDefaultValue(B, superR, R, Ty))
       return *D;
 
     // If our super region is a field or element itself, walk up the region
     // hierarchy to see if there is a default value installed in an ancestor.
-    if (isa<FieldRegion>(superR) || isa<ElementRegion>(superR)) {
-      superR = cast<SubRegion>(superR)->getSuperRegion();
+    if (const SubRegion *SR = dyn_cast<SubRegion>(superR)) {
+      superR = SR->getSuperRegion();
       continue;
     }
-
     break;
   }
 
@@ -1211,7 +1168,7 @@ SVal RegionStoreManager::RetrieveFieldOrElementCommon(Store store,
   }
 
   // All other values are symbolic.
-  return ValMgr.getRegionValueSymbolVal(R);
+  return svalBuilder.getRegionValueSymbolVal(R);
 }
 
 SVal RegionStoreManager::RetrieveObjCIvar(Store store, const ObjCIvarRegion* R){
@@ -1227,7 +1184,7 @@ SVal RegionStoreManager::RetrieveObjCIvar(Store store, const ObjCIvarRegion* R){
   // Check if the super region has a default binding.
   if (const Optional<SVal> &V = getDefaultBinding(B, superR)) {
     if (SymbolRef parentSym = V->getAsSymbol())
-      return ValMgr.getDerivedRegionValueSymbolVal(parentSym, R);
+      return svalBuilder.getDerivedRegionValueSymbolVal(parentSym, R);
 
     // Other cases: give up.
     return UnknownVal();
@@ -1251,7 +1208,7 @@ SVal RegionStoreManager::RetrieveVar(Store store, const VarRegion *R) {
 
   if (isa<UnknownSpaceRegion>(MS) ||
       isa<StackArgumentsSpaceRegion>(MS))
-    return ValMgr.getRegionValueSymbolVal(R);
+    return svalBuilder.getRegionValueSymbolVal(R);
 
   if (isa<GlobalsSpaceRegion>(MS)) {
     if (isa<NonStaticGlobalSpaceRegion>(MS)) {
@@ -1263,22 +1220,21 @@ SVal RegionStoreManager::RetrieveVar(Store store, const VarRegion *R) {
         if (Init)
           if (const IntegerLiteral *IL =
               dyn_cast<IntegerLiteral>(Init->IgnoreParenCasts())) {
-            const nonloc::ConcreteInt &V = ValMgr.makeIntVal(IL);
-            return ValMgr.getSValuator().EvalCast(V, Init->getType(),
-                                                  IL->getType());
+            const nonloc::ConcreteInt &V = svalBuilder.makeIntVal(IL);
+            return svalBuilder.evalCast(V, Init->getType(), IL->getType());
           }
       }
 
       if (const Optional<SVal> &V = RetrieveDerivedDefaultValue(B, MS, R, CT))
         return V.getValue();
 
-      return ValMgr.getRegionValueSymbolVal(R);
+      return svalBuilder.getRegionValueSymbolVal(R);
     }
 
     if (T->isIntegerType())
-      return ValMgr.makeIntVal(0, T);
+      return svalBuilder.makeIntVal(0, T);
     if (T->isPointerType())
-      return ValMgr.makeNull();
+      return svalBuilder.makeNull();
 
     return UnknownVal();
   }
@@ -1288,35 +1244,37 @@ SVal RegionStoreManager::RetrieveVar(Store store, const VarRegion *R) {
 
 SVal RegionStoreManager::RetrieveLazySymbol(const TypedRegion *R) {
   // All other values are symbolic.
-  return ValMgr.getRegionValueSymbolVal(R);
+  return svalBuilder.getRegionValueSymbolVal(R);
 }
 
 SVal RegionStoreManager::RetrieveStruct(Store store, const TypedRegion* R) {
   QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
-  return ValMgr.makeLazyCompoundVal(store, R);
+  return svalBuilder.makeLazyCompoundVal(store, R);
 }
 
 SVal RegionStoreManager::RetrieveArray(Store store, const TypedRegion * R) {
-  assert(isa<ConstantArrayType>(R->getValueType()));
-  return ValMgr.makeLazyCompoundVal(store, R);
+  assert(Ctx.getAsConstantArrayType(R->getValueType()));
+  return svalBuilder.makeLazyCompoundVal(store, R);
 }
 
 //===----------------------------------------------------------------------===//
 // Binding values to regions.
 //===----------------------------------------------------------------------===//
 
-Store RegionStoreManager::Remove(Store store, Loc L) {
+StoreRef RegionStoreManager::Remove(Store store, Loc L) {
   if (isa<loc::MemRegionVal>(L))
     if (const MemRegion* R = cast<loc::MemRegionVal>(L).getRegion())
-      return Remove(GetRegionBindings(store), R).getRoot();
+      return StoreRef(removeBinding(GetRegionBindings(store),
+                                    R).getRootWithoutRetain(),
+                      *this);
 
-  return store;
+  return StoreRef(store, *this);
 }
 
-Store RegionStoreManager::Bind(Store store, Loc L, SVal V) {
+StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
   if (isa<loc::ConcreteInt>(L))
-    return store;
+    return StoreRef(store, *this);
 
   // If we get here, the location should be a region.
   const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
@@ -1355,11 +1313,12 @@ Store RegionStoreManager::Bind(Store store, Loc L, SVal V) {
 
   // Perform the binding.
   RegionBindings B = GetRegionBindings(store);
-  return Add(B, R, BindingKey::Direct, V).getRoot();
+  return StoreRef(addBinding(B, R, BindingKey::Direct,
+                             V).getRootWithoutRetain(), *this);
 }
 
-Store RegionStoreManager::BindDecl(Store store, const VarRegion *VR,
-                                   SVal InitVal) {
+StoreRef RegionStoreManager::BindDecl(Store store, const VarRegion *VR,
+                                      SVal InitVal) {
 
   QualType T = VR->getDecl()->getType();
 
@@ -1368,43 +1327,43 @@ Store RegionStoreManager::BindDecl(Store store, const VarRegion *VR,
   if (T->isStructureOrClassType())
     return BindStruct(store, VR, InitVal);
 
-  return Bind(store, ValMgr.makeLoc(VR), InitVal);
+  return Bind(store, svalBuilder.makeLoc(VR), InitVal);
 }
 
 // FIXME: this method should be merged into Bind().
-Store RegionStoreManager::BindCompoundLiteral(Store store,
-                                              const CompoundLiteralExpr *CL,
-                                              const LocationContext *LC,
-                                              SVal V) {
+StoreRef RegionStoreManager::BindCompoundLiteral(Store store,
+                                                 const CompoundLiteralExpr *CL,
+                                                 const LocationContext *LC,
+                                                 SVal V) {
   return Bind(store, loc::MemRegionVal(MRMgr.getCompoundLiteralRegion(CL, LC)),
               V);
 }
 
-
-Store RegionStoreManager::setImplicitDefaultValue(Store store,
-                                                  const MemRegion *R,
-                                                  QualType T) {
+StoreRef RegionStoreManager::setImplicitDefaultValue(Store store,
+                                                     const MemRegion *R,
+                                                     QualType T) {
   RegionBindings B = GetRegionBindings(store);
   SVal V;
 
-  if (Loc::IsLocType(T))
-    V = ValMgr.makeNull();
+  if (Loc::isLocType(T))
+    V = svalBuilder.makeNull();
   else if (T->isIntegerType())
-    V = ValMgr.makeZeroVal(T);
+    V = svalBuilder.makeZeroVal(T);
   else if (T->isStructureOrClassType() || T->isArrayType()) {
     // Set the default value to a zero constant when it is a structure
     // or array.  The type doesn't really matter.
-    V = ValMgr.makeZeroVal(Ctx.IntTy);
+    V = svalBuilder.makeZeroVal(Ctx.IntTy);
   }
   else {
-    return store;
+    return StoreRef(store, *this);
   }
 
-  return Add(B, R, BindingKey::Default, V).getRoot();
+  return StoreRef(addBinding(B, R, BindingKey::Default,
+                             V).getRootWithoutRetain(), *this);
 }
 
-Store RegionStoreManager::BindArray(Store store, const TypedRegion* R,
-                                    SVal Init) {
+StoreRef RegionStoreManager::BindArray(Store store, const TypedRegion* R,
+                                       SVal Init) {
 
   const ArrayType *AT =cast<ArrayType>(Ctx.getCanonicalType(R->getValueType()));
   QualType ElementTy = AT->getElementType();
@@ -1419,7 +1378,7 @@ Store RegionStoreManager::BindArray(Store store, const TypedRegion* R,
 
     // Treat the string as a lazy compound value.
     nonloc::LazyCompoundVal LCV =
-      cast<nonloc::LazyCompoundVal>(ValMgr.makeLazyCompoundVal(store, S));
+      cast<nonloc::LazyCompoundVal>(svalBuilder.makeLazyCompoundVal(store, S));
     return CopyLazyBindings(LCV, store, R);
   }
 
@@ -1436,35 +1395,36 @@ Store RegionStoreManager::BindArray(Store store, const TypedRegion* R,
   nonloc::CompoundVal::iterator VI = CV.begin(), VE = CV.end();
   uint64_t i = 0;
 
+  StoreRef newStore(store, *this);
   for (; Size.hasValue() ? i < Size.getValue() : true ; ++i, ++VI) {
     // The init list might be shorter than the array length.
     if (VI == VE)
       break;
 
-    SVal Idx = ValMgr.makeArrayIndex(i);
+    const NonLoc &Idx = svalBuilder.makeArrayIndex(i);
     const ElementRegion *ER = MRMgr.getElementRegion(ElementTy, Idx, R, Ctx);
 
     if (ElementTy->isStructureOrClassType())
-      store = BindStruct(store, ER, *VI);
+      newStore = BindStruct(newStore.getStore(), ER, *VI);
     else if (ElementTy->isArrayType())
-      store = BindArray(store, ER, *VI);
+      newStore = BindArray(newStore.getStore(), ER, *VI);
     else
-      store = Bind(store, ValMgr.makeLoc(ER), *VI);
+      newStore = Bind(newStore.getStore(), svalBuilder.makeLoc(ER), *VI);
   }
 
   // If the init list is shorter than the array length, set the
   // array default value.
   if (Size.hasValue() && i < Size.getValue())
-    store = setImplicitDefaultValue(store, R, ElementTy);
+    newStore = setImplicitDefaultValue(newStore.getStore(), R, ElementTy);
 
-  return store;
+  return newStore;
 }
 
-Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
-                                     SVal V) {
+StoreRef RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
+                                        SVal V) {
 
   if (!Features.supportsFields())
-    return store;
+    return StoreRef(store, *this);
 
   QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
@@ -1473,7 +1433,7 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   RecordDecl* RD = RT->getDecl();
 
   if (!RD->isDefinition())
-    return store;
+    return StoreRef(store, *this);
 
   // Handle lazy compound values.
   if (const nonloc::LazyCompoundVal *LCV=dyn_cast<nonloc::LazyCompoundVal>(&V))
@@ -1491,7 +1451,8 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   nonloc::CompoundVal::iterator VI = CV.begin(), VE = CV.end();
 
   RecordDecl::field_iterator FI, FE;
-
+  StoreRef newStore(store, *this);
+  
   for (FI = RD->field_begin(), FE = RD->field_end(); FI != FE; ++FI, ++VI) {
 
     if (VI == VE)
@@ -1501,36 +1462,61 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
     const FieldRegion* FR = MRMgr.getFieldRegion(*FI, R);
 
     if (FTy->isArrayType())
-      store = BindArray(store, FR, *VI);
+      newStore = BindArray(newStore.getStore(), FR, *VI);
     else if (FTy->isStructureOrClassType())
-      store = BindStruct(store, FR, *VI);
+      newStore = BindStruct(newStore.getStore(), FR, *VI);
     else
-      store = Bind(store, ValMgr.makeLoc(FR), *VI);
+      newStore = Bind(newStore.getStore(), svalBuilder.makeLoc(FR), *VI);
   }
 
   // There may be fewer values in the initialize list than the fields of struct.
   if (FI != FE) {
-    RegionBindings B = GetRegionBindings(store);
-    B = Add(B, R, BindingKey::Default, ValMgr.makeIntVal(0, false));
-    store = B.getRoot();
+    RegionBindings B = GetRegionBindings(newStore.getStore());
+    B = addBinding(B, R, BindingKey::Default, svalBuilder.makeIntVal(0, false));
+    newStore = StoreRef(B.getRootWithoutRetain(), *this);
   }
 
-  return store;
+  return newStore;
 }
 
-Store RegionStoreManager::KillStruct(Store store, const TypedRegion* R,
+StoreRef RegionStoreManager::KillStruct(Store store, const TypedRegion* R,
                                      SVal DefaultVal) {
+  BindingKey key = BindingKey::Make(R, BindingKey::Default);
+  
+  // The BindingKey may be "invalid" if we cannot handle the region binding
+  // explicitly.  One example is something like array[index], where index
+  // is a symbolic value.  In such cases, we want to invalidate the entire
+  // array, as the index assignment could have been to any element.  In
+  // the case of nested symbolic indices, we need to march up the region
+  // hierarchy untile we reach a region whose binding we can reason about.
+  const SubRegion *subReg = R;
+
+  while (!key.isValid()) {
+    if (const SubRegion *tmp = dyn_cast<SubRegion>(subReg->getSuperRegion())) {
+      subReg = tmp;
+      key = BindingKey::Make(tmp, BindingKey::Default);
+    }
+    else
+      break;
+  }                                 
+
+  // Remove the old bindings, using 'subReg' as the root of all regions
+  // we will invalidate.
   RegionBindings B = GetRegionBindings(store);
   llvm::OwningPtr<RegionStoreSubRegionMap>
     SubRegions(getRegionStoreSubRegionMap(store));
-  RemoveSubRegionBindings(B, R, *SubRegions);
+  RemoveSubRegionBindings(B, subReg, *SubRegions);
 
   // Set the default value of the struct region to "unknown".
-  return Add(B, R, BindingKey::Default, DefaultVal).getRoot();
+  if (!key.isValid())
+    return StoreRef(B.getRootWithoutRetain(), *this);
+  
+  return StoreRef(addBinding(B, key, DefaultVal).getRootWithoutRetain(), *this);
 }
 
-Store RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
-                                           Store store, const TypedRegion *R) {
+StoreRef RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
+                                              Store store,
+                                              const TypedRegion *R) {
 
   // Nuke the old bindings stemming from R.
   RegionBindings B = GetRegionBindings(store);
@@ -1543,7 +1529,8 @@ Store RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
 
   // Now copy the bindings.  This amounts to just binding 'V' to 'R'.  This
   // results in a zero-copy algorithm.
-  return Add(B, R, BindingKey::Direct, V).getRoot();
+  return StoreRef(addBinding(B, R, BindingKey::Direct,
+                             V).getRootWithoutRetain(), *this);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1551,38 +1538,42 @@ Store RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
 //===----------------------------------------------------------------------===//
 
 
-RegionBindings RegionStoreManager::Add(RegionBindings B, BindingKey K, SVal V) {
+RegionBindings RegionStoreManager::addBinding(RegionBindings B, BindingKey K,
+                                              SVal V) {
   if (!K.isValid())
     return B;
-  return RBFactory.Add(B, K, V);
+  return RBFactory.add(B, K, V);
 }
 
-RegionBindings RegionStoreManager::Add(RegionBindings B, const MemRegion *R,
-                                       BindingKey::Kind k, SVal V) {
-  return Add(B, BindingKey::Make(R, k), V);
+RegionBindings RegionStoreManager::addBinding(RegionBindings B,
+                                              const MemRegion *R,
+                                              BindingKey::Kind k, SVal V) {
+  return addBinding(B, BindingKey::Make(R, k), V);
 }
 
-const SVal *RegionStoreManager::Lookup(RegionBindings B, BindingKey K) {
+const SVal *RegionStoreManager::lookup(RegionBindings B, BindingKey K) {
   if (!K.isValid())
     return NULL;
   return B.lookup(K);
 }
 
-const SVal *RegionStoreManager::Lookup(RegionBindings B,
+const SVal *RegionStoreManager::lookup(RegionBindings B,
                                        const MemRegion *R,
                                        BindingKey::Kind k) {
-  return Lookup(B, BindingKey::Make(R, k));
+  return lookup(B, BindingKey::Make(R, k));
 }
 
-RegionBindings RegionStoreManager::Remove(RegionBindings B, BindingKey K) {
+RegionBindings RegionStoreManager::removeBinding(RegionBindings B,
+                                                 BindingKey K) {
   if (!K.isValid())
     return B;
-  return RBFactory.Remove(B, K);
+  return RBFactory.remove(B, K);
 }
 
-RegionBindings RegionStoreManager::Remove(RegionBindings B, const MemRegion *R,
-                                          BindingKey::Kind k){
-  return Remove(B, BindingKey::Make(R, k));
+RegionBindings RegionStoreManager::removeBinding(RegionBindings B,
+                                                 const MemRegion *R,
+                                                BindingKey::Kind k){
+  return removeBinding(B, BindingKey::Make(R, k));
 }
 
 //===----------------------------------------------------------------------===//
@@ -1590,17 +1581,18 @@ RegionBindings RegionStoreManager::Remove(RegionBindings B, const MemRegion *R,
 //===----------------------------------------------------------------------===//
 
 namespace {
-class RemoveDeadBindingsWorker :
-  public ClusterAnalysis<RemoveDeadBindingsWorker> {
+class removeDeadBindingsWorker :
+  public ClusterAnalysis<removeDeadBindingsWorker> {
   llvm::SmallVector<const SymbolicRegion*, 12> Postponed;
   SymbolReaper &SymReaper;
   const StackFrameContext *CurrentLCtx;
 
 public:
-  RemoveDeadBindingsWorker(RegionStoreManager &rm, GRStateManager &stateMgr,
+  removeDeadBindingsWorker(RegionStoreManager &rm, GRStateManager &stateMgr,
                            RegionBindings b, SymbolReaper &symReaper,
                            const StackFrameContext *LCtx)
-    : ClusterAnalysis<RemoveDeadBindingsWorker>(rm, stateMgr, b),
+    : ClusterAnalysis<removeDeadBindingsWorker>(rm, stateMgr, b,
+                                                /* includeGlobals = */ false),
       SymReaper(symReaper), CurrentLCtx(LCtx) {}
 
   // Called by ClusterAnalysis.
@@ -1613,7 +1605,7 @@ public:
 };
 }
 
-void RemoveDeadBindingsWorker::VisitAddedToCluster(const MemRegion *baseR,
+void removeDeadBindingsWorker::VisitAddedToCluster(const MemRegion *baseR,
                                                    RegionCluster &C) {
 
   if (const VarRegion *VR = dyn_cast<VarRegion>(baseR)) {
@@ -1647,13 +1639,13 @@ void RemoveDeadBindingsWorker::VisitAddedToCluster(const MemRegion *baseR,
   }
 }
 
-void RemoveDeadBindingsWorker::VisitCluster(const MemRegion *baseR,
+void removeDeadBindingsWorker::VisitCluster(const MemRegion *baseR,
                                             BindingKey *I, BindingKey *E) {
   for ( ; I != E; ++I)
     VisitBindingKey(*I);
 }
 
-void RemoveDeadBindingsWorker::VisitBinding(SVal V) {
+void removeDeadBindingsWorker::VisitBinding(SVal V) {
   // Is it a LazyCompoundVal?  All referenced regions are live as well.
   if (const nonloc::LazyCompoundVal *LCS =
       dyn_cast<nonloc::LazyCompoundVal>(&V)) {
@@ -1678,7 +1670,7 @@ void RemoveDeadBindingsWorker::VisitBinding(SVal V) {
     SymReaper.markLive(*SI);
 }
 
-void RemoveDeadBindingsWorker::VisitBindingKey(BindingKey K) {
+void removeDeadBindingsWorker::VisitBindingKey(BindingKey K) {
   const MemRegion *R = K.getRegion();
 
   // Mark this region "live" by adding it to the worklist.  This will cause
@@ -1707,11 +1699,11 @@ void RemoveDeadBindingsWorker::VisitBindingKey(BindingKey K) {
   }
 
   // Visit the data binding for K.
-  if (const SVal *V = RM.Lookup(B, K))
+  if (const SVal *V = RM.lookup(B, K))
     VisitBinding(*V);
 }
 
-bool RemoveDeadBindingsWorker::UpdatePostponed() {
+bool removeDeadBindingsWorker::UpdatePostponed() {
   // See if any postponed SymbolicRegions are actually live now, after
   // having done a scan.
   bool changed = false;
@@ -1729,13 +1721,13 @@ bool RemoveDeadBindingsWorker::UpdatePostponed() {
   return changed;
 }
 
-Store RegionStoreManager::RemoveDeadBindings(Store store,
-                                             const StackFrameContext *LCtx,
-                                             SymbolReaper& SymReaper,
+StoreRef RegionStoreManager::removeDeadBindings(Store store,
+                                                const StackFrameContext *LCtx,
+                                                SymbolReaper& SymReaper,
                            llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
 {
   RegionBindings B = GetRegionBindings(store);
-  RemoveDeadBindingsWorker W(*this, StateMgr, B, SymReaper, LCtx);
+  removeDeadBindingsWorker W(*this, StateMgr, B, SymReaper, LCtx);
   W.GenerateClusters();
 
   // Enqueue the region roots onto the worklist.
@@ -1756,7 +1748,7 @@ Store RegionStoreManager::RemoveDeadBindings(Store store,
       continue;
 
     // Remove the dead entry.
-    B = Remove(B, K);
+    B = removeBinding(B, K);
 
     // Mark all non-live symbols that this binding references as dead.
     if (const SymbolicRegion* SymR = dyn_cast<SymbolicRegion>(K.getRegion()))
@@ -1768,23 +1760,27 @@ Store RegionStoreManager::RemoveDeadBindings(Store store,
       SymReaper.maybeDead(*SI);
   }
 
-  return B.getRoot();
+  return StoreRef(B.getRootWithoutRetain(), *this);
 }
 
 
-Store RegionStoreManager::EnterStackFrame(const GRState *state,
-                                          const StackFrameContext *frame) {
+StoreRef RegionStoreManager::enterStackFrame(const GRState *state,
+                                             const StackFrameContext *frame) {
   FunctionDecl const *FD = cast<FunctionDecl>(frame->getDecl());
-  FunctionDecl::param_const_iterator PI = FD->param_begin();
-  Store store = state->getStore();
+  FunctionDecl::param_const_iterator PI = FD->param_begin(), 
+                                     PE = FD->param_end();
+  StoreRef store = StoreRef(state->getStore(), *this);
 
   if (CallExpr const *CE = dyn_cast<CallExpr>(frame->getCallSite())) {
     CallExpr::const_arg_iterator AI = CE->arg_begin(), AE = CE->arg_end();
 
-    // Copy the arg expression value to the arg variables.
-    for (; AI != AE; ++AI, ++PI) {
+    // Copy the arg expression value to the arg variables.  We check that
+    // PI != PE because the actual number of arguments may be different than
+    // the function declaration.
+    for (; AI != AE && PI != PE; ++AI, ++PI) {
       SVal ArgVal = state->getSVal(*AI);
-      store = Bind(store, ValMgr.makeLoc(MRMgr.getVarRegion(*PI,frame)),ArgVal);
+      store = Bind(store.getStore(),
+                   svalBuilder.makeLoc(MRMgr.getVarRegion(*PI, frame)), ArgVal);
     }
   } else if (const CXXConstructExpr *CE =
                dyn_cast<CXXConstructExpr>(frame->getCallSite())) {
@@ -1794,10 +1790,11 @@ Store RegionStoreManager::EnterStackFrame(const GRState *state,
     // Copy the arg expression value to the arg variables.
     for (; AI != AE; ++AI, ++PI) {
       SVal ArgVal = state->getSVal(*AI);
-      store = Bind(store, ValMgr.makeLoc(MRMgr.getVarRegion(*PI,frame)),ArgVal);
+      store = Bind(store.getStore(),
+                   svalBuilder.makeLoc(MRMgr.getVarRegion(*PI,frame)), ArgVal);
     }
   } else
-    llvm_unreachable("Unhandled call expression.");
+    assert(isa<CXXDestructorDecl>(frame->getDecl()));
 
   return store;
 }
diff --git a/lib/StaticAnalyzer/Core/SValBuilder.cpp b/lib/StaticAnalyzer/Core/SValBuilder.cpp
new file mode 100644
index 000000000000..b0fd497e5719
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -0,0 +1,310 @@
+// SValBuilder.cpp - Basic class for all SValBuilder implementations -*- C++ -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines SValBuilder, the base class for all (complete) SValBuilder
+//  implementations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SVals.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/BasicValueFactory.h"
+
+using namespace clang;
+using namespace ento;
+
+//===----------------------------------------------------------------------===//
+// Basic SVal creation.
+//===----------------------------------------------------------------------===//
+
+DefinedOrUnknownSVal SValBuilder::makeZeroVal(QualType T) {
+  if (Loc::isLocType(T))
+    return makeNull();
+
+  if (T->isIntegerType())
+    return makeIntVal(0, T);
+
+  // FIXME: Handle floats.
+  // FIXME: Handle structs.
+  return UnknownVal();
+}
+
+
+NonLoc SValBuilder::makeNonLoc(const SymExpr *lhs, BinaryOperator::Opcode op,
+                                const llvm::APSInt& v, QualType T) {
+  // The Environment ensures we always get a persistent APSInt in
+  // BasicValueFactory, so we don't need to get the APSInt from
+  // BasicValueFactory again.
+  assert(!Loc::isLocType(T));
+  return nonloc::SymExprVal(SymMgr.getSymIntExpr(lhs, op, v, T));
+}
+
+NonLoc SValBuilder::makeNonLoc(const SymExpr *lhs, BinaryOperator::Opcode op,
+                                const SymExpr *rhs, QualType T) {
+  assert(SymMgr.getType(lhs) == SymMgr.getType(rhs));
+  assert(!Loc::isLocType(T));
+  return nonloc::SymExprVal(SymMgr.getSymSymExpr(lhs, op, rhs, T));
+}
+
+
+SVal SValBuilder::convertToArrayIndex(SVal V) {
+  if (V.isUnknownOrUndef())
+    return V;
+
+  // Common case: we have an appropriately sized integer.
+  if (nonloc::ConcreteInt* CI = dyn_cast<nonloc::ConcreteInt>(&V)) {
+    const llvm::APSInt& I = CI->getValue();
+    if (I.getBitWidth() == ArrayIndexWidth && I.isSigned())
+      return V;
+  }
+
+  return evalCastNL(cast<NonLoc>(V), ArrayIndexTy);
+}
+
+DefinedOrUnknownSVal 
+SValBuilder::getRegionValueSymbolVal(const TypedRegion* R) {
+  QualType T = R->getValueType();
+
+  if (!SymbolManager::canSymbolicate(T))
+    return UnknownVal();
+
+  SymbolRef sym = SymMgr.getRegionValueSymbol(R);
+
+  if (Loc::isLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
+
+DefinedOrUnknownSVal SValBuilder::getConjuredSymbolVal(const void *SymbolTag,
+                                                        const Expr *E,
+                                                        unsigned Count) {
+  QualType T = E->getType();
+
+  if (!SymbolManager::canSymbolicate(T))
+    return UnknownVal();
+
+  SymbolRef sym = SymMgr.getConjuredSymbol(E, Count, SymbolTag);
+
+  if (Loc::isLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
+
+DefinedOrUnknownSVal SValBuilder::getConjuredSymbolVal(const void *SymbolTag,
+                                                        const Expr *E,
+                                                        QualType T,
+                                                        unsigned Count) {
+  
+  if (!SymbolManager::canSymbolicate(T))
+    return UnknownVal();
+
+  SymbolRef sym = SymMgr.getConjuredSymbol(E, T, Count, SymbolTag);
+
+  if (Loc::isLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
+
+DefinedSVal SValBuilder::getMetadataSymbolVal(const void *SymbolTag,
+                                               const MemRegion *MR,
+                                               const Expr *E, QualType T,
+                                               unsigned Count) {
+  assert(SymbolManager::canSymbolicate(T) && "Invalid metadata symbol type");
+
+  SymbolRef sym = SymMgr.getMetadataSymbol(MR, E, T, Count, SymbolTag);
+
+  if (Loc::isLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
+
+DefinedOrUnknownSVal
+SValBuilder::getDerivedRegionValueSymbolVal(SymbolRef parentSymbol,
+                                             const TypedRegion *R) {
+  QualType T = R->getValueType();
+
+  if (!SymbolManager::canSymbolicate(T))
+    return UnknownVal();
+
+  SymbolRef sym = SymMgr.getDerivedSymbol(parentSymbol, R);
+
+  if (Loc::isLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
+
+DefinedSVal SValBuilder::getFunctionPointer(const FunctionDecl* FD) {
+  return loc::MemRegionVal(MemMgr.getFunctionTextRegion(FD));
+}
+
+DefinedSVal SValBuilder::getBlockPointer(const BlockDecl *D,
+                                          CanQualType locTy,
+                                          const LocationContext *LC) {
+  const BlockTextRegion *BC =
+    MemMgr.getBlockTextRegion(D, locTy, LC->getAnalysisContext());
+  const BlockDataRegion *BD = MemMgr.getBlockDataRegion(BC, LC);
+  return loc::MemRegionVal(BD);
+}
+
+//===----------------------------------------------------------------------===//
+
+SVal SValBuilder::evalBinOp(const GRState *ST, BinaryOperator::Opcode Op,
+                          SVal L, SVal R, QualType T) {
+
+  if (L.isUndef() || R.isUndef())
+    return UndefinedVal();
+
+  if (L.isUnknown() || R.isUnknown())
+    return UnknownVal();
+
+  if (isa<Loc>(L)) {
+    if (isa<Loc>(R))
+      return evalBinOpLL(ST, Op, cast<Loc>(L), cast<Loc>(R), T);
+
+    return evalBinOpLN(ST, Op, cast<Loc>(L), cast<NonLoc>(R), T);
+  }
+
+  if (isa<Loc>(R)) {
+    // Support pointer arithmetic where the addend is on the left
+    // and the pointer on the right.
+    assert(Op == BO_Add);
+
+    // Commute the operands.
+    return evalBinOpLN(ST, Op, cast<Loc>(R), cast<NonLoc>(L), T);
+  }
+
+  return evalBinOpNN(ST, Op, cast<NonLoc>(L), cast<NonLoc>(R), T);
+}
+
+DefinedOrUnknownSVal SValBuilder::evalEQ(const GRState *ST,
+                                       DefinedOrUnknownSVal L,
+                                       DefinedOrUnknownSVal R) {
+  return cast<DefinedOrUnknownSVal>(evalBinOp(ST, BO_EQ, L, R,
+                                              Context.IntTy));
+}
+
+// FIXME: should rewrite according to the cast kind.
+SVal SValBuilder::evalCast(SVal val, QualType castTy, QualType originalTy) {
+  if (val.isUnknownOrUndef() || castTy == originalTy)
+    return val;
+
+  // For const casts, just propagate the value.
+  if (!castTy->isVariableArrayType() && !originalTy->isVariableArrayType())
+    if (Context.hasSameUnqualifiedType(castTy, originalTy))
+      return val;
+
+  // Check for casts to real or complex numbers.  We don't handle these at all
+  // right now.
+  if (castTy->isFloatingType() || castTy->isAnyComplexType())
+    return UnknownVal();
+  
+  // Check for casts from integers to integers.
+  if (castTy->isIntegerType() && originalTy->isIntegerType())
+    return evalCastNL(cast<NonLoc>(val), castTy);
+
+  // Check for casts from pointers to integers.
+  if (castTy->isIntegerType() && Loc::isLocType(originalTy))
+    return evalCastL(cast<Loc>(val), castTy);
+
+  // Check for casts from integers to pointers.
+  if (Loc::isLocType(castTy) && originalTy->isIntegerType()) {
+    if (nonloc::LocAsInteger *LV = dyn_cast<nonloc::LocAsInteger>(&val)) {
+      if (const MemRegion *R = LV->getLoc().getAsRegion()) {
+        StoreManager &storeMgr = StateMgr.getStoreManager();
+        R = storeMgr.castRegion(R, castTy);
+        return R ? SVal(loc::MemRegionVal(R)) : UnknownVal();
+      }
+      return LV->getLoc();
+    }
+    goto DispatchCast;
+  }
+
+  // Just pass through function and block pointers.
+  if (originalTy->isBlockPointerType() || originalTy->isFunctionPointerType()) {
+    assert(Loc::isLocType(castTy));
+    return val;
+  }
+
+  // Check for casts from array type to another type.
+  if (originalTy->isArrayType()) {
+    // We will always decay to a pointer.
+    val = StateMgr.ArrayToPointer(cast<Loc>(val));
+
+    // Are we casting from an array to a pointer?  If so just pass on
+    // the decayed value.
+    if (castTy->isPointerType())
+      return val;
+
+    // Are we casting from an array to an integer?  If so, cast the decayed
+    // pointer value to an integer.
+    assert(castTy->isIntegerType());
+
+    // FIXME: Keep these here for now in case we decide soon that we
+    // need the original decayed type.
+    //    QualType elemTy = cast<ArrayType>(originalTy)->getElementType();
+    //    QualType pointerTy = C.getPointerType(elemTy);
+    return evalCastL(cast<Loc>(val), castTy);
+  }
+
+  // Check for casts from a region to a specific type.
+  if (const MemRegion *R = val.getAsRegion()) {
+    // FIXME: We should handle the case where we strip off view layers to get
+    //  to a desugared type.
+
+    if (!Loc::isLocType(castTy)) {
+      // FIXME: There can be gross cases where one casts the result of a function
+      // (that returns a pointer) to some other value that happens to fit
+      // within that pointer value.  We currently have no good way to
+      // model such operations.  When this happens, the underlying operation
+      // is that the caller is reasoning about bits.  Conceptually we are
+      // layering a "view" of a location on top of those bits.  Perhaps
+      // we need to be more lazy about mutual possible views, even on an
+      // SVal?  This may be necessary for bit-level reasoning as well.
+      return UnknownVal();
+    }
+
+    // We get a symbolic function pointer for a dereference of a function
+    // pointer, but it is of function type. Example:
+
+    //  struct FPRec {
+    //    void (*my_func)(int * x);
+    //  };
+    //
+    //  int bar(int x);
+    //
+    //  int f1_a(struct FPRec* foo) {
+    //    int x;
+    //    (*foo->my_func)(&x);
+    //    return bar(x)+1; // no-warning
+    //  }
+
+    assert(Loc::isLocType(originalTy) || originalTy->isFunctionType() ||
+           originalTy->isBlockPointerType() || castTy->isReferenceType());
+
+    StoreManager &storeMgr = StateMgr.getStoreManager();
+
+    // Delegate to store manager to get the result of casting a region to a
+    // different type.  If the MemRegion* returned is NULL, this expression
+    // Evaluates to UnknownVal.
+    R = storeMgr.castRegion(R, castTy);
+    return R ? SVal(loc::MemRegionVal(R)) : UnknownVal();
+  }
+
+DispatchCast:
+  // All other cases.
+  return isa<Loc>(val) ? evalCastL(cast<Loc>(val), castTy)
+                       : evalCastNL(cast<NonLoc>(val), castTy);
+}
diff --git a/lib/Checker/SVals.cpp b/lib/StaticAnalyzer/Core/SVals.cpp
index 97ba74e94878..4614e349dece 100644
--- a/lib/Checker/SVals.cpp
+++ b/lib/StaticAnalyzer/Core/SVals.cpp
@@ -12,10 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/AST/ExprObjC.h"
 #include "clang/Basic/IdentifierTable.h"
 
 using namespace clang;
+using namespace ento;
 using llvm::dyn_cast;
 using llvm::cast;
 using llvm::APSInt;
@@ -66,7 +68,7 @@ SymbolRef SVal::getAsLocSymbol() const {
     return X->getLoc().getAsLocSymbol();
 
   if (const loc::MemRegionVal *X = dyn_cast<loc::MemRegionVal>(this)) {
-    const MemRegion *R = X->StripCasts();
+    const MemRegion *R = X->stripCasts();
     if (const SymbolicRegion *SymR = dyn_cast<SymbolicRegion>(R))
       return SymR->getSymbol();
   }
@@ -126,7 +128,7 @@ const MemRegion *SVal::getAsRegion() const {
   return 0;
 }
 
-const MemRegion *loc::MemRegionVal::StripCasts() const {
+const MemRegion *loc::MemRegionVal::stripCasts() const {
   const MemRegion *R = getRegion();
   return R ?  R->StripCasts() : NULL;
 }
@@ -221,11 +223,11 @@ bool SVal::isZeroConstant() const {
 // Transfer function dispatch for Non-Locs.
 //===----------------------------------------------------------------------===//
 
-SVal nonloc::ConcreteInt::evalBinOp(ValueManager &ValMgr,
+SVal nonloc::ConcreteInt::evalBinOp(SValBuilder &svalBuilder,
                                     BinaryOperator::Opcode Op,
                                     const nonloc::ConcreteInt& R) const {
   const llvm::APSInt* X =
-    ValMgr.getBasicValueFactory().EvaluateAPSInt(Op, getValue(), R.getValue());
+    svalBuilder.getBasicValueFactory().evalAPSInt(Op, getValue(), R.getValue());
 
   if (X)
     return nonloc::ConcreteInt(*X);
@@ -234,26 +236,27 @@ SVal nonloc::ConcreteInt::evalBinOp(ValueManager &ValMgr,
 }
 
 nonloc::ConcreteInt
-nonloc::ConcreteInt::evalComplement(ValueManager &ValMgr) const {
-  return ValMgr.makeIntVal(~getValue());
+nonloc::ConcreteInt::evalComplement(SValBuilder &svalBuilder) const {
+  return svalBuilder.makeIntVal(~getValue());
 }
 
-nonloc::ConcreteInt nonloc::ConcreteInt::evalMinus(ValueManager &ValMgr) const {
-  return ValMgr.makeIntVal(-getValue());
+nonloc::ConcreteInt
+nonloc::ConcreteInt::evalMinus(SValBuilder &svalBuilder) const {
+  return svalBuilder.makeIntVal(-getValue());
 }
 
 //===----------------------------------------------------------------------===//
 // Transfer function dispatch for Locs.
 //===----------------------------------------------------------------------===//
 
-SVal loc::ConcreteInt::EvalBinOp(BasicValueFactory& BasicVals,
+SVal loc::ConcreteInt::evalBinOp(BasicValueFactory& BasicVals,
                                  BinaryOperator::Opcode Op,
                                  const loc::ConcreteInt& R) const {
 
   assert (Op == BO_Add || Op == BO_Sub ||
           (Op >= BO_LT && Op <= BO_NE));
 
-  const llvm::APSInt* X = BasicVals.EvaluateAPSInt(Op, getValue(), R.getValue());
+  const llvm::APSInt* X = BasicVals.evalAPSInt(Op, getValue(), R.getValue());
 
   if (X)
     return loc::ConcreteInt(*X);
@@ -270,7 +273,7 @@ void SVal::dump() const { dumpToStream(llvm::errs()); }
 void SVal::dumpToStream(llvm::raw_ostream& os) const {
   switch (getBaseKind()) {
     case UnknownKind:
-      os << "Invalid";
+      os << "Unknown";
       break;
     case NonLocKind:
       cast<NonLoc>(this)->dumpToStream(os);
@@ -288,11 +291,16 @@ void SVal::dumpToStream(llvm::raw_ostream& os) const {
 
 void NonLoc::dumpToStream(llvm::raw_ostream& os) const {
   switch (getSubKind()) {
-    case nonloc::ConcreteIntKind:
-      os << cast<nonloc::ConcreteInt>(this)->getValue().getZExtValue();
-      if (cast<nonloc::ConcreteInt>(this)->getValue().isUnsigned())
-        os << 'U';
+    case nonloc::ConcreteIntKind: {
+      const nonloc::ConcreteInt& C = *cast<nonloc::ConcreteInt>(this);
+      if (C.getValue().isUnsigned())
+        os << C.getValue().getZExtValue();
+      else
+        os << C.getValue().getSExtValue();
+      os << ' ' << (C.getValue().isUnsigned() ? 'U' : 'S')
+         << C.getValue().getBitWidth() << 'b';
       break;
+    }
     case nonloc::SymbolValKind:
       os << '$' << cast<nonloc::SymbolVal>(this)->getSymbol();
       break;
@@ -342,11 +350,27 @@ void Loc::dumpToStream(llvm::raw_ostream& os) const {
       os << cast<loc::ConcreteInt>(this)->getValue().getZExtValue() << " (Loc)";
       break;
     case loc::GotoLabelKind:
-      os << "&&" << cast<loc::GotoLabel>(this)->getLabel()->getID()->getName();
+      os << "&&" << cast<loc::GotoLabel>(this)->getLabel()->getName();
       break;
     case loc::MemRegionKind:
       os << '&' << cast<loc::MemRegionVal>(this)->getRegion()->getString();
       break;
+    case loc::ObjCPropRefKind: {
+      const ObjCPropertyRefExpr *E = cast<loc::ObjCPropRef>(this)->getPropRefExpr();
+      os << "objc-prop{";
+      if (E->isSuperReceiver())
+        os << "super.";
+      else if (E->getBase())
+        os << "<base>.";
+
+      if (E->isImplicitProperty())
+        os << E->getImplicitPropertyGetter()->getSelector().getAsString();
+      else
+        os << E->getExplicitProperty()->getName();
+
+      os << "}";
+      break;
+    }
     default:
       assert(false && "Pretty-printing not implemented for this Loc.");
       break;
diff --git a/lib/Checker/SimpleConstraintManager.cpp b/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
index 04496e1e3332..e0b61ab58009 100644
--- a/lib/Checker/SimpleConstraintManager.cpp
+++ b/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
@@ -13,12 +13,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "SimpleConstraintManager.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/PathSensitive/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Checker.h"
 
 namespace clang {
 
+namespace ento {
+
 SimpleConstraintManager::~SimpleConstraintManager() {}
 
 bool SimpleConstraintManager::canReasonAbout(SVal X) const {
@@ -55,22 +57,22 @@ bool SimpleConstraintManager::canReasonAbout(SVal X) const {
   return true;
 }
 
-const GRState *SimpleConstraintManager::Assume(const GRState *state,
+const GRState *SimpleConstraintManager::assume(const GRState *state,
                                                DefinedSVal Cond,
                                                bool Assumption) {
   if (isa<NonLoc>(Cond))
-    return Assume(state, cast<NonLoc>(Cond), Assumption);
+    return assume(state, cast<NonLoc>(Cond), Assumption);
   else
-    return Assume(state, cast<Loc>(Cond), Assumption);
+    return assume(state, cast<Loc>(Cond), Assumption);
 }
 
-const GRState *SimpleConstraintManager::Assume(const GRState *state, Loc cond,
+const GRState *SimpleConstraintManager::assume(const GRState *state, Loc cond,
                                                bool assumption) {
-  state = AssumeAux(state, cond, assumption);
-  return SU.ProcessAssume(state, cond, assumption);
+  state = assumeAux(state, cond, assumption);
+  return SU.processAssume(state, cond, assumption);
 }
 
-const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
+const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
                                                   Loc Cond, bool Assumption) {
 
   BasicValueFactory &BasicVals = state->getBasicVals();
@@ -91,9 +93,9 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
       if (const SymbolicRegion *SymR = dyn_cast<SymbolicRegion>(SubR)) {
         const llvm::APSInt &zero = BasicVals.getZeroWithPtrWidth();
         if (Assumption)
-          return AssumeSymNE(state, SymR->getSymbol(), zero, zero);
+          return assumeSymNE(state, SymR->getSymbol(), zero, zero);
         else
-          return AssumeSymEQ(state, SymR->getSymbol(), zero, zero);
+          return assumeSymEQ(state, SymR->getSymbol(), zero, zero);
       }
       SubR = dyn_cast<SubRegion>(SubR->getSuperRegion());
     }
@@ -112,16 +114,16 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
   } // end switch
 }
 
-const GRState *SimpleConstraintManager::Assume(const GRState *state,
+const GRState *SimpleConstraintManager::assume(const GRState *state,
                                                NonLoc cond,
                                                bool assumption) {
-  state = AssumeAux(state, cond, assumption);
-  return SU.ProcessAssume(state, cond, assumption);
+  state = assumeAux(state, cond, assumption);
+  return SU.processAssume(state, cond, assumption);
 }
 
 static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   // FIXME: This should probably be part of BinaryOperator, since this isn't
-  // the only place it's used. (This code was copied from SimpleSValuator.cpp.)
+  // the only place it's used. (This code was copied from SimpleSValBuilder.cpp.)
   switch (op) {
   default:
     assert(false && "Invalid opcode.");
@@ -134,7 +136,7 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   }
 }
 
-const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
+const GRState *SimpleConstraintManager::assumeAux(const GRState *state,
                                                   NonLoc Cond,
                                                   bool Assumption) {
 
@@ -159,9 +161,9 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
     QualType T =  SymMgr.getType(sym);
     const llvm::APSInt &zero = BasicVals.getValue(0, T);
     if (Assumption)
-      return AssumeSymNE(state, sym, zero, zero);
+      return assumeSymNE(state, sym, zero, zero);
     else
-      return AssumeSymEQ(state, sym, zero, zero);
+      return assumeSymEQ(state, sym, zero, zero);
   }
 
   case nonloc::SymExprValKind: {
@@ -179,14 +181,14 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
       QualType T = SymMgr.getType(SE);
       const llvm::APSInt &zero = BasicVals.getValue(0, T);
       op = (Assumption ? BO_NE : BO_EQ);
-      return AssumeSymRel(state, SE, op, zero);
+      return assumeSymRel(state, SE, op, zero);
     }
 
     // From here on out, op is the real comparison we'll be testing.
     if (!Assumption)
       op = NegateComparison(op);
   
-    return AssumeSymRel(state, SE->getLHS(), op, SE->getRHS());
+    return assumeSymRel(state, SE->getLHS(), op, SE->getRHS());
   }
 
   case nonloc::ConcreteIntKind: {
@@ -196,12 +198,12 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
   }
 
   case nonloc::LocAsIntegerKind:
-    return AssumeAux(state, cast<nonloc::LocAsInteger>(Cond).getLoc(),
+    return assumeAux(state, cast<nonloc::LocAsInteger>(Cond).getLoc(),
                      Assumption);
   } // end switch
 }
 
-const GRState *SimpleConstraintManager::AssumeSymRel(const GRState *state,
+const GRState *SimpleConstraintManager::assumeSymRel(const GRState *state,
                                                      const SymExpr *LHS,
                                                      BinaryOperator::Opcode op,
                                                      const llvm::APSInt& Int) {
@@ -259,41 +261,43 @@ const GRState *SimpleConstraintManager::AssumeSymRel(const GRState *state,
   ASTContext &Ctx = StateMgr.getContext();
 
   QualType T = Sym->getType(Ctx);
-  assert(T->isIntegerType() || Loc::IsLocType(T));
+  assert(T->isIntegerType() || Loc::isLocType(T));
   unsigned bitwidth = Ctx.getTypeSize(T);
-  bool isSymUnsigned = T->isUnsignedIntegerType() || Loc::IsLocType(T);
+  bool isSymUnsigned = T->isUnsignedIntegerType() || Loc::isLocType(T);
 
   // Convert the adjustment.
   Adjustment.setIsUnsigned(isSymUnsigned);
-  Adjustment.extOrTrunc(bitwidth);
+  Adjustment = Adjustment.extOrTrunc(bitwidth);
 
   // Convert the right-hand side integer.
   llvm::APSInt ConvertedInt(Int, isSymUnsigned);
-  ConvertedInt.extOrTrunc(bitwidth);
+  ConvertedInt = ConvertedInt.extOrTrunc(bitwidth);
 
   switch (op) {
   default:
-    // No logic yet for other operators.  Assume the constraint is feasible.
+    // No logic yet for other operators.  assume the constraint is feasible.
     return state;
 
   case BO_EQ:
-    return AssumeSymEQ(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymEQ(state, Sym, ConvertedInt, Adjustment);
 
   case BO_NE:
-    return AssumeSymNE(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymNE(state, Sym, ConvertedInt, Adjustment);
 
   case BO_GT:
-    return AssumeSymGT(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymGT(state, Sym, ConvertedInt, Adjustment);
 
   case BO_GE:
-    return AssumeSymGE(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymGE(state, Sym, ConvertedInt, Adjustment);
 
   case BO_LT:
-    return AssumeSymLT(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymLT(state, Sym, ConvertedInt, Adjustment);
 
   case BO_LE:
-    return AssumeSymLE(state, Sym, ConvertedInt, Adjustment);
+    return assumeSymLE(state, Sym, ConvertedInt, Adjustment);
   } // end switch
 }
 
-}  // end of namespace clang
+} // end of namespace ento
+
+} // end of namespace clang
diff --git a/lib/Checker/SimpleConstraintManager.h b/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
index 96811b3e36e6..a2952afa7351 100644
--- a/lib/Checker/SimpleConstraintManager.h
+++ b/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
@@ -11,18 +11,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_ANALYSIS_SIMPLE_CONSTRAINT_MANAGER_H
-#define LLVM_CLANG_ANALYSIS_SIMPLE_CONSTRAINT_MANAGER_H
+#ifndef LLVM_CLANG_GR_SIMPLE_CONSTRAINT_MANAGER_H
+#define LLVM_CLANG_GR_SIMPLE_CONSTRAINT_MANAGER_H
 
-#include "clang/Checker/PathSensitive/ConstraintManager.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ConstraintManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 
 namespace clang {
 
+namespace ento {
+
 class SimpleConstraintManager : public ConstraintManager {
-  GRSubEngine &SU;
+  SubEngine &SU;
 public:
-  SimpleConstraintManager(GRSubEngine &subengine) : SU(subengine) {}
+  SimpleConstraintManager(SubEngine &subengine) : SU(subengine) {}
   virtual ~SimpleConstraintManager();
 
   //===------------------------------------------------------------------===//
@@ -31,14 +33,14 @@ public:
 
   bool canReasonAbout(SVal X) const;
 
-  const GRState *Assume(const GRState *state, DefinedSVal Cond,
+  const GRState *assume(const GRState *state, DefinedSVal Cond,
                         bool Assumption);
 
-  const GRState *Assume(const GRState *state, Loc Cond, bool Assumption);
+  const GRState *assume(const GRState *state, Loc Cond, bool Assumption);
 
-  const GRState *Assume(const GRState *state, NonLoc Cond, bool Assumption);
+  const GRState *assume(const GRState *state, NonLoc Cond, bool Assumption);
 
-  const GRState *AssumeSymRel(const GRState *state,
+  const GRState *assumeSymRel(const GRState *state,
                               const SymExpr *LHS,
                               BinaryOperator::Opcode op,
                               const llvm::APSInt& Int);
@@ -51,27 +53,27 @@ protected:
 
   // Each of these is of the form "$sym+Adj <> V", where "<>" is the comparison
   // operation for the method being invoked.
-  virtual const GRState *AssumeSymNE(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymNE(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *AssumeSymEQ(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymEQ(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *AssumeSymLT(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymLT(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *AssumeSymGT(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymGT(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *AssumeSymLE(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymLE(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
-  virtual const GRState *AssumeSymGE(const GRState *state, SymbolRef sym,
+  virtual const GRState *assumeSymGE(const GRState *state, SymbolRef sym,
                                      const llvm::APSInt& V,
                                      const llvm::APSInt& Adjustment) = 0;
 
@@ -79,11 +81,13 @@ protected:
   // Internal implementation.
   //===------------------------------------------------------------------===//
 
-  const GRState *AssumeAux(const GRState *state, Loc Cond,bool Assumption);
+  const GRState *assumeAux(const GRState *state, Loc Cond,bool Assumption);
 
-  const GRState *AssumeAux(const GRState *state, NonLoc Cond, bool Assumption);
+  const GRState *assumeAux(const GRState *state, NonLoc Cond, bool Assumption);
 };
 
-}  // end clang namespace
+} // end GR namespace
+
+} // end clang namespace
 
 #endif
diff --git a/lib/Checker/SimpleSValuator.cpp b/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index 782cd4f5e68b..9a46bd6d69a0 100644
--- a/lib/Checker/SimpleSValuator.cpp
+++ b/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -1,4 +1,4 @@
-// SimpleSValuator.cpp - A basic SValuator ------------------------*- C++ -*--//
+// SimpleSValBuilder.cpp - A basic SValBuilder -----------------------*- C++ -*-
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,35 +7,38 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines SimpleSValuator, a basic implementation of SValuator.
+//  This file defines SimpleSValBuilder, a basic implementation of SValBuilder.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/SValuator.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SValBuilder.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 
 using namespace clang;
+using namespace ento;
 
 namespace {
-class SimpleSValuator : public SValuator {
+class SimpleSValBuilder : public SValBuilder {
 protected:
-  virtual SVal EvalCastNL(NonLoc val, QualType castTy);
-  virtual SVal EvalCastL(Loc val, QualType castTy);
+  virtual SVal evalCastNL(NonLoc val, QualType castTy);
+  virtual SVal evalCastL(Loc val, QualType castTy);
 
 public:
-  SimpleSValuator(ValueManager &valMgr) : SValuator(valMgr) {}
-  virtual ~SimpleSValuator() {}
-
-  virtual SVal EvalMinus(NonLoc val);
-  virtual SVal EvalComplement(NonLoc val);
-  virtual SVal EvalBinOpNN(const GRState *state, BinaryOperator::Opcode op,
+  SimpleSValBuilder(llvm::BumpPtrAllocator &alloc, ASTContext &context,
+                    GRStateManager &stateMgr)
+                    : SValBuilder(alloc, context, stateMgr) {}
+  virtual ~SimpleSValBuilder() {}
+
+  virtual SVal evalMinus(NonLoc val);
+  virtual SVal evalComplement(NonLoc val);
+  virtual SVal evalBinOpNN(const GRState *state, BinaryOperator::Opcode op,
                            NonLoc lhs, NonLoc rhs, QualType resultTy);
-  virtual SVal EvalBinOpLL(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLL(const GRState *state, BinaryOperator::Opcode op,
                            Loc lhs, Loc rhs, QualType resultTy);
-  virtual SVal EvalBinOpLN(const GRState *state, BinaryOperator::Opcode op,
+  virtual SVal evalBinOpLN(const GRState *state, BinaryOperator::Opcode op,
                            Loc lhs, NonLoc rhs, QualType resultTy);
 
-  /// getKnownValue - Evaluates a given SVal. If the SVal has only one possible
+  /// getKnownValue - evaluates a given SVal. If the SVal has only one possible
   ///  (integer) value, that value is returned. Otherwise, returns NULL.
   virtual const llvm::APSInt *getKnownValue(const GRState *state, SVal V);
   
@@ -44,35 +47,34 @@ public:
 };
 } // end anonymous namespace
 
-SValuator *clang::CreateSimpleSValuator(ValueManager &valMgr) {
-  return new SimpleSValuator(valMgr);
+SValBuilder *ento::createSimpleSValBuilder(llvm::BumpPtrAllocator &alloc,
+                                           ASTContext &context,
+                                           GRStateManager &stateMgr) {
+  return new SimpleSValBuilder(alloc, context, stateMgr);
 }
 
 //===----------------------------------------------------------------------===//
 // Transfer function for Casts.
 //===----------------------------------------------------------------------===//
 
-SVal SimpleSValuator::EvalCastNL(NonLoc val, QualType castTy) {
+SVal SimpleSValBuilder::evalCastNL(NonLoc val, QualType castTy) {
 
-  bool isLocType = Loc::IsLocType(castTy);
+  bool isLocType = Loc::isLocType(castTy);
 
   if (nonloc::LocAsInteger *LI = dyn_cast<nonloc::LocAsInteger>(&val)) {
     if (isLocType)
       return LI->getLoc();
 
     // FIXME: Correctly support promotions/truncations.
-    ASTContext &Ctx = ValMgr.getContext();
-    unsigned castSize = Ctx.getTypeSize(castTy);
+    unsigned castSize = Context.getTypeSize(castTy);
     if (castSize == LI->getNumBits())
       return val;
-
-    return ValMgr.makeLocAsInteger(LI->getLoc(), castSize);
+    return makeLocAsInteger(LI->getLoc(), castSize);
   }
 
   if (const SymExpr *se = val.getAsSymbolicExpression()) {
-    ASTContext &Ctx = ValMgr.getContext();
-    QualType T = Ctx.getCanonicalType(se->getType(Ctx));
-    if (T == Ctx.getCanonicalType(castTy))
+    QualType T = Context.getCanonicalType(se->getType(Context));
+    if (T == Context.getCanonicalType(castTy))
       return val;
     
     // FIXME: Remove this hack when we support symbolic truncation/extension.
@@ -95,16 +97,16 @@ SVal SimpleSValuator::EvalCastNL(NonLoc val, QualType castTy) {
     return UnknownVal();
 
   llvm::APSInt i = cast<nonloc::ConcreteInt>(val).getValue();
-  i.setIsUnsigned(castTy->isUnsignedIntegerType() || Loc::IsLocType(castTy));
-  i.extOrTrunc(ValMgr.getContext().getTypeSize(castTy));
+  i.setIsUnsigned(castTy->isUnsignedIntegerType() || Loc::isLocType(castTy));
+  i = i.extOrTrunc(Context.getTypeSize(castTy));
 
   if (isLocType)
-    return ValMgr.makeIntLocVal(i);
+    return makeIntLocVal(i);
   else
-    return ValMgr.makeIntVal(i);
+    return makeIntVal(i);
 }
 
-SVal SimpleSValuator::EvalCastL(Loc val, QualType castTy) {
+SVal SimpleSValBuilder::evalCastL(Loc val, QualType castTy) {
 
   // Casts from pointers -> pointers, just return the lval.
   //
@@ -112,7 +114,7 @@ SVal SimpleSValuator::EvalCastL(Loc val, QualType castTy) {
   //   can be introduced by the frontend for corner cases, e.g
   //   casting from va_list* to __builtin_va_list&.
   //
-  if (Loc::IsLocType(castTy) || castTy->isReferenceType())
+  if (Loc::isLocType(castTy) || castTy->isReferenceType())
     return val;
 
   // FIXME: Handle transparent unions where a value can be "transparently"
@@ -121,15 +123,15 @@ SVal SimpleSValuator::EvalCastL(Loc val, QualType castTy) {
     return UnknownVal();
 
   if (castTy->isIntegerType()) {
-    unsigned BitWidth = ValMgr.getContext().getTypeSize(castTy);
+    unsigned BitWidth = Context.getTypeSize(castTy);
 
     if (!isa<loc::ConcreteInt>(val))
-      return ValMgr.makeLocAsInteger(val, BitWidth);
+      return makeLocAsInteger(val, BitWidth);
 
     llvm::APSInt i = cast<loc::ConcreteInt>(val).getValue();
-    i.setIsUnsigned(castTy->isUnsignedIntegerType() || Loc::IsLocType(castTy));
-    i.extOrTrunc(BitWidth);
-    return ValMgr.makeIntVal(i);
+    i.setIsUnsigned(castTy->isUnsignedIntegerType() || Loc::isLocType(castTy));
+    i = i.extOrTrunc(BitWidth);
+    return makeIntVal(i);
   }
 
   // All other cases: return 'UnknownVal'.  This includes casting pointers
@@ -142,19 +144,19 @@ SVal SimpleSValuator::EvalCastL(Loc val, QualType castTy) {
 // Transfer function for unary operators.
 //===----------------------------------------------------------------------===//
 
-SVal SimpleSValuator::EvalMinus(NonLoc val) {
+SVal SimpleSValBuilder::evalMinus(NonLoc val) {
   switch (val.getSubKind()) {
   case nonloc::ConcreteIntKind:
-    return cast<nonloc::ConcreteInt>(val).evalMinus(ValMgr);
+    return cast<nonloc::ConcreteInt>(val).evalMinus(*this);
   default:
     return UnknownVal();
   }
 }
 
-SVal SimpleSValuator::EvalComplement(NonLoc X) {
+SVal SimpleSValBuilder::evalComplement(NonLoc X) {
   switch (X.getSubKind()) {
   case nonloc::ConcreteIntKind:
-    return cast<nonloc::ConcreteInt>(X).evalComplement(ValMgr);
+    return cast<nonloc::ConcreteInt>(X).evalComplement(*this);
   default:
     return UnknownVal();
   }
@@ -191,7 +193,7 @@ static BinaryOperator::Opcode ReverseComparison(BinaryOperator::Opcode op) {
   }
 }
 
-SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
+SVal SimpleSValBuilder::MakeSymIntVal(const SymExpr *LHS,
                                     BinaryOperator::Opcode op,
                                     const llvm::APSInt &RHS,
                                     QualType resultTy) {
@@ -205,7 +207,7 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
   case BO_Mul:
     // a*0 and a*1
     if (RHS == 0)
-      return ValMgr.makeIntVal(0, resultTy);
+      return makeIntVal(0, resultTy);
     else if (RHS == 1)
       isIdempotent = true;
     break;
@@ -223,7 +225,7 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
       // This is also handled elsewhere.
       return UndefinedVal();
     else if (RHS == 1)
-      return ValMgr.makeIntVal(0, resultTy);
+      return makeIntVal(0, resultTy);
     break;
   case BO_Add:
   case BO_Sub:
@@ -237,7 +239,7 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
   case BO_And:
     // a&0 and a&(~0)
     if (RHS == 0)
-      return ValMgr.makeIntVal(0, resultTy);
+      return makeIntVal(0, resultTy);
     else if (RHS.isAllOnesValue())
       isIdempotent = true;
     break;
@@ -246,27 +248,26 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
     if (RHS == 0)
       isIdempotent = true;
     else if (RHS.isAllOnesValue()) {
-      BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
-      const llvm::APSInt &Result = BVF.Convert(resultTy, RHS);
+      const llvm::APSInt &Result = BasicVals.Convert(resultTy, RHS);
       return nonloc::ConcreteInt(Result);
     }
     break;
   }
 
   // Idempotent ops (like a*1) can still change the type of an expression.
-  // Wrap the LHS up in a NonLoc again and let EvalCastNL do the dirty work.
+  // Wrap the LHS up in a NonLoc again and let evalCastNL do the dirty work.
   if (isIdempotent) {
     if (SymbolRef LHSSym = dyn_cast<SymbolData>(LHS))
-      return EvalCastNL(nonloc::SymbolVal(LHSSym), resultTy);
-    return EvalCastNL(nonloc::SymExprVal(LHS), resultTy);
+      return evalCastNL(nonloc::SymbolVal(LHSSym), resultTy);
+    return evalCastNL(nonloc::SymExprVal(LHS), resultTy);
   }
 
   // If we reach this point, the expression cannot be simplified.
   // Make a SymExprVal for the entire thing.
-  return ValMgr.makeNonLoc(LHS, op, RHS, resultTy);
+  return makeNonLoc(LHS, op, RHS, resultTy);
 }
 
-SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpNN(const GRState *state,
                                   BinaryOperator::Opcode op,
                                   NonLoc lhs, NonLoc rhs,
                                   QualType resultTy)  {
@@ -278,17 +279,17 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
       case BO_EQ:
       case BO_LE:
       case BO_GE:
-        return ValMgr.makeTruthVal(true, resultTy);
+        return makeTruthVal(true, resultTy);
       case BO_LT:
       case BO_GT:
       case BO_NE:
-        return ValMgr.makeTruthVal(false, resultTy);
+        return makeTruthVal(false, resultTy);
       case BO_Xor:
       case BO_Sub:
-        return ValMgr.makeIntVal(0, resultTy);
+        return makeIntVal(0, resultTy);
       case BO_Or:
       case BO_And:
-        return EvalCastNL(lhs, resultTy);
+        return evalCastNL(lhs, resultTy);
     }
 
   while (1) {
@@ -299,23 +300,22 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
       Loc lhsL = cast<nonloc::LocAsInteger>(lhs).getLoc();
       switch (rhs.getSubKind()) {
         case nonloc::LocAsIntegerKind:
-          return EvalBinOpLL(state, op, lhsL,
+          return evalBinOpLL(state, op, lhsL,
                              cast<nonloc::LocAsInteger>(rhs).getLoc(),
                              resultTy);
         case nonloc::ConcreteIntKind: {
           // Transform the integer into a location and compare.
-          ASTContext& Ctx = ValMgr.getContext();
           llvm::APSInt i = cast<nonloc::ConcreteInt>(rhs).getValue();
           i.setIsUnsigned(true);
-          i.extOrTrunc(Ctx.getTypeSize(Ctx.VoidPtrTy));
-          return EvalBinOpLL(state, op, lhsL, ValMgr.makeLoc(i), resultTy);
+          i = i.extOrTrunc(Context.getTypeSize(Context.VoidPtrTy));
+          return evalBinOpLL(state, op, lhsL, makeLoc(i), resultTy);
         }
         default:
           switch (op) {
             case BO_EQ:
-              return ValMgr.makeTruthVal(false, resultTy);
+              return makeTruthVal(false, resultTy);
             case BO_NE:
-              return ValMgr.makeTruthVal(true, resultTy);
+              return makeTruthVal(true, resultTy);
             default:
               // This case also handles pointer arithmetic.
               return UnknownVal();
@@ -358,7 +358,7 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
         case BO_XorAssign:
         case BO_OrAssign:
         case BO_Comma:
-          assert(false && "'=' and ',' operators handled by GRExprEngine.");
+          assert(false && "'=' and ',' operators handled by ExprEngine.");
           return UnknownVal();
         case BO_PtrMemD:
         case BO_PtrMemI:
@@ -372,8 +372,8 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
         case BO_NE:
           // Negate the comparison and make a value.
           opc = NegateComparison(opc);
-          assert(symIntExpr->getType(ValMgr.getContext()) == resultTy);
-          return ValMgr.makeNonLoc(symIntExpr->getLHS(), opc,
+          assert(symIntExpr->getType(Context) == resultTy);
+          return makeNonLoc(symIntExpr->getLHS(), opc,
                                    symIntExpr->getRHS(), resultTy);
         }
       }
@@ -388,23 +388,20 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
       if (BinaryOperator::isAdditiveOp(op)) {
         BinaryOperator::Opcode lop = symIntExpr->getOpcode();
         if (BinaryOperator::isAdditiveOp(lop)) {
-          BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
-
           // resultTy may not be the best type to convert to, but it's
           // probably the best choice in expressions with mixed type
           // (such as x+1U+2LL). The rules for implicit conversions should
           // choose a reasonable type to preserve the expression, and will
           // at least match how the value is going to be used.
           const llvm::APSInt &first =
-            BVF.Convert(resultTy, symIntExpr->getRHS());
+            BasicVals.Convert(resultTy, symIntExpr->getRHS());
           const llvm::APSInt &second =
-            BVF.Convert(resultTy, rhsInt->getValue());
-
+            BasicVals.Convert(resultTy, rhsInt->getValue());
           const llvm::APSInt *newRHS;
           if (lop == op)
-            newRHS = BVF.EvaluateAPSInt(BO_Add, first, second);
+            newRHS = BasicVals.evalAPSInt(BO_Add, first, second);
           else
-            newRHS = BVF.EvaluateAPSInt(BO_Sub, first, second);
+            newRHS = BasicVals.evalAPSInt(BO_Sub, first, second);
           return MakeSymIntVal(symIntExpr->getLHS(), lop, *newRHS, resultTy);
         }
       }
@@ -416,7 +413,7 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
       const nonloc::ConcreteInt& lhsInt = cast<nonloc::ConcreteInt>(lhs);
 
       if (isa<nonloc::ConcreteInt>(rhs)) {
-        return lhsInt.evalBinOp(ValMgr, op, cast<nonloc::ConcreteInt>(rhs));
+        return lhsInt.evalBinOp(*this, op, cast<nonloc::ConcreteInt>(rhs));
       } else {
         const llvm::APSInt& lhsValue = lhsInt.getValue();
         
@@ -461,17 +458,13 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
     case nonloc::SymbolValKind: {
       nonloc::SymbolVal *slhs = cast<nonloc::SymbolVal>(&lhs);
       SymbolRef Sym = slhs->getSymbol();
-
-      ASTContext& Ctx = ValMgr.getContext();
-
       // Does the symbol simplify to a constant?  If so, "fold" the constant
       // by setting 'lhs' to a ConcreteInt and try again.
-      if (Sym->getType(Ctx)->isIntegerType())
+      if (Sym->getType(Context)->isIntegerType())
         if (const llvm::APSInt *Constant = state->getSymVal(Sym)) {
           // The symbol evaluates to a constant. If necessary, promote the
           // folded constant (LHS) to the result type.
-          BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
-          const llvm::APSInt &lhs_I = BVF.Convert(resultTy, *Constant);
+          const llvm::APSInt &lhs_I = BasicVals.Convert(resultTy, *Constant);
           lhs = nonloc::ConcreteInt(lhs_I);
           
           // Also promote the RHS (if necessary).
@@ -483,7 +476,8 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
           // Other operators: do an implicit conversion.  This shouldn't be
           // necessary once we support truncation/extension of symbolic values.
           if (nonloc::ConcreteInt *rhs_I = dyn_cast<nonloc::ConcreteInt>(&rhs)){
-            rhs = nonloc::ConcreteInt(BVF.Convert(resultTy, rhs_I->getValue()));
+            rhs = nonloc::ConcreteInt(BasicVals.Convert(resultTy,
+                                                        rhs_I->getValue()));
           }
           
           continue;
@@ -492,11 +486,10 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
       // Is the RHS a symbol we can simplify?
       if (const nonloc::SymbolVal *srhs = dyn_cast<nonloc::SymbolVal>(&rhs)) {
         SymbolRef RSym = srhs->getSymbol();
-        if (RSym->getType(Ctx)->isIntegerType()) {
+        if (RSym->getType(Context)->isIntegerType()) {
           if (const llvm::APSInt *Constant = state->getSymVal(RSym)) {
             // The symbol evaluates to a constant.
-            BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
-            const llvm::APSInt &rhs_I = BVF.Convert(resultTy, *Constant);
+            const llvm::APSInt &rhs_I = BasicVals.Convert(resultTy, *Constant);
             rhs = nonloc::ConcreteInt(rhs_I);
           }
         }
@@ -515,13 +508,13 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
 }
 
 // FIXME: all this logic will change if/when we have MemRegion::getLocation().
-SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpLL(const GRState *state,
                                   BinaryOperator::Opcode op,
                                   Loc lhs, Loc rhs,
                                   QualType resultTy) {
   // Only comparisons and subtractions are valid operations on two pointers.
   // See [C99 6.5.5 through 6.5.14] or [C++0x 5.6 through 5.15].
-  // However, if a pointer is casted to an integer, EvalBinOpNN may end up
+  // However, if a pointer is casted to an integer, evalBinOpNN may end up
   // calling this function with another operation (PR7527). We don't attempt to
   // model this for now, but it could be useful, particularly when the
   // "location" is actually an integer value that's been passed through a void*.
@@ -535,15 +528,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       assert(false && "Unimplemented operation for two identical values");
       return UnknownVal();
     case BO_Sub:
-      return ValMgr.makeZeroVal(resultTy);
+      return makeZeroVal(resultTy);
     case BO_EQ:
     case BO_LE:
     case BO_GE:
-      return ValMgr.makeTruthVal(true, resultTy);
+      return makeTruthVal(true, resultTy);
     case BO_NE:
     case BO_LT:
     case BO_GT:
-      return ValMgr.makeTruthVal(false, resultTy);
+      return makeTruthVal(false, resultTy);
     }
   }
 
@@ -559,15 +552,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       default:
         break;
       case BO_Sub:
-        return EvalCastL(lhs, resultTy);
+        return evalCastL(lhs, resultTy);
       case BO_EQ:
       case BO_LE:
       case BO_LT:
-        return ValMgr.makeTruthVal(false, resultTy);
+        return makeTruthVal(false, resultTy);
       case BO_NE:
       case BO_GT:
       case BO_GE:
-        return ValMgr.makeTruthVal(true, resultTy);
+        return makeTruthVal(true, resultTy);
       }
     }
     // There may be two labels for the same location, and a function region may
@@ -587,15 +580,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         return UnknownVal();
 
       const llvm::APSInt &lVal = cast<loc::ConcreteInt>(lhs).getValue();
-      return ValMgr.makeNonLoc(rSym, ReverseComparison(op), lVal, resultTy);
+      return makeNonLoc(rSym, ReverseComparison(op), lVal, resultTy);
     }
 
     // If both operands are constants, just perform the operation.
     if (loc::ConcreteInt *rInt = dyn_cast<loc::ConcreteInt>(&rhs)) {
-      BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
-      SVal ResultVal = cast<loc::ConcreteInt>(lhs).EvalBinOp(BVF, op, *rInt);
+      SVal ResultVal = cast<loc::ConcreteInt>(lhs).evalBinOp(BasicVals, op,
+                                                             *rInt);
       if (Loc *Result = dyn_cast<Loc>(&ResultVal))
-        return EvalCastL(*Result, resultTy);
+        return evalCastL(*Result, resultTy);
       else
         return UnknownVal();
     }
@@ -612,11 +605,11 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       case BO_EQ:
       case BO_GT:
       case BO_GE:
-        return ValMgr.makeTruthVal(false, resultTy);
+        return makeTruthVal(false, resultTy);
       case BO_NE:
       case BO_LT:
       case BO_LE:
-        return ValMgr.makeTruthVal(true, resultTy);
+        return makeTruthVal(true, resultTy);
       }
     }
 
@@ -640,15 +633,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         default:
           break;
         case BO_Sub:
-          return EvalCastL(lhs, resultTy);
+          return evalCastL(lhs, resultTy);
         case BO_EQ:
         case BO_LT:
         case BO_LE:
-          return ValMgr.makeTruthVal(false, resultTy);
+          return makeTruthVal(false, resultTy);
         case BO_NE:
         case BO_GT:
         case BO_GE:
-          return ValMgr.makeTruthVal(true, resultTy);
+          return makeTruthVal(true, resultTy);
         }
       }
 
@@ -676,9 +669,9 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       default:
         return UnknownVal();
       case BO_EQ:
-        return ValMgr.makeTruthVal(false, resultTy);
+        return makeTruthVal(false, resultTy);
       case BO_NE:
-        return ValMgr.makeTruthVal(true, resultTy);
+        return makeTruthVal(true, resultTy);
       }
     }
 
@@ -705,7 +698,7 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         NonLoc *LeftIndex = dyn_cast<NonLoc>(&LeftIndexVal);
         if (!LeftIndex)
           return UnknownVal();
-        LeftIndexVal = EvalCastNL(*LeftIndex, resultTy);
+        LeftIndexVal = evalCastNL(*LeftIndex, resultTy);
         LeftIndex = dyn_cast<NonLoc>(&LeftIndexVal);
         if (!LeftIndex)
           return UnknownVal();
@@ -715,14 +708,14 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         NonLoc *RightIndex = dyn_cast<NonLoc>(&RightIndexVal);
         if (!RightIndex)
           return UnknownVal();
-        RightIndexVal = EvalCastNL(*RightIndex, resultTy);
+        RightIndexVal = evalCastNL(*RightIndex, resultTy);
         RightIndex = dyn_cast<NonLoc>(&RightIndexVal);
         if (!RightIndex)
           return UnknownVal();
 
         // Actually perform the operation.
-        // EvalBinOpNN expects the two indexes to already be the right type.
-        return EvalBinOpNN(state, op, *LeftIndex, *RightIndex, resultTy);
+        // evalBinOpNN expects the two indexes to already be the right type.
+        return evalBinOpNN(state, op, *LeftIndex, *RightIndex, resultTy);
       }
 
       // If the element indexes aren't comparable, see if the raw offsets are.
@@ -731,24 +724,24 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
 
       if (LeftOffset.getRegion() != NULL &&
           LeftOffset.getRegion() == RightOffset.getRegion()) {
-        int64_t left = LeftOffset.getByteOffset();
-        int64_t right = RightOffset.getByteOffset();
+        CharUnits left = LeftOffset.getOffset();
+        CharUnits right = RightOffset.getOffset();
 
         switch (op) {
         default:
           return UnknownVal();
         case BO_LT:
-          return ValMgr.makeTruthVal(left < right, resultTy);
+          return makeTruthVal(left < right, resultTy);
         case BO_GT:
-          return ValMgr.makeTruthVal(left > right, resultTy);
+          return makeTruthVal(left > right, resultTy);
         case BO_LE:
-          return ValMgr.makeTruthVal(left <= right, resultTy);
+          return makeTruthVal(left <= right, resultTy);
         case BO_GE:
-          return ValMgr.makeTruthVal(left >= right, resultTy);
+          return makeTruthVal(left >= right, resultTy);
         case BO_EQ:
-          return ValMgr.makeTruthVal(left == right, resultTy);
+          return makeTruthVal(left == right, resultTy);
         case BO_NE:
-          return ValMgr.makeTruthVal(left != right, resultTy);
+          return makeTruthVal(left != right, resultTy);
         }
       }
 
@@ -786,9 +779,9 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       // We know for sure that the two fields are not the same, since that
       // would have given us the same SVal.
       if (op == BO_EQ)
-        return ValMgr.makeTruthVal(false, resultTy);
+        return makeTruthVal(false, resultTy);
       if (op == BO_NE)
-        return ValMgr.makeTruthVal(true, resultTy);
+        return makeTruthVal(true, resultTy);
 
       // Iterate through the fields and see which one comes first.
       // [C99 6.7.2.1.13] "Within a structure object, the non-bit-field
@@ -798,9 +791,9 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       for (RecordDecl::field_iterator I = RD->field_begin(),
            E = RD->field_end(); I!=E; ++I) {
         if (*I == LeftFD)
-          return ValMgr.makeTruthVal(leftFirst, resultTy);
+          return makeTruthVal(leftFirst, resultTy);
         if (*I == RightFD)
-          return ValMgr.makeTruthVal(!leftFirst, resultTy);
+          return makeTruthVal(!leftFirst, resultTy);
       }
 
       assert(false && "Fields not found in parent record's definition");
@@ -812,9 +805,14 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
   }
 }
 
-SVal SimpleSValuator::EvalBinOpLN(const GRState *state,
+SVal SimpleSValBuilder::evalBinOpLN(const GRState *state,
                                   BinaryOperator::Opcode op,
                                   Loc lhs, NonLoc rhs, QualType resultTy) {
+  
+  // Special case: rhs is a zero constant.
+  if (rhs.isZeroConstant())
+    return lhs;
+  
   // Special case: 'rhs' is an integer that has the same width as a pointer and
   // we are using the integer location in a comparison.  Normally this cannot be
   // triggered, but transfer functions like those for OSCommpareAndSwapBarrier32
@@ -823,13 +821,13 @@ SVal SimpleSValuator::EvalBinOpLN(const GRState *state,
   if (BinaryOperator::isComparisonOp(op)) {
     if (nonloc::ConcreteInt *rhsInt = dyn_cast<nonloc::ConcreteInt>(&rhs)) {
       const llvm::APSInt *x = &rhsInt->getValue();
-      ASTContext &ctx = ValMgr.getContext();
+      ASTContext &ctx = Context;
       if (ctx.getTypeSize(ctx.VoidPtrTy) == x->getBitWidth()) {
         // Convert the signedness of the integer (if necessary).
         if (x->isSigned())
-          x = &ValMgr.getBasicValueFactory().getValue(*x, true);
+          x = &getBasicValueFactory().getValue(*x, true);
 
-        return EvalBinOpLL(state, op, lhs, loc::ConcreteInt(*x), resultTy);
+        return evalBinOpLL(state, op, lhs, loc::ConcreteInt(*x), resultTy);
       }
     }
   }
@@ -845,7 +843,7 @@ SVal SimpleSValuator::EvalBinOpLN(const GRState *state,
 
       // Convert the bitwidth of rightI.  This should deal with overflow
       // since we are dealing with concrete values.
-      rightI.extOrTrunc(leftI.getBitWidth());
+      rightI = rightI.extOrTrunc(leftI.getBitWidth());
 
       // Offset the increment by the pointer size.
       llvm::APSInt Multiplicand(rightI.getBitWidth(), /* isUnsigned */ true);
@@ -862,17 +860,45 @@ SVal SimpleSValuator::EvalBinOpLN(const GRState *state,
         default:
           llvm_unreachable("Invalid pointer arithmetic operation");
       }
-      return loc::ConcreteInt(ValMgr.getBasicValueFactory().getValue(rightI));
+      return loc::ConcreteInt(getBasicValueFactory().getValue(rightI));
     }
   }
-  
 
-  // Delegate remaining pointer arithmetic to the StoreManager.
-  return state->getStateManager().getStoreManager().EvalBinOp(op, lhs,
-                                                              rhs, resultTy);
+  // Handle cases where 'lhs' is a region.
+  if (const MemRegion *region = lhs.getAsRegion()) {
+    rhs = cast<NonLoc>(convertToArrayIndex(rhs));
+    SVal index = UnknownVal();
+    const MemRegion *superR = 0;
+    QualType elementType;
+
+    if (const ElementRegion *elemReg = dyn_cast<ElementRegion>(region)) {
+      index = evalBinOpNN(state, BO_Add, elemReg->getIndex(), rhs,
+                          getArrayIndexType());
+      superR = elemReg->getSuperRegion();
+      elementType = elemReg->getElementType();
+    }
+    else if (isa<SubRegion>(region)) {
+      superR = region;
+      index = rhs;
+      if (const PointerType *PT = resultTy->getAs<PointerType>()) {
+        elementType = PT->getPointeeType();
+      }
+      else {
+        const ObjCObjectPointerType *OT =
+          resultTy->getAs<ObjCObjectPointerType>();
+        elementType = OT->getPointeeType();
+      }
+    }
+
+    if (NonLoc *indexV = dyn_cast<NonLoc>(&index)) {
+      return loc::MemRegionVal(MemMgr.getElementRegion(elementType, *indexV,
+                                                       superR, getContext()));
+    }
+  }
+  return UnknownVal();  
 }
 
-const llvm::APSInt *SimpleSValuator::getKnownValue(const GRState *state,
+const llvm::APSInt *SimpleSValBuilder::getKnownValue(const GRState *state,
                                                    SVal V) {
   if (V.isUnknownOrUndef())
     return NULL;
diff --git a/lib/Checker/Store.cpp b/lib/StaticAnalyzer/Core/Store.cpp
index 1cb5cd70cae6..722517097c73 100644
--- a/lib/Checker/Store.cpp
+++ b/lib/StaticAnalyzer/Core/Store.cpp
@@ -11,25 +11,26 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/Store.h"
-#include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/Store.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/GRState.h"
 #include "clang/AST/CharUnits.h"
 
 using namespace clang;
+using namespace ento;
 
 StoreManager::StoreManager(GRStateManager &stateMgr)
-  : ValMgr(stateMgr.getValueManager()), StateMgr(stateMgr),
-    MRMgr(ValMgr.getRegionManager()), Ctx(stateMgr.getContext()) {}
+  : svalBuilder(stateMgr.getSValBuilder()), StateMgr(stateMgr),
+    MRMgr(svalBuilder.getRegionManager()), Ctx(stateMgr.getContext()) {}
 
-Store StoreManager::EnterStackFrame(const GRState *state,
-                                    const StackFrameContext *frame) {
-  return state->getStore();
+StoreRef StoreManager::enterStackFrame(const GRState *state,
+                                       const StackFrameContext *frame) {
+  return StoreRef(state->getStore(), *this);
 }
 
 const MemRegion *StoreManager::MakeElementRegion(const MemRegion *Base,
                                               QualType EleTy, uint64_t index) {
-  SVal idx = ValMgr.makeArrayIndex(index);
-  return MRMgr.getElementRegion(EleTy, idx, Base, ValMgr.getContext());
+  NonLoc idx = svalBuilder.makeArrayIndex(index);
+  return MRMgr.getElementRegion(EleTy, idx, Base, svalBuilder.getContext());
 }
 
 // FIXME: Merge with the implementation of the same method in MemRegion.cpp
@@ -43,14 +44,18 @@ static bool IsCompleteType(ASTContext &Ctx, QualType Ty) {
   return true;
 }
 
+StoreRef StoreManager::BindDefault(Store store, const MemRegion *R, SVal V) {
+  return StoreRef(store, *this);
+}
+
 const ElementRegion *StoreManager::GetElementZeroRegion(const MemRegion *R, 
                                                         QualType T) {
-  SVal idx = ValMgr.makeZeroArrayIndex();
+  NonLoc idx = svalBuilder.makeZeroArrayIndex();
   assert(!T.isNull());
   return MRMgr.getElementRegion(T, idx, R, Ctx);
 }
 
-const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy) {
+const MemRegion *StoreManager::castRegion(const MemRegion *R, QualType CastToTy) {
 
   ASTContext& Ctx = StateMgr.getContext();
 
@@ -73,7 +78,7 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
 
   // Now assume we are casting from pointer to pointer. Other cases should
   // already be handled.
-  QualType PointeeTy = CastToTy->getAs<PointerType>()->getPointeeType();
+  QualType PointeeTy = CastToTy->getPointeeType();
   QualType CanonPointeeTy = Ctx.getCanonicalType(PointeeTy);
 
   // Handle casts to void*.  We just pass the region through.
@@ -113,7 +118,8 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
     case MemRegion::FieldRegionKind:
     case MemRegion::ObjCIvarRegionKind:
     case MemRegion::VarRegionKind:
-    case MemRegion::CXXObjectRegionKind:
+    case MemRegion::CXXTempObjectRegionKind:
+    case MemRegion::CXXBaseObjectRegionKind:
       return MakeElementRegion(R, PointeeTy);
 
     case MemRegion::ElementRegionKind: {
@@ -145,7 +151,7 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
       if (!baseR)
         return NULL;
 
-      CharUnits off = CharUnits::fromQuantity(rawOff.getByteOffset());
+      CharUnits off = rawOff.getOffset();
 
       if (off.isZero()) {
         // Edge case: we are at 0 bytes off the beginning of baseR.  We
@@ -211,7 +217,7 @@ SVal StoreManager::CastRetrievedVal(SVal V, const TypedRegion *R,
   if (castTy.isNull())
     return V;
   
-  ASTContext &Ctx = ValMgr.getContext();
+  ASTContext &Ctx = svalBuilder.getContext();
 
   if (performTestOnly) {  
     // Automatically translate references to pointers.
@@ -219,14 +225,14 @@ SVal StoreManager::CastRetrievedVal(SVal V, const TypedRegion *R,
     if (const ReferenceType *RT = T->getAs<ReferenceType>())
       T = Ctx.getPointerType(RT->getPointeeType());
     
-    assert(ValMgr.getContext().hasSameUnqualifiedType(castTy, T));
+    assert(svalBuilder.getContext().hasSameUnqualifiedType(castTy, T));
     return V;
   }
   
   if (const Loc *L = dyn_cast<Loc>(&V))
-    return ValMgr.getSValuator().EvalCastL(*L, castTy);
+    return svalBuilder.evalCastL(*L, castTy);
   else if (const NonLoc *NL = dyn_cast<NonLoc>(&V))
-    return ValMgr.getSValuator().EvalCastNL(*NL, castTy);
+    return svalBuilder.evalCastNL(*NL, castTy);
   
   return V;
 }
@@ -267,7 +273,7 @@ SVal StoreManager::getLValueFieldOrIvar(const Decl* D, SVal Base) {
   return loc::MemRegionVal(MRMgr.getFieldRegion(cast<FieldDecl>(D), BaseR));
 }
 
-SVal StoreManager::getLValueElement(QualType elementType, SVal Offset, 
+SVal StoreManager::getLValueElement(QualType elementType, NonLoc Offset, 
                                     SVal Base) {
 
   // If the base is an unknown or undefined value, just return it back.
@@ -283,7 +289,7 @@ SVal StoreManager::getLValueElement(QualType elementType, SVal Offset,
   const ElementRegion *ElemR = dyn_cast<ElementRegion>(BaseRegion);
 
   // Convert the offset to the appropriate size and signedness.
-  Offset = ValMgr.convertToArrayIndex(Offset);
+  Offset = cast<NonLoc>(svalBuilder.convertToArrayIndex(Offset));
 
   if (!ElemR) {
     //
@@ -308,7 +314,7 @@ SVal StoreManager::getLValueElement(QualType elementType, SVal Offset,
 
   // Only allow non-integer offsets if the base region has no offset itself.
   // FIXME: This is a somewhat arbitrary restriction. We should be using
-  // SValuator here to add the two offsets without checking their types.
+  // SValBuilder here to add the two offsets without checking their types.
   if (!isa<nonloc::ConcreteInt>(Offset)) {
     if (isa<ElementRegion>(BaseRegion->StripCasts()))
       return UnknownVal();
@@ -322,8 +328,8 @@ SVal StoreManager::getLValueElement(QualType elementType, SVal Offset,
   assert(BaseIdxI.isSigned());
 
   // Compute the new index.
-  SVal NewIdx = nonloc::ConcreteInt(
-                      ValMgr.getBasicValueFactory().getValue(BaseIdxI + OffI));
+  nonloc::ConcreteInt NewIdx(svalBuilder.getBasicValueFactory().getValue(BaseIdxI +
+                                                                    OffI));
 
   // Construct the new ElementRegion.
   const MemRegion *ArrayR = ElemR->getSuperRegion();
diff --git a/lib/Checker/SymbolManager.cpp b/lib/StaticAnalyzer/Core/SymbolManager.cpp
index 3b1bb6d98d95..c1ca1cfcdc07 100644
--- a/lib/Checker/SymbolManager.cpp
+++ b/lib/StaticAnalyzer/Core/SymbolManager.cpp
@@ -8,16 +8,17 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines SymbolManager, a class that manages symbolic values
-//  created for use by GRExprEngine and related classes.
+//  created for use by ExprEngine and related classes.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/PathSensitive/SymbolManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
-#include "clang/Checker/PathSensitive/MemRegion.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
+using namespace ento;
 
 void SymExpr::dump() const {
   dumpToStream(llvm::errs());
@@ -232,13 +233,15 @@ QualType SymbolRegionValue::getType(ASTContext& C) const {
 SymbolManager::~SymbolManager() {}
 
 bool SymbolManager::canSymbolicate(QualType T) {
-  if (Loc::IsLocType(T))
+  T = T.getCanonicalType();
+
+  if (Loc::isLocType(T))
     return true;
 
   if (T->isIntegerType())
     return T->isScalarType();
 
-  if (T->isRecordType())
+  if (T->isRecordType() && !T->isUnionType())
     return true;
 
   return false;
diff --git a/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
new file mode 100644
index 000000000000..230b6a1087dc
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
@@ -0,0 +1,70 @@
+//===--- TextPathDiagnostics.cpp - Text Diagnostics for Paths ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the TextPathDiagnostics object.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace clang;
+using namespace ento;
+using namespace llvm;
+
+namespace {
+
+/// \brief Simple path diagnostic client used for outputting as diagnostic notes
+/// the sequence of events.
+class TextPathDiagnostics : public PathDiagnosticClient {
+  const std::string OutputFile;
+  Diagnostic &Diag;
+
+public:
+  TextPathDiagnostics(const std::string& output, Diagnostic &diag)
+    : OutputFile(output), Diag(diag) {}
+
+  void HandlePathDiagnostic(const PathDiagnostic* D);
+
+  void FlushDiagnostics(llvm::SmallVectorImpl<std::string> *FilesMade) { }
+  
+  virtual llvm::StringRef getName() const {
+    return "TextPathDiagnostics";
+  }
+
+  PathGenerationScheme getGenerationScheme() const { return Minimal; }
+  bool supportsLogicalOpControlFlow() const { return true; }
+  bool supportsAllBlockEdges() const { return true; }
+  virtual bool useVerboseDescription() const { return true; }
+};
+
+} // end anonymous namespace
+
+PathDiagnosticClient*
+ento::createTextPathDiagnosticClient(const std::string& out,
+                                     const Preprocessor &PP) {
+  return new TextPathDiagnostics(out, PP.getDiagnostics());
+}
+
+void TextPathDiagnostics::HandlePathDiagnostic(const PathDiagnostic* D) {
+  if (!D)
+    return;
+
+  if (D->empty()) {
+    delete D;
+    return;
+  }
+
+  for (PathDiagnostic::const_iterator I=D->begin(), E=D->end(); I != E; ++I) {
+    unsigned diagID = Diag.getDiagnosticIDs()->getCustomDiagID(
+                                           DiagnosticIDs::Note, I->getString());
+    Diag.Report(I->getLocation().asLocation(), diagID);
+  }
+}
diff --git a/lib/Checker/AnalysisConsumer.cpp b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index ad5ccb503b24..dbfebccee745 100644
--- a/lib/Checker/AnalysisConsumer.cpp
+++ b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/AnalysisConsumer.h"
+#include "AnalysisConsumer.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
@@ -20,26 +20,33 @@
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/Analyses/UninitializedValues.h"
 #include "clang/Analysis/CFG.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
-#include "clang/Checker/ManagerRegistry.h"
-#include "clang/Checker/BugReporter/PathDiagnostic.h"
-#include "clang/Checker/PathSensitive/AnalysisManager.h"
-#include "clang/Checker/BugReporter/BugReporter.h"
-#include "clang/Checker/PathSensitive/GRExprEngine.h"
-#include "clang/Checker/PathSensitive/GRTransferFuncs.h"
-#include "clang/Checker/PathDiagnosticClients.h"
-#include "GRExprEngineExperimentalChecks.h"
-#include "GRExprEngineInternalChecks.h"
+#include "clang/StaticAnalyzer/Frontend/CheckerRegistration.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Checkers/LocalCheckers.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/TransferFuncs.h"
+#include "clang/StaticAnalyzer/Core/PathDiagnosticClients.h"
+
+// FIXME: Restructure checker registration.
+#include "../Checkers/ClangSACheckers.h"
+#include "../Checkers/ExperimentalChecks.h"
+#include "../Checkers/InternalChecks.h"
+#include "../Checkers/BasicObjCFoundationChecks.h"
+
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/AnalyzerOptions.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
-#include "llvm/System/Program.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
 #include "llvm/ADT/OwningPtr.h"
 
 using namespace clang;
+using namespace ento;
 
 static ExplodedNode::Auditor* CreateUbiViz();
 
@@ -48,11 +55,11 @@ static ExplodedNode::Auditor* CreateUbiViz();
 //===----------------------------------------------------------------------===//
 
 static PathDiagnosticClient*
-CreatePlistHTMLDiagnosticClient(const std::string& prefix,
+createPlistHTMLDiagnosticClient(const std::string& prefix,
                                 const Preprocessor &PP) {
-  llvm::sys::Path F(prefix);
-  PathDiagnosticClient *PD = CreateHTMLDiagnosticClient(F.getDirname(), PP);
-  return CreatePlistDiagnosticClient(prefix, PP, PD);
+  PathDiagnosticClient *PD =
+    createHTMLDiagnosticClient(llvm::sys::path::parent_path(prefix), PP);
+  return createPlistDiagnosticClient(prefix, PP, PD);
 }
 
 //===----------------------------------------------------------------------===//
@@ -75,7 +82,6 @@ private:
   Actions ObjCMethodActions;
   Actions ObjCImplementationActions;
   Actions CXXMethodActions;
-  TUActions TranslationUnitActions; // Remove this.
 
 public:
   ASTContext* Ctx;
@@ -89,6 +95,7 @@ public:
   StoreManagerCreator CreateStoreMgr;
   ConstraintManagerCreator CreateConstraintMgr;
 
+  llvm::OwningPtr<CheckerManager> checkerMgr;
   llvm::OwningPtr<AnalysisManager> Mgr;
 
   AnalysisConsumer(const Preprocessor& pp,
@@ -108,32 +115,27 @@ public:
         case PD_##NAME: PD = CREATEFN(OutDir, PP); break;
 #include "clang/Frontend/Analyses.def"
       }
+    } else if (Opts.AnalysisDiagOpt == PD_TEXT) {
+      // Create the text client even without a specified output file since
+      // it just uses diagnostic notes.
+      PD = createTextPathDiagnosticClient("", PP);
     }
 
     // Create the analyzer component creators.
-    if (ManagerRegistry::StoreMgrCreator != 0) {
-      CreateStoreMgr = ManagerRegistry::StoreMgrCreator;
-    }
-    else {
-      switch (Opts.AnalysisStoreOpt) {
-      default:
-        assert(0 && "Unknown store manager.");
+    switch (Opts.AnalysisStoreOpt) {
+    default:
+      assert(0 && "Unknown store manager.");
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATEFN)           \
-        case NAME##Model: CreateStoreMgr = CREATEFN; break;
+      case NAME##Model: CreateStoreMgr = CREATEFN; break;
 #include "clang/Frontend/Analyses.def"
-      }
     }
 
-    if (ManagerRegistry::ConstraintMgrCreator != 0)
-      CreateConstraintMgr = ManagerRegistry::ConstraintMgrCreator;
-    else {
-      switch (Opts.AnalysisConstraintsOpt) {
-      default:
-        assert(0 && "Unknown store manager.");
+    switch (Opts.AnalysisConstraintsOpt) {
+    default:
+      assert(0 && "Unknown store manager.");
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATEFN)     \
-        case NAME##Model: CreateConstraintMgr = CREATEFN; break;
+      case NAME##Model: CreateConstraintMgr = CREATEFN; break;
 #include "clang/Frontend/Analyses.def"
-      }
     }
   }
 
@@ -143,15 +145,21 @@ public:
 
     SourceManager &SM = Mgr->getASTContext().getSourceManager();
     PresumedLoc Loc = SM.getPresumedLoc(D->getLocation());
-    llvm::errs() << "ANALYZE: " << Loc.getFilename();
+    if (Loc.isValid()) {
+      llvm::errs() << "ANALYZE: " << Loc.getFilename();
 
-    if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
-      const NamedDecl *ND = cast<NamedDecl>(D);
-      llvm::errs() << ' ' << ND << '\n';
-    }
-    else if (isa<BlockDecl>(D)) {
-      llvm::errs() << ' ' << "block(line:" << Loc.getLine() << ",col:"
-                   << Loc.getColumn() << '\n';
+      if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
+        const NamedDecl *ND = cast<NamedDecl>(D);
+        llvm::errs() << ' ' << ND << '\n';
+      }
+      else if (isa<BlockDecl>(D)) {
+        llvm::errs() << ' ' << "block(line:" << Loc.getLine() << ",col:"
+                     << Loc.getColumn() << '\n';
+      }
+      else if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
+        Selector S = MD->getSelector();
+        llvm::errs() << ' ' << S.getAsString();
+      }
     }
   }
 
@@ -161,28 +169,30 @@ public:
     CXXMethodActions.push_back(action);
   }
 
-  void addTranslationUnitAction(TUAction action) {
-    TranslationUnitActions.push_back(action);
-  }
-
   void addObjCImplementationAction(CodeAction action) {
     ObjCImplementationActions.push_back(action);
   }
 
   virtual void Initialize(ASTContext &Context) {
     Ctx = &Context;
+    checkerMgr.reset(registerCheckers(Opts, PP.getDiagnostics()));
     Mgr.reset(new AnalysisManager(*Ctx, PP.getDiagnostics(),
                                   PP.getLangOptions(), PD,
                                   CreateStoreMgr, CreateConstraintMgr,
+                                  checkerMgr.get(),
                                   /* Indexer */ 0, 
                                   Opts.MaxNodes, Opts.MaxLoop,
                                   Opts.VisualizeEGDot, Opts.VisualizeEGUbi,
                                   Opts.PurgeDead, Opts.EagerlyAssume,
                                   Opts.TrimGraph, Opts.InlineCall,
-                                  Opts.UnoptimizedCFG));
+                                  Opts.UnoptimizedCFG, Opts.CFGAddImplicitDtors,
+                                  Opts.CFGAddInitializers,
+                                  Opts.EagerlyTrimEGraph));
   }
 
   virtual void HandleTranslationUnit(ASTContext &C);
+  void HandleDeclContext(ASTContext &C, DeclContext *dc);
+
   void HandleCode(Decl *D, Actions& actions);
 };
 } // end anonymous namespace
@@ -191,70 +201,65 @@ public:
 // AnalysisConsumer implementation.
 //===----------------------------------------------------------------------===//
 
-void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
-
-  TranslationUnitDecl *TU = C.getTranslationUnitDecl();
-
-  for (DeclContext::decl_iterator I = TU->decls_begin(), E = TU->decls_end();
+void AnalysisConsumer::HandleDeclContext(ASTContext &C, DeclContext *dc) {
+  BugReporter BR(*Mgr);
+  for (DeclContext::decl_iterator I = dc->decls_begin(), E = dc->decls_end();
        I != E; ++I) {
     Decl *D = *I;
+    checkerMgr->runCheckersOnASTDecl(D, *Mgr, BR);
 
     switch (D->getKind()) {
-    case Decl::CXXConstructor:
-    case Decl::CXXDestructor:
-    case Decl::CXXConversion:
-    case Decl::CXXMethod:
-    case Decl::Function: {
-      FunctionDecl* FD = cast<FunctionDecl>(D);
-      
-      if (FD->isThisDeclarationADefinition()) {
-        if (!Opts.AnalyzeSpecificFunction.empty() &&
-            FD->getDeclName().getAsString() != Opts.AnalyzeSpecificFunction)
-          break;
-        DisplayFunction(FD);
-        HandleCode(FD, FunctionActions);
-      }
-      break;
-    }
-
-    case Decl::ObjCMethod: {
-      ObjCMethodDecl* MD = cast<ObjCMethodDecl>(D);
-      
-      if (MD->isThisDeclarationADefinition()) {
-        if (!Opts.AnalyzeSpecificFunction.empty() &&
-            Opts.AnalyzeSpecificFunction != MD->getSelector().getAsString())
-          break;
-        DisplayFunction(MD);
-        HandleCode(MD, ObjCMethodActions);
+      case Decl::Namespace: {
+        HandleDeclContext(C, cast<NamespaceDecl>(D));
+        break;
       }
-      break;
-    }
-
-    case Decl::ObjCImplementation: {
-      ObjCImplementationDecl* ID = cast<ObjCImplementationDecl>(*I);
-      HandleCode(ID, ObjCImplementationActions);
-
-      for (ObjCImplementationDecl::method_iterator MI = ID->meth_begin(), 
-             ME = ID->meth_end(); MI != ME; ++MI) {
-        if ((*MI)->isThisDeclarationADefinition()) {
+      case Decl::CXXConstructor:
+      case Decl::CXXDestructor:
+      case Decl::CXXConversion:
+      case Decl::CXXMethod:
+      case Decl::Function: {
+        FunctionDecl* FD = cast<FunctionDecl>(D);
+        // We skip function template definitions, as their semantics is
+        // only determined when they are instantiated.
+        if (FD->isThisDeclarationADefinition() &&
+            !FD->isDependentContext()) {
           if (!Opts.AnalyzeSpecificFunction.empty() &&
-             Opts.AnalyzeSpecificFunction != (*MI)->getSelector().getAsString())
+              FD->getDeclName().getAsString() != Opts.AnalyzeSpecificFunction)
             break;
-          HandleCode(*MI, ObjCMethodActions);
+          DisplayFunction(FD);
+          HandleCode(FD, FunctionActions);
         }
+        break;
       }
-      break;
-    }
-
-    default:
-      break;
+        
+      case Decl::ObjCImplementation: {
+        ObjCImplementationDecl* ID = cast<ObjCImplementationDecl>(*I);
+        HandleCode(ID, ObjCImplementationActions);
+        
+        for (ObjCImplementationDecl::method_iterator MI = ID->meth_begin(), 
+             ME = ID->meth_end(); MI != ME; ++MI) {
+          if ((*MI)->isThisDeclarationADefinition()) {
+            if (!Opts.AnalyzeSpecificFunction.empty() &&
+                Opts.AnalyzeSpecificFunction != (*MI)->getSelector().getAsString())
+              break;
+            DisplayFunction(*MI);
+            HandleCode(*MI, ObjCMethodActions);
+          }
+        }
+        break;
+      }
+        
+      default:
+        break;
     }
-  }
+  }  
+}
 
-  for (TUActions::iterator I = TranslationUnitActions.begin(),
-                           E = TranslationUnitActions.end(); I != E; ++I) {
-    (*I)(*this, *Mgr, *TU);
-  }
+void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
+  BugReporter BR(*Mgr);
+  TranslationUnitDecl *TU = C.getTranslationUnitDecl();
+  checkerMgr->runCheckersOnASTDecl(TU, *Mgr, BR);
+  HandleDeclContext(C, TU);
 
   // Explicitly destroy the PathDiagnosticClient.  This will flush its output.
   // FIXME: This should be replaced with something that doesn't rely on
@@ -297,6 +302,12 @@ void AnalysisConsumer::HandleCode(Decl *D, Actions& actions) {
   if (D->hasBody() && Opts.AnalyzeNestedBlocks)
     FindBlocks(cast<DeclContext>(D), WL);
 
+  BugReporter BR(*Mgr);
+  for (llvm::SmallVectorImpl<Decl*>::iterator WI=WL.begin(), WE=WL.end();
+       WI != WE; ++WI)
+    if ((*WI)->hasBody())
+      checkerMgr->runCheckersOnASTBody(*WI, *Mgr, BR);
+
   for (Actions::iterator I = actions.begin(), E = actions.end(); I != E; ++I)
     for (llvm::SmallVectorImpl<Decl*>::iterator WI=WL.begin(), WE=WL.end();
          WI != WE; ++WI)
@@ -307,14 +318,6 @@ void AnalysisConsumer::HandleCode(Decl *D, Actions& actions) {
 // Analyses
 //===----------------------------------------------------------------------===//
 
-static void ActionWarnDeadStores(AnalysisConsumer &C, AnalysisManager& mgr,
-                                 Decl *D) {
-  if (LiveVariables *L = mgr.getLiveVariables(D)) {
-    BugReporter BR(mgr);
-    CheckDeadStores(*mgr.getCFG(D), *L, mgr.getParentMap(D), BR);
-  }
-}
-
 static void ActionWarnUninitVals(AnalysisConsumer &C, AnalysisManager& mgr,
                                  Decl *D) {
   if (CFG* c = mgr.getCFG(D)) {
@@ -323,32 +326,33 @@ static void ActionWarnUninitVals(AnalysisConsumer &C, AnalysisManager& mgr,
 }
 
 
-static void ActionGRExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
+static void ActionExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
                                Decl *D,
-                               GRTransferFuncs* tf) {
+                               TransferFuncs* tf) {
 
-  llvm::OwningPtr<GRTransferFuncs> TF(tf);
+  llvm::OwningPtr<TransferFuncs> TF(tf);
 
   // Construct the analysis engine.  We first query for the LiveVariables
   // information to see if the CFG is valid.
   // FIXME: Inter-procedural analysis will need to handle invalid CFGs.
   if (!mgr.getLiveVariables(D))
     return;
-  GRExprEngine Eng(mgr, TF.take());
+  ExprEngine Eng(mgr, TF.take());
 
   if (C.Opts.EnableExperimentalInternalChecks)
     RegisterExperimentalInternalChecks(Eng);
 
-  RegisterAppleChecks(Eng, *D);
+  RegisterNSErrorChecks(Eng.getBugReporter(), Eng, *D);
 
   if (C.Opts.EnableExperimentalChecks)
     RegisterExperimentalChecks(Eng);
 
-  // Enable idempotent operation checking if it was explicitly turned on, or if
-  // we are running experimental checks (i.e. everything)
-  if (C.Opts.IdempotentOps || C.Opts.EnableExperimentalChecks
-      || C.Opts.EnableExperimentalInternalChecks)
-    RegisterIdempotentOperationChecker(Eng);
+  if (C.Opts.BufferOverflows)
+    RegisterArrayBoundCheckerV2(Eng);
+
+  // Enable AnalyzerStatsChecker if it was given as an argument
+  if (C.Opts.AnalyzerStats)
+    RegisterAnalyzerStatsChecker(Eng);
 
   // Set the graph auditor.
   llvm::OwningPtr<ExplodedNode::Auditor> Auditor;
@@ -375,11 +379,11 @@ static void ActionGRExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
 static void ActionObjCMemCheckerAux(AnalysisConsumer &C, AnalysisManager& mgr,
                                   Decl *D, bool GCEnabled) {
 
-  GRTransferFuncs* TF = MakeCFRefCountTF(mgr.getASTContext(),
+  TransferFuncs* TF = MakeCFRefCountTF(mgr.getASTContext(),
                                          GCEnabled,
                                          mgr.getLangOptions());
 
-  ActionGRExprEngine(C, mgr, D, TF);
+  ActionExprEngine(C, mgr, D, TF);
 }
 
 static void ActionObjCMemChecker(AnalysisConsumer &C, AnalysisManager& mgr,
@@ -403,69 +407,11 @@ static void ActionObjCMemChecker(AnalysisConsumer &C, AnalysisManager& mgr,
  }
 }
 
-static void ActionDisplayLiveVariables(AnalysisConsumer &C,
-                                       AnalysisManager& mgr, Decl *D) {
-  if (LiveVariables* L = mgr.getLiveVariables(D)) {
-    L->dumpBlockLiveness(mgr.getSourceManager());
-  }
-}
-
-static void ActionCFGDump(AnalysisConsumer &C, AnalysisManager& mgr, Decl *D) {
-  if (CFG *cfg = mgr.getCFG(D)) {
-    cfg->dump(mgr.getLangOptions());
-  }
-}
-
-static void ActionCFGView(AnalysisConsumer &C, AnalysisManager& mgr, Decl *D) {
-  if (CFG *cfg = mgr.getCFG(D)) {
-    cfg->viewCFG(mgr.getLangOptions());
-  }
-}
-
-static void ActionSecuritySyntacticChecks(AnalysisConsumer &C,
-                                          AnalysisManager &mgr, Decl *D) {
-  BugReporter BR(mgr);
-  CheckSecuritySyntaxOnly(D, BR);
-}
-
-static void ActionLLVMConventionChecker(AnalysisConsumer &C,
-                                        AnalysisManager &mgr,
-                                        TranslationUnitDecl &TU) {
-  BugReporter BR(mgr);
-  CheckLLVMConventions(TU, BR);
-}
-
-static void ActionWarnObjCDealloc(AnalysisConsumer &C, AnalysisManager& mgr,
-                                  Decl *D) {
-  if (mgr.getLangOptions().getGCMode() == LangOptions::GCOnly)
-    return;
-  BugReporter BR(mgr);
-  CheckObjCDealloc(cast<ObjCImplementationDecl>(D), mgr.getLangOptions(), BR);
-}
-
-static void ActionWarnObjCUnusedIvars(AnalysisConsumer &C, AnalysisManager& mgr,
-                                      Decl *D) {
-  BugReporter BR(mgr);
-  CheckObjCUnusedIvar(cast<ObjCImplementationDecl>(D), BR);
-}
-
-static void ActionWarnObjCMethSigs(AnalysisConsumer &C, AnalysisManager& mgr,
-                                   Decl *D) {
-  BugReporter BR(mgr);
-  CheckObjCInstMethSignature(cast<ObjCImplementationDecl>(D), BR);
-}
-
-static void ActionWarnSizeofPointer(AnalysisConsumer &C, AnalysisManager &mgr,
-                                    Decl *D) {
-  BugReporter BR(mgr);
-  CheckSizeofPointer(D, BR);
-}
-
 //===----------------------------------------------------------------------===//
 // AnalysisConsumer creation.
 //===----------------------------------------------------------------------===//
 
-ASTConsumer* clang::CreateAnalysisConsumer(const Preprocessor& pp,
+ASTConsumer* ento::CreateAnalysisConsumer(const Preprocessor& pp,
                                            const std::string& OutDir,
                                            const AnalyzerOptions& Opts) {
   llvm::OwningPtr<AnalysisConsumer> C(new AnalysisConsumer(pp, OutDir, Opts));
diff --git a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
new file mode 100644
index 000000000000..646fe97564b8
--- /dev/null
+++ b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
@@ -0,0 +1,41 @@
+//===--- AnalysisConsumer.h - Front-end Analysis Engine Hooks ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This header contains the functions necessary for a front-end to run various
+// analyses.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_GR_ANALYSISCONSUMER_H
+#define LLVM_CLANG_GR_ANALYSISCONSUMER_H
+
+#include <string>
+
+namespace clang {
+
+class AnalyzerOptions;
+class ASTConsumer;
+class Preprocessor;
+class Diagnostic;
+
+namespace ento {
+class CheckerManager;
+
+/// CreateAnalysisConsumer - Creates an ASTConsumer to run various code
+/// analysis passes.  (The set of analyses run is controlled by command-line
+/// options.)
+ASTConsumer* CreateAnalysisConsumer(const Preprocessor &pp,
+                                    const std::string &output,
+                                    const AnalyzerOptions& Opts);
+
+} // end GR namespace
+
+} // end clang namespace
+
+#endif
diff --git a/lib/StaticAnalyzer/Frontend/CMakeLists.txt b/lib/StaticAnalyzer/Frontend/CMakeLists.txt
new file mode 100644
index 000000000000..cd9ac1b0c7e8
--- /dev/null
+++ b/lib/StaticAnalyzer/Frontend/CMakeLists.txt
@@ -0,0 +1,20 @@
+set(LLVM_NO_RTTI 1)
+
+set(LLVM_USED_LIBS clangBasic clangLex clangAST clangFrontend clangRewrite)
+
+include_directories( ${CMAKE_CURRENT_BINARY_DIR}/../Checkers )
+
+add_clang_library(clangStaticAnalyzerFrontend
+  AnalysisConsumer.cpp
+  CheckerRegistration.cpp
+  FrontendActions.cpp
+  )
+
+add_dependencies(clangStaticAnalyzerFrontend
+  clangStaticAnalyzerCheckers
+  clangStaticAnalyzerCore
+  ClangAttrClasses
+  ClangAttrList
+  ClangDeclNodes
+  ClangStmtNode
+  )
diff --git a/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp b/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
new file mode 100644
index 000000000000..6625729eafe4
--- /dev/null
+++ b/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
@@ -0,0 +1,50 @@
+//===--- CheckerRegistration.cpp - Registration for the Analyzer Checkers -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the registration function for the analyzer checkers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Frontend/CheckerRegistration.h"
+#include "../Checkers/ClangSACheckerProvider.h"
+#include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/CheckerProvider.h"
+#include "clang/Frontend/AnalyzerOptions.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallVector.h"
+
+using namespace clang;
+using namespace ento;
+
+CheckerManager *ento::registerCheckers(const AnalyzerOptions &opts,
+                                       Diagnostic &diags) {
+  llvm::OwningPtr<CheckerManager> checkerMgr(new CheckerManager());
+
+  llvm::SmallVector<CheckerOptInfo, 8> checkerOpts;
+  for (unsigned i = 0, e = opts.CheckersControlList.size(); i != e; ++i) {
+    const std::pair<std::string, bool> &opt = opts.CheckersControlList[i];
+    checkerOpts.push_back(CheckerOptInfo(opt.first.c_str(), opt.second));
+  }
+
+  llvm::OwningPtr<CheckerProvider> provider(createClangSACheckerProvider());
+  provider->registerCheckers(*checkerMgr,
+                             checkerOpts.data(), checkerOpts.size());
+
+  // FIXME: Load CheckerProviders from plugins.
+
+  for (unsigned i = 0, e = checkerOpts.size(); i != e; ++i) {
+    if (checkerOpts[i].isUnclaimed())
+      diags.Report(diag::warn_unkwown_analyzer_checker)
+          << checkerOpts[i].getName();
+  }
+
+  return checkerMgr.take();
+}
diff --git a/lib/Checker/FrontendActions.cpp b/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
index d9a54a021bc9..a59cc6888fdb 100644
--- a/lib/Checker/FrontendActions.cpp
+++ b/lib/StaticAnalyzer/Frontend/FrontendActions.cpp
@@ -7,10 +7,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Checker/FrontendActions.h"
+#include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "clang/Frontend/CompilerInstance.h"
-#include "clang/Checker/AnalysisConsumer.h"
+#include "AnalysisConsumer.h"
 using namespace clang;
+using namespace ento;
 
 ASTConsumer *AnalysisAction::CreateASTConsumer(CompilerInstance &CI,
                                                llvm::StringRef InFile) {
diff --git a/lib/StaticAnalyzer/Frontend/Makefile b/lib/StaticAnalyzer/Frontend/Makefile
new file mode 100644
index 000000000000..2698120d9098
--- /dev/null
+++ b/lib/StaticAnalyzer/Frontend/Makefile
@@ -0,0 +1,19 @@
+##===- clang/lib/StaticAnalyzer/Frontend/Makefile ----------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+# Starting point into the static analyzer land for the driver.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangStaticAnalyzerFrontend
+
+CPP.Flags += -I${PROJ_OBJ_DIR}/../Checkers
+
+include $(CLANG_LEVEL)/Makefile
diff --git a/lib/Checker/Makefile b/lib/StaticAnalyzer/Makefile
index 4ec6f65a0bb1..c166f063f925 100644
--- a/lib/Checker/Makefile
+++ b/lib/StaticAnalyzer/Makefile
@@ -1,4 +1,4 @@
-##===- clang/lib/Checker/Makefile --------------------------*- Makefile -*-===##
+##===- clang/lib/StaticAnalyzer/Makefile -------------------*- Makefile -*-===##
 # 
 #                     The LLVM Compiler Infrastructure
 #
@@ -12,7 +12,7 @@
 ##===----------------------------------------------------------------------===##
 
 CLANG_LEVEL := ../..
-LIBRARYNAME := clangChecker
+DIRS := Checkers Frontend
+PARALLEL_DIRS := Core
 
 include $(CLANG_LEVEL)/Makefile
-
diff --git a/lib/StaticAnalyzer/README.txt b/lib/StaticAnalyzer/README.txt
new file mode 100644
index 000000000000..1406eca8c85f
--- /dev/null
+++ b/lib/StaticAnalyzer/README.txt
@@ -0,0 +1,139 @@
+//===----------------------------------------------------------------------===//
+// Clang Static Analyzer
+//===----------------------------------------------------------------------===//
+
+= Library Structure =
+
+The analyzer library has two layers: a (low-level) static analysis
+engine (GRExprEngine.cpp and friends), and some static checkers
+(*Checker.cpp).  The latter are built on top of the former via the
+Checker and CheckerVisitor interfaces (Checker.h and
+CheckerVisitor.h).  The Checker interface is designed to be minimal
+and simple for checker writers, and attempts to isolate them from much
+of the gore of the internal analysis engine.
+
+= How It Works =
+
+The analyzer is inspired by several foundational research papers ([1],
+[2]).  (FIXME: kremenek to add more links)
+
+In a nutshell, the analyzer is basically a source code simulator that
+traces out possible paths of execution.  The state of the program
+(values of variables and expressions) is encapsulated by the state
+(GRState).  A location in the program is called a program point
+(ProgramPoint), and the combination of state and program point is a
+node in an exploded graph (ExplodedGraph).  The term "exploded" comes
+from exploding the control-flow edges in the control-flow graph (CFG).
+
+Conceptually the analyzer does a reachability analysis through the
+ExplodedGraph.  We start at a root node, which has the entry program
+point and initial state, and then simulate transitions by analyzing
+individual expressions.  The analysis of an expression can cause the
+state to change, resulting in a new node in the ExplodedGraph with an
+updated program point and an updated state.  A bug is found by hitting
+a node that satisfies some "bug condition" (basically a violation of a
+checking invariant).
+
+The analyzer traces out multiple paths by reasoning about branches and
+then bifurcating the state: on the true branch the conditions of the
+branch are assumed to be true and on the false branch the conditions
+of the branch are assumed to be false.  Such "assumptions" create
+constraints on the values of the program, and those constraints are
+recorded in the GRState object (and are manipulated by the
+ConstraintManager).  If assuming the conditions of a branch would
+cause the constraints to be unsatisfiable, the branch is considered
+infeasible and that path is not taken.  This is how we get
+path-sensitivity.  We reduce exponential blow-up by caching nodes.  If
+a new node with the same state and program point as an existing node
+would get generated, the path "caches out" and we simply reuse the
+existing node.  Thus the ExplodedGraph is not a DAG; it can contain
+cycles as paths loop back onto each other and cache out.
+
+GRState and ExplodedNodes are basically immutable once created.  Once
+one creates a GRState, you need to create a new one to get a new
+GRState.  This immutability is key since the ExplodedGraph represents
+the behavior of the analyzed program from the entry point.  To
+represent these efficiently, we use functional data structures (e.g.,
+ImmutableMaps) which share data between instances.
+
+Finally, individual Checkers work by also manipulating the analysis
+state.  The analyzer engine talks to them via a visitor interface.
+For example, the PreVisitCallExpr() method is called by GRExprEngine
+to tell the Checker that we are about to analyze a CallExpr, and the
+checker is asked to check for any preconditions that might not be
+satisfied.  The checker can do nothing, or it can generate a new
+GRState and ExplodedNode which contains updated checker state.  If it
+finds a bug, it can tell the BugReporter object about the bug,
+providing it an ExplodedNode which is the last node in the path that
+triggered the problem.
+
+= Notes about C++ =
+
+Since now constructors are seen before the variable that is constructed 
+in the CFG, we create a temporary object as the destination region that 
+is constructed into. See ExprEngine::VisitCXXConstructExpr().
+
+In ExprEngine::processCallExit(), we always bind the object region to the
+evaluated CXXConstructExpr. Then in VisitDeclStmt(), we compute the
+corresponding lazy compound value if the variable is not a reference, and
+bind the variable region to the lazy compound value. If the variable
+is a reference, just use the object region as the initilizer value.
+
+Before entering a C++ method (or ctor/dtor), the 'this' region is bound
+to the object region. In ctors, we synthesize 'this' region with  
+CXXRecordDecl*, which means we do not use type qualifiers. In methods, we
+synthesize 'this' region with CXXMethodDecl*, which has getThisType() 
+taking type qualifiers into account. It does not matter we use qualified
+'this' region in one method and unqualified 'this' region in another
+method, because we only need to ensure the 'this' region is consistent 
+when we synthesize it and create it directly from CXXThisExpr in a single
+method call.
+
+= Working on the Analyzer =
+
+If you are interested in bringing up support for C++ expressions, the
+best place to look is the visitation logic in GRExprEngine, which
+handles the simulation of individual expressions.  There are plenty of
+examples there of how other expressions are handled.
+
+If you are interested in writing checkers, look at the Checker and
+CheckerVisitor interfaces (Checker.h and CheckerVisitor.h).  Also look
+at the files named *Checker.cpp for examples on how you can implement
+these interfaces.
+
+= Debugging the Analyzer =
+
+There are some useful command-line options for debugging.  For example:
+
+$ clang -cc1 -help | grep analyze
+ -analyze-function <value>
+ -analyzer-display-progress
+ -analyzer-viz-egraph-graphviz
+ ...
+
+The first allows you to specify only analyzing a specific function.
+The second prints to the console what function is being analyzed.  The
+third generates a graphviz dot file of the ExplodedGraph.  This is
+extremely useful when debugging the analyzer and viewing the
+simulation results.
+
+Of course, viewing the CFG (Control-Flow Graph) is also useful:
+
+$ clang -cc1 -help | grep cfg
+ -cfg-add-implicit-dtors Add C++ implicit destructors to CFGs for all analyses
+ -cfg-add-initializers   Add C++ initializers to CFGs for all analyses
+ -cfg-dump               Display Control-Flow Graphs
+ -cfg-view               View Control-Flow Graphs using GraphViz
+ -unoptimized-cfg        Generate unoptimized CFGs for all analyses
+
+-cfg-dump dumps a textual representation of the CFG to the console,
+and -cfg-view creates a GraphViz representation.
+
+= References =
+
+[1] Precise interprocedural dataflow analysis via graph reachability,
+    T Reps, S Horwitz, and M Sagiv, POPL '95,
+    http://portal.acm.org/citation.cfm?id=199462
+
+[2] A memory model for static analysis of C programs, Z Xu, T
+    Kremenek, and J Zhang, http://lcs.ios.ac.cn/~xzx/memmodel.pdf